xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feat/mode_function

This commit is contained in:
Ganlin Zhao 2022-06-28 10:42:58 +08:00
parent 4d05053e66 a22ccb3912
commit e9fbe1805b
113 changed files with 4938 additions and 1272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Jenkinsfile2
View File

@ -338,6 +338,14 @@ pipeline {
changeRequest()
}
steps {
script {
def linux_node_ip = sh (
script: 'ip addr|grep 192|grep -v virbr|awk "{print \\\$2}"|sed "s/\\/.*//"',
returnStdout: true
).trim()
echo "${linux_node_ip}"
echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}"
}
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
timeout(time: 120, unit: 'MINUTES'){
pre_test()

1
include/client/taos.h
View File

@ -128,6 +128,7 @@ typedef struct setConfRet {
DLL_EXPORT void taos_cleanup(void);
DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...);
DLL_EXPORT setConfRet taos_set_config(const char *config);
DLL_EXPORT int taos_init(void);
DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);

2
include/common/tdatablock.h
View File

@ -236,7 +236,7 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t*
int8_t needCompress);
const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData);
void blockDebugShowData(const SArray* dataBlocks, const char* flag);
void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag);
// for debug
char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf);

1
include/common/tdataformat.h
View File

@ -78,7 +78,6 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag);
int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag);
int32_t tTagToValArray(const STag *pTag, SArray **ppArray);
void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove
void debugCheckTags(STag *pTag); // TODO: remove
// STRUCT =================
struct STColumn {

2
include/common/tmsg.h
View File

@ -1886,7 +1886,7 @@ typedef struct SVCreateStbReq {
int8_t rollup;
SSchemaWrapper schemaRow;
SSchemaWrapper schemaTag;
SRSmaParam pRSmaParam;
SRSmaParam rsmaParam;
} SVCreateStbReq;
int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq);

14
include/libs/nodes/plannodes.h
View File

@ -24,6 +24,8 @@ extern "C" {
#include "querynodes.h"
#include "tname.h"
#define SLOT_NAME_LEN TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN
typedef struct SLogicNode {
ENodeType type;
SNodeList* pTargets; // SColumnNode
@ -74,8 +76,8 @@ typedef struct SScanLogicNode {
int16_t tsColId;
double filesFactor;
SArray* pSmaIndexes;
SNodeList* pPartTags;
bool partSort;
SNodeList* pGroupTags;
bool groupSort;
} SScanLogicNode;
typedef struct SJoinLogicNode {
@ -100,6 +102,7 @@ typedef struct SProjectLogicNode {
typedef struct SIndefRowsFuncLogicNode {
SLogicNode node;
SNodeList* pFuncs;
bool isTailFunc;
} SIndefRowsFuncLogicNode;
typedef struct SInterpFuncLogicNode {
@ -138,6 +141,7 @@ typedef struct SMergeLogicNode {
SNodeList* pInputs;
int32_t numOfChannels;
int32_t srcGroupId;
bool groupSort;
} SMergeLogicNode;
typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType;
@ -184,6 +188,7 @@ typedef struct SFillLogicNode {
typedef struct SSortLogicNode {
SLogicNode node;
SNodeList* pSortKeys;
bool groupSort;
} SSortLogicNode;
typedef struct SPartitionLogicNode {
@ -230,6 +235,7 @@ typedef struct SSlotDescNode {
bool reserve;
bool output;
bool tag;
char name[SLOT_NAME_LEN];
} SSlotDescNode;
typedef struct SDataBlockDescNode {
@ -279,7 +285,8 @@ typedef struct STableScanPhysiNode {
double ratio;
int32_t dataRequired;
SNodeList* pDynamicScanFuncs;
SNodeList* pPartitionTags;
SNodeList* pGroupTags;
bool groupSort;
int64_t interval;
int64_t offset;
int64_t sliding;
@ -353,6 +360,7 @@ typedef struct SMergePhysiNode {
SNodeList* pTargets;
int32_t numOfChannels;
int32_t srcGroupId;
bool groupSort;
} SMergePhysiNode;
typedef struct SWinodwPhysiNode {

1
include/libs/nodes/querynodes.h
View File

@ -259,6 +259,7 @@ typedef struct SSelectStmt {
bool hasTailFunc;
bool hasInterpFunc;
bool hasLastRowFunc;
bool groupSort;
} SSelectStmt;
typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;

2
include/libs/transport/trpc.h
View File

@ -69,7 +69,7 @@ typedef struct SRpcMsg {
} SRpcMsg;
typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *rf);
typedef bool (*RpcRfp)(int32_t code);
typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType);
typedef struct SRpcInit {
char localFqdn[TSDB_FQDN_LEN];

3
include/os/osSystem.h
View File

@ -29,6 +29,9 @@ extern "C" {
#define tcgetattr TCGETATTR_FUNC_TAOS_FORBID
#endif
#define TAOS_CONSOLE_PROMPT_HEADER "taos> "
#define TAOS_CONSOLE_PROMPT_CONTINUE " -> "
typedef struct TdCmd *TdCmdPtr;
TdCmdPtr taosOpenCmd(const char* cmd);

5
source/client/src/clientEnv.c
View File

@ -84,9 +84,12 @@ void closeTransporter(STscObj *pTscObj) {
rpcClose(pTscObj->pAppInfo->pTransporter);
}
static bool clientRpcRfp(int32_t code) {
static bool clientRpcRfp(int32_t code, tmsg_t msgType) {
if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) {
return false;
}
return true;
} else {
return false;

13
source/client/src/clientMain.c
View File

@ -81,6 +81,19 @@ void taos_cleanup(void) {
taosCloseLog();
}
static setConfRet taos_set_config_imp(const char *config){
setConfRet ret = {SET_CONF_RET_SUCC, {0}};
// TODO: need re-implementation
return ret;
}
setConfRet taos_set_config(const char *config){
// TODO pthread_mutex_lock(&setConfMutex);
setConfRet ret = taos_set_config_imp(config);
// pthread_mutex_unlock(&setConfMutex);
return ret;
}
TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db);
if (user == NULL) {

4
source/common/src/tdatablock.c
View File

@ -1605,7 +1605,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
return buf;
}
void blockDebugShowData(const SArray* dataBlocks, const char* flag) {
void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) {
char pBuf[128] = {0};
int32_t sz = taosArrayGetSize(dataBlocks);
for (int32_t i = 0; i < sz; i++) {
@ -1613,7 +1613,7 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
int32_t rows = pDataBlock->info.rows;
printf("%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId);
printf("%s |block type %d |child id %d|group id %zX\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId);
for (int32_t j = 0; j < rows; j++) {
printf("%s |", flag);
for (int32_t k = 0; k < numOfCols; k++) {

16
source/common/src/tdataformat.c
View File

@ -862,21 +862,6 @@ void debugPrintSTag(STag *pTag, const char *tag, int32_t ln) {
printf("\n");
}
void debugCheckTags(STag *pTag) {
switch (pTag->flags) {
case 0x0:
case 0x20:
case 0x40:
case 0x60:
break;
default:
ASSERT(0);
}
ASSERT(pTag->nTag <= 128 && pTag->nTag >= 0);
ASSERT(pTag->ver <= 512 && pTag->ver >= 0); // temp condition for pTag->ver
}
static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
int32_t n = 0;
@ -999,7 +984,6 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) {
debugPrintSTag(*ppTag, __func__, __LINE__);
#endif
debugCheckTags(*ppTag); // TODO: remove this line after debug
return code;
_err:

4
source/common/src/tmsg.c
View File

@ -4763,7 +4763,7 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) {
if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaRow) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1;
if (pReq->rollup) {
if (tEncodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1;
if (tEncodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1;
}
tEndEncode(pCoder);
@ -4779,7 +4779,7 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) {
if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaRow) < 0) return -1;
if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1;
if (pReq->rollup) {
if (tDecodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1;
if (tDecodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1;
}
tEndDecode(pCoder);

5
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -248,9 +248,12 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) {
}
}
static bool rpcRfp(int32_t code) {
static bool rpcRfp(int32_t code, tmsg_t msgType) {
if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) {
return false;
}
return true;
} else {
return false;

4
source/dnode/mnode/impl/inc/mndTrans.h
View File

@ -39,8 +39,10 @@ typedef struct {
int32_t id;
int32_t errCode;
int32_t acceptableCode;
ETrnStage stage;
int32_t retryCode;
ETrnAct actionType;
ETrnStage stage;
int8_t reserved;
int8_t rawWritten;
int8_t msgSent;
int8_t msgReceived;

2
source/dnode/mnode/impl/src/mndShow.c
View File

@ -234,7 +234,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
if (retrieveReq.user[0] != 0) {
memcpy(pReq->info.conn.user, retrieveReq.user, TSDB_USER_LEN);
} else {
memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, TSDB_USER_LEN);
memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1);
}
if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) {
return -1;

20
source/dnode/mnode/impl/src/mndStb.c
View File

@ -427,17 +427,17 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
req.schemaTag.pSchema = pStb->pTags;
if (req.rollup) {
req.pRSmaParam.maxdelay[0] = pStb->maxdelay[0];
req.pRSmaParam.maxdelay[1] = pStb->maxdelay[1];
req.rsmaParam.maxdelay[0] = pStb->maxdelay[0];
req.rsmaParam.maxdelay[1] = pStb->maxdelay[1];
if (pStb->ast1Len > 0) {
if (mndConvertRsmaTask(&req.pRSmaParam.qmsg[0], &req.pRSmaParam.qmsgLen[0], pStb->pAst1, pStb->uid,
STREAM_TRIGGER_WINDOW_CLOSE, req.pRSmaParam.watermark[0]) < 0) {
if (mndConvertRsmaTask(&req.rsmaParam.qmsg[0], &req.rsmaParam.qmsgLen[0], pStb->pAst1, pStb->uid,
STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[0]) < 0) {
goto _err;
}
}
if (pStb->ast2Len > 0) {
if (mndConvertRsmaTask(&req.pRSmaParam.qmsg[1], &req.pRSmaParam.qmsgLen[1], pStb->pAst2, pStb->uid,
STREAM_TRIGGER_WINDOW_CLOSE, req.pRSmaParam.watermark[1]) < 0) {
if (mndConvertRsmaTask(&req.rsmaParam.qmsg[1], &req.rsmaParam.qmsgLen[1], pStb->pAst2, pStb->uid,
STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[1]) < 0) {
goto _err;
}
}
@ -470,12 +470,12 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
tEncoderClear(&encoder);
*pContLen = contLen;
taosMemoryFreeClear(req.pRSmaParam.qmsg[0]);
taosMemoryFreeClear(req.pRSmaParam.qmsg[1]);
taosMemoryFreeClear(req.rsmaParam.qmsg[0]);
taosMemoryFreeClear(req.rsmaParam.qmsg[1]);
return pHead;
_err:
taosMemoryFreeClear(req.pRSmaParam.qmsg[0]);
taosMemoryFreeClear(req.pRSmaParam.qmsg[1]);
taosMemoryFreeClear(req.rsmaParam.qmsg[0]);
taosMemoryFreeClear(req.rsmaParam.qmsg[1]);
return NULL;
}

90
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -15,15 +15,15 @@
#define _DEFAULT_SOURCE
#include "mndTrans.h"
#include "mndPrivilege.h"
#include "mndConsumer.h"
#include "mndDb.h"
#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndSync.h"
#include "mndUser.h"
#define TRANS_VER_NUMBER 1
#define TRANS_ARRAY_SIZE 8
#define TRANS_VER_NUMBER 1
#define TRANS_ARRAY_SIZE 8
#define TRANS_RESERVE_SIZE 64
static SSdbRaw *mndTransActionEncode(STrans *pTrans);
@ -55,7 +55,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans);
static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); }
static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans);
static int32_t mndProcessTransReq(SRpcMsg *pReq);
static int32_t mndProcessTransTimer(SRpcMsg *pReq);
static int32_t mndProcessTtl(SRpcMsg *pReq);
static int32_t mndProcessKillTransReq(SRpcMsg *pReq);
@ -73,7 +73,7 @@ int32_t mndInitTrans(SMnode *pMnode) {
.deleteFp = (SdbDeleteFp)mndTransActionDelete,
};
mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq);
mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransTimer);
mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndRetrieveTrans);
@ -139,8 +139,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER)
SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER)
SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER)
SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER)
if (pAction->actionType == TRANS_ACTION_RAW) {
int32_t len = sdbGetRawTotalSize(pAction->pRaw);
SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER)
@ -163,8 +165,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER)
SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER)
SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER)
SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER)
if (pAction->actionType == TRANS_ACTION_RAW) {
int32_t len = sdbGetRawTotalSize(pAction->pRaw);
SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER)
@ -187,8 +191,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER)
SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER)
SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER)
SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER)
if (pAction->actionType == TRANS_ACTION_RAW) {
int32_t len = sdbGetRawTotalSize(pAction->pRaw);
SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER)
@ -291,10 +297,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER)
SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER)
SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER)
SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER)
SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER)
action.actionType = actionType;
SDB_GET_INT8(pRaw, dataPos, &stage, _OVER)
action.stage = stage;
SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER)
if (action.actionType == TRANS_ACTION_RAW) {
SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER)
SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER)
@ -324,10 +332,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER)
SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER)
SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER)
SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER)
SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER)
action.actionType = actionType;
SDB_GET_INT8(pRaw, dataPos, &stage, _OVER)
action.stage = stage;
SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER)
if (action.actionType == TRANS_ACTION_RAW) {
SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER)
SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER)
@ -357,10 +367,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER)
SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER)
SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER)
SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER)
SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER)
action.actionType = actionType;
SDB_GET_INT8(pRaw, dataPos, &stage, _OVER)
action.stage = stage;
SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER)
if (action.actionType) {
SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER)
SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER)
@ -463,15 +475,25 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) {
if (fp) {
(*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
}
pTrans->startFunc = 0;
}
return 0;
}
static void mndTransDropData(STrans *pTrans) {
mndTransDropActions(pTrans->redoActions);
mndTransDropActions(pTrans->undoActions);
mndTransDropActions(pTrans->commitActions);
if (pTrans->redoActions != NULL) {
mndTransDropActions(pTrans->redoActions);
pTrans->redoActions = NULL;
}
if (pTrans->undoActions != NULL) {
mndTransDropActions(pTrans->undoActions);
pTrans->undoActions = NULL;
}
if (pTrans->commitActions != NULL) {
mndTransDropActions(pTrans->commitActions);
pTrans->commitActions = NULL;
}
if (pTrans->rpcRsp != NULL) {
taosMemoryFree(pTrans->rpcRsp);
pTrans->rpcRsp = NULL;
@ -492,6 +514,7 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) {
if (fp) {
(*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
}
pTrans->stopFunc = 0;
}
mndTransDropData(pTrans);
@ -805,7 +828,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
sendRsp = true;
}
} else {
if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 3) {
if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 6) {
if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR;
sendRsp = true;
}
@ -875,8 +898,8 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) {
pAction->errCode = pRsp->code;
}
mDebug("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x", transId, mndTransStr(pAction->stage), action,
pRsp->code, pAction->acceptableCode);
mDebug("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId,
mndTransStr(pAction->stage), action, pRsp->code, pAction->acceptableCode, pAction->retryCode);
mndTransExecute(pMnode, pTrans);
_OVER:
@ -884,6 +907,21 @@ _OVER:
return 0;
}
static void mndTransResetAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) {
pAction->rawWritten = 0;
pAction->msgSent = 0;
pAction->msgReceived = 0;
if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR ||
pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) {
pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps;
mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage),
pAction->id, pAction->epSet.inUse);
} else {
mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), pAction->id);
}
pAction->errCode = 0;
}
static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) {
int32_t numOfActions = taosArrayGetSize(pArray);
@ -894,18 +932,7 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray)
continue;
if (pAction->rawWritten && (pAction->errCode == 0 || pAction->errCode == pAction->acceptableCode)) continue;
pAction->rawWritten = 0;
pAction->msgSent = 0;
pAction->msgReceived = 0;
if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR ||
pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) {
pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps;
mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage),
action, pAction->epSet.inUse);
} else {
mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action);
}
pAction->errCode = 0;
mndTransResetAction(pMnode, pTrans, pAction);
}
}
@ -1112,9 +1139,9 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
if (pAction->msgReceived) {
if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) {
code = pAction->errCode;
pAction->msgSent = 0;
pAction->msgReceived = 0;
mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action);
mndTransResetAction(pMnode, pTrans, pAction);
} else {
mDebug("trans:%d, %s:%d execute successfully", pTrans->id, mndTransStr(pAction->stage), action);
}
} else {
code = TSDB_CODE_ACTION_IN_PROGRESS;
@ -1123,6 +1150,8 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
if (pAction->rawWritten) {
if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) {
code = pAction->errCode;
} else {
mDebug("trans:%d, %s:%d write successfully", pTrans->id, mndTransStr(pAction->stage), action);
}
}
}
@ -1156,9 +1185,16 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
} else if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
mDebug("trans:%d, %s:%d is in progress and wait it finish", pTrans->id, mndTransStr(pAction->stage), pAction->id);
break;
} else if (code == pAction->retryCode) {
mDebug("trans:%d, %s:%d receive code:0x%x and retry", pTrans->id, mndTransStr(pAction->stage), pAction->id, code);
taosMsleep(300);
action--;
continue;
} else {
terrno = code;
pTrans->code = code;
mDebug("trans:%d, %s:%d receive code:0x%x and wait another schedule, failedTimes:%d", pTrans->id,
mndTransStr(pAction->stage), pAction->id, code, pTrans->failedTimes);
break;
}
}
@ -1343,7 +1379,7 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
mndTransSendRpcRsp(pMnode, pTrans);
}
static int32_t mndProcessTransReq(SRpcMsg *pReq) {
static int32_t mndProcessTransTimer(SRpcMsg *pReq) {
mndTransPullup(pReq->info.node);
return 0;
}

115
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -15,10 +15,10 @@
#define _DEFAULT_SOURCE
#include "mndVgroup.h"
#include "mndPrivilege.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndTrans.h"
#include "mndUser.h"
@ -896,6 +896,8 @@ int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD
action.pCont = pHead;
action.contLen = contLen;
action.msgType = TDMT_VND_ALTER_CONFIRM;
// incorrect redirect result will cause this erro
action.retryCode = TSDB_CODE_VND_INVALID_VGROUP_ID;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pHead);
@ -942,6 +944,8 @@ static int32_t mndAddSetVnodeStandByAction(SMnode *pMnode, STrans *pTrans, SDbOb
action.contLen = contLen;
action.msgType = TDMT_SYNC_SET_VNODE_STANDBY;
action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED;
// Keep retrying until the target vnode is not the leader
action.retryCode = TSDB_CODE_SYN_IS_LEADER;
if (isRedo) {
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
@ -1003,7 +1007,7 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
mInfo("vgId:%d, will add 1 vnodes", pVgroup->vgId);
if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1;
if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[1], true) != 0) return -1;
if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[newVg.replica - 1], true) != 0) return -1;
if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg, TDMT_VND_ALTER_REPLICA) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
@ -1017,10 +1021,19 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) return -1;
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL;
{
SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) return -1;
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL;
}
{
SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) return -1;
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL;
}
mInfo("vgId:%d, vgroup info after move, replica:%d", newVg.vgId, newVg.replica);
for (int32_t i = 0; i < newVg.replica; ++i) {
@ -1168,10 +1181,19 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld3->id) != 0) goto _OVER;
}
pRaw = mndVgroupActionEncode(&newVg);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL;
{
pRaw = mndVgroupActionEncode(&newVg);
if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER;
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL;
}
{
pRaw = mndVgroupActionEncode(&newVg);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL;
}
mInfo("vgId:%d, vgroup info after redistribute, replica:%d", newVg.vgId, newVg.replica);
for (int32_t i = 0; i < newVg.replica; ++i) {
@ -1229,7 +1251,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) {
}
if (req.dnodeId1 == pVgroup->vnodeGid[0].dnodeId) {
terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
// terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
code = 0;
goto _OVER;
}
@ -1351,7 +1374,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) {
}
if (pNew1 == NULL && pOld1 == NULL && pNew2 == NULL && pOld2 == NULL && pNew3 == NULL && pOld3 == NULL) {
terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
// terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
code = 0;
goto _OVER;
}
@ -1424,13 +1448,25 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, S
} else {
}
SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
if (pVgRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw);
return -1;
{
SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
if (pVgRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw);
return -1;
}
sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
}
{
SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
if (pVgRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw);
return -1;
}
sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
}
sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
}
return 0;
@ -1538,12 +1574,23 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD
if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pDst->id) != 0) return -1;
if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pSrc->id) != 0) return -1;
SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) {
sdbFreeRaw(pRaw);
return -1;
{
SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) {
sdbFreeRaw(pRaw);
return -1;
}
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
}
{
SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) {
sdbFreeRaw(pRaw);
return -1;
}
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
}
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mInfo("vgId:%d, vgroup info after balance, replica:%d", newVg.vgId, newVg.replica);
for (int32_t i = 0; i < newVg.replica; ++i) {
@ -1552,7 +1599,8 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD
return 0;
}
static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst) {
static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst,
SHashObj *pBalancedVgroups) {
void *pIter = NULL;
int32_t code = -1;
SSdb *pSdb = pMnode->pSdb;
@ -1561,6 +1609,10 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (taosHashGet(pBalancedVgroups, &pVgroup->vgId, sizeof(int32_t)) != NULL) {
sdbRelease(pSdb, pVgroup);
continue;
}
bool existInSrc = false;
bool existInDst = false;
@ -1577,6 +1629,9 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno
SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName);
code = mndSetBalanceVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, pSrc, pDst);
if (code == 0) {
code = taosHashPut(pBalancedVgroups, &pVgroup->vgId, sizeof(int32_t), &pVgroup->vgId, sizeof(int32_t));
}
mndReleaseDb(pMnode, pDb);
sdbRelease(pSdb, pVgroup);
sdbCancelFetch(pSdb, pIter);
@ -1590,6 +1645,10 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) {
int32_t code = -1;
int32_t numOfVgroups = 0;
STrans *pTrans = NULL;
SHashObj *pBalancedVgroups = NULL;
pBalancedVgroups = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
if (pBalancedVgroups == NULL) goto _OVER;
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq);
if (pTrans == NULL) goto _OVER;
@ -1613,18 +1672,18 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) {
pDst->id, dstScore);
if (srcScore > dstScore - 0.000001) {
code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst);
code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst, pBalancedVgroups);
if (code == 0) {
pSrc->numOfVnodes--;
pDst->numOfVnodes++;
numOfVgroups++;
continue;
} else {
mError("trans:%d, failed to balance vgroup from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id);
return -1;
mDebug("trans:%d, no vgroup need to balance from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id);
break;
}
} else {
mDebug("trans:%d, no vgroup need to balance vgroup any more", pTrans->id);
mDebug("trans:%d, no vgroup need to balance any more", pTrans->id);
break;
}
}

5
source/dnode/vnode/inc/vnode.h
View File

@ -199,15 +199,20 @@ typedef struct {
uint64_t groupId;
} STableKeyInfo;
#define TABLE_ROLLUP_ON ((int8_t)0x1)
#define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0)
#define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON)
struct SMetaEntry {
int64_t version;
int8_t type;
int8_t flags; // TODO: need refactor?
tb_uid_t uid;
char *name;
union {
struct {
SSchemaWrapper schemaRow;
SSchemaWrapper schemaTag;
SRSmaParam rsmaParam;
} stbEntry;
struct {
int64_t ctime;

11
source/dnode/vnode/src/inc/sma.h
View File

@ -176,15 +176,18 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) {
static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
void *tdFreeRSmaInfo(SRSmaInfo *pInfo);
void *tdFreeRSmaInfo(SRSmaInfo *pInfo);
int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
int32_t tdProcessRSmaRestoreImpl(SSma *pSma);
int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg);
int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg);
int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
// smaFileUtil ================
#define TD_FILE_HEAD_SIZE 512
typedef struct STFInfo STFInfo;
typedef struct STFile STFile;
@ -220,12 +223,14 @@ int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte);
int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence);
int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte);
int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset);
int64_t tdGetTFileSize(STFile *pTFile, int64_t *size);
int32_t tdRemoveTFile(STFile *pTFile);
int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo);
int32_t tdUpdateTFileHeader(STFile *pTFile);
void tdUpdateTFileMagic(STFile *pTFile, void *pCksm);
void tdCloseTFile(STFile *pTFile);
void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName);
void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName);
#ifdef __cplusplus
}

18
source/dnode/vnode/src/meta/metaEntry.c
View File

@ -24,22 +24,25 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
if (tEncodeCStr(pCoder, pME->name) < 0) return -1;
if (pME->type == TSDB_SUPER_TABLE) {
if (tEncodeI8(pCoder, pME->flags) < 0) return -1; // TODO: need refactor?
if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
if (TABLE_IS_ROLLUP(pME->flags)) {
if (tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1;
}
} else if (pME->type == TSDB_CHILD_TABLE) {
if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1;
if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1;
if (tEncodeI32(pCoder, pME->ctbEntry.commentLen) < 0) return -1;
if (tEncodeI32v(pCoder, pME->ctbEntry.commentLen) < 0) return -1;
if (pME->ctbEntry.commentLen > 0){
if (tEncodeCStr(pCoder, pME->ctbEntry.comment) < 0) return -1;
}
if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1;
debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug
if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1;
} else if (pME->type == TSDB_NORMAL_TABLE) {
if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1;
if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1;
if (tEncodeI32(pCoder, pME->ntbEntry.commentLen) < 0) return -1;
if (tEncodeI32v(pCoder, pME->ntbEntry.commentLen) < 0) return -1;
if (pME->ntbEntry.commentLen > 0){
if (tEncodeCStr(pCoder, pME->ntbEntry.comment) < 0) return -1;
}
@ -64,23 +67,26 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
if (tDecodeCStr(pCoder, &pME->name) < 0) return -1;
if (pME->type == TSDB_SUPER_TABLE) {
if (tDecodeI8(pCoder, &pME->flags) < 0) return -1; // TODO: need refactor?
if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow) < 0) return -1;
if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
if (TABLE_IS_ROLLUP(pME->flags)) {
if (tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1;
}
} else if (pME->type == TSDB_CHILD_TABLE) {
if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ctbEntry.commentLen) < 0) return -1;
if (tDecodeI32v(pCoder, &pME->ctbEntry.commentLen) < 0) return -1;
if (pME->ctbEntry.commentLen > 0){
if (tDecodeCStr(pCoder, &pME->ctbEntry.comment) < 0)
return -1;
}
if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1;
if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1; // (TODO)
debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug
} else if (pME->type == TSDB_NORMAL_TABLE) {
if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ntbEntry.commentLen) < 0) return -1;
if (tDecodeI32v(pCoder, &pME->ntbEntry.commentLen) < 0) return -1;
if (pME->ntbEntry.commentLen > 0){
if (tDecodeCStr(pCoder, &pME->ntbEntry.comment) < 0) return -1;
}

2
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -342,6 +342,7 @@ SMStbCursor *metaOpenStbCursor(SMeta *pMeta, tb_uid_t suid) {
pStbCur = (SMStbCursor *)taosMemoryCalloc(1, sizeof(*pStbCur));
if (pStbCur == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
@ -351,6 +352,7 @@ SMStbCursor *metaOpenStbCursor(SMeta *pMeta, tb_uid_t suid) {
ret = tdbTbcOpen(pMeta->pSuidIdx, &pStbCur->pCur, NULL);
if (ret < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
metaULock(pMeta);
taosMemoryFree(pStbCur);
return NULL;

4
source/dnode/vnode/src/meta/metaTable.c
View File

@ -139,6 +139,10 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
me.name = pReq->name;
me.stbEntry.schemaRow = pReq->schemaRow;
me.stbEntry.schemaTag = pReq->schemaTag;
if (pReq->rollup) {
TABLE_SET_ROLLUP(me.flags);
me.stbEntry.rsmaParam = pReq->rsmaParam;
}
if (metaHandleEntry(pMeta, &me) < 0) goto _err;

19
source/dnode/vnode/src/sma/smaEnv.c
View File

@ -156,6 +156,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
static void tdDestroyTSmaStat(STSmaStat *pStat) {
if (pStat) {
smaDebug("destroy tsma stat");
tDestroyTSma(pStat->pTSma);
taosMemoryFreeClear(pStat->pTSma);
taosMemoryFreeClear(pStat->pTSchema);
@ -170,15 +171,12 @@ static void *tdFreeTSmaStat(STSmaStat *pStat) {
static void tdDestroyRSmaStat(SRSmaStat *pStat) {
if (pStat) {
smaDebug("vgId:%d, %s:%d free rsma stat", SMA_VID(pStat->pSma), __func__, __LINE__);
smaDebug("vgId:%d destroy rsma stat", SMA_VID(pStat->pSma));
// step 1: set persistence task cancelled
atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED);
// step 2: clean timer
// step 2: stop the persistence timer
taosTmrStopA(&RSMA_TMR_ID(pStat));
if (RSMA_TMR_HANDLE(pStat)) {
taosTmrCleanUp(RSMA_TMR_HANDLE(pStat));
}
// step 3: wait the persistence thread to finish
int32_t nLoops = 0;
@ -194,7 +192,6 @@ static void tdDestroyRSmaStat(SRSmaStat *pStat) {
sched_yield();
nLoops = 0;
}
taosMsleep(1000); // TODO: remove this line when release
}
}
@ -219,7 +216,11 @@ static void tdDestroyRSmaStat(SRSmaStat *pStat) {
sched_yield();
nLoops = 0;
}
taosMsleep(1000); // TODO: remove this line when release
}
// step 6: cleanup the timer handle
if (RSMA_TMR_HANDLE(pStat)) {
taosTmrCleanUp(RSMA_TMR_HANDLE(pStat));
}
}
}
@ -245,16 +246,12 @@ void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) {
int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
if (pSmaStat) {
if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
smaDebug("%s:%d destroy tsma stat", __func__, __LINE__);
tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat));
} else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
smaDebug("%s:%d destroy rsma stat", __func__, __LINE__);
tdDestroyRSmaStat(SMA_RSMA_STAT(pSmaStat));
} else {
ASSERT(0);
}
} else {
smaDebug("%s:%d no need to destroy rsma stat", __func__, __LINE__);
}
return TSDB_CODE_SUCCESS;
}

30
source/dnode/vnode/src/sma/smaOpen.c
View File

@ -18,6 +18,7 @@
static int32_t smaEvalDays(SRetention *r, int8_t precision);
static int32_t smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type);
static int32_t rsmaRestore(SSma *pSma);
#define SMA_SET_KEEP_CFG(l) \
do { \
@ -100,6 +101,9 @@ int32_t smaOpen(SVnode *pVnode) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pVnode->pSma = pSma;
pSma->pVnode = pVnode;
taosThreadMutexInit(&pSma->mutex, NULL);
pSma->locked = false;
@ -117,17 +121,22 @@ int32_t smaOpen(SVnode *pVnode) {
ASSERT(0);
}
}
// restore the rsma
#if 0
if (rsmaRestore(pSma) < 0) {
goto _err;
}
#endif
}
pVnode->pSma = pSma;
return 0;
_err:
taosMemoryFreeClear(pSma);
return -1;
}
int32_t smaCloseEnv(SSma *pSma) {
if(pSma) {
if (pSma) {
SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma));
SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma));
}
@ -153,13 +162,12 @@ int32_t smaClose(SSma *pSma) {
/**
* @brief rsma env restore
*
* @param pSma
* @return int32_t
*
* @param pSma
* @return int32_t
*/
int32_t smaRestore(SSma *pSma) {
if (!pSma) return 0;
// iterate all stables to restore the rsma env
return TSDB_CODE_SUCCESS;
static int32_t rsmaRestore(SSma *pSma) {
ASSERT(VND_IS_RSMA(pSma->pVnode));
return tdProcessRSmaRestoreImpl(pSma);
}

407
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -15,10 +15,14 @@
#include "sma.h"
#define RSMA_QTASK_PERSIST_MS 7200000
#define RSMA_QTASKINFO_PERSIST_MS 7200000
#define RSMA_QTASKINFO_BUFSIZE 32768
typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T;
static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"};
typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
typedef struct SRSmaQTaskFIter SRSmaQTaskFIter;
static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *handle,
@ -27,6 +31,13 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
tb_uid_t suid, int8_t level);
static void tdRSmaFetchTrigger(void *param, void *tmrId);
static void tdRSmaPersistTrigger(void *param, void *tmrId);
static void *tdRSmaPersistExec(void *param);
static void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName);
static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskFIter *pIter, STFile *pTFile);
static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskFIter *pIter, bool *isFinish);
static int32_t tdRSmaQTaskInfoIterNext(SRSmaQTaskFIter *pIter, SRSmaQTaskInfoItem *pItem, bool *isEnd);
static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem);
struct SRSmaInfoItem {
SRSmaInfo *pRsmaInfo;
@ -45,14 +56,38 @@ struct SRSmaInfo {
SRSmaInfoItem items[TSDB_RETENTION_L2];
};
struct SRSmaQTaskInfoItem {
int32_t len;
int8_t type;
int64_t suid;
void *qTaskInfo;
};
struct SRSmaQTaskFIter {
STFile *pTFile;
int64_t offset;
int64_t fsize;
int32_t nBytes;
int32_t nAlloc;
char *buf;
// ------------
int32_t nBufPos;
};
static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) {
return lenWithHead - sizeof(int32_t) - sizeof(int8_t) - sizeof(int64_t);
}
static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskFIter *pIter) { taosMemoryFreeClear(pIter->buf); }
static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) {
// Note: free/kill may in RC
qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle);
if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) {
smaDebug("vgId:%d, %s:%d free qTaskInfo_t %p of level %d", vgId, __func__, __LINE__, otaskHandle, level);
smaDebug("vgId:%d, free qTaskInfo_t %p of level %d", vgId, otaskHandle, level);
qDestroyTask(otaskHandle);
} else {
smaDebug("vgId:%d, %s:%d not free qTaskInfo_t %p of level %d", vgId, __func__, __LINE__, otaskHandle, level);
smaDebug("vgId:%d, not free qTaskInfo_t %p of level %d", vgId, otaskHandle, level);
}
}
@ -89,7 +124,7 @@ static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) {
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) {
static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) {
SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
SRSmaInfo *pRSmaInfo = NULL;
@ -230,26 +265,21 @@ _err:
}
/**
* @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam.
* @brief for rsam create or restore
*
* @param pTsdb
* @param pMeta
* @param pReq
* @param pSma
* @param param
* @param suid
* @param tbName
* @return int32_t
*/
int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
SSma *pSma = pVnode->pSma;
if (!pReq->rollup) {
smaTrace("vgId:%d, return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
return TSDB_CODE_SUCCESS;
}
SMeta *pMeta = pVnode->pMeta;
SMsgCb *pMsgCb = &pVnode->msgCb;
SRSmaParam *param = &pReq->pRSmaParam;
int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName) {
SVnode *pVnode = pSma->pVnode;
SMeta *pMeta = pVnode->pMeta;
SMsgCb *pMsgCb = &pVnode->msgCb;
if ((param->qmsgLen[0] == 0) && (param->qmsgLen[1] == 0)) {
smaWarn("vgId:%d, no qmsg1/qmsg2 for rollup stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
smaDebug("vgId:%d, no qmsg1/qmsg2 for rollup table %s %" PRIi64, SMA_VID(pSma), tbName, suid);
return TSDB_CODE_SUCCESS;
}
@ -262,10 +292,10 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
SRSmaInfo *pRSmaInfo = NULL;
pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t));
pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
if (pRSmaInfo) {
ASSERT(0); // TODO: free original pRSmaInfo is exists abnormally
smaWarn("vgId:%d, rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
smaDebug("vgId:%d, rsma info already exists for table %s, %" PRIi64, SMA_VID(pSma), tbName, suid);
return TSDB_CODE_SUCCESS;
}
@ -289,14 +319,14 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
.vnode = pVnode,
};
STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), pReq->suid, -1);
STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, -1);
if (!pTSchema) {
terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
goto _err;
}
pRSmaInfo->pTSchema = pTSchema;
pRSmaInfo->pSma = pSma;
pRSmaInfo->suid = pReq->suid;
pRSmaInfo->suid = suid;
if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) {
goto _err;
@ -306,16 +336,16 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
goto _err;
}
if (taosHashPut(RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) {
if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) {
goto _err;
} else {
smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid);
smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), suid);
}
// start the persist timer
if (TASK_TRIGGER_STAT_INIT ==
atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_INIT, TASK_TRIGGER_STAT_ACTIVE)) {
taosTmrStart(tdRSmaPersistTrigger, RSMA_QTASK_PERSIST_MS, pStat, RSMA_TMR_HANDLE(pStat));
taosTmrStart(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pStat, RSMA_TMR_HANDLE(pStat));
}
return TSDB_CODE_SUCCESS;
@ -325,6 +355,24 @@ _err:
return TSDB_CODE_FAILED;
}
/**
* @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam.
*
* @param pTsdb
* @param pMeta
* @param pReq
* @return int32_t
*/
int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
SSma *pSma = pVnode->pSma;
if (!pReq->rollup) {
smaTrace("vgId:%d, return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
return TSDB_CODE_SUCCESS;
}
return tdProcessRSmaCreateImpl(pSma, &pReq->rsmaParam, pReq->suid, pReq->name);
}
/**
* @brief store suid/[uids], prefer to use array and then hash
*
@ -487,7 +535,7 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
#if 1
char flag[10] = {0};
snprintf(flag, 10, "level %" PRIi8, pItem->level);
blockDebugShowData(pResult, flag);
blockDebugShowDataBlocks(pResult, flag);
#endif
STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2);
SSubmitReq *pReq = NULL;
@ -502,8 +550,10 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
}
taosMemoryFreeClear(pReq);
} else if (terrno == 0) {
smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level);
} else {
smaDebug("vgId:%d, no rsma %" PRIi8 " data generated since %s", SMA_VID(pSma), pItem->level, tstrerror(terrno));
smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, tstrerror(terrno));
}
tdDestroySDataBlockArray(pResult);
@ -526,16 +576,16 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
if (rsmaTriggerStat == TASK_TRIGGER_STAT_CANCELLED || rsmaTriggerStat == TASK_TRIGGER_STAT_FINISHED) {
smaDebug("vgId:%d, %s:%d level %" PRIi8 " not fetch since stat is cancelled for table suid:%" PRIi64, SMA_VID(pSma),
__func__, __LINE__, pItem->level, pItem->pRsmaInfo->suid);
smaDebug("vgId:%d, level %" PRIi8 " not fetch since stat is cancelled for table suid:%" PRIi64, SMA_VID(pSma),
pItem->level, pItem->pRsmaInfo->suid);
return;
}
int8_t fetchTriggerStat =
atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
if (fetchTriggerStat == TASK_TRIGGER_STAT_ACTIVE) {
smaDebug("vgId:%d, %s:%d level %" PRIi8 " stat is active for table suid:%" PRIi64, SMA_VID(pSma), __func__,
__LINE__, pItem->level, pItem->pRsmaInfo->suid);
smaDebug("vgId:%d, level %" PRIi8 " stat is active for table suid:%" PRIi64, SMA_VID(pSma), pItem->level,
pItem->pRsmaInfo->suid);
tdRefSmaStat(pSma, (SSmaStat *)pStat);
@ -546,13 +596,13 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
} else {
smaDebug("vgId:%d, %s:%d level %" PRIi8 " stat is inactive for table suid:%" PRIi64, SMA_VID(pSma), __func__,
__LINE__, pItem->level, pItem->pRsmaInfo->suid);
smaDebug("vgId:%d, level %" PRIi8 " stat is inactive for table suid:%" PRIi64, SMA_VID(pSma), pItem->level,
pItem->pRsmaInfo->suid);
}
}
static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem,
tb_uid_t suid, int8_t level) {
static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, tb_uid_t suid,
int8_t level) {
if (!pItem || !pItem->taskInfo) {
smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
return TSDB_CODE_SUCCESS;
@ -568,7 +618,7 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3
tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SUBMIT_BLOCK);
atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
smaDebug("vgId:%d, %s:%d process rsma insert", SMA_VID(pSma), __func__, __LINE__);
smaDebug("vgId:%d, process rsma insert", SMA_VID(pSma));
SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat);
@ -645,7 +695,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
return TSDB_CODE_SUCCESS;
}
void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName) {
static void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName) {
tdGetVndFileName(vid, "rsma", tdQTaskInfoFname[ftype], outputName);
}
@ -661,18 +711,6 @@ static void *tdRSmaPersistExec(void *param) {
goto _end;
}
#if 0
SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t));
if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) {
ASSERT(0);
} else {
for (int32_t i = 0; i < taosArrayGetSize(suidList); ++i) {
tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
smaDebug("suid [%d] is %" PRIi64, i, suid);
}
}
#endif
void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL);
if (!infoHash) {
goto _end;
@ -700,7 +738,7 @@ static void *tdRSmaPersistExec(void *param) {
}
char *pOutput = NULL;
int32_t len = 0;
int8_t type = 0;
int8_t type = (int8_t)(i + 1);
if (qSerializeTaskStatus(taskInfo, &pOutput, &len) < 0) {
smaError("vgId:%d, table %" PRIi64 " level %d serialize rsma task failed since %s", vid, pRSmaInfo->suid, i + 1,
terrstr(terrno));
@ -736,8 +774,9 @@ static void *tdRSmaPersistExec(void *param) {
isFileCreated = true;
}
len += (sizeof(len) + sizeof(pRSmaInfo->suid));
len += (sizeof(len) + sizeof(type) + sizeof(pRSmaInfo->suid));
tdAppendTFile(&tFile, &len, sizeof(len), &toffset);
tdAppendTFile(&tFile, &type, sizeof(type), &toffset);
tdAppendTFile(&tFile, &pRSmaInfo->suid, sizeof(pRSmaInfo->suid), &toffset);
tdAppendTFile(&tFile, pOutput, len, &toffset);
@ -833,25 +872,275 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
if (TASK_TRIGGER_STAT_CANCELLED != atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
TASK_TRIGGER_STAT_CANCELLED,
TASK_TRIGGER_STAT_FINISHED)) {
smaDebug("%s:%d rsma persistence start since active", __func__, __LINE__);
smaDebug("rsma persistence start since active");
// start persist task
tdRSmaPersistTask(pRSmaStat);
taosTmrReset(tdRSmaPersistTrigger, RSMA_QTASK_PERSIST_MS, pRSmaStat, pRSmaStat->tmrHandle, &pRSmaStat->tmrId);
taosTmrReset(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pRSmaStat, pRSmaStat->tmrHandle,
&pRSmaStat->tmrId);
} else {
atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
}
} break;
case TASK_TRIGGER_STAT_CANCELLED: {
atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED);
smaDebug("%s:%d rsma persistence not start since cancelled and finished", __func__, __LINE__);
smaDebug("rsma persistence not start since cancelled and finished");
} break;
case TASK_TRIGGER_STAT_INACTIVE: {
smaDebug("%s:%d rsma persistence not start since inactive", __func__, __LINE__);
smaDebug("rsma persistence not start since inactive");
} break;
case TASK_TRIGGER_STAT_INIT: {
smaDebug("%s:%d rsma persistence not start since init", __func__, __LINE__);
smaDebug("rsma persistence not start since init");
} break;
default: {
smaWarn("%s:%d rsma persistence not start since unknown stat %" PRIi8, __func__, __LINE__, tmrStat);
smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat);
ASSERT(0);
} break;
}
}
}
int32_t tdProcessRSmaRestoreImpl(SSma *pSma) {
SVnode *pVnode = pSma->pVnode;
// step 1: iterate all stables to restore the rsma env
SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t));
if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) {
smaError("vgId:%d, failed to restore rsma since get stb id list error: %s", TD_VID(pVnode), terrstr());
return TSDB_CODE_FAILED;
}
if (taosArrayGetSize(suidList) == 0) {
smaDebug("vgId:%d no need to restore rsma since empty stb id list", TD_VID(pVnode));
return TSDB_CODE_SUCCESS;
}
SMetaReader mr = {0};
metaReaderInit(&mr, SMA_META(pSma), 0);
for (int32_t i = 0; i < taosArrayGetSize(suidList); ++i) {
tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
smaDebug("suid [%d] is %" PRIi64, i, suid);
if (metaGetTableEntryByUid(&mr, suid) < 0) {
smaError("vgId:%d failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr());
goto _err;
}
ASSERT(mr.me.type == TSDB_SUPER_TABLE);
ASSERT(mr.me.uid == suid);
if (TABLE_IS_ROLLUP(mr.me.flags)) {
SRSmaParam *param = &mr.me.stbEntry.rsmaParam;
for (int i = 0; i < 2; ++i) {
smaDebug("vgId: %d table:%" PRIi64 " maxdelay[%d]:%" PRIi64 " watermark[%d]:%" PRIi64, TD_VID(pSma->pVnode),
suid, i, param->maxdelay[i], i, param->watermark[i]);
}
if (tdProcessRSmaCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name) < 0) {
smaError("vgId:%d failed to retore rsma env for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr());
goto _err;
}
}
}
// step 2: retrieve qtaskinfo object from the rsma/qtaskinfo file and restore
STFile tFile = {0};
char qTaskInfoFName[TSDB_FILENAME_LEN];
tdRSmaQTaskGetFName(TD_VID(pVnode), TD_QTASK_CUR_FILE, qTaskInfoFName);
if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) {
goto _err;
}
if (tdOpenTFile(&tFile, TD_FILE_READ) < 0) {
goto _err;
}
SRSmaQTaskFIter fIter = {0};
if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) {
goto _err;
}
SRSmaQTaskInfoItem infoItem = {0};
bool isEnd = false;
int32_t code = 0;
while ((code = tdRSmaQTaskInfoIterNext(&fIter, &infoItem, &isEnd)) == 0) {
if (isEnd) {
break;
}
if ((code = tdRSmaQTaskInfoItemRestore(pSma, &infoItem)) < 0) {
break;
}
}
tdRSmaQTaskInfoIterDestroy(&fIter);
if (code < 0) {
goto _err;
}
metaReaderClear(&mr);
taosArrayDestroy(suidList);
return TSDB_CODE_SUCCESS;
_err:
ASSERT(0);
metaReaderClear(&mr);
taosArrayDestroy(suidList);
smaError("failed to restore rsma info since %s", terrstr());
return TSDB_CODE_FAILED;
}
static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem) {
SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv);
SRSmaInfo *pRSmaInfo = NULL;
void *qTaskInfo = NULL;
pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &infoItem->suid, sizeof(infoItem->suid));
if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
smaDebug("vgId:%d, no restore as no rsma info for suid:%" PRIu64, SMA_VID(pSma), infoItem->suid);
return TSDB_CODE_SUCCESS;
}
if (infoItem->type == 1) {
qTaskInfo = pRSmaInfo->items[0].taskInfo;
} else if (infoItem->type == 2) {
qTaskInfo = pRSmaInfo->items[1].taskInfo;
} else {
ASSERT(0);
}
if (!qTaskInfo) {
smaDebug("vgId:%d, no restore as NULL rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), infoItem->suid);
return TSDB_CODE_SUCCESS;
}
if (qDeserializeTaskStatus(qTaskInfo, infoItem->qTaskInfo, infoItem->len) < 0) {
smaError("vgId:%d, restore rsma failed for suid:%" PRIi64 " level %d since %s", SMA_VID(pSma), infoItem->suid,
infoItem->type, terrstr(terrno));
return TSDB_CODE_FAILED;
}
smaDebug("vgId:%d, restore rsma success for suid:%" PRIi64 " level %d", SMA_VID(pSma), infoItem->suid,
infoItem->type);
return TSDB_CODE_SUCCESS;
}
static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskFIter *pIter, STFile *pTFile) {
memset(pIter, 0, sizeof(*pIter));
pIter->pTFile = pTFile;
pIter->offset = TD_FILE_HEAD_SIZE;
if (tdGetTFileSize(pTFile, &pIter->fsize) < 0) {
return TSDB_CODE_FAILED;
}
if ((pIter->fsize - TD_FILE_HEAD_SIZE) < RSMA_QTASKINFO_BUFSIZE) {
pIter->nAlloc = pIter->fsize - TD_FILE_HEAD_SIZE;
} else {
pIter->nAlloc = RSMA_QTASKINFO_BUFSIZE;
}
if (pIter->nAlloc < TD_FILE_HEAD_SIZE) {
pIter->nAlloc = TD_FILE_HEAD_SIZE;
}
pIter->buf = taosMemoryMalloc(pIter->nAlloc);
if (!pIter->buf) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskFIter *pIter, bool *isFinish) {
STFile *pTFile = pIter->pTFile;
int64_t nBytes = RSMA_QTASKINFO_BUFSIZE;
if (pIter->offset >= pIter->fsize) {
*isFinish = true;
return TSDB_CODE_SUCCESS;
}
if ((pIter->fsize - pIter->offset) < RSMA_QTASKINFO_BUFSIZE) {
nBytes = pIter->fsize - pIter->offset;
}
if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
if (tdReadTFile(pTFile, pIter->buf, nBytes) != nBytes) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
int32_t infoLen = 0;
taosDecodeFixedI32(pIter->buf, &infoLen);
if (infoLen > nBytes) {
ASSERT(infoLen > RSMA_QTASKINFO_BUFSIZE);
pIter->nAlloc = infoLen;
void *pBuf = taosMemoryRealloc(pIter->buf, infoLen);
if (!pBuf) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
pIter->buf = pBuf;
nBytes = infoLen;
if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
if (tdReadTFile(pTFile, pIter->buf, nBytes) != nBytes) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
}
pIter->offset += nBytes;
pIter->nBytes = nBytes;
pIter->nBufPos = 0;
return TSDB_CODE_SUCCESS;
}
static int32_t tdRSmaQTaskInfoIterNext(SRSmaQTaskFIter *pIter, SRSmaQTaskInfoItem *pItem, bool *isEnd) {
while (1) {
// block iter
bool isFinish = false;
if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
if (isFinish) {
*isEnd = true;
return TSDB_CODE_SUCCESS;
}
// consume the block
int32_t qTaskInfoLenWithHead = 0;
pIter->buf = taosDecodeFixedI32(pIter->buf, &qTaskInfoLenWithHead);
if (qTaskInfoLenWithHead < 0) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return TSDB_CODE_FAILED;
}
while (1) {
if ((pIter->nBufPos + qTaskInfoLenWithHead) <= pIter->nBytes) {
pIter->buf = taosDecodeFixedI8(pIter->buf, &pItem->type);
pIter->buf = taosDecodeFixedI64(pIter->buf, &pItem->suid);
pItem->qTaskInfo = pIter->buf;
pItem->len = tdRSmaQTaskInfoContLen(qTaskInfoLenWithHead);
// do the restore job
printf("%s:%d ###### restore the qtask info offset:%" PRIi64 "\n", __func__, __LINE__, pIter->offset);
pIter->buf = POINTER_SHIFT(pIter->buf, pItem->len);
pIter->nBufPos += qTaskInfoLenWithHead;
pIter->buf = taosDecodeFixedI32(pIter->buf, &qTaskInfoLenWithHead);
continue;
}
// prepare and load next block in the file
pIter->offset -= (pIter->nBytes - pIter->nBufPos);
break;
}
}
return TSDB_CODE_SUCCESS;
}

7
source/dnode/vnode/src/sma/smaUtil.c
View File

@ -17,8 +17,6 @@
// smaFileUtil ================
#define TD_FILE_HEAD_SIZE 512
#define TD_FILE_STATE_OK 0
#define TD_FILE_STATE_BAD 1
@ -71,6 +69,11 @@ int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence) {
return loffset;
}
int64_t tdGetTFileSize(STFile *pTFile, int64_t *size) {
ASSERT(TD_FILE_OPENED(pTFile));
return taosFStatFile(pTFile->pFile, size, NULL);
}
int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte) {
ASSERT(TD_FILE_OPENED(pTFile));

3
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -2886,6 +2886,9 @@ int32_t tsdbGetCtbIdList(SMeta* pMeta, int64_t suid, SArray* list) {
*/
int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list) {
SMStbCursor* pCur = metaOpenStbCursor(pMeta, suid);
if(!pCur) {
return TSDB_CODE_FAILED;
}
while (1) {
tb_uid_t id = metaStbCursorNext(pCur);

2
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -172,9 +172,9 @@ void vnodeClose(SVnode *pVnode) {
vnodeSyncClose(pVnode);
vnodeQueryClose(pVnode);
walClose(pVnode->pWal);
smaCloseEx(pVnode->pSma);
tqClose(pVnode->pTq);
if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
smaCloseEx(pVnode->pSma);
metaClose(pVnode->pMeta);
vnodeCloseBufPool(pVnode);
// destroy handle

2
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -299,7 +299,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRp
void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
// TODO
blockDebugShowData(data, __func__);
blockDebugShowDataBlocks(data, __func__);
tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data);
}

11
source/libs/executor/inc/executorimpl.h
View File

@ -403,7 +403,7 @@ typedef struct SIntervalAggOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
SOptrBasicInfo binfo; // basic info
SAggSupporter aggSup; // aggregate supporter
SExprSupp scalarSupp; // supporter for perform scalar function
SGroupResInfo groupResInfo; // multiple results build supporter
SInterval interval; // interval info
int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator.
@ -730,15 +730,12 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo*
SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, SSDataBlock* pInputBlock,
SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream);
STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream);
SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
@ -811,7 +808,7 @@ int32_t getMaximumIdleDurationSec();
* nOptrWithVal: *nOptrWithVal save the number of optr with value
* return: result code, 0 means success
*/
int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length);
int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length, int32_t *nOptrWithVal);
/*
* ops: root operator, created by caller

7
source/libs/executor/inc/tsort.h
View File

@ -136,6 +136,13 @@ bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId);
*/
void* tsortGetValue(STupleHandle* pVHandle, int32_t colId);
/**
*
* @param pVHandle
* @return
*/
uint64_t tsortGetGroupId(STupleHandle* pVHandle);
/**
*
* @param pSortHandle

7
source/libs/executor/src/executil.c
View File

@ -159,7 +159,12 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) {
}
SArray* createSortInfo(SNodeList* pNodeList) {
size_t numOfCols = LIST_LENGTH(pNodeList);
size_t numOfCols = 0;
if (pNodeList != NULL) {
numOfCols = LIST_LENGTH(pNodeList);
} else {
numOfCols = 0;
}
SArray* pList = taosArrayInit(numOfCols, sizeof(SBlockOrderInfo));
if (pList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;

8
source/libs/executor/src/executorMain.c
View File

@ -222,7 +222,13 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) {
return TSDB_CODE_INVALID_PARA;
}
return encodeOperator(pTaskInfo->pRoot, pOutput, len);
int32_t nOptrWithVal = 0;
int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal);
if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) {
taosMemoryFreeClear(*pOutput);
*len = 0;
}
return code;
}
int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) {

31
source/libs/executor/src/executorimpl.c
View File

@ -4224,8 +4224,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
bool isStream = (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type);
pOptr =
createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTaskInfo, isStream);
pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pIntervalPhyNode,
pTaskInfo, isStream);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) {
SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode;
@ -4269,17 +4269,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pOptr = createGroupSortOperatorInfo(ops[0], (SGroupSortPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == type) {
SMergePhysiNode* pMergePhyNode = (SMergePhysiNode*)pPhyNode;
SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc;
SSDataBlock* pResBlock = createResDataBlock(pDescNode);
SArray* sortInfo = createSortInfo(pMergePhyNode->pMergeKeys);
int32_t numOfOutputCols = 0;
SArray* pColList =
extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0);
SSDataBlock* pInputDataBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc);
pOptr = createMultiwaySortMergeOperatorInfo(ops, size, pInputDataBlock, pResBlock, sortInfo, pColList, pTaskInfo);
pOptr = createMultiwayMergeOperatorInfo(ops, size, pMergePhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION == type) {
SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
@ -4472,12 +4462,12 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa
return 0;
}
int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) {
int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length, int32_t* nOptrWithVal) {
int32_t code = TDB_CODE_SUCCESS;
char* pCurrent = NULL;
int32_t currLength = 0;
if (ops->fpSet.encodeResultRow) {
if (result == NULL || length == NULL) {
if (result == NULL || length == NULL || nOptrWithVal == NULL) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
code = ops->fpSet.encodeResultRow(ops, &pCurrent, &currLength);
@ -4488,8 +4478,13 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) {
*result = NULL;
}
return code;
} else if (currLength == 0) {
ASSERT(!pCurrent);
goto _downstream;
}
++(*nOptrWithVal);
ASSERT(currLength >= 0);
if (*result == NULL) {
@ -4516,8 +4511,10 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) {
taosMemoryFree(pCurrent);
*length = *(int32_t*)(*result);
}
_downstream:
for (int32_t i = 0; i < ops->numOfDownstream; ++i) {
code = encodeOperator(ops->pDownstream[i], result, length);
code = encodeOperator(ops->pDownstream[i], result, length, nOptrWithVal);
if (code != TDB_CODE_SUCCESS) {
return code;
}

10
source/libs/executor/src/scanoperator.c
View File

@ -448,7 +448,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
qDebug("%s start to repeat ascending order scan data blocks due to query func required", GET_TASKID(pTaskInfo));
for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) {
STimeWindow* pWin = &pTableScanInfo->cond.twindows[i];
qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
}
// do prepare for the next round table scan operation
tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
@ -467,7 +467,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo));
for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) {
STimeWindow* pWin = &pTableScanInfo->cond.twindows[i];
qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
}
while (pTableScanInfo->scanTimes < total) {
@ -492,7 +492,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
GET_TASKID(pTaskInfo));
for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) {
STimeWindow* pWin = &pTableScanInfo->cond.twindows[i];
qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
}
tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
pTableScanInfo->curTWinIdx = 0;
@ -2051,7 +2051,7 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle
qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId);
return TSDB_CODE_SUCCESS;
}
code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pPartitionTags);
code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pGroupTags);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -2455,7 +2455,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
if (pTableScanNode->pPartitionTags) {
if (pTableScanNode->pGroupTags) {
taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid);
}

117
source/libs/executor/src/sortoperator.c
View File

@ -142,7 +142,8 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
SSDataBlock* loadNextDataBlock(void* param) {
SOperatorInfo* pOperator = (SOperatorInfo*)param;
return pOperator->fpSet.getNextFn(pOperator);
SSDataBlock* pBlock = pOperator->fpSet.getNextFn(pOperator);
return pBlock;
}
// todo refactor: merged with fetch fp
@ -442,7 +443,7 @@ void destroyGroupSortOperatorInfo(void* param, int32_t numOfOutput) {
SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
SExecTaskInfo* pTaskInfo) {
SGroupSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupSortOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL /* || rowSize > 100 * 1024 * 1024*/) {
goto _error;
}
@ -474,8 +475,8 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort
pOperator->exprSupp.numOfExprs = numOfCols;
pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, NULL,
NULL, getGroupSortExplainExecInfo);
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo,
NULL, NULL, getGroupSortExplainExecInfo);
int32_t code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -493,7 +494,7 @@ _error:
//=====================================================================================
// Multiway Sort Merge operator
typedef struct SMultiwaySortMergeOperatorInfo {
typedef struct SMultiwayMergeOperatorInfo {
SOptrBasicInfo binfo;
int32_t bufPageSize;
@ -503,14 +504,17 @@ typedef struct SMultiwaySortMergeOperatorInfo {
SSortHandle* pSortHandle;
SArray* pColMatchInfo; // for index map from table scan output
SSDataBlock* pInputBlock;
int64_t startTs; // sort start time
uint64_t groupId;
} SMultiwaySortMergeOperatorInfo;
SSDataBlock* pInputBlock;
int64_t startTs; // sort start time
bool groupSort;
bool hasGroupId;
uint64_t groupId;
STupleHandle* prefetchedTuple;
} SMultiwayMergeOperatorInfo;
int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) {
SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
if (OPTR_IS_OPENED(pOperator)) {
return TSDB_CODE_SUCCESS;
@ -524,8 +528,9 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
pInfo->pInputBlock, pTaskInfo->id.str);
tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL);
tsortSetCompareGroupId(pInfo->pSortHandle, true);
tsortSetCompareGroupId(pInfo->pSortHandle, pInfo->groupSort);
for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) {
SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
ps->param = pOperator->pDownstream[i];
@ -547,8 +552,8 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
SArray* pColMatchInfo, SOperatorInfo* pOperator) {
SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
blockDataCleanup(pDataBlock);
@ -560,18 +565,47 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
blockDataEnsureCapacity(p, capacity);
while (1) {
STupleHandle* pTupleHandle = tsortNextTuple(pHandle);
STupleHandle* pTupleHandle = NULL;
if (pInfo->groupSort) {
if (pInfo->prefetchedTuple == NULL) {
pTupleHandle = tsortNextTuple(pHandle);
} else {
pTupleHandle = pInfo->prefetchedTuple;
pInfo->groupId = tsortGetGroupId(pTupleHandle);
pInfo->prefetchedTuple = NULL;
}
}
else {
pTupleHandle = tsortNextTuple(pHandle);
pInfo->groupId = 0;
}
if (pTupleHandle == NULL) {
break;
}
appendOneRowToDataBlock(p, pTupleHandle);
if (pInfo->groupSort)
{
uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle);
if (!pInfo->hasGroupId) {
pInfo->groupId = tupleGroupId;
pInfo->hasGroupId = true;
appendOneRowToDataBlock(p, pTupleHandle);
} else if (pInfo->groupId == tupleGroupId) {
appendOneRowToDataBlock(p, pTupleHandle);
} else {
pInfo->prefetchedTuple = pTupleHandle;
break;
}
} else {
appendOneRowToDataBlock(p, pTupleHandle);
}
if (p->info.rows >= capacity) {
break;
}
}
if (p->info.rows > 0) {// todo extract method
if (p->info.rows > 0) { // todo extract method
blockDataEnsureCapacity(pDataBlock, p->info.rows);
int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
for (int32_t i = 0; i < numOfCols; ++i) {
@ -593,13 +627,13 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
}
SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) {
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
int32_t code = pOperator->fpSet._openFn(pOperator);
if (code != TSDB_CODE_SUCCESS) {
@ -608,7 +642,6 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes,
pOperator->resultInfo.capacity, pInfo->pColMatchInfo, pOperator);
if (pBlock != NULL) {
pOperator->resultInfo.totalRows += pBlock->info.rows;
} else {
@ -617,8 +650,8 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
return pBlock;
}
void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) {
SMultiwaySortMergeOperatorInfo* pInfo = (SMultiwaySortMergeOperatorInfo*)param;
void destroyMultiwayMergeOperatorInfo(void* param, int32_t numOfOutput) {
SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)param;
pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
pInfo->pInputBlock = blockDataDestroy(pInfo->pInputBlock);
@ -626,11 +659,11 @@ void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) {
taosArrayDestroy(pInfo->pColMatchInfo);
}
int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) {
int32_t getMultiwayMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) {
ASSERT(pOptr != NULL);
SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo));
SMultiwaySortMergeOperatorInfo* pOperatorInfo = (SMultiwaySortMergeOperatorInfo*)pOptr->info;
SMultiwayMergeOperatorInfo* pOperatorInfo = (SMultiwayMergeOperatorInfo*)pOptr->info;
*pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle);
*pOptrExplain = pInfo;
@ -638,24 +671,35 @@ int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrEx
return TSDB_CODE_SUCCESS;
}
SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams,
SSDataBlock* pInputBlock, SSDataBlock* pResBlock, SArray* pSortInfo,
SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) {
SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
int32_t rowSize = pResBlock->info.rowSize;
SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size_t numStreams,
SMergePhysiNode* pMergePhyNode, SExecTaskInfo* pTaskInfo) {
SPhysiNode* pPhyNode = (SPhysiNode*)pMergePhyNode;
SMultiwayMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwayMergeOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc;
SSDataBlock* pResBlock = createResDataBlock(pDescNode);
int32_t rowSize = pResBlock->info.rowSize;
if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) {
goto _error;
}
SArray* pSortInfo = createSortInfo(pMergePhyNode->pMergeKeys);
int32_t numOfOutputCols = 0;
SArray* pColMatchColInfo =
extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0);
SSDataBlock* pInputBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc);
initResultSizeInfo(pOperator, 1024);
pInfo->groupSort = pMergePhyNode->groupSort;
pInfo->binfo.pRes = pResBlock;
pInfo->pSortInfo = pSortInfo;
pInfo->pColMatchInfo = pColMatchColInfo;
pInfo->pInputBlock = pInputBlock;
pOperator->name = "MultiwaySortMerge";
pOperator->name = "MultiwayMerge";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
@ -667,9 +711,8 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams,
// one additional is reserved for merged result.
pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1);
pOperator->fpSet =
createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL,
destroyMultiwaySortMergeOperatorInfo, NULL, NULL, getMultiwaySortMergeExplainExecInfo);
pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, NULL,
destroyMultiwayMergeOperatorInfo, NULL, NULL, getMultiwayMergeExplainExecInfo);
int32_t code = appendDownstream(pOperator, downStreams, numStreams);
if (code != TSDB_CODE_SUCCESS) {

32
source/libs/executor/src/timewindowoperator.c
View File

@ -969,6 +969,12 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
getTableScanInfo(pOperator, &pInfo->order, &scanFlag);
if (pInfo->scalarSupp.pExprInfo != NULL) {
SExprSupp* pExprSup =& pInfo->scalarSupp;
projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx,
pExprSup->numOfExprs, NULL);
}
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, scanFlag, true);
hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL);
@ -1381,6 +1387,11 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
continue;
}
if (pInfo->scalarSupp.pExprInfo != NULL) {
SExprSupp* pExprSup = &pInfo->scalarSupp;
projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
}
// The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the
// caller. Note that all the time window are not close till now.
// the pDataBlock are always the same one, no need to call this again
@ -1498,18 +1509,27 @@ void increaseTs(SqlFunctionCtx* pCtx) {
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream) {
STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream) {
SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
pInfo->win = pTaskInfo->window;
pInfo->order = TSDB_ORDER_ASC;
pInfo->interval = *pInterval;
pInfo->win = pTaskInfo->window;
pInfo->order = TSDB_ORDER_ASC;
pInfo->interval = *pInterval;
pInfo->execModel = pTaskInfo->execModel;
pInfo->twAggSup = *pTwAggSupp;
pInfo->twAggSup = *pTwAggSupp;
if (pPhyNode->window.pExprs != NULL) {
int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar);
int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
}
pInfo->primaryTsIndex = primaryTsSlotId;
@ -2473,7 +2493,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
if (IS_FINAL_OP(pInfo)) {
int32_t childIndex = getChildIndex(pBlock);
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
SIntervalAggOperatorInfo* pChildInfo = pChildOp->info;
SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
SExprSupp* pChildSup = &pChildOp->exprSupp;
doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildInfo->primaryTsIndex,

4
source/libs/executor/src/tsort.c
View File

@ -752,6 +752,10 @@ void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) {
}
}
uint64_t tsortGetGroupId(STupleHandle* pVHandle) {
return pVHandle->pBlock->info.groupId;
}
SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) {
SSortExecInfo info = {0};

7
source/libs/function/src/udfd.c
View File

@ -110,7 +110,7 @@ static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
static int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf);
static int32_t udfdConnectToMnode();
static int32_t udfdLoadUdf(char *udfName, SUdf *udf);
static bool udfdRpcRfp(int32_t code);
static bool udfdRpcRfp(int32_t code, tmsg_t msgType);
static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet);
static int32_t udfdOpenClientRpc();
static int32_t udfdCloseClientRpc();
@ -546,9 +546,12 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
}
return 0;
}
static bool udfdRpcRfp(int32_t code) {
static bool udfdRpcRfp(int32_t code, tmsg_t msgType) {
if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) {
return false;
}
return true;
} else {
return false;

6
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -351,7 +351,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
COPY_SCALAR_FIELD(watermark);
COPY_SCALAR_FIELD(tsColId);
COPY_SCALAR_FIELD(filesFactor);
CLONE_NODE_LIST_FIELD(pPartTags);
CLONE_NODE_LIST_FIELD(pGroupTags);
return TSDB_CODE_SUCCESS;
}
@ -401,6 +401,7 @@ static int32_t logicMergeCopy(const SMergeLogicNode* pSrc, SMergeLogicNode* pDst
CLONE_NODE_LIST_FIELD(pInputs);
COPY_SCALAR_FIELD(numOfChannels);
COPY_SCALAR_FIELD(srcGroupId);
COPY_SCALAR_FIELD(groupSort);
return TSDB_CODE_SUCCESS;
}
@ -436,6 +437,7 @@ static int32_t logicFillCopy(const SFillLogicNode* pSrc, SFillLogicNode* pDst) {
static int32_t logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
CLONE_NODE_LIST_FIELD(pSortKeys);
COPY_SCALAR_FIELD(groupSort);
return TSDB_CODE_SUCCESS;
}
@ -500,7 +502,7 @@ static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhy
COPY_SCALAR_FIELD(ratio);
COPY_SCALAR_FIELD(dataRequired);
CLONE_NODE_LIST_FIELD(pDynamicScanFuncs);
CLONE_NODE_LIST_FIELD(pPartitionTags);
CLONE_NODE_LIST_FIELD(pGroupTags);
COPY_SCALAR_FIELD(interval);
COPY_SCALAR_FIELD(offset);
COPY_SCALAR_FIELD(sliding);

37
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -234,6 +234,8 @@ const char* nodesNodeName(ENodeType type) {
return "PhysiMerge";
case QUERY_NODE_PHYSICAL_PLAN_SORT:
return "PhysiSort";
case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT:
return "PhysiGroupSort";
case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
return "PhysiHashInterval";
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL:
@ -539,7 +541,7 @@ static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols";
static const char* jkScanLogicPlanTableId = "TableId";
static const char* jkScanLogicPlanTableType = "TableType";
static const char* jkScanLogicPlanTagCond = "TagCond";
static const char* jkScanLogicPlanPartTags = "PartTags";
static const char* jkScanLogicPlanGroupTags = "GroupTags";
static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
const SScanLogicNode* pNode = (const SScanLogicNode*)pObj;
@ -561,7 +563,7 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
code = tjsonAddObject(pJson, jkScanLogicPlanTagCond, nodeToJson, pNode->pTagCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkScanLogicPlanPartTags, pNode->pPartTags);
code = nodeListToJson(pJson, jkScanLogicPlanGroupTags, pNode->pGroupTags);
}
return code;
@ -588,7 +590,7 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
code = jsonToNodeObject(pJson, jkScanLogicPlanTagCond, &pNode->pTagCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkScanLogicPlanPartTags, &pNode->pPartTags);
code = jsonToNodeList(pJson, jkScanLogicPlanGroupTags, &pNode->pGroupTags);
}
return code;
@ -1430,7 +1432,8 @@ static const char* jkTableScanPhysiPlanTriggerType = "triggerType";
static const char* jkTableScanPhysiPlanWatermark = "watermark";
static const char* jkTableScanPhysiPlanTsColId = "tsColId";
static const char* jkTableScanPhysiPlanFilesFactor = "FilesFactor";
static const char* jkTableScanPhysiPlanPartitionTags = "PartitionTags";
static const char* jkTableScanPhysiPlanGroupTags = "GroupTags";
static const char* jkTableScanPhysiPlanGroupSort = "GroupSort";
static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj;
@ -1485,7 +1488,10 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
code = tjsonAddDoubleToObject(pJson, jkTableScanPhysiPlanFilesFactor, pNode->filesFactor);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkTableScanPhysiPlanPartitionTags, pNode->pPartitionTags);
code = nodeListToJson(pJson, jkTableScanPhysiPlanGroupTags, pNode->pGroupTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanGroupSort, pNode->groupSort);
}
return code;
@ -1544,7 +1550,10 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
code = tjsonGetDoubleValue(pJson, jkTableScanPhysiPlanFilesFactor, &pNode->filesFactor);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkTableScanPhysiPlanPartitionTags, &pNode->pPartitionTags);
code = jsonToNodeList(pJson, jkTableScanPhysiPlanGroupTags, &pNode->pGroupTags);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanGroupSort, &pNode->groupSort);
}
return code;
@ -1725,6 +1734,7 @@ static const char* jkMergePhysiPlanMergeKeys = "MergeKeys";
static const char* jkMergePhysiPlanTargets = "Targets";
static const char* jkMergePhysiPlanNumOfChannels = "NumOfChannels";
static const char* jkMergePhysiPlanSrcGroupId = "SrcGroupId";
static const char* jkMergePhysiPlanGroupSort = "GroupSort";
static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) {
const SMergePhysiNode* pNode = (const SMergePhysiNode*)pObj;
@ -1742,6 +1752,9 @@ static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkMergePhysiPlanSrcGroupId, pNode->srcGroupId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkMergePhysiPlanGroupSort, pNode->groupSort);
}
return code;
}
@ -1762,6 +1775,9 @@ static int32_t jsonToPhysiMergeNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkMergePhysiPlanSrcGroupId, &pNode->srcGroupId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkMergePhysiPlanGroupSort, &pNode->groupSort);
}
return code;
}
@ -3369,6 +3385,7 @@ static const char* jkSlotDescSlotId = "SlotId";
static const char* jkSlotDescDataType = "DataType";
static const char* jkSlotDescReserve = "Reserve";
static const char* jkSlotDescOutput = "Output";
static const char* jkSlotDescName = "Name";
static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) {
const SSlotDescNode* pNode = (const SSlotDescNode*)pObj;
@ -3383,6 +3400,9 @@ static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkSlotDescOutput, pNode->output);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkSlotDescName, pNode->name);
}
return code;
}
@ -3400,6 +3420,9 @@ static int32_t jsonToSlotDescNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkSlotDescOutput, &pNode->output);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkSlotDescName, pNode->name);
}
return code;
}
@ -4137,6 +4160,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN_MERGE:
return physiMergeNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_SORT:
case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT:
return physiSortNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL:
@ -4280,6 +4304,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_MERGE:
return jsonToPhysiMergeNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_SORT:
case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT:
return jsonToPhysiSortNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL:

3
source/libs/nodes/src/nodesTraverseFuncs.c
View File

@ -500,7 +500,8 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_SORT: {
case QUERY_NODE_PHYSICAL_PLAN_SORT:
case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: {
SSortPhysiNode* pSort = (SSortPhysiNode*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {

9
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -288,6 +288,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SMergePhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_SORT:
return makeNode(type, sizeof(SSortPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT:
return makeNode(type, sizeof(SGroupSortPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
return makeNode(type, sizeof(SIntervalPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL:
@ -709,7 +711,7 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pLogicNode->pTagCond);
nodesDestroyNode(pLogicNode->pTagIndexCond);
taosArrayDestroy(pLogicNode->pSmaIndexes);
nodesDestroyList(pLogicNode->pPartTags);
nodesDestroyList(pLogicNode->pGroupTags);
break;
}
case QUERY_NODE_LOGIC_PLAN_JOIN: {
@ -813,7 +815,7 @@ void nodesDestroyNode(SNode* pNode) {
STableScanPhysiNode* pPhyNode = (STableScanPhysiNode*)pNode;
destroyScanPhysiNode((SScanPhysiNode*)pNode);
nodesDestroyList(pPhyNode->pDynamicScanFuncs);
nodesDestroyList(pPhyNode->pPartitionTags);
nodesDestroyList(pPhyNode->pGroupTags);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_PROJECT: {
@ -850,7 +852,8 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyList(pPhyNode->pTargets);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_SORT: {
case QUERY_NODE_PHYSICAL_PLAN_SORT:
case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: {
SSortPhysiNode* pPhyNode = (SSortPhysiNode*)pNode;
destroyPhysiNode((SPhysiNode*)pPhyNode);
nodesDestroyList(pPhyNode->pExprs);

87
source/libs/parser/src/parTranslater.c
View File

@ -2456,90 +2456,6 @@ static int32_t rewriteUniqueStmt(STranslateContext* pCxt, SSelectStmt* pSelect)
return cxt.pTranslateCxt->errCode;
}
typedef struct SRwriteTailCxt {
STranslateContext* pTranslateCxt;
int64_t limit;
int64_t offset;
} SRwriteTailCxt;
static EDealRes rewriteTailFunc(SNode** pNode, void* pContext) {
SRwriteTailCxt* pCxt = pContext;
if (QUERY_NODE_FUNCTION == nodeType(*pNode)) {
SFunctionNode* pFunc = (SFunctionNode*)*pNode;
if (FUNCTION_TYPE_TAIL == pFunc->funcType) {
pCxt->limit = ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 1))->datum.i;
if (3 == LIST_LENGTH(pFunc->pParameterList)) {
pCxt->offset = ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 2))->datum.i;
}
SNode* pExpr = nodesListGetNode(pFunc->pParameterList, 0);
strcpy(((SExprNode*)pExpr)->aliasName, ((SExprNode*)*pNode)->aliasName);
NODES_CLEAR_LIST(pFunc->pParameterList);
nodesDestroyNode(*pNode);
*pNode = pExpr;
return DEAL_RES_IGNORE_CHILD;
}
}
return DEAL_RES_CONTINUE;
}
static int32_t createLimieNode(SRwriteTailCxt* pCxt, SLimitNode** pOutput) {
*pOutput = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT);
if (NULL == *pOutput) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pOutput)->limit = pCxt->limit;
(*pOutput)->offset = pCxt->offset;
return TSDB_CODE_SUCCESS;
}
static SNode* createOrderByExpr(STranslateContext* pCxt) {
SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR);
if (NULL == pOrder) {
return NULL;
}
pCxt->errCode = createPrimaryKeyCol(pCxt, &pOrder->pExpr);
if (TSDB_CODE_SUCCESS != pCxt->errCode) {
nodesDestroyNode((SNode*)pOrder);
return NULL;
}
pOrder->order = ORDER_DESC;
pOrder->nullOrder = NULL_ORDER_FIRST;
return (SNode*)pOrder;
}
/* case 1:
* in: select tail(expr, k, f) from t where_clause
* out: select expr from t where_clause order by _rowts desc limit k offset f
*
* case 2:
* in: select tail(expr, k, f) from t where_clause partition_by_clause
* out: select expr from t where_clause partition_by_clause sort by _rowts desc limit k offset f
*
* case 3:
* in: select tail(expr, k, f) from t where_clause order_by_clause limit_clause
* out: select expr from (
* select expr from t where_clause order by _rowts desc limit k offset f
* ) order_by_clause limit_clause
*
* case 4:
* in: select tail(expr, k, f) from t where_clause partition_by_clause limit_clause
* out:
*/
static int32_t rewriteTailStmt(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (!pSelect->hasTailFunc) {
return TSDB_CODE_SUCCESS;
}
SRwriteTailCxt cxt = {.pTranslateCxt = pCxt, .limit = -1, .offset = -1};
nodesRewriteExprs(pSelect->pProjectionList, rewriteTailFunc, &cxt);
int32_t code = nodesListMakeStrictAppend(&pSelect->pOrderByList, createOrderByExpr(pCxt));
if (TSDB_CODE_SUCCESS == code) {
code = createLimieNode(&cxt, &pSelect->pLimit);
}
pSelect->hasIndefiniteRowsFunc = false;
return code;
}
typedef struct SReplaceOrderByAliasCxt {
STranslateContext* pTranslateCxt;
SNodeList* pProjectionList;
@ -2616,9 +2532,6 @@ static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect
if (TSDB_CODE_SUCCESS == code) {
code = rewriteUniqueStmt(pCxt, pSelect);
}
// if (TSDB_CODE_SUCCESS == code) {
// code = rewriteTailStmt(pCxt, pSelect);
// }
if (TSDB_CODE_SUCCESS == code) {
code = rewriteTimelineFunc(pCxt, pSelect);
}

5
source/libs/parser/src/parUtil.c
View File

@ -337,7 +337,11 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) {
static bool isValidateTag(char* input) {
if (!input) return false;
for (size_t i = 0; i < strlen(input); ++i) {
#ifdef WINDOWS
if (input[i] < 0x20 || input[i] > 0x7E) return false;
#else
if (isprint(input[i]) == 0) return false;
#endif
}
return true;
}
@ -377,6 +381,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, SMs
char* jsonKey = item->string;
if (!isValidateTag(jsonKey)) {
fprintf(stdout,"%s(%d) %s %08" PRId64 "\n", __FILE__, __LINE__,__func__,taosGetSelfPthreadId());fflush(stdout);
retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey);
goto end;
}

4
source/libs/planner/src/planLogicCreater.c
View File

@ -507,6 +507,8 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt
return TSDB_CODE_OUT_OF_MEMORY;
}
pIdfRowsFunc->isTailFunc = pSelect->hasTailFunc;
// indefinite rows functions and _select_values functions
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pFuncs);
if (TSDB_CODE_SUCCESS == code) {
@ -733,6 +735,8 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
return TSDB_CODE_OUT_OF_MEMORY;
}
pSort->groupSort = pSelect->groupSort;
int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets);
if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) {
code = nodesListMakeStrictAppend(&pSort->node.pTargets,

516
source/libs/planner/src/planOptimizer.c
View File

File diff suppressed because it is too large Load Diff

40
source/libs/planner/src/planPhysiCreater.c
View File

@ -41,8 +41,12 @@ typedef struct SPhysiPlanContext {
static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) {
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
SColumnNode* pCol = (SColumnNode*)pNode;
if (NULL != pStmtName && '\0' != pStmtName[0]) {
return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName);
if (NULL != pStmtName) {
if ('\0' != pStmtName[0]) {
return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName);
} else {
return sprintf(pKey, "%s", pCol->node.aliasName);
}
}
if ('\0' == pCol->tableAlias[0]) {
return sprintf(pKey, "%s", pCol->colName);
@ -56,11 +60,13 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) {
return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName);
}
static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId, bool output, bool reserve) {
static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const char* pName, const SNode* pNode, int16_t slotId,
bool output, bool reserve) {
SSlotDescNode* pSlot = (SSlotDescNode*)nodesMakeNode(QUERY_NODE_SLOT_DESC);
if (NULL == pSlot) {
return NULL;
}
strcpy(pSlot->name, pName);
pSlot->slotId = slotId;
pSlot->dataType = ((SExprNode*)pNode)->resType;
pSlot->reserve = reserve;
@ -99,10 +105,8 @@ static int32_t putSlotToHashImpl(int16_t dataBlockId, int16_t slotId, const char
return taosHashPut(pHash, pName, len, &index, sizeof(SSlotIndex));
}
static int32_t putSlotToHash(int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) {
char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
int32_t len = getSlotKey(pNode, NULL, name);
return putSlotToHashImpl(dataBlockId, slotId, name, len, pHash);
static int32_t putSlotToHash(const char* pName, int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) {
return putSlotToHashImpl(dataBlockId, slotId, pName, strlen(pName), pHash);
}
static int32_t createDataBlockDescHash(SPhysiPlanContext* pCxt, int32_t capacity, int16_t dataBlockId,
@ -131,9 +135,11 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD
int16_t slotId = 0;
SNode* pNode = NULL;
FOREACH(pNode, pList) {
code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, slotId, true, false));
char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
getSlotKey(pNode, NULL, name);
code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pNode, slotId, true, false));
if (TSDB_CODE_SUCCESS == code) {
code = putSlotToHash(pDataBlockDesc->dataBlockId, slotId, pNode, pHash);
code = putSlotToHash(name, pDataBlockDesc->dataBlockId, slotId, pNode, pHash);
}
if (TSDB_CODE_SUCCESS == code) {
pDataBlockDesc->totalRowSize += ((SExprNode*)pNode)->resType.bytes;
@ -196,7 +202,8 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList,
int32_t len = getSlotKey(pExpr, pStmtName, name);
SSlotIndex* pIndex = taosHashGet(pHash, name, len);
if (NULL == pIndex) {
code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pExpr, nextSlotId, output, reserve));
code =
nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pExpr, nextSlotId, output, reserve));
if (TSDB_CODE_SUCCESS == code) {
code = putSlotToHashImpl(pDataBlockDesc->dataBlockId, nextSlotId, name, len, pHash);
}
@ -513,12 +520,13 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp
tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName);
pTableScan->dataRequired = pScanLogicNode->dataRequired;
pTableScan->pDynamicScanFuncs = nodesCloneList(pScanLogicNode->pDynamicScanFuncs);
pTableScan->pPartitionTags = nodesCloneList(pScanLogicNode->pPartTags);
pTableScan->pGroupTags = nodesCloneList(pScanLogicNode->pGroupTags);
if ((NULL != pScanLogicNode->pDynamicScanFuncs && NULL == pTableScan->pDynamicScanFuncs) ||
(NULL != pScanLogicNode->pPartTags && NULL == pTableScan->pPartitionTags)) {
(NULL != pScanLogicNode->pGroupTags && NULL == pTableScan->pGroupTags)) {
nodesDestroyNode((SNode*)pTableScan);
return TSDB_CODE_OUT_OF_MEMORY;
}
pTableScan->groupSort = pScanLogicNode->groupSort;
pTableScan->interval = pScanLogicNode->interval;
pTableScan->offset = pScanLogicNode->offset;
pTableScan->sliding = pScanLogicNode->sliding;
@ -1170,8 +1178,9 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr
static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SSortLogicNode* pSortLogicNode,
SPhysiNode** pPhyNode) {
SSortPhysiNode* pSort =
(SSortPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pSortLogicNode, QUERY_NODE_PHYSICAL_PLAN_SORT);
SSortPhysiNode* pSort = (SSortPhysiNode*)makePhysiNode(
pCxt, (SLogicNode*)pSortLogicNode,
pSortLogicNode->groupSort ? QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT : QUERY_NODE_PHYSICAL_PLAN_SORT);
if (NULL == pSort) {
return TSDB_CODE_OUT_OF_MEMORY;
}
@ -1185,7 +1194,7 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) {
code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pSort->pExprs);
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, pSort->pExprs, pChildTupe);
code = pushdownDataBlockSlots(pCxt, pSort->pExprs, pChildTupe);
}
}
@ -1322,6 +1331,7 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM
pMerge->numOfChannels = pMergeLogicNode->numOfChannels;
pMerge->srcGroupId = pMergeLogicNode->srcGroupId;
pMerge->groupSort = pMergeLogicNode->groupSort;
int32_t code = addDataBlockSlots(pCxt, pMergeLogicNode->pInputs, pMerge->node.pOutputDataBlockDesc);

48
source/libs/planner/src/planSpliter.c
View File

@ -362,7 +362,7 @@ static int32_t stbSplGetNumOfVgroups(SLogicNode* pNode) {
}
static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode,
SNodeList* pMergeKeys, SLogicNode* pPartChild) {
SNodeList* pMergeKeys, SLogicNode* pPartChild, bool groupSort) {
SMergeLogicNode* pMerge = (SMergeLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE);
if (NULL == pMerge) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -371,6 +371,7 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla
pMerge->srcGroupId = pCxt->groupId;
pMerge->node.precision = pPartChild->precision;
pMerge->pMergeKeys = pMergeKeys;
pMerge->groupSort = groupSort;
int32_t code = TSDB_CODE_SUCCESS;
pMerge->pInputs = nodesCloneList(pPartChild->pTargets);
@ -430,7 +431,7 @@ static int32_t stbSplSplitIntervalForBatch(SSplitContext* pCxt, SStableSplitInfo
SNodeList* pMergeKeys = NULL;
code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk, &pMergeKeys);
if (TSDB_CODE_SUCCESS == code) {
code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow);
code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow, false);
}
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pMergeKeys);
@ -497,12 +498,16 @@ static int32_t stbSplSplitSessionForStream(SSplitContext* pCxt, SStableSplitInfo
return code;
}
static void splSetTableScanType(SLogicNode* pNode, EScanType scanType) {
static void stbSplSetTableMergeScan(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
((SScanLogicNode*)pNode)->scanType = scanType;
SScanLogicNode* pScan = (SScanLogicNode*)pNode;
pScan->scanType = SCAN_TYPE_TABLE_MERGE;
if (NULL != pScan->pGroupTags) {
pScan->groupSort = true;
}
} else {
if (1 == LIST_LENGTH(pNode->pChildren)) {
splSetTableScanType((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), scanType);
stbSplSetTableMergeScan((SLogicNode*)nodesListGetNode(pNode->pChildren, 0));
}
}
}
@ -515,7 +520,7 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl
int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys);
if (TSDB_CODE_SUCCESS == code) {
code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild);
code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild, true);
}
if (TSDB_CODE_SUCCESS == code) {
@ -524,13 +529,10 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl
}
if (TSDB_CODE_SUCCESS == code) {
splSetTableScanType(pChild, SCAN_TYPE_TABLE_MERGE);
++(pCxt->groupId);
}
if (TSDB_CODE_SUCCESS == code) {
stbSplSetTableMergeScan(pChild);
pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT);
++(pCxt->groupId);
} else {
nodesDestroyList(pMergeKeys);
}
@ -560,7 +562,7 @@ static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
return ((SScanLogicNode*)pNode)->pPartTags;
return ((SScanLogicNode*)pNode)->pGroupTags;
} else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) {
return ((SPartitionLogicNode*)pNode)->pPartitionKeys;
} else {
@ -775,6 +777,7 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOut
pPartSort->node.pChildren = pChildren;
splSetParent((SLogicNode*)pPartSort);
pPartSort->pSortKeys = pSortKeys;
pPartSort->groupSort = pSort->groupSort;
code = stbSplCreateMergeKeys(pPartSort->pSortKeys, pPartSort->node.pTargets, &pMergeKeys);
}
@ -789,12 +792,29 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOut
return code;
}
static void stbSplSetScanPartSort(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
SScanLogicNode* pScan = (SScanLogicNode*)pNode;
if (NULL != pScan->pGroupTags) {
pScan->groupSort = true;
}
} else {
if (1 == LIST_LENGTH(pNode->pChildren)) {
stbSplSetScanPartSort((SLogicNode*)nodesListGetNode(pNode->pChildren, 0));
}
}
}
static int32_t stbSplSplitSortNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
SLogicNode* pPartSort = NULL;
SNodeList* pMergeKeys = NULL;
bool groupSort = ((SSortLogicNode*)pInfo->pSplitNode)->groupSort;
int32_t code = stbSplCreatePartSortNode((SSortLogicNode*)pInfo->pSplitNode, &pPartSort, &pMergeKeys);
if (TSDB_CODE_SUCCESS == code) {
code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pPartSort);
code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pPartSort, groupSort);
}
if (TSDB_CODE_SUCCESS == code && groupSort) {
stbSplSetScanPartSort(pPartSort);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,
@ -829,7 +849,7 @@ static int32_t stbSplSplitScanNodeForJoin(SSplitContext* pCxt, SLogicSubplan* pS
SNodeList* pMergeKeys = NULL;
int32_t code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pScan), &pMergeKeys);
if (TSDB_CODE_SUCCESS == code) {
code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan);
code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan, false);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeStrictAppend(&pSubplan->pChildren,

8
source/libs/planner/test/planBasicTest.cpp
View File

@ -67,6 +67,14 @@ TEST_F(PlanBasicTest, tailFunc) {
run("SELECT TAIL(c1, 10) FROM t1");
run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10");
run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 PARTITION BY c1");
run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 ORDER BY 1");
run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 LIMIT 5");
run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 PARTITION BY c1 LIMIT 5");
}
TEST_F(PlanBasicTest, interpFunc) {

4
source/libs/planner/test/planTestMain.cpp
View File

@ -76,6 +76,7 @@ static void parseArg(int argc, char* argv[]) {
static struct option long_options[] = {
{"dump", optional_argument, NULL, 'd'},
{"skipSql", required_argument, NULL, 's'},
{"limitSql", required_argument, NULL, 'i'},
{"log", required_argument, NULL, 'l'},
{0, 0, 0, 0}
};
@ -88,6 +89,9 @@ static void parseArg(int argc, char* argv[]) {
case 's':
setSkipSqlNum(optarg);
break;
case 'i':
setLimitSqlNum(optarg);
break;
case 'l':
setLogLevel(optarg);
break;

30
source/libs/planner/test/planTestUtil.cpp
View File

@ -51,6 +51,7 @@ enum DumpModule {
DumpModule g_dumpModule = DUMP_MODULE_NOTHING;
int32_t g_skipSql = 0;
int32_t g_limitSql = 0;
int32_t g_logLevel = 131;
void setDumpModule(const char* pModule) {
@ -76,28 +77,33 @@ void setDumpModule(const char* pModule) {
}
void setSkipSqlNum(const char* pNum) { g_skipSql = stoi(pNum); }
void setLimitSqlNum(const char* pNum) { g_limitSql = stoi(pNum); }
void setLogLevel(const char* pLogLevel) { g_logLevel = stoi(pLogLevel); }
int32_t getLogLevel() { return g_logLevel; }
class PlannerTestBaseImpl {
public:
PlannerTestBaseImpl() : sqlNo_(0) {}
PlannerTestBaseImpl() : sqlNo_(0), sqlNum_(0) {}
void useDb(const string& user, const string& db) {
caseEnv_.acctId_ = 0;
caseEnv_.user_ = user;
caseEnv_.db_ = db;
caseEnv_.nsql_ = g_skipSql;
caseEnv_.numOfSkipSql_ = g_skipSql;
caseEnv_.numOfLimitSql_ = g_limitSql;
}
void run(const string& sql) {
++sqlNo_;
if (caseEnv_.nsql_ > 0) {
--(caseEnv_.nsql_);
if (caseEnv_.numOfSkipSql_ > 0) {
--(caseEnv_.numOfSkipSql_);
return;
}
if (caseEnv_.numOfLimitSql_ > 0 && caseEnv_.numOfLimitSql_ == sqlNum_) {
return;
}
++sqlNum_;
reset();
try {
@ -134,7 +140,7 @@ class PlannerTestBaseImpl {
}
void prepare(const string& sql) {
if (caseEnv_.nsql_ > 0) {
if (caseEnv_.numOfSkipSql_ > 0) {
return;
}
@ -148,7 +154,7 @@ class PlannerTestBaseImpl {
}
void bindParams(TAOS_MULTI_BIND* pParams, int32_t colIdx) {
if (caseEnv_.nsql_ > 0) {
if (caseEnv_.numOfSkipSql_ > 0) {
return;
}
@ -161,8 +167,8 @@ class PlannerTestBaseImpl {
}
void exec() {
if (caseEnv_.nsql_ > 0) {
--(caseEnv_.nsql_);
if (caseEnv_.numOfSkipSql_ > 0) {
--(caseEnv_.numOfSkipSql_);
return;
}
@ -197,9 +203,10 @@ class PlannerTestBaseImpl {
int32_t acctId_;
string user_;
string db_;
int32_t nsql_;
int32_t numOfSkipSql_;
int32_t numOfLimitSql_;
caseEnv() : nsql_(0) {}
caseEnv() : numOfSkipSql_(0) {}
};
struct stmtEnv {
@ -401,6 +408,7 @@ class PlannerTestBaseImpl {
stmtEnv stmtEnv_;
stmtRes res_;
int32_t sqlNo_;
int32_t sqlNum_;
};
PlannerTestBase::PlannerTestBase() : impl_(new PlannerTestBaseImpl()) {}

1
source/libs/planner/test/planTestUtil.h
View File

@ -43,6 +43,7 @@ class PlannerTestBase : public testing::Test {
extern void setDumpModule(const char* pModule);
extern void setSkipSqlNum(const char* pNum);
extern void setLimitSqlNum(const char* pNum);
extern void setLogLevel(const char* pLogLevel);
extern int32_t getLogLevel();

25
source/libs/sync/inc/syncRaftEntry.h
View File

@ -56,6 +56,31 @@ void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj);
void syncEntryLog(const SSyncRaftEntry* pObj);
void syncEntryLog2(char* s, const SSyncRaftEntry* pObj);
//-----------------------------------
typedef struct SRaftEntryCache {
SHashObj* pEntryHash;
int32_t maxCount;
int32_t currentCount;
TdThreadMutex mutex;
SSyncNode* pSyncNode;
} SRaftEntryCache;
SRaftEntryCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount);
void raftCacheDestroy(SRaftEntryCache* pCache);
int32_t raftCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry);
int32_t raftCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry);
int32_t raftCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry);
int32_t raftCacheDelEntry(struct SRaftEntryCache* pCache, SyncIndex index);
int32_t raftCacheGetAndDel(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry);
int32_t raftCacheClear(struct SRaftEntryCache* pCache);
cJSON* raftCache2Json(SRaftEntryCache* pObj);
char* raftCache2Str(SRaftEntryCache* pObj);
void raftCachePrint(SRaftEntryCache* pObj);
void raftCachePrint2(char* s, SRaftEntryCache* pObj);
void raftCacheLog(SRaftEntryCache* pObj);
void raftCacheLog2(char* s, SRaftEntryCache* pObj);
#ifdef __cplusplus
}
#endif

44
source/libs/sync/inc/syncSnapshot.h
View File

@ -35,12 +35,13 @@ extern "C" {
#define SYNC_SNAPSHOT_RETRY_MS 5000
//---------------------------------------------------
typedef struct SSyncSnapshotSender {
bool start;
int32_t seq;
int32_t ack;
void * pReader;
void * pCurrentBlock;
void *pReader;
void *pCurrentBlock;
int32_t blockLen;
SSnapshot snapshot;
SSyncCfg lastConfig;
@ -59,33 +60,36 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot
void snapshotSenderStop(SSyncSnapshotSender *pSender);
int32_t snapshotSend(SSyncSnapshotSender *pSender);
int32_t snapshotReSend(SSyncSnapshotSender *pSender);
cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender);
char * snapshotSender2Str(SSyncSnapshotSender *pSender);
char * snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event);
cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender);
char *snapshotSender2Str(SSyncSnapshotSender *pSender);
char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event);
//---------------------------------------------------
typedef struct SSyncSnapshotReceiver {
bool start;
int32_t ack;
void * pWriter;
SyncTerm term;
SyncTerm privateTerm;
SSnapshot snapshot;
SSyncNode *pSyncNode;
bool start;
int32_t ack;
void *pWriter;
SyncTerm term;
SyncTerm privateTerm;
SSnapshot snapshot;
SRaftId fromId;
SSyncNode *pSyncNode;
} SSyncSnapshotReceiver;
SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId);
void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver);
void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg);
bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver);
void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply);
cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver);
char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver);
char * snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event);
void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg);
bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver);
void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply);
cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver);
char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver);
char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event);
//---------------------------------------------------
// on message
int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg);
int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg);

11
source/libs/sync/src/syncMain.c
View File

@ -1355,11 +1355,16 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
int32_t userStrLen = strlen(str);
SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0};
if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
}
SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore);
SyncIndex logLastIndex = SYNC_INDEX_INVALID;
SyncIndex logBeginIndex = SYNC_INDEX_INVALID;
if (pSyncNode->pLogStore != NULL) {
logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore);
}
char* pCfgStr = syncCfg2SimpleStr(&(pSyncNode->pRaftCfg->cfg));
char* printStr = "";

244
source/libs/sync/src/syncRaftEntry.c
View File

@ -180,3 +180,247 @@ void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) {
sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized);
taosMemoryFree(serialized);
}
//-----------------------------------
SRaftEntryCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount) {
SRaftEntryCache* pCache = taosMemoryMalloc(sizeof(SRaftEntryCache));
if (pCache == NULL) {
sError("vgId:%d raft cache create error", pSyncNode->vgId);
return NULL;
}
pCache->pEntryHash =
taosHashInit(sizeof(SyncIndex), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
if (pCache->pEntryHash == NULL) {
sError("vgId:%d raft cache create hash error", pSyncNode->vgId);
return NULL;
}
taosThreadMutexInit(&(pCache->mutex), NULL);
pCache->maxCount = maxCount;
pCache->currentCount = 0;
pCache->pSyncNode = pSyncNode;
return pCache;
}
void raftCacheDestroy(SRaftEntryCache* pCache) {
if (pCache != NULL) {
taosThreadMutexLock(&(pCache->mutex));
taosHashCleanup(pCache->pEntryHash);
taosThreadMutexUnlock(&(pCache->mutex));
taosThreadMutexDestroy(&(pCache->mutex));
taosMemoryFree(pCache);
}
}
// success, return 1
// max count, return 0
// error, return -1
int32_t raftCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry) {
taosThreadMutexLock(&(pCache->mutex));
if (pCache->currentCount >= pCache->maxCount) {
taosThreadMutexUnlock(&(pCache->mutex));
return 0;
}
taosHashPut(pCache->pEntryHash, &(pEntry->index), sizeof(pEntry->index), pEntry, pEntry->bytes);
++(pCache->currentCount);
do {
char eventLog[128];
snprintf(eventLog, sizeof(eventLog), "raft cache add, type:%s,%d, type2:%s,%d, index:%ld, bytes:%d",
TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType,
pEntry->index, pEntry->bytes);
syncNodeEventLog(pCache->pSyncNode, eventLog);
} while (0);
taosThreadMutexUnlock(&(pCache->mutex));
return 1;
}
// success, return 0
// error, return -1
// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST
int32_t raftCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) {
if (ppEntry == NULL) {
return -1;
}
*ppEntry = NULL;
taosThreadMutexLock(&(pCache->mutex));
void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index));
if (pTmp != NULL) {
SSyncRaftEntry* pEntry = pTmp;
*ppEntry = taosMemoryMalloc(pEntry->bytes);
memcpy(*ppEntry, pTmp, pEntry->bytes);
do {
char eventLog[128];
snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld",
TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType),
(*ppEntry)->originalRpcType, (*ppEntry)->index);
syncNodeEventLog(pCache->pSyncNode, eventLog);
} while (0);
taosThreadMutexUnlock(&(pCache->mutex));
return 0;
}
taosThreadMutexUnlock(&(pCache->mutex));
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
return -1;
}
// success, return 0
// error, return -1
// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST
int32_t raftCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) {
if (ppEntry == NULL) {
return -1;
}
*ppEntry = NULL;
taosThreadMutexLock(&(pCache->mutex));
void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index));
if (pTmp != NULL) {
SSyncRaftEntry* pEntry = pTmp;
*ppEntry = pEntry;
do {
char eventLog[128];
snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld",
TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType),
(*ppEntry)->originalRpcType, (*ppEntry)->index);
syncNodeEventLog(pCache->pSyncNode, eventLog);
} while (0);
taosThreadMutexUnlock(&(pCache->mutex));
return 0;
}
taosThreadMutexUnlock(&(pCache->mutex));
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
return -1;
}
int32_t raftCacheDelEntry(struct SRaftEntryCache* pCache, SyncIndex index) {
taosThreadMutexLock(&(pCache->mutex));
taosHashRemove(pCache->pEntryHash, &index, sizeof(index));
--(pCache->currentCount);
taosThreadMutexUnlock(&(pCache->mutex));
return 0;
}
int32_t raftCacheGetAndDel(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) {
if (ppEntry == NULL) {
return -1;
}
*ppEntry = NULL;
taosThreadMutexLock(&(pCache->mutex));
void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index));
if (pTmp != NULL) {
SSyncRaftEntry* pEntry = pTmp;
*ppEntry = taosMemoryMalloc(pEntry->bytes);
memcpy(*ppEntry, pTmp, pEntry->bytes);
do {
char eventLog[128];
snprintf(eventLog, sizeof(eventLog), "raft cache get-and-del, type:%s,%d, type2:%s,%d, index:%ld",
TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType),
(*ppEntry)->originalRpcType, (*ppEntry)->index);
syncNodeEventLog(pCache->pSyncNode, eventLog);
} while (0);
taosHashRemove(pCache->pEntryHash, &index, sizeof(index));
--(pCache->currentCount);
taosThreadMutexUnlock(&(pCache->mutex));
return 0;
}
taosThreadMutexUnlock(&(pCache->mutex));
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
return -1;
}
int32_t raftCacheClear(struct SRaftEntryCache* pCache) {
taosThreadMutexLock(&(pCache->mutex));
taosHashClear(pCache->pEntryHash);
pCache->currentCount = 0;
taosThreadMutexUnlock(&(pCache->mutex));
return 0;
}
//-----------------------------------
cJSON* raftCache2Json(SRaftEntryCache* pCache) {
char u64buf[128] = {0};
cJSON* pRoot = cJSON_CreateObject();
if (pCache != NULL) {
taosThreadMutexLock(&(pCache->mutex));
snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode);
cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount);
cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount);
cJSON* pEntries = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "entries", pEntries);
SSyncRaftEntry* pIter = (SSyncRaftEntry*)taosHashIterate(pCache->pEntryHash, NULL);
if (pIter != NULL) {
SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter;
cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry));
}
while (pIter) {
pIter = taosHashIterate(pCache->pEntryHash, pIter);
if (pIter != NULL) {
SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter;
cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry));
}
}
taosThreadMutexUnlock(&(pCache->mutex));
}
cJSON* pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SRaftEntryCache", pRoot);
return pJson;
}
char* raftCache2Str(SRaftEntryCache* pCache) {
cJSON* pJson = raftCache2Json(pCache);
char* serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
void raftCachePrint(SRaftEntryCache* pCache) {
char* serialized = raftCache2Str(pCache);
printf("raftCachePrint | len:%lu | %s \n", strlen(serialized), serialized);
fflush(NULL);
taosMemoryFree(serialized);
}
void raftCachePrint2(char* s, SRaftEntryCache* pCache) {
char* serialized = raftCache2Str(pCache);
printf("raftCachePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized);
fflush(NULL);
taosMemoryFree(serialized);
}
void raftCacheLog(SRaftEntryCache* pCache) {
char* serialized = raftCache2Str(pCache);
sTrace("raftCacheLog | len:%lu | %s", strlen(serialized), serialized);
taosMemoryFree(serialized);
}
void raftCacheLog2(char* s, SRaftEntryCache* pCache) {
if (gRaftDetailLog) {
char* serialized = raftCache2Str(pCache);
sTraceLong("raftCacheLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
taosMemoryFree(serialized);
}
}

12
source/libs/sync/src/syncRespMgr.c
View File

@ -32,11 +32,13 @@ SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) {
}
void syncRespMgrDestroy(SSyncRespMgr *pObj) {
taosThreadMutexLock(&(pObj->mutex));
taosHashCleanup(pObj->pRespHash);
taosThreadMutexUnlock(&(pObj->mutex));
taosThreadMutexDestroy(&(pObj->mutex));
taosMemoryFree(pObj);
if (pObj != NULL) {
taosThreadMutexLock(&(pObj->mutex));
taosHashCleanup(pObj->pRespHash);
taosThreadMutexUnlock(&(pObj->mutex));
taosThreadMutexDestroy(&(pObj->mutex));
taosMemoryFree(pObj);
}
}
int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) {

194
source/libs/sync/src/syncSnapshot.c
View File

@ -21,6 +21,7 @@
#include "syncUtil.h"
#include "wal.h"
//----------------------------------
static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm,
SyncSnapshotSend *pBeginMsg);
@ -49,7 +50,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI
pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &(pSender->snapshot));
pSender->finish = false;
} else {
sError("snapshotSenderCreate cannot create sender");
sError("vgId:%d cannot create snapshot sender", pSyncNode->vgId);
}
return pSender;
@ -57,39 +58,59 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI
void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {
if (pSender != NULL) {
// free current block
if (pSender->pCurrentBlock != NULL) {
taosMemoryFree(pSender->pCurrentBlock);
pSender->pCurrentBlock = NULL;
}
// close reader
if (pSender->pReader != NULL) {
int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader);
ASSERT(ret == 0);
pSender->pReader = NULL;
}
// free sender
taosMemoryFree(pSender);
}
}
bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; }
// begin send snapshot (current term, seq begin)
// begin send snapshot by snapshot, pReader
void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void *pReader) {
ASSERT(!snapshotSenderIsStart(pSender));
pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN;
pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID;
// init snapshot and reader
ASSERT(pSender->pReader == NULL);
pSender->pReader = pReader;
pSender->snapshot = snapshot;
// init current block
if (pSender->pCurrentBlock != NULL) {
taosMemoryFree(pSender->pCurrentBlock);
}
pSender->blockLen = 0;
// update term
pSender->term = pSender->pSyncNode->pRaftStore->currentTerm;
++(pSender->privateTerm);
// update state
pSender->finish = false;
pSender->start = true;
pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN;
pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID;
// init last config
if (pSender->snapshot.lastConfigIndex != SYNC_INDEX_INVALID) {
int32_t code = 0;
SSyncRaftEntry *pEntry = NULL;
bool getLastConfig = false;
code = pSender->pSyncNode->pLogStore->syncLogGetEntry(pSender->pSyncNode->pLogStore,
pSender->snapshot.lastConfigIndex, &pEntry);
bool getLastConfig = false;
if (code == 0) {
ASSERT(pEntry != NULL);
@ -112,29 +133,29 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void
}
}
// last config not found in wal, update to -1
if (!getLastConfig) {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "snapshot sender update lcindex from %ld to -1",
pSender->snapshot.lastConfigIndex);
pSender->snapshot.lastConfigIndex = -1;
char *eventLog = snapshotSender2SimpleStr(pSender, logBuf);
syncNodeEventLog(pSender->pSyncNode, eventLog);
taosMemoryFree(eventLog);
SyncIndex oldLastConfigIndex = pSender->snapshot.lastConfigIndex;
SyncIndex newLastConfigIndex = SYNC_INDEX_INVALID;
pSender->snapshot.lastConfigIndex = SYNC_INDEX_INVALID;
memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg));
// event log
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "snapshot sender update lcindex from %ld to %ld", oldLastConfigIndex,
newLastConfigIndex);
char *eventLog = snapshotSender2SimpleStr(pSender, logBuf);
syncNodeEventLog(pSender->pSyncNode, eventLog);
taosMemoryFree(eventLog);
} while (0);
}
} else {
// no last config
memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg));
}
pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS;
pSender->term = pSender->pSyncNode->pRaftStore->currentTerm;
++(pSender->privateTerm);
pSender->finish = false;
pSender->start = true;
// build begin msg
SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId);
pMsg->srcId = pSender->pSyncNode->myRaftId;
@ -151,40 +172,39 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void
SRpcMsg rpcMsg;
syncSnapshotSend2RpcMsg(pMsg, &rpcMsg);
syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg);
char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender send");
syncNodeEventLog(pSender->pSyncNode, eventLog);
taosMemoryFree(eventLog);
syncSnapshotSendDestroy(pMsg);
// event log
do {
char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender send");
syncNodeEventLog(pSender->pSyncNode, eventLog);
taosMemoryFree(eventLog);
} while (0);
}
void snapshotSenderStop(SSyncSnapshotSender *pSender) {
// close reader
if (pSender->pReader != NULL) {
int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader);
ASSERT(ret == 0);
pSender->pReader = NULL;
}
// free current block
if (pSender->pCurrentBlock != NULL) {
taosMemoryFree(pSender->pCurrentBlock);
pSender->pCurrentBlock = NULL;
pSender->blockLen = 0;
}
// update flag
pSender->start = false;
if (gRaftDetailLog) {
char *s = snapshotSender2Str(pSender);
sInfo("snapshotSenderStop %s", s);
taosMemoryFree(s);
}
}
// when sender receiver ack, call this function to send msg from seq
// when sender receive ack, call this function to send msg from seq
// seq = ack + 1, already updated
int32_t snapshotSend(SSyncSnapshotSender *pSender) {
// free memory last time (seq - 1)
// free memory last time (current seq - 1)
if (pSender->pCurrentBlock != NULL) {
taosMemoryFree(pSender->pCurrentBlock);
pSender->pCurrentBlock = NULL;
@ -198,7 +218,7 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) {
if (pSender->blockLen > 0) {
// has read data
} else {
// read finish
// read finish, update seq to end
pSender->seq = SYNC_SNAPSHOT_SEQ_END;
}
@ -219,25 +239,28 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) {
SRpcMsg rpcMsg;
syncSnapshotSend2RpcMsg(pMsg, &rpcMsg);
syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg);
if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) {
char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender finish");
syncNodeEventLog(pSender->pSyncNode, eventLog);
taosMemoryFree(eventLog);
} else {
char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender sending");
syncNodeEventLog(pSender->pSyncNode, eventLog);
taosMemoryFree(eventLog);
}
syncSnapshotSendDestroy(pMsg);
// event log
do {
char *eventLog = NULL;
if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) {
eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender finish");
} else {
eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender sending");
}
syncNodeEventLog(pSender->pSyncNode, eventLog);
taosMemoryFree(eventLog);
} while (0);
return 0;
}
// send snapshot data from cache
int32_t snapshotReSend(SSyncSnapshotSender *pSender) {
if (pSender->pCurrentBlock != NULL) {
// send current block data
if (pSender->pCurrentBlock != NULL && pSender->blockLen > 0) {
// build msg
SyncSnapshotSend *pMsg = syncSnapshotSendBuild(pSender->blockLen, pSender->pSyncNode->vgId);
pMsg->srcId = pSender->pSyncNode->myRaftId;
pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex];
@ -249,16 +272,20 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) {
pMsg->seq = pSender->seq;
memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen);
// send msg
SRpcMsg rpcMsg;
syncSnapshotSend2RpcMsg(pMsg, &rpcMsg);
syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg);
char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender resend");
syncNodeEventLog(pSender->pSyncNode, eventLog);
taosMemoryFree(eventLog);
syncSnapshotSendDestroy(pMsg);
// event log
do {
char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender resend");
syncNodeEventLog(pSender->pSyncNode, eventLog);
taosMemoryFree(eventLog);
} while (0);
}
return 0;
}
@ -294,7 +321,6 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) {
snprintf(u64buf, sizeof(u64buf), "%lu", pSender->snapshot.lastApplyTerm);
cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf);
cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot);
snprintf(u64buf, sizeof(u64buf), "%lu", pSender->sendingMS);
cJSON_AddStringToObject(pRoot, "sendingMS", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode);
@ -314,17 +340,17 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) {
char *snapshotSender2Str(SSyncSnapshotSender *pSender) {
cJSON *pJson = snapshotSender2Json(pSender);
char * serialized = cJSON_Print(pJson);
char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event) {
int32_t len = 256;
char * s = taosMemoryMalloc(len);
char *s = taosMemoryMalloc(len);
SRaftId destId = pSender->pSyncNode->replicasId[pSender->replicaIndex];
char host[128];
char host[64];
uint16_t port;
syncUtilU642Addr(destId.addr, host, sizeof(host), &port);
@ -355,12 +381,12 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from
pReceiver->term = pSyncNode->pRaftStore->currentTerm;
pReceiver->privateTerm = 0;
pReceiver->snapshot.data = NULL;
pReceiver->snapshot.lastApplyIndex = -1;
pReceiver->snapshot.lastApplyIndex = SYNC_INDEX_INVALID;
pReceiver->snapshot.lastApplyTerm = 0;
pReceiver->snapshot.lastConfigIndex = -1;
pReceiver->snapshot.lastConfigIndex = SYNC_INDEX_INVALID;
} else {
sInfo("snapshotReceiverCreate cannot create receiver");
sError("vgId:%d cannot create snapshot receiver", pSyncNode->vgId);
}
return pReceiver;
@ -368,29 +394,55 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from
void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) {
if (pReceiver != NULL) {
// close writer
if (pReceiver->pWriter != NULL) {
int32_t ret =
pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false);
ASSERT(ret == 0);
pReceiver->pWriter = NULL;
}
// free receiver
taosMemoryFree(pReceiver);
}
}
bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceiver->start; }
// begin receive snapshot msg (current term, seq begin)
// static do start
// receive first snapshot data
// privateTerm, pBeginMsg
static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm,
SyncSnapshotSend *pBeginMsg) {
// update state
pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm;
pReceiver->privateTerm = privateTerm;
pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN;
pReceiver->fromId = pBeginMsg->srcId;
// update snapshot
pReceiver->snapshot.lastApplyIndex = pBeginMsg->lastIndex;
pReceiver->snapshot.lastApplyTerm = pBeginMsg->lastTerm;
pReceiver->snapshot.lastConfigIndex = pBeginMsg->lastConfigIndex;
// write data
ASSERT(pReceiver->pWriter == NULL);
int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStartWrite(pReceiver->pSyncNode->pFsm, &(pReceiver->pWriter));
ASSERT(ret == 0);
}
static void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) {
// force close, abandon incomplete data
if (pReceiver->pWriter != NULL) {
int32_t ret =
pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false);
ASSERT(ret == 0);
pReceiver->pWriter = NULL;
}
pReceiver->start = false;
}
// if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver
// if already start, force close, start again
void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg) {
@ -413,12 +465,6 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTer
snapshotReceiverDoStart(pReceiver, privateTerm, pBeginMsg);
pReceiver->start = true;
}
if (gRaftDetailLog) {
char *s = snapshotReceiver2Str(pReceiver);
sInfo("snapshotReceiverStart %s", s);
taosMemoryFree(s);
}
}
void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) {
@ -434,12 +480,6 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) {
if (apply) {
// ++(pReceiver->privateTerm);
}
if (gRaftDetailLog) {
char *s = snapshotReceiver2Str(pReceiver);
sInfo("snapshotReceiverStop %s", s);
taosMemoryFree(s);
}
}
cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) {
@ -461,7 +501,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) {
cJSON_AddStringToObject(pFromId, "addr", u64buf);
{
uint64_t u64 = pReceiver->fromId.addr;
cJSON * pTmp = pFromId;
cJSON *pTmp = pFromId;
char host[128] = {0};
uint16_t port;
syncUtilU642Addr(u64, host, sizeof(host), &port);
@ -494,14 +534,14 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) {
char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) {
cJSON *pJson = snapshotReceiver2Json(pReceiver);
char * serialized = cJSON_Print(pJson);
char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event) {
int32_t len = 256;
char * s = taosMemoryMalloc(len);
char *s = taosMemoryMalloc(len);
SRaftId fromId = pReceiver->fromId;
char host[128];

14
source/libs/sync/test/CMakeLists.txt
View File

@ -17,6 +17,7 @@ add_executable(syncVotesRespondTest "")
add_executable(syncIndexMgrTest "")
add_executable(syncLogStoreTest "")
add_executable(syncEntryTest "")
add_executable(syncEntryCacheTest "")
add_executable(syncRequestVoteTest "")
add_executable(syncRequestVoteReplyTest "")
add_executable(syncAppendEntriesTest "")
@ -129,6 +130,10 @@ target_sources(syncEntryTest
PRIVATE
"syncEntryTest.cpp"
)
target_sources(syncEntryCacheTest
PRIVATE
"syncEntryCacheTest.cpp"
)
target_sources(syncRequestVoteTest
PRIVATE
"syncRequestVoteTest.cpp"
@ -362,6 +367,11 @@ target_include_directories(syncEntryTest
"${TD_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories(syncEntryCacheTest
PUBLIC
"${TD_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories(syncRequestVoteTest
PUBLIC
"${TD_SOURCE_DIR}/include/libs/sync"
@ -610,6 +620,10 @@ target_link_libraries(syncEntryTest
sync
gtest_main
)
target_link_libraries(syncEntryCacheTest
sync
gtest_main
)
target_link_libraries(syncRequestVoteTest
sync
gtest_main

162
source/libs/sync/test/syncEntryCacheTest.cpp Normal file
View File

@ -0,0 +1,162 @@
#include <stdio.h>
#include "syncEnv.h"
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftLog.h"
#include "syncRaftStore.h"
#include "syncUtil.h"
void logTest() {
sTrace("--- sync log test: trace");
sDebug("--- sync log test: debug");
sInfo("--- sync log test: info");
sWarn("--- sync log test: warn");
sError("--- sync log test: error");
sFatal("--- sync log test: fatal");
}
SSyncRaftEntry* createEntry(int i) {
int32_t dataLen = 20;
SSyncRaftEntry* pEntry = syncEntryBuild(dataLen);
assert(pEntry != NULL);
pEntry->msgType = 88;
pEntry->originalRpcType = 99;
pEntry->seqNum = 3;
pEntry->isWeak = true;
pEntry->term = 100 + i;
pEntry->index = i;
snprintf(pEntry->data, dataLen, "value%d", i);
return pEntry;
}
SSyncNode* createFakeNode() {
SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode));
ASSERT(pSyncNode != NULL);
memset(pSyncNode, 0, sizeof(SSyncNode));
return pSyncNode;
}
SRaftEntryCache* createCache(int maxCount) {
SSyncNode* pSyncNode = createFakeNode();
ASSERT(pSyncNode != NULL);
SRaftEntryCache* pCache = raftCacheCreate(pSyncNode, maxCount);
ASSERT(pCache != NULL);
return pCache;
}
void test1() {
int32_t code = 0;
SRaftEntryCache* pCache = createCache(5);
for (int i = 0; i < 5; ++i) {
SSyncRaftEntry* pEntry = createEntry(i);
code = raftCachePutEntry(pCache, pEntry);
ASSERT(code == 1);
syncEntryDestory(pEntry);
}
raftCacheLog2((char*)"==test1 write 5 entries==", pCache);
SyncIndex index;
index = 1;
code = raftCacheDelEntry(pCache, index);
ASSERT(code == 0);
index = 3;
code = raftCacheDelEntry(pCache, index);
ASSERT(code == 0);
raftCacheLog2((char*)"==test1 delete 1,3==", pCache);
code = raftCacheClear(pCache);
ASSERT(code == 0);
raftCacheLog2((char*)"==clear all==", pCache);
}
void test2() {
int32_t code = 0;
SRaftEntryCache* pCache = createCache(5);
for (int i = 0; i < 5; ++i) {
SSyncRaftEntry* pEntry = createEntry(i);
code = raftCachePutEntry(pCache, pEntry);
ASSERT(code == 1);
syncEntryDestory(pEntry);
}
raftCacheLog2((char*)"==test2 write 5 entries==", pCache);
SyncIndex index;
index = 1;
SSyncRaftEntry* pEntry;
code = raftCacheGetEntry(pCache, index, &pEntry);
ASSERT(code == 0);
syncEntryDestory(pEntry);
syncEntryLog2((char*)"==test2 get entry 1==", pEntry);
index = 2;
code = raftCacheGetEntryP(pCache, index, &pEntry);
ASSERT(code == 0);
syncEntryLog2((char*)"==test2 get entry pointer 2==", pEntry);
// not found
index = 8;
code = raftCacheGetEntry(pCache, index, &pEntry);
ASSERT(code == -1 && terrno == TSDB_CODE_WAL_LOG_NOT_EXIST);
sTrace("==test2 get entry 8 not found==");
// not found
index = 9;
code = raftCacheGetEntryP(pCache, index, &pEntry);
ASSERT(code == -1 && terrno == TSDB_CODE_WAL_LOG_NOT_EXIST);
sTrace("==test2 get entry pointer 9 not found==");
}
void test3() {
int32_t code = 0;
SRaftEntryCache* pCache = createCache(5);
for (int i = 0; i < 5; ++i) {
SSyncRaftEntry* pEntry = createEntry(i);
code = raftCachePutEntry(pCache, pEntry);
ASSERT(code == 1);
syncEntryDestory(pEntry);
}
for (int i = 6; i < 10; ++i) {
SSyncRaftEntry* pEntry = createEntry(i);
code = raftCachePutEntry(pCache, pEntry);
ASSERT(code == 0);
syncEntryDestory(pEntry);
}
raftCacheLog2((char*)"==test3 write 10 entries, max count is 5==", pCache);
}
void test4() {
int32_t code = 0;
SRaftEntryCache* pCache = createCache(5);
for (int i = 0; i < 5; ++i) {
SSyncRaftEntry* pEntry = createEntry(i);
code = raftCachePutEntry(pCache, pEntry);
ASSERT(code == 1);
syncEntryDestory(pEntry);
}
raftCacheLog2((char*)"==test4 write 5 entries==", pCache);
SyncIndex index;
index = 3;
SSyncRaftEntry* pEntry;
code = raftCacheGetAndDel(pCache, index, &pEntry);
ASSERT(code == 0);
syncEntryLog2((char*)"==test4 get-and-del entry 3==", pEntry);
raftCacheLog2((char*)"==test4 after get-and-del entry 3==", pCache);
}
int main(int argc, char** argv) {
gRaftDetailLog = true;
tsAsyncLog = 0;
sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE + DEBUG_DEBUG;
test1();
test2();
test3();
test4();
return 0;
}

2
source/libs/transport/inc/transportInt.h
View File

@ -52,7 +52,7 @@ typedef struct {
char user[TSDB_UNI_LEN]; // meter ID
void (*cfp)(void* parent, SRpcMsg*, SEpSet*);
bool (*retry)(int32_t code);
bool (*retry)(int32_t code, tmsg_t msgType);
int index;
void* parent;

5
source/libs/transport/src/trans.c
View File

@ -164,11 +164,6 @@ void rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) {
transSetDefaultAddr(thandle, ip, fqdn);
}
// void rpcSetMsgTraceId(SRpcMsg* pMsg, STraceId uid) {
// SRpcHandleInfo* pInfo = &pMsg->info;
// pInfo->traceId = uid;
//}
int32_t rpcInit() {
// impl later
return 0;

2
source/libs/transport/src/transCli.c
View File

@ -1030,7 +1030,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
*/
STransConnCtx* pCtx = pMsg->ctx;
int32_t code = pResp->code;
if (pTransInst->retry != NULL && pTransInst->retry(code)) {
if (pTransInst->retry != NULL && pTransInst->retry(code, pResp->msgType - 1)) {
pMsg->sent = 0;
pCtx->retryCnt += 1;
if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {

9
source/os/src/osSystem.c
View File

@ -18,6 +18,10 @@
#include "os.h"
#if defined(WINDOWS)
BOOL WINAPI CtrlHandler(DWORD fdwCtrlType) {
printf("\n" TAOS_CONSOLE_PROMPT_HEADER);
return TRUE;
}
#elif defined(_TD_DARWIN_64)
#else
#include <dlfcn.h>
@ -124,7 +128,7 @@ int taosSetConsoleEcho(bool on) {
void taosSetTerminalMode() {
#if defined(WINDOWS)
// assert(0);
SetConsoleCtrlHandler(CtrlHandler, TRUE);
#else
struct termios newtio;
@ -158,7 +162,6 @@ void taosSetTerminalMode() {
int32_t taosGetOldTerminalMode() {
#if defined(WINDOWS)
// assert(0);
#else
/* Make sure stdin is a terminal. */
if (!isatty(STDIN_FILENO)) {
@ -176,7 +179,7 @@ int32_t taosGetOldTerminalMode() {
void taosResetTerminalMode() {
#if defined(WINDOWS)
// assert(0);
SetConsoleCtrlHandler(CtrlHandler, FALSE);
#else
if (tcsetattr(0, TCSANOW, &oldtio) != 0) {
fprintf(stderr, "Fail to reset the terminal properties!\n");

2
tests/parallel_test/container_build.sh
View File

@ -52,7 +52,7 @@ fi
docker run \
-v $REP_MOUNT_PARAM \
--rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_TOOLS=true;make -j $THREAD_COUNT"
--rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true;make -j $THREAD_COUNT"
ret=$?
exit $ret

10
tests/pytest/util/common.py
View File

@ -637,5 +637,13 @@ class TDCom:
column_value_str = ", ".join(str(v) for v in column_value_list)
insert_sql = f'insert into {dbname}.{tbname} values ({column_value_str});'
tsql.execute(insert_sql)
def getOneRow(self, location, containElm):
res_list = list()
if 0 <= location < tdSql.queryRows:
for row in tdSql.queryResult:
if row[location] == containElm:
res_list.append(row)
return res_list
else:
tdLog.exit(f"getOneRow out of range: row_index={location} row_count={self.query_row}")
tdCom = TDCom()

107
tests/pytest/util/constant.py
View File

@ -1,5 +1,76 @@
# -*- coding: utf-8 -*-
# basic type
TAOS_DATA_TYPE = [
"INT", "BIGINT", "SMALLINT", "TINYINT", "INT UNSIGNED", "BIGINT UNSIGNED", "SMALLINT UNSIGNED", "TINYINT UNSIGNED",
"FLOAT", "DOUBLE",
"BOOL",
"BINARY", "NCHAR", "VARCHAR",
"TIMESTAMP",
# "MEDIUMBLOB", "BLOB", # add in 3.x
# "DECIMAL", "NUMERIC", # add in 3.x
"JSON", # only for tag
]
TAOS_NUM_TYPE = [
"INT", "BIGINT", "SMALLINT", "TINYINT", "INT UNSIGNED", "BIGINT UNSIGNED", "SMALLINT UNSIGNED", "TINYINT UNSIGNED", "FLOAT", "DOUBLE",
# "DECIMAL", "NUMERIC", # add in 3.x
]
TAOS_CHAR_TYPE = [
"BINARY", "NCHAR", "VARCHAR",
]
TAOS_BOOL_TYPE = ["BOOL",]
TAOS_TS_TYPE = ["TIMESTAMP",]
TAOS_BIN_TYPE = [
"MEDIUMBLOB", "BLOB", # add in 3.x
]
TAOS_TIME_INIT = ["b", "u", "a", "s", "m", "h", "d", "w", "n", "y"]
TAOS_PRECISION = ["ms", "us", "ns"]
PRECISION_DEFAULT = "ms"
PRECISION = PRECISION_DEFAULT
TAOS_KEYWORDS = [
"ABORT", "CREATE", "IGNORE", "NULL", "STAR",
"ACCOUNT", "CTIME", "IMMEDIATE", "OF", "STATE",
"ACCOUNTS", "DATABASE", "IMPORT", "OFFSET", "STATEMENT",
"ADD", "DATABASES", "IN", "OR", "STATE_WINDOW",
"AFTER", "DAYS", "INITIALLY", "ORDER", "STORAGE",
"ALL", "DBS", "INSERT", "PARTITIONS", "STREAM",
"ALTER", "DEFERRED", "INSTEAD", "PASS", "STREAMS",
"AND", "DELIMITERS", "INT", "PLUS", "STRING",
"AS", "DESC", "INTEGER", "PPS", "SYNCDB",
"ASC", "DESCRIBE", "INTERVAL", "PRECISION", "TABLE",
"ATTACH", "DETACH", "INTO", "PREV", "TABLES",
"BEFORE", "DISTINCT", "IS", "PRIVILEGE", "TAG",
"BEGIN", "DIVIDE", "ISNULL", "QTIME", "TAGS",
"BETWEEN", "DNODE", "JOIN", "QUERIES", "TBNAME",
"BIGINT", "DNODES", "KEEP", "QUERY", "TIMES",
"BINARY", "DOT", "KEY", "QUORUM", "TIMESTAMP",
"BITAND", "DOUBLE", "KILL", "RAISE", "TINYINT",
"BITNOT", "DROP", "LE", "REM", "TOPIC",
"BITOR", "EACH", "LIKE", "REPLACE", "TOPICS",
"BLOCKS", "END", "LIMIT", "REPLICA", "TRIGGER",
"BOOL", "EQ", "LINEAR", "RESET", "TSERIES",
"BY", "EXISTS", "LOCAL", "RESTRICT", "UMINUS",
"CACHE", "EXPLAIN", "LP", "ROW", "UNION",
"CACHELAST", "FAIL", "LSHIFT", "RP", "UNSIGNED",
"CASCADE", "FILE", "LT", "RSHIFT", "UPDATE",
"CHANGE", "FILL", "MATCH", "SCORES", "UPLUS",
"CLUSTER", "FLOAT", "MAXROWS", "SELECT", "USE",
"COLON", "FOR", "MINROWS", "SEMI", "USER",
"COLUMN", "FROM", "MINUS", "SESSION", "USERS",
"COMMA", "FSYNC", "MNODES", "SET", "USING",
"COMP", "GE", "MODIFY", "SHOW", "VALUES",
"COMPACT", "GLOB", "MODULES", "SLASH", "VARIABLE",
"CONCAT", "GRANTS", "NCHAR", "SLIDING", "VARIABLES",
"CONFLICT", "GROUP", "NE", "SLIMIT", "VGROUPS",
"CONNECTION", "GT", "NONE", "SMALLINT", "VIEW",
"CONNECTIONS", "HAVING", "NOT", "SOFFSET", "VNODES",
"CONNS", "ID", "NOTNULL", "STABLE", "WAL",
"COPY", "IF", "NOW", "STABLES", "WHERE",
]
# basic data type boundary
TINYINT_MAX = 127
TINYINT_MIN = -128
@ -11,7 +82,7 @@ SMALLINT_MAX = 32767
SMALLINT_MIN = -32768
SMALLINT_UN_MAX = 65535
MALLINT_UN_MIN = 0
SMALLINT_UN_MIN = 0
INT_MAX = 2147483647
INT_MIN = -2147483648
@ -33,8 +104,8 @@ DOUBLE_MIN = -1.7E+308
# schema boundary
BINARY_LENGTH_MAX = 16374
NCAHR_LENGTH_MAX_ = 4093
DBNAME_LENGTH_MAX_ = 64
NCAHR_LENGTH_MAX = 4093
DBNAME_LENGTH_MAX = 64
STBNAME_LENGTH_MAX = 192
STBNAME_LENGTH_MIN = 1
@ -66,4 +137,32 @@ MNODE_SHM_SIZE_DEFAULT = 6292480
VNODE_SHM_SIZE_MAX = 2147483647
VNODE_SHM_SIZE_MIN = 6292480
VNODE_SHM_SIZE_DEFAULT = 31458304
VNODE_SHM_SIZE_DEFAULT = 31458304
# time_init
TIME_MS = 1
TIME_US = TIME_MS/1000
TIME_NS = TIME_US/1000
TIME_S = 1000 * TIME_MS
TIME_M = 60 * TIME_S
TIME_H = 60 * TIME_M
TIME_D = 24 * TIME_H
TIME_W = 7 * TIME_D
TIME_N = 30 * TIME_D
TIME_Y = 365 * TIME_D
# session parameters
INTERVAL_MIN = 1 * TIME_MS if PRECISION == PRECISION_DEFAULT else 1 * TIME_US
# streams and related agg-function
SMA_INDEX_FUNCTIONS = ["MIN", "MAX"]
ROLLUP_FUNCTIONS = ["AVG", "SUM", "MIN", "MAX", "LAST", "FIRST"]
SMA_WATMARK_MAXDELAY_INIT = ['a', "s", "m"]
WATERMARK_MAX = 900000
WATERMARK_MIN = 0
MAX_DELAY_MAX = 900000
MAX_DELAY_MIN = 1

39
tests/pytest/util/sql.py
View File

@ -21,6 +21,7 @@ import psutil
import shutil
import pandas as pd
from util.log import *
from util.constant import *
def _parse_datetime(timestr):
try:
@ -117,8 +118,7 @@ class TDSql:
col_name_list = []
col_type_list = []
self.cursor.execute(sql)
self.queryCols = self.cursor.description
for query_col in self.queryCols:
for query_col in self.cursor.description:
col_name_list.append(query_col[0])
col_type_list.append(query_col[1])
except Exception as e:
@ -301,6 +301,41 @@ class TDSql:
args = (caller.filename, caller.lineno, self.sql, elm, expect_elm)
tdLog.exit("%s(%d) failed: sql:%s, elm:%s == expect_elm:%s" % args)
def get_times(self, time_str, precision="ms"):
caller = inspect.getframeinfo(inspect.stack()[1][0])
if time_str[-1] not in TAOS_TIME_INIT:
tdLog.exit(f"{caller.filename}({caller.lineno}) failed: {time_str} not a standard taos time init")
if precision not in TAOS_PRECISION:
tdLog.exit(f"{caller.filename}({caller.lineno}) failed: {precision} not a standard taos time precision")
if time_str[-1] == TAOS_TIME_INIT[0]:
times = int(time_str[:-1]) * TIME_NS
if time_str[-1] == TAOS_TIME_INIT[1]:
times = int(time_str[:-1]) * TIME_US
if time_str[-1] == TAOS_TIME_INIT[2]:
times = int(time_str[:-1]) * TIME_MS
if time_str[-1] == TAOS_TIME_INIT[3]:
times = int(time_str[:-1]) * TIME_S
if time_str[-1] == TAOS_TIME_INIT[4]:
times = int(time_str[:-1]) * TIME_M
if time_str[-1] == TAOS_TIME_INIT[5]:
times = int(time_str[:-1]) * TIME_H
if time_str[-1] == TAOS_TIME_INIT[6]:
times = int(time_str[:-1]) * TIME_D
if time_str[-1] == TAOS_TIME_INIT[7]:
times = int(time_str[:-1]) * TIME_W
if time_str[-1] == TAOS_TIME_INIT[8]:
times = int(time_str[:-1]) * TIME_N
if time_str[-1] == TAOS_TIME_INIT[9]:
times = int(time_str[:-1]) * TIME_Y
if precision == "ms":
return int(times)
elif precision == "us":
return int(times*1000)
elif precision == "ns":
return int(times*1000*1000)
def taosdStatus(self, state):
tdLog.sleep(5)
pstate = 0

12
tests/script/jenkins/basic.txt
View File

@ -22,15 +22,19 @@
# ---- dnode
./test.sh -f tsim/dnode/balance_replica1.sim
#./test.sh -f tsim/dnode/balance_replica3.sim
./test.sh -f tsim/dnode/create_dnode.sim
./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim
./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim
./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim
#./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
#./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim
./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim
# ---- insert
./test.sh -f tsim/insert/basic0.sim

26
tests/script/test-all.bat
View File

@ -11,14 +11,17 @@ if not "%2" == "" (
)
for /F "usebackq tokens=*" %%i in (!caseFile!) do (
set line=%%i
if "!line:~,9!" == "./test.sh" (
set /a a+=1
echo !a! Processing %%i
call :GetTimeSeconds !time!
set time1=!_timeTemp!
echo Start at !time!
call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt
if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. )
call :CheckSkipCase %%i
if !skipCase! == false (
if "!line:~,9!" == "./test.sh" (
set /a a+=1
echo !a! Processing %%i
call :GetTimeSeconds !time!
set time1=!_timeTemp!
echo Start at !time!
call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt
if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. )
)
)
)
exit !exitNum!
@ -56,3 +59,10 @@ for %%a in (%tt%) do (
)
set /a _timeTemp=(%hh%*60+%mm%)*60+%ss%
goto :eof
:CheckSkipCase
set skipCase=false
if "%*" == "./test.sh -f tsim/query/scalarFunction.sim" ( set skipCase=true )
if "%*" == "./test.sh -f tsim/stream/distributeInterval0.sim" ( set skipCase=true )
if "%*" == "./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim" ( set skipCase=true )
:goto eof

1
tests/script/tsim/dnode/balance_replica1.sim
View File

@ -117,7 +117,6 @@ if $rows != 6 then
return -1
endi
return
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT

362
tests/script/tsim/dnode/balance_replica3.sim Normal file
View File

@ -0,0 +1,362 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/deploy.sh -n dnode5 -i 5
system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
system sh/exec.sh -n dnode4 -s start
sql connect
print =============== step1 create dnode2
sql create dnode $hostname port 7200
sql create dnode $hostname port 7300
sql create dnode $hostname port 7400
sql create dnode $hostname port 7500
$x = 0
step1:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data34 $data35
print ===> $data40 $data41 $data42 $data43 $data44 $data45
if $rows != 5 then
return -1
endi
if $data(1)[4] != ready then
goto step1
endi
if $data(2)[4] != ready then
goto step1
endi
if $data(3)[4] != ready then
goto step1
endi
if $data(4)[4] != ready then
goto step1
endi
print =============== step2: create db
sql create database d1 vgroups 4 replica 3
print =============== step32 wait vgroup2
$x = 0
step32:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(2)[4] == leader then
$leaderExist = 1
endi
if $data(2)[6] == leader then
$leaderExist = 1
endi
if $data(2)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step32
endi
print =============== step33 wait vgroup3
$x = 0
step33:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(3)[4] == leader then
$leaderExist = 1
endi
if $data(3)[6] == leader then
$leaderExist = 1
endi
if $data(3)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step33
endi
print =============== step34 wait vgroup4
$x = 0
step34:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
if $data(4)[6] == leader then
$leaderExist = 1
endi
if $data(4)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step34
endi
print =============== step35 wait vgroup5
$x = 0
step35:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
if $data(4)[6] == leader then
$leaderExist = 1
endi
if $data(4)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step35
endi
print =============== step36: create table
sql use d1
sql create table d1.st (ts timestamp, i int) tags (j int)
sql create table d1.c1 using st tags(1)
sql create table d1.c2 using st tags(1)
sql create table d1.c3 using st tags(1)
sql create table d1.c4 using st tags(1)
sql create table d1.c5 using st tags(1)
sql create table d1.c6 using st tags(1)
sql show d1.tables
if $rows != 6 then
return -1
endi
print =============== step4: start dnode5
system sh/exec.sh -n dnode5 -s start
$x = 0
step4:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data34 $data35
print ===> $data40 $data41 $data42 $data43 $data44 $data45
if $rows != 5 then
return -1
endi
if $data(1)[4] != ready then
goto step4
endi
if $data(2)[4] != ready then
goto step4
endi
if $data(3)[4] != ready then
goto step4
endi
if $data(4)[4] != ready then
goto step4
endi
if $data(5)[4] != ready then
goto step4
endi
print =============== step5: balance
sql balance vgroup
print =============== step62 wait vgroup2
$x = 0
step62:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(2)[4] == leader then
$leaderExist = 1
endi
if $data(2)[6] == leader then
$leaderExist = 1
endi
if $data(2)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step62
endi
print =============== step63 wait vgroup3
$x = 0
step63:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(3)[4] == leader then
$leaderExist = 1
endi
if $data(3)[6] == leader then
$leaderExist = 1
endi
if $data(3)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step63
endi
print =============== step64 wait vgroup4
$x = 0
step64:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
if $data(4)[6] == leader then
$leaderExist = 1
endi
if $data(4)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step64
endi
print =============== step65 wait vgroup5
$x = 0
step65:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
if $data(4)[6] == leader then
$leaderExist = 1
endi
if $data(4)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step65
endi
print =============== step7: select data
sql show d1.tables
print rows $rows
if $rows != 6 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT

263
tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
View File

@ -28,7 +28,7 @@ step1:
sql show dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
if $rows != 3 then
if $rows != 5 then
return -1
endi
if $data(1)[4] != ready then
@ -49,6 +49,128 @@ endi
print =============== step3 create database
sql create database d1 vgroups 4 replica 3
print =============== step32 wait vgroup2
$x = 0
step32:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(2)[4] == leader then
$leaderExist = 1
endi
if $data(2)[6] == leader then
$leaderExist = 1
endi
if $data(2)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step32
endi
print =============== step33 wait vgroup3
$x = 0
step33:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(3)[4] == leader then
$leaderExist = 1
endi
if $data(3)[6] == leader then
$leaderExist = 1
endi
if $data(3)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step33
endi
print =============== step34 wait vgroup4
$x = 0
step34:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
if $data(4)[6] == leader then
$leaderExist = 1
endi
if $data(4)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step34
endi
print =============== step35 wait vgroup5
$x = 0
step35:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
if $data(4)[6] == leader then
$leaderExist = 1
endi
if $data(4)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step35
endi
print =============== step36: create table
sql use d1
sql create table d1.st (ts timestamp, i int) tags (j int)
sql create table d1.c1 using st tags(1)
@ -57,21 +179,6 @@ if $rows != 1 then
return -1
endi
sql show d1.vgroups
print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4]
if $rows != 1 then
return -1
endi
if $data(2)[3] != 2 then
return -1
endi
if $data(2)[5] != 3 then
return -1
endi
if $data(2)[7] != 4 then
return -1
endi
print =============== step4: drop dnode 2
system sh/exec.sh -n dnode5 -s start
$x = 0
@ -85,7 +192,7 @@ step4:
sql show dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
if $rows != 3 then
if $rows != 5 then
return -1
endi
if $data(1)[4] != ready then
@ -115,21 +222,133 @@ if $rows != 4 then
return -1
endi
print show d1.vgroups
print =============== step62 wait vgroup2
$x = 0
step62:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4]
if $rows != 1 then
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(2)[4] == leader then
$leaderExist = 1
endi
if $data(2)[6] == leader then
$leaderExist = 1
endi
if $data(2)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step62
endi
print =============== step63 wait vgroup3
$x = 0
step63:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(3)[4] == leader then
$leaderExist = 1
endi
if $data(3)[6] == leader then
$leaderExist = 1
endi
if $data(3)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step63
endi
print =============== step6: select data
print =============== step64 wait vgroup4
$x = 0
step64:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
if $data(4)[6] == leader then
$leaderExist = 1
endi
if $data(4)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step64
endi
print =============== step35 wait vgroup5
$x = 0
step65:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $rows != 4 then
return -1
endi
if $data(4)[4] == leader then
$leaderExist = 1
endi
if $data(4)[6] == leader then
$leaderExist = 1
endi
if $data(4)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step65
endi
print =============== step7: select data
print ===> $data00 $data01
sql show d1.tables
if $rows != 1 then
return -1
endi
return
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT

98
tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim
View File

@ -49,6 +49,50 @@ endi
print =============== step3 create database
sql create database d1 vgroups 1 replica 3
$leaderExist = 0
$leaderVnode = 0
$follower1 = 0
$follower2 = 0
$x = 0
step3:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
if $rows != 1 then
return -1
endi
if $data(2)[4] == leader then
$leaderExist = 1
$leaderVnode = 2
$follower1 = 3
$follower2 = 4
endi
if $data(2)[6] == leader then
$leaderExist = 1
$leaderVnode = 3
$follower1 = 2
$follower2 = 4
endi
if $data(2)[8] == leader then
$leaderExist = 1
$leaderVnode = 4
$follower1 = 2
$follower2 = 3
endi
if $leaderExist != 1 then
goto step3
endi
print leader $leaderVnode
print follower1 $follower1
print follower2 $follower2
sql use d1
sql create table d1.st (ts timestamp, i int) tags (j int)
sql create table d1.c1 using st tags(1)
@ -72,13 +116,6 @@ if $data(2)[7] != 4 then
return -1
endi
system sh/exec.sh -n dnode4 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT
return
print =============== step4: drop dnode 2
system sh/exec.sh -n dnode5 -s start
$x = 0
@ -92,7 +129,7 @@ step4:
sql show dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
if $rows != 3 then
if $rows != 5 then
return -1
endi
if $data(1)[4] != ready then
@ -116,8 +153,11 @@ sql drop dnode 2
print show dnodes;
sql show dnodes;
print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4]
print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4]
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data34 $data35
print ===> $data40 $data41 $data42 $data43 $data44 $data45
if $rows != 4 then
return -1
endi
@ -128,9 +168,9 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4]
if $rows != 1 then
return -1
endi
#if $data(2)[3] != 3 then
# return -1
#endi
if $data(2)[3] != 5 then
return -1
endi
print =============== step6: select data
sql show d1.tables
@ -138,7 +178,37 @@ if $rows != 1 then
return -1
endi
return
$leaderExist = 0
$leaderVnode = 0
$follower1 = 0
$follower2 = 0
$x = 0
step6:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
if $rows != 1 then
return -1
endi
if $data(2)[4] == leader then
$leaderExist = 1
endi
if $data(2)[6] == leader then
$leaderExist = 1
endi
if $data(2)[8] == leader then
$leaderExist = 1
endi
if $leaderExist != 1 then
goto step6
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT

178
tests/script/tsim/dnode/redistribute_vgroup_replica1.sim Normal file
View File

@ -0,0 +1,178 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
sql connect
sql create user u1 pass 'taosdata'
print =============== step1 create dnode2
sql create dnode $hostname port 7200
sql create dnode $hostname port 7300
sql create dnode $hostname port 7400
$x = 0
step1:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data34 $data35
if $rows != 4 then
return -1
endi
if $data(1)[4] != ready then
goto step1
endi
if $data(2)[4] != ready then
goto step1
endi
print =============== step2: create db
sql create database d1 vgroups 1 replica 1
system sh/exec.sh -n dnode3 -s start
system sh/exec.sh -n dnode4 -s start
$x = 0
step2:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data34 $data35
if $rows != 4 then
return -1
endi
if $data(1)[4] != ready then
goto step2
endi
if $data(2)[4] != ready then
goto step2
endi
if $data(3)[4] != ready then
goto step2
endi
if $data(4)[4] != ready then
goto step2
endi
print =============== step3: create database
sql use d1
sql create table d1.st (ts timestamp, i int) tags (j int)
sql create table d1.c1 using st tags(1)
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step40:
print redistribute vgroup 2 dnode 3
sql redistribute vgroup 2 dnode 3
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step41:
print redistribute vgroup 2 dnode 4
sql redistribute vgroup 2 dnode 4
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step42:
print redistribute vgroup 2 dnode 2
sql redistribute vgroup 2 dnode 2
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step43:
print redistribute vgroup 2 dnode 3
sql redistribute vgroup 2 dnode 3
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step44:
print redistribute vgroup 2 dnode 4
sql redistribute vgroup 2 dnode 4
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step45:
print redistribute vgroup 2 dnode 2
sql redistribute vgroup 2 dnode 2
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step46:
print redistribute vgroup 2 dnode 3
sql redistribute vgroup 2 dnode 3
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step47:
print redistribute vgroup 2 dnode 2
sql redistribute vgroup 2 dnode 2
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT
system sh/exec.sh -n dnode4 -s stop -x SIGINT

4
tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
View File

@ -65,7 +65,7 @@ sql_error redistribute vgroup 3 dnode 6 dnode 3 dnode 4
# vgroup not exist
sql_error redistribute vgroup 3 dnode 5 dnode 3 dnode 4
# un changed
sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4
# sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4
# no enought vnodes
sql_error redistribute vgroup 2 dnode 1 dnode 3 dnode 4
# offline vnodes
@ -176,8 +176,6 @@ if $rows != 1 then
return -1
endi
return
print =============== step32:
print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5
sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5

245
tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim Normal file
View File

@ -0,0 +1,245 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/deploy.sh -n dnode5 -i 5
system sh/deploy.sh -n dnode6 -i 6
system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode6 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
system sh/exec.sh -n dnode4 -s start
sql connect
sql create user u1 pass 'taosdata'
print =============== step1 create dnode2
sql create dnode $hostname port 7200
sql create dnode $hostname port 7300
sql create dnode $hostname port 7400
sql create dnode $hostname port 7500
sql create dnode $hostname port 7600
$x = 0
step1:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data34 $data35
print ===> $data40 $data41 $data42 $data43 $data44 $data45
if $rows != 6 then
return -1
endi
if $data(1)[4] != ready then
goto step1
endi
if $data(2)[4] != ready then
goto step1
endi
if $data(3)[4] != ready then
goto step1
endi
if $data(4)[4] != ready then
goto step1
endi
print =============== step2: create db
sql create database d1 vgroups 1 replica 3
system sh/exec.sh -n dnode5 -s start
system sh/exec.sh -n dnode6 -s start
$x = 0
step2:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data34 $data35
print ===> $data40 $data41 $data42 $data43 $data44 $data45
if $rows != 6 then
return -1
endi
if $data(1)[4] != ready then
goto step2
endi
if $data(2)[4] != ready then
goto step2
endi
if $data(3)[4] != ready then
goto step2
endi
if $data(4)[4] != ready then
goto step2
endi
if $data(5)[4] != ready then
goto step2
endi
if $data(6)[4] != ready then
goto step2
endi
print =============== step3: create database
$leaderExist = 0
$leaderVnode = 0
$follower1 = 0
$follower2 = 0
$x = 0
step3:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
if $rows != 1 then
return -1
endi
if $data(2)[4] == leader then
$leaderExist = 1
$leaderVnode = 2
$follower1 = 3
$follower2 = 4
endi
if $data(2)[6] == leader then
$leaderExist = 1
$leaderVnode = 3
$follower1 = 2
$follower2 = 4
endi
if $data(2)[8] == leader then
$leaderExist = 1
$leaderVnode = 4
$follower1 = 2
$follower2 = 3
endi
if $leaderExist != 1 then
goto step3
endi
print leader $leaderVnode
print follower1 $follower1
print follower2 $follower2
sql use d1
sql create table d1.st (ts timestamp, i int) tags (j int)
sql create table d1.c1 using st tags(1)
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step40:
print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode 6
sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode 6
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step41:
print redistribute vgroup 2 dnode 5 dnode $follower2 dnode $follower1
sql redistribute vgroup 2 dnode 5 dnode $follower2 dnode $follower1
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step42:
print redistribute vgroup 2 dnode $follower2 dnode 6 dnode $leaderVnode
sql redistribute vgroup 2 dnode $follower2 dnode 6 dnode $leaderVnode
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step43:
print redistribute vgroup 2 dnode $follower1 dnode 5 dnode 6
sql redistribute vgroup 2 dnode $follower1 dnode 5 dnode 6
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step44:
print redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode
sql redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step45:
print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 6
sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 6
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step46:
print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5
sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step47:
print redistribute vgroup 2 dnode $follower1 dnode 6 dnode $leaderVnode
sql redistribute vgroup 2 dnode $follower1 dnode 6 dnode $leaderVnode
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT
system sh/exec.sh -n dnode4 -s stop -x SIGINT
system sh/exec.sh -n dnode5 -s stop -x SIGINT

263
tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim Normal file
View File

@ -0,0 +1,263 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/deploy.sh -n dnode5 -i 5
system sh/deploy.sh -n dnode6 -i 6
system sh/deploy.sh -n dnode7 -i 7
system sh/deploy.sh -n dnode8 -i 8
system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode6 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode7 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode8 -c transPullupInterval -v 1
system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
system sh/exec.sh -n dnode4 -s start
sql connect
sql create user u1 pass 'taosdata'
print =============== step1 create dnode2
sql create dnode $hostname port 7200
sql create dnode $hostname port 7300
sql create dnode $hostname port 7400
sql create dnode $hostname port 7500
sql create dnode $hostname port 7600
sql create dnode $hostname port 7700
sql create dnode $hostname port 7800
$x = 0
step1:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data34 $data35
print ===> $data40 $data41 $data42 $data43 $data44 $data45
if $rows != 8 then
return -1
endi
if $data(1)[4] != ready then
goto step1
endi
if $data(2)[4] != ready then
goto step1
endi
if $data(3)[4] != ready then
goto step1
endi
if $data(4)[4] != ready then
goto step1
endi
print =============== step2: create db
sql create database d1 vgroups 1 replica 3
system sh/exec.sh -n dnode5 -s start
system sh/exec.sh -n dnode6 -s start
system sh/exec.sh -n dnode7 -s start
system sh/exec.sh -n dnode8 -s start
$x = 0
step2:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data34 $data35
print ===> $data40 $data41 $data42 $data43 $data44 $data45
if $rows != 8 then
return -1
endi
if $data(1)[4] != ready then
goto step2
endi
if $data(2)[4] != ready then
goto step2
endi
if $data(3)[4] != ready then
goto step2
endi
if $data(4)[4] != ready then
goto step2
endi
if $data(5)[4] != ready then
goto step2
endi
if $data(6)[4] != ready then
goto step2
endi
if $data(7)[4] != ready then
goto step2
endi
if $data(8)[4] != ready then
goto step2
endi
print =============== step3: create database
$leaderExist = 0
$leaderVnode = 0
$follower1 = 0
$follower2 = 0
$x = 0
step3:
$x = $x + 1
sleep 1000
if $x == 60 then
print ====> db not ready!
return -1
endi
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
if $rows != 1 then
return -1
endi
if $data(2)[4] == leader then
$leaderExist = 1
$leaderVnode = 2
$follower1 = 3
$follower2 = 4
endi
if $data(2)[6] == leader then
$leaderExist = 1
$leaderVnode = 3
$follower1 = 2
$follower2 = 4
endi
if $data(2)[8] == leader then
$leaderExist = 1
$leaderVnode = 4
$follower1 = 2
$follower2 = 3
endi
if $leaderExist != 1 then
goto step3
endi
print leader $leaderVnode
print follower1 $follower1
print follower2 $follower2
sql use d1
sql create table d1.st (ts timestamp, i int) tags (j int)
sql create table d1.c1 using st tags(1)
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step40:
print redistribute vgroup 2 dnode 7 dnode 5 dnode 6
sql redistribute vgroup 2 dnode 7 dnode 5 dnode 6
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step41:
print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1
sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step42:
print redistribute vgroup 2 dnode 7 dnode 6 dnode 8
sql redistribute vgroup 2 dnode 7 dnode 6 dnode 8
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step43:
print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5
sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
return
print =============== step44:
print redistribute vgroup 2 dnode 6 dnode 7 dnode $leaderVnode
sql redistribute vgroup 2 dnode 6 dnode 7 dnode $leaderVnode
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step45:
print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 8
sql redistribute vgroup 2dnode $follower1 dnode $follower2 dnode 8
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step46:
print redistribute vgroup 2 dnode $leaderVnode dnode 6 dnode 5
sql redistribute vgroup 2 dnode $leaderVnode dnode 6 dnode 5
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
print =============== step47:
print redistribute vgroup 2 dnode $follower1 dnode 7 dnode 8
sql redistribute vgroup 2 dnode $follower1 dnode 7 dnode 8
sql show d1.vgroups
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
sql show d1.tables
if $rows != 1 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT
system sh/exec.sh -n dnode4 -s stop -x SIGINT
system sh/exec.sh -n dnode5 -s stop -x SIGINT

2
tests/script/tsim/stream/distributeInterval0.sim
View File

@ -41,7 +41,7 @@ sql create table ts1 using st tags(1,1,1);
sql create table ts2 using st tags(2,2,2);
sql create table ts3 using st tags(3,2,2);
sql create table ts4 using st tags(4,2,2);
sql create stream stream_t1 trigger at_once into streamtST1 as select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s);
sql create stream stream_t1 trigger at_once watermark 1d into streamtST1 as select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s);
sleep 1000

291
tests/script/tsim/stream/distributeIntervalRetrive0.sim Normal file
View File

@ -0,0 +1,291 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
sql create dnode $hostname2 port 7200
system sh/exec.sh -n dnode2 -s start
print ===== step1
$x = 0
step1:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
if $rows != 2 then
return -1
endi
if $data(1)[4] != ready then
goto step1
endi
if $data(2)[4] != ready then
goto step1
endi
print ===== step2
sql create database test vgroups 4;
sql use test;
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
sql create table ts1 using st tags(1,1,1);
sql create table ts2 using st tags(2,2,2);
sql create table ts3 using st tags(3,2,2);
sql create table ts4 using st tags(4,2,2);
sql create stream stream_t1 trigger at_once into streamtST1 as select _wstartts, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s);
sleep 1000
sql insert into ts1 values(1648791213001,1,12,3,1.0);
sql insert into ts2 values(1648791213001,1,12,3,1.0);
sql insert into ts1 values(1648791213002,NULL,NULL,NULL,NULL);
sql insert into ts2 values(1648791213002,NULL,NULL,NULL,NULL);
sql insert into ts1 values(1648791223002,2,2,3,1.1);
sql insert into ts1 values(1648791233003,3,2,3,2.1);
sql insert into ts2 values(1648791243004,4,2,43,73.1);
sql insert into ts1 values(1648791213002,24,22,23,4.1) (1648791243005,4,20,3,3.1);
sleep 1000
sql insert into ts3 values(1648791213001,12,12,13,14.1) (1648791243005,14,30,30,30.1);
$loop_count = 0
loop1:
sleep 1000
sql select * from streamtST1;
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $data01 != 5 then
print =====data01=$data01
goto loop1
endi
if $data02 != 14 then
print =====data02=$data02
goto loop1
endi
# row 1
if $data11 != 1 then
print =====data11=$data11
goto loop1
endi
if $data12 != 2 then
print =====data12=$data12
goto loop1
endi
#row2
if $data21 != 1 then
print =====data21=$data21
goto loop1
endi
if $data22 != 3 then
print =====data22=$data22
goto loop1
endi
#row 3
if $data31 != 3 then
print =====data31=$data31
goto loop1
endi
if $data32 != 22 then
print =====data32=$data32
goto loop1
endi
print loop1 over
sql insert into ts1 values(1648791223008,4,2,30,3.1) (1648791213009,4,2,3,3.1) (1648791233010,4,2,3,3.1) (1648791243011,4,2,3,3.1)(1648791243012,34,32,33,3.1);
$loop_count = 0
loop2:
sleep 1000
sql select * from streamtST1;
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $data01 != 6 then
print =====data01=$data01
goto loop2
endi
if $data02 != 18 then
print =====data02=$data02
goto loop2
endi
# row 1
if $data11 != 2 then
print =====data11=$data11
goto loop2
endi
if $data12 != 6 then
print =====data12=$data12
goto loop2
endi
#row2
if $data21 != 2 then
print =====data21=$data21
goto loop2
endi
if $data22 != 7 then
print =====data22=$data22
goto loop2
endi
#row 3
if $data31 != 5 then
print =====data31=$data31
goto loop2
endi
if $data32 != 60 then
print =====data32=$data32
goto loop2
endi
print loop2 over
sql insert into ts4 values(1648791223008,4,2,30,3.1) (1648791213009,4,2,3,3.1) (1648791233010,4,2,3,3.1);
$loop_count = 0
loop3:
sleep 1000
sql select * from streamtST1;
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $data01 != 7 then
print =====data01=$data01
goto loop3
endi
if $data02 != 22 then
print =====data02=$data02
goto loop3
endi
# row 1
if $data11 != 3 then
print =====data11=$data11
goto loop3
endi
if $data12 != 10 then
print =====data12=$data12
goto loop3
endi
#row2
if $data21 != 3 then
print =====data21=$data21
goto loop3
endi
if $data22 != 11 then
print =====data22=$data22
goto loop3
endi
#row 3
if $data31 != 5 then
print =====data31=$data31
goto loop3
endi
if $data32 != 60 then
print =====data32=$data32
goto loop3
endi
print loop3 over
$loop_count = 0
loop4:
sleep 1000
sql select * from streamtST1;
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
# row 0
if $data01 != 7 then
print =====data01=$data01
goto loop4
endi
if $data02 != 22 then
print =====data02=$data02
goto loop4
endi
# row 1
if $data11 != 3 then
print =====data11=$data11
goto loop4
endi
if $data12 != 10 then
print =====data12=$data12
goto loop4
endi
#row2
if $data21 != 3 then
print =====data21=$data21
goto loop4
endi
if $data22 != 11 then
print =====data22=$data22
goto loop4
endi
#row 3
if $data31 != 5 then
print =====data31=$data31
goto loop4
endi
if $data32 != 60 then
print =====data32=$data32
goto loop4
endi
print loop4 over
system sh/stop_dnodes.sh

0
tests/script/tsim/stream/distributesession0.sim → tests/script/tsim/stream/distributeSession0.sim
View File

515
tests/system-test/1-insert/alter_table.py
View File

@ -16,273 +16,272 @@ import string
from util.log import *
from util.cases import *
from util.sql import *
from util import constant
from util.common import *
from util.sqlset import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.setsql = TDSetSql()
self.ntbname = 'ntb'
self.stbname = 'stb'
self.binary_length = 20 # the length of binary for column_dict
self.nchar_length = 20 # the length of nchar for column_dict
self.column_dict = {
'ts' : 'timestamp',
'col1': 'tinyint',
'col2': 'smallint',
'col3': 'int',
'col4': 'bigint',
'col5': 'tinyint unsigned',
'col6': 'smallint unsigned',
'col7': 'int unsigned',
'col8': 'bigint unsigned',
'col9': 'float',
'col10': 'double',
'col11': 'bool',
'col12': f'binary({self.binary_length})',
'col13': f'nchar({self.nchar_length})'
}
self.tag_dict = {
'ts_tag' : 'timestamp',
't1': 'tinyint',
't2': 'smallint',
't3': 'int',
't4': 'bigint',
't5': 'tinyint unsigned',
't6': 'smallint unsigned',
't7': 'int unsigned',
't8': 'bigint unsigned',
't9': 'float',
't10': 'double',
't11': 'bool',
't12': f'binary({self.binary_length})',
't13': f'nchar({self.nchar_length})'
}
self.tag_list = [
f'now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据"'
]
self.tbnum = 1
self.values_list = [
f'now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据"'
]
self.column_add_dict = {
'col_time' : 'timestamp',
'col_tinyint' : 'tinyint',
'col_smallint' : 'smallint',
'col_int' : 'int',
'col_bigint' : 'bigint',
'col_untinyint' : 'tinyint unsigned',
'col_smallint' : 'smallint unsigned',
'col_int' : 'int unsigned',
'col_bigint' : 'bigint unsigned',
'col_bool' : 'bool',
'col_float' : 'float',
'col_double' : 'double',
'col_binary' : f'binary({constant.BINARY_LENGTH_MAX})',
'col_nchar' : f'nchar({constant.NCAHR_LENGTH_MAX})'
def get_long_name(self, length, mode="mixed"):
"""
generate long name
mode could be numbers/letters/letters_mixed/mixed
"""
if mode == "numbers":
population = string.digits
elif mode == "letters":
population = string.ascii_letters.lower()
elif mode == "letters_mixed":
population = string.ascii_letters.upper() + string.ascii_letters.lower()
else:
population = string.ascii_letters.lower() + string.digits
return "".join(random.choices(population, k=length))
def alter_tb_tag_check(self):
tag_tinyint = random.randint(-127,129)
tag_int = random.randint(-2147483648,2147483647)
tag_smallint = random.randint(-32768,32768)
tag_bigint = random.randint(-2147483648,2147483647)
tag_untinyint = random.randint(0,256)
tag_unsmallint = random.randint(0,65536)
tag_unint = random.randint(0,4294967296)
tag_unbigint = random.randint(0,2147483647)
tag_binary = self.get_long_name(length=10, mode="letters")
tag_nchar = self.get_long_name(length=10, mode="letters")
dbname = self.get_long_name(length=10, mode="letters")
tdSql.execute(f'create database if not exists {dbname}')
stbname = self.get_long_name(length=3, mode="letters")
tbname = self.get_long_name(length=3, mode="letters")
tdSql.execute(f'create stable if not exists {dbname}.{stbname} (col_ts timestamp, c1 int) tags (tag_ts timestamp, t1 tinyint, t2 smallint, t3 int, \
t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 bool,t12 binary(20),t13 nchar(20))')
tdSql.execute(f'create table if not exists {dbname}.{tbname} using {dbname}.{stbname} tags(now, 1, 2, 3, 4, 5, 6, 7, 8, 9.9, 10.1, True,"abc123","涛思数据")')
tdSql.execute(f'insert into {dbname}.{tbname} values(now, 1)')
tdSql.execute(f'alter table {dbname}.{tbname} set tag tag_ts = 1640966400000')
tdSql.execute(f'alter table {dbname}.{tbname} set tag `t1` = 11')
tdSql.query(f'select * from {dbname}.{stbname}')
tdSql.checkData(0,3,11)
tdSql.execute(f'alter table {dbname}.{tbname} set tag t1 = {tag_tinyint}')
tdSql.execute(f'alter table {dbname}.{tbname} set tag t2 = {tag_smallint}')
tdSql.execute(f'alter table {dbname}.{tbname} set tag t3 = {tag_int}')
tdSql.execute(f'alter table {dbname}.{tbname} set tag t4 = {tag_bigint}')
tdSql.execute(f'alter table {dbname}.{tbname} set tag t5 = {tag_untinyint}')
tdSql.execute(f'alter table {dbname}.{tbname} set tag t6 = {tag_unsmallint}')
tdSql.execute(f'alter table {dbname}.{tbname} set tag t7 = {tag_unint}')
tdSql.execute(f'alter table {dbname}.{tbname} set tag t8 = {tag_unbigint}')
tdSql.execute(f'alter table {dbname}.{tbname} set tag t11 = false')
tdSql.execute(f'alter table {dbname}.{tbname} set tag t12 = "{tag_binary}"')
tdSql.execute(f'alter table {dbname}.{tbname} set tag t13 = "{tag_nchar}"')
tdSql.query(f'select * from {dbname}.{stbname}')
# bug TD-15899
tdSql.checkData(0,2,'2022-01-01 00:00:00.000')
tdSql.checkData(0,3,tag_tinyint)
tdSql.checkData(0,4,tag_smallint)
tdSql.checkData(0,5,tag_int)
tdSql.checkData(0,6,tag_bigint)
tdSql.checkData(0,7,tag_untinyint)
tdSql.checkData(0,8,tag_unsmallint)
tdSql.checkData(0,9,tag_unint)
tdSql.checkData(0,10,tag_unbigint)
}
def alter_check_ntb(self):
tdSql.checkData(0,13,False)
tdSql.checkData(0,14,tag_binary)
tdSql.checkData(0,15,tag_nchar)
tdSql.prepare()
tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict))
for i in self.values_list:
tdSql.execute(f'insert into {self.ntbname} values({i})')
for key,values in self.column_add_dict.items():
tdSql.execute(f'alter table {self.ntbname} add column {key} {values}')
tdSql.query(f'describe {self.ntbname}')
tdSql.checkRows(len(self.column_dict)+1)
tdSql.query(f'select {key} from {self.ntbname}')
tdSql.checkRows(len(self.values_list))
tdSql.execute(f'alter table {self.ntbname} drop column {key}')
tdSql.query(f'describe {self.ntbname}')
tdSql.checkRows(len(self.column_dict))
tdSql.error(f'select {key} from {self.ntbname} ')
for key,values in self.column_dict.items():
if 'binary' in values.lower():
v = f'binary({self.binary_length+1})'
v_error = f'binary({self.binary_length-1})'
tdSql.error(f'alter table {self.ntbname} modify column {key} {v_error}')
tdSql.execute(f'alter table {self.ntbname} modify column {key} {v}')
tdSql.query(f'describe {self.ntbname}')
result = tdCom.getOneRow(1,'VARCHAR')
tdSql.checkEqual(result[0][2],self.binary_length+1)
elif 'nchar' in values.lower():
v = f'nchar({self.binary_length+1})'
v_error = f'nchar({self.binary_length-1})'
tdSql.error(f'alter table {self.ntbname} modify column {key} {v_error}')
tdSql.execute(f'alter table {self.ntbname} modify column {key} {v}')
tdSql.query(f'describe {self.ntbname}')
result = tdCom.getOneRow(1,'NCHAR')
tdSql.checkEqual(result[0][2],self.binary_length+1)
else:
for v in self.column_dict.values():
tdSql.error(f'alter table {self.ntbname} modify column {key} {v}')
for key,values in self.column_dict.items():
rename_str = f'{tdCom.getLongName(constant.COL_NAME_LENGTH_MAX,"letters")}'
tdSql.execute(f'alter table {self.ntbname} rename column {key} {rename_str}')
tdSql.query(f'select {rename_str} from {self.ntbname}')
tdSql.checkRows(1)
def alter_check_tb(self):
tag_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX)
tag_smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX)
tag_int = random.randint(constant.INT_MIN,constant.INT_MAX)
tag_bigint = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX)
tag_untinyint = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX)
tag_unsmallint = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX)
tag_unint = random.randint(constant.INT_UN_MIN,constant.INT_MAX)
tag_unbigint = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX)
tag_bool = random.randint(0,100)%2
tag_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX)
tag_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300))
tag_binary = tdCom.getLongName(self.binary_length)
tag_nchar = tdCom.getLongName(self.binary_length)
modify_column_dict = {
'ts1' : 'timestamp',
'c1': 'tinyint',
'c2': 'smallint',
'c3': 'int',
'c4': 'bigint',
'c5': 'tinyint unsigned',
'c6': 'smallint unsigned',
'c7': 'int unsigned',
'c8': 'bigint unsigned',
'c9': 'float',
'c10': 'double',
'c11': 'bool',
'c12': f'binary({self.binary_length})',
'c13': f'nchar({self.nchar_length})'
}
tdSql.prepare()
tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict))
for i in range(self.tbnum):
tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})')
for j in self.values_list:
tdSql.execute(f'insert into {self.stbname}_{i} values({j})')
for i in range(self.tbnum):
for k,v in modify_column_dict.items():
tdSql.error(f'alter table {self.stbname}_{i} add column {k} {v}')
for k in self.column_dict.keys():
tdSql.error(f'alter table {self.stbname}_{i} drop column {k}')
for k,v in self.column_dict.items():
if 'binary' in v.lower():
values = [f'binary({self.binary_length+1})', f'binary({self.binary_length-1})']
for j in values:
tdSql.error(f'alter table {self.stbname}_{i} modify {k} {j}')
elif 'nchar' in v.lower():
values = [f'nchar({self.nchar_length+1})', f'binary({self.nchar_length-1})']
for j in values:
tdSql.error(f'alter table {self.stbname}_{i} modify {k} {j}')
else:
for values in self.column_dict.values():
tdSql.error(f'alter table {self.stbname}_{i} modify column {k} {values}')
for k,v in self.tag_dict.items():
if v.lower() == 'tinyint':
self.tag_check(i,k,tag_tinyint)
elif v.lower() == 'smallint':
self.tag_check(i,k,tag_smallint)
elif v.lower() == 'int':
self.tag_check(i,k,tag_int)
elif v.lower() == 'bigint':
self.tag_check(i,k,tag_bigint)
elif v.lower() == 'tinyint unsigned':
self.tag_check(i,k,tag_untinyint)
elif v.lower() == 'smallint unsigned':
self.tag_check(i,k,tag_unsmallint)
elif v.lower() == 'int unsigned':
self.tag_check(i,k,tag_unint)
elif v.lower() == 'bigint unsigned':
self.tag_check(i,k,tag_unbigint)
elif v.lower() == 'bool':
self.tag_check(i,k,tag_bool)
elif v.lower() == 'float':
tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = {tag_float}')
tdSql.query(f'select {k} from {self.stbname}_{i}')
if abs(tdSql.queryResult[0][0] - tag_float)/tag_float<=0.0001:
tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0])
else:
tdLog.exit(f'select {k} from {self.stbname}_{i},data check failure')
elif v.lower() == 'double':
tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = {tag_double}')
tdSql.query(f'select {k} from {self.stbname}_{i}')
if abs(tdSql.queryResult[0][0] - tag_double)/tag_double<=0.0001:
tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0])
else:
tdLog.exit(f'select {k} from {self.stbname}_{i},data check failure')
elif 'binary' in v.lower():
tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = "{tag_binary}"')
tdSql.query(f'select {k} from {self.stbname}_{i}')
tdSql.checkData(0,0,tag_binary)
elif 'nchar' in v.lower():
tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = "{tag_nchar}"')
tdSql.query(f'select {k} from {self.stbname}_{i}')
tdSql.checkData(0,0,tag_nchar)
def tag_check(self,tb_no,tag,values):
tdSql.execute(f'alter table {self.stbname}_{tb_no} set tag {tag} = {values}')
tdSql.query(f'select {tag} from {self.stbname}_{tb_no}')
tdSql.checkData(0,0,values)
def alter_check_stb(self):
tdSql.prepare()
tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict))
for i in range(self.tbnum):
tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})')
for j in self.values_list:
tdSql.execute(f'insert into {self.stbname}_{i} values({j})')
for key,values in self.column_add_dict.items():
tdSql.execute(f'alter table {self.stbname} add column {key} {values}')
tdSql.query(f'describe {self.stbname}')
tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)+1)
for i in range(self.tbnum):
tdSql.query(f'describe {self.stbname}_{i}')
tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)+1)
tdSql.query(f'select {key} from {self.stbname}_{i}')
tdSql.checkRows(len(self.values_list))
tdSql.execute(f'alter table {self.stbname} drop column {key}')
tdSql.query(f'describe {self.stbname}')
tdSql.checkRows(len(self.column_dict)+len(self.tag_dict))
for i in range(self.tbnum):
tdSql.query(f'describe {self.stbname}_{i}')
tdSql.checkRows(len(self.column_dict)+len(self.tag_dict))
tdSql.error(f'select {key} from {self.stbname} ')
for key,values in self.column_dict.items():
if 'binary' in values.lower():
v = f'binary({self.binary_length+1})'
v_error = f'binary({self.binary_length-1})'
tdSql.error(f'alter table {self.stbname} modify column {key} {v_error}')
tdSql.execute(f'alter table {self.stbname} modify column {key} {v}')
tdSql.query(f'describe {self.stbname}')
result = tdCom.getOneRow(1,'VARCHAR')
tdSql.checkEqual(result[0][2],self.binary_length+1)
for i in range(self.tbnum):
tdSql.query(f'describe {self.stbname}_{i}')
result = tdCom.getOneRow(1,'VARCHAR')
tdSql.checkEqual(result[0][2],self.binary_length+1)
elif 'nchar' in values.lower():
v = f'nchar({self.binary_length+1})'
v_error = f'nchar({self.binary_length-1})'
tdSql.error(f'alter table {self.stbname} modify column {key} {v_error}')
tdSql.execute(f'alter table {self.stbname} modify column {key} {v}')
tdSql.query(f'describe {self.stbname}')
result = tdCom.getOneRow(1,'NCHAR')
tdSql.checkEqual(result[0][2],self.binary_length+1)
for i in range(self.tbnum):
tdSql.query(f'describe {self.stbname}')
result = tdCom.getOneRow(1,'NCHAR')
tdSql.checkEqual(result[0][2],self.binary_length+1)
else:
for v in self.column_dict.values():
tdSql.error(f'alter table {self.stbname} modify column {key} {v}')
# bug TD-16211 insert length more than setting binary and nchar
# error_tag_binary = self.get_long_name(length=21, mode="letters")
# error_tag_nchar = self.get_long_name(length=21, mode="letters")
# tdSql.error(f'alter table {dbname}.{tbname} set tag t12 = "{error_tag_binary}"')
# tdSql.error(f'alter table {dbname}.{tbname} set tag t13 = "{error_tag_nchar}"')
error_tag_binary = self.get_long_name(length=25, mode="letters")
error_tag_nchar = self.get_long_name(length=25, mode="letters")
tdSql.error(f'alter table {dbname}.{tbname} set tag t12 = "{error_tag_binary}"')
tdSql.error(f'alter table {dbname}.{tbname} set tag t13 = "{error_tag_nchar}"')
# bug TD-16210 modify binary to nchar
tdSql.error(f'alter table {dbname}.{tbname} modify tag t12 nchar(10)')
tdSql.execute(f"drop database {dbname}")
def alter_ntb_column_check(self):
'''
alter ntb column check
'''
dbname = self.get_long_name(length=10, mode="letters")
tdSql.execute(f'create database if not exists {dbname}')
tbname = self.get_long_name(length=3, mode="letters")
tdLog.info('------------------normal table column check---------------------')
tdLog.info(f'-----------------create normal table {tbname}-------------------')
tdSql.execute(f'create table if not exists {dbname}.{tbname} (ts timestamp, c1 tinyint, c2 smallint, c3 int, \
c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 bool,c12 binary(20),c13 nchar(20))')
tdSql.execute(f'insert into {dbname}.{tbname} values (now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")')
# bug TD-15757
tdSql.execute(f'alter table {dbname}.{tbname} add column c14 int')
tdSql.query(f'select c14 from {dbname}.{tbname}')
tdSql.checkRows(1)
tdSql.execute(f'alter table {dbname}.{tbname} add column `c15` int')
tdSql.query(f'select c15 from {dbname}.{tbname}')
tdSql.checkRows(1)
tdSql.query(f'describe {dbname}.{tbname}')
tdSql.checkRows(16)
tdSql.execute(f'alter table {dbname}.{tbname} drop column c14')
tdSql.query(f'describe {dbname}.{tbname}')
tdSql.checkRows(15)
tdSql.execute(f'alter table {dbname}.{tbname} drop column `c15`')
tdSql.query(f'describe {dbname}.{tbname}')
tdSql.checkRows(14)
#! TD-16422
# tdSql.execute(f'alter table {dbname}.{tbname} add column c16 binary(10)')
# tdSql.query(f'describe {dbname}.{tbname}')
# tdSql.checkRows(15)
# tdSql.checkEqual(tdSql.queryResult[14][2],10)
# tdSql.execute(f'alter table {dbname}.{tbname} drop column c16')
# tdSql.execute(f'alter table {dbname}.{tbname} add column c16 nchar(10)')
# tdSql.query(f'describe {dbname}.{tbname}')
# tdSql.checkRows(15)
# tdSql.checkEqual(tdSql.queryResult[14][2],10)
# tdSql.execute(f'alter table {dbname}.{tbname} drop column c16')
tdSql.execute(f'alter table {dbname}.{tbname} modify column c12 binary(30)')
tdSql.query(f'describe {dbname}.{tbname}')
tdSql.checkData(12,2,30)
tdSql.execute(f'alter table {dbname}.{tbname} modify column `c12` binary(35)')
tdSql.query(f'describe {dbname}.{tbname}')
tdSql.checkData(12,2,35)
tdSql.error(f'alter table {dbname}.{tbname} modify column c12 binary(34)')
tdSql.error(f'alter table {dbname}.{tbname} modify column c12 nchar(10)')
tdSql.error(f'alter table {dbname}.{tbname} modify column c12 int')
tdSql.execute(f'alter table {dbname}.{tbname} modify column c13 nchar(30)')
tdSql.query(f'describe {dbname}.{tbname}')
tdSql.checkData(13,2,30)
tdSql.execute(f'alter table {dbname}.{tbname} modify column `c13` nchar(35)')
tdSql.query(f'describe {dbname}.{tbname}')
tdSql.checkData(13,2,35)
tdSql.error(f'alter table {dbname}.{tbname} modify column c13 nchar(34)')
tdSql.error(f'alter table {dbname}.{tbname} modify column c13 binary(10)')
tdSql.execute(f'alter table {dbname}.{tbname} rename column c1 c21')
tdSql.query(f'describe {dbname}.{tbname}')
tdSql.checkData(1,0,'c21')
# !bug TD-16423
# tdSql.error(f'select c1 from {dbname}.{tbname}')
# tdSql.query(f'select c21 from {dbname}.{tbname}')
# tdSql.checkData(0,1,1)
tdSql.execute(f'alter table {dbname}.{tbname} rename column `c21` c1')
tdSql.query(f'describe {dbname}.{tbname}')
tdSql.checkData(1,0,'c1')
# !bug TD-16423
# tdSql.error(f'select c1 from {dbname}.{tbname}')
# tdSql.query(f'select c1 from {dbname}.{tbname}')
# tdSql.checkData(0,1,1)
tdSql.error(f'alter table {dbname}.{tbname} modify column c1 bigint')
tdSql.error(f'alter table {dbname}.{tbname} modify column c1 double')
tdSql.error(f'alter table {dbname}.{tbname} modify column c4 int')
tdSql.error(f'alter table {dbname}.{tbname} modify column `c1` double')
tdSql.error(f'alter table {dbname}.{tbname} modify column c9 double')
tdSql.error(f'alter table {dbname}.{tbname} modify column c10 float')
tdSql.error(f'alter table {dbname}.{tbname} modify column c1 bool')
tdSql.error(f'alter table {dbname}.{tbname} modify column c1 binary(10)')
tdSql.execute(f'drop database {dbname}')
def alter_stb_column_check(self):
dbname = self.get_long_name(length=10, mode="letters")
tdSql.execute(f'create database if not exists {dbname}')
stbname = self.get_long_name(length=3, mode="letters")
tbname = self.get_long_name(length=3, mode="letters")
tdSql.execute(f'create database if not exists {dbname}')
tdSql.execute(f'use {dbname}')
tdSql.execute(
f'create table {stbname} (ts timestamp, c1 tinyint, c2 smallint, c3 int, \
c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 bool,c12 binary(20),c13 nchar(20)) tags(t0 int) ')
tdSql.execute(f'create table {tbname} using {stbname} tags(1)')
tdSql.execute(f'insert into {tbname} values (now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")')
tdSql.execute(f'alter table {stbname} add column c14 int')
tdSql.query(f'select c14 from {stbname}')
tdSql.checkRows(1)
tdSql.execute(f'alter table {stbname} add column `c15` int')
tdSql.query(f'select c15 from {stbname}')
tdSql.checkRows(1)
tdSql.query(f'describe {stbname}')
tdSql.checkRows(17)
tdSql.execute(f'alter table {stbname} drop column c14')
tdSql.query(f'describe {stbname}')
tdSql.checkRows(16)
tdSql.execute(f'alter table {stbname} drop column `c15`')
tdSql.query(f'describe {stbname}')
tdSql.checkRows(15)
tdSql.execute(f'alter table {stbname} modify column c12 binary(30)')
tdSql.query(f'describe {stbname}')
tdSql.checkData(12,2,30)
tdSql.execute(f'alter table {stbname} modify column `c12` binary(35)')
tdSql.query(f'describe {stbname}')
tdSql.checkData(12,2,35)
tdSql.error(f'alter table {stbname} modify column `c12` binary(34)')
tdSql.execute(f'alter table {stbname} modify column c13 nchar(30)')
tdSql.query(f'describe {stbname}')
tdSql.checkData(13,2,30)
tdSql.error(f'alter table {stbname} modify column c13 nchar(29)')
tdSql.error(f'alter table {stbname} rename column c1 c21')
tdSql.error(f'alter table {stbname} modify column c1 int')
tdSql.error(f'alter table {stbname} modify column c4 int')
tdSql.error(f'alter table {stbname} modify column c8 int')
tdSql.error(f'alter table {stbname} modify column c1 unsigned int')
tdSql.error(f'alter table {stbname} modify column c9 double')
tdSql.error(f'alter table {stbname} modify column c10 float')
tdSql.error(f'alter table {stbname} modify column c11 int')
tdSql.execute(f'drop database {dbname}')
def alter_stb_tag_check(self):
dbname = self.get_long_name(length=10, mode="letters")
tdSql.execute(f'create database if not exists {dbname}')
stbname = self.get_long_name(length=3, mode="letters")
tbname = self.get_long_name(length=3, mode="letters")
tdSql.execute(f'create database if not exists {dbname}')
tdSql.execute(f'use {dbname}')
tdSql.execute(
f'create table {stbname} (ts timestamp, c1 int) tags(ts_tag timestamp, t1 tinyint, t2 smallint, t3 int, \
t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 bool,t12 binary(20),t13 nchar(20)) ')
tdSql.execute(f'create table {tbname} using {stbname} tags(now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")')
tdSql.execute(f'insert into {tbname} values(now,1)')
tdSql.execute(f'alter table {stbname} add tag t14 int')
tdSql.query(f'select t14 from {stbname}')
tdSql.checkRows(1)
tdSql.execute(f'alter table {stbname} add tag `t15` int')
tdSql.query(f'select t14 from {stbname}')
tdSql.checkRows(1)
tdSql.query(f'describe {stbname}')
tdSql.checkRows(18)
tdSql.execute(f'alter table {stbname} drop tag t14')
tdSql.query(f'describe {stbname}')
tdSql.checkRows(17)
tdSql.execute(f'alter table {stbname} drop tag `t15`')
tdSql.query(f'describe {stbname}')
tdSql.checkRows(16)
tdSql.execute(f'alter table {stbname} modify tag t12 binary(30)')
tdSql.query(f'describe {stbname}')
tdSql.checkData(14,2,30)
tdSql.execute(f'alter table {stbname} modify tag `t12` binary(35)')
tdSql.query(f'describe {stbname}')
tdSql.checkData(14,2,35)
tdSql.error(f'alter table {stbname} modify tag `t12` binary(34)')
tdSql.execute(f'alter table {stbname} modify tag t13 nchar(30)')
tdSql.query(f'describe {stbname}')
tdSql.checkData(15,2,30)
tdSql.error(f'alter table {stbname} modify tag t13 nchar(29)')
tdSql.execute(f'alter table {stbname} rename tag t1 t21')
tdSql.query(f'describe {stbname}')
tdSql.checkData(3,0,'t21')
tdSql.execute(f'alter table {stbname} rename tag `t21` t1')
tdSql.query(f'describe {stbname}')
tdSql.checkData(3,0,'t1')
for i in ['bigint','unsigned int','float','double','binary(10)','nchar(10)']:
for j in [1,2,3]:
tdSql.error(f'alter table {stbname} modify tag t{j} {i}')
for i in ['int','unsigned int','float','binary(10)','nchar(10)']:
tdSql.error(f'alter table {stbname} modify tag t8 {i}')
tdSql.error(f'alter table {stbname} modify tag t4 int')
tdSql.execute(f'drop database {dbname}')
def run(self):
self.alter_tb_tag_check()
self.alter_ntb_column_check()
self.alter_stb_column_check()
self.alter_stb_tag_check()
self.alter_check_ntb()
self.alter_check_tb()
self.alter_check_stb()
def stop(self):
tdSql.close()

80
tests/system-test/1-insert/create_retentions.py
View File

@ -21,9 +21,9 @@ SINT_UN_COL = "c_sint_un"
BINT_UN_COL = "c_bint_un"
INT_UN_COL = "c_int_un"
BINARY_COL = "c8"
NCHAR_COL = "c9"
TS_COL = "c10"
BINARY_COL = "c_binary"
NCHAR_COL = "c_nchar"
TS_COL = "c_ts"
NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
CHAR_COL = [ BINARY_COL, NCHAR_COL, ]
@ -51,12 +51,28 @@ class DataSet:
binary_data : List[str] = None
nchar_data : List[str] = None
def __post_init__(self):
self.ts_data = []
self.int_data = []
self.bint_data = []
self.sint_data = []
self.tint_data = []
self.int_un_data = []
self.bint_un_data = []
self.sint_un_data = []
self.tint_un_data = []
self.float_data = []
self.double_data = []
self.bool_data = []
self.binary_data = []
self.nchar_data = []
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
tdSql.init(conn.cursor(), False)
@property
def create_databases_sql_err(self):
@ -87,28 +103,28 @@ class TDTestCase:
@property
def create_stable_sql_err(self):
return [
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) watermark 1s maxdelay 1m",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) watermark 1s max_delay 1m",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(count) watermark 1min",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay -1s",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay -1s",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark -1m",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 1m ",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) maxdelay 1m ",
# f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 1m ",
# f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) max_delay 1m ",
f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} binary(16)) tags (tag1 int) rollup(avg) watermark 1s",
f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) rollup(avg) maxdelay 1m",
# f"create table ntb_1 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) rollup(avg) watermark 1s maxdelay 1s",
f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) rollup(avg) max_delay 1m",
# f"create table ntb_1 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) rollup(avg) watermark 1s max_delay 1s",
# f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) " ,
# f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) " ,
# f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) " ,
# f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) " ,
# watermark, maxdelay: [0, 900000], [ms, s, m, ?]
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 1u",
# watermark, max_delay: [0, 900000], [ms, s, m, ?]
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1u",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 1b",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 900001ms",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 16m",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 901s",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 1h",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 0.2h",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 16m",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 901s",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1h",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 0.2h",
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 0.002d",
]
@ -117,11 +133,11 @@ class TDTestCase:
def create_stable_sql_current(self):
return [
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(avg)",
f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 5s maxdelay 1m",
f"create stable stb3 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(max) watermark 5s maxdelay 1m",
f"create stable stb4 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum) watermark 5s maxdelay 1m",
# f"create stable stb5 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(last) watermark 5s maxdelay 1m",
# f"create stable stb6 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s maxdelay 1m",
f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 5s max_delay 1m",
f"create stable stb3 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(max) watermark 5s max_delay 1m",
f"create stable stb4 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum) watermark 5s max_delay 1m",
# f"create stable stb5 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(last) watermark 5s max_delay 1m",
# f"create stable stb6 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s max_delay 1m",
]
def test_create_stb(self):
@ -135,7 +151,7 @@ class TDTestCase:
tdSql.checkRows(len(self.create_stable_sql_current))
# tdSql.execute("use db") # because db is a noraml database, not a rollup database, should not be able to create a rollup database
# tdSql.error(f"create stable nor_db_rollup_stb ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) file_factor 5.0")
# tdSql.error(f"create stable nor_db_rollup_stb ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 5s max_delay 1m")
def test_create_databases(self):
@ -177,21 +193,6 @@ class TDTestCase:
def __data_set(self, rows):
data_set = DataSet()
# neg_data_set = DataSet()
data_set.ts_data = []
data_set.int_data = []
data_set.bint_data = []
data_set.sint_data = []
data_set.tint_data = []
data_set.int_un_data = []
data_set.bint_un_data = []
data_set.sint_un_data = []
data_set.tint_un_data = []
data_set.float_data = []
data_set.double_data = []
data_set.bool_data = []
data_set.binary_data = []
data_set.nchar_data = []
for i in range(rows):
data_set.ts_data.append(NOW + 1 * (rows - i))
@ -226,6 +227,7 @@ class TDTestCase:
return data_set
def __insert_data(self):
tdLog.printNoPrefix("==========step: start inser data into tables now.....")
data = self.__data_set(rows=self.rows)
# now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
@ -264,10 +266,10 @@ class TDTestCase:
def run(self):
self.rows = 10
tdSql.prepare()
tdLog.printNoPrefix("==========step0:all check")
# self.all_test()
self.all_test()
tdLog.printNoPrefix("==========step1:create table in normal database")
tdSql.prepare()

417
tests/system-test/1-insert/time_range_wise.py
View File

@ -17,25 +17,34 @@ TINT_COL = "c_tint"
FLOAT_COL = "c_float"
DOUBLE_COL = "c_double"
BOOL_COL = "c_bool"
TINT_UN_COL = "c_tint_un"
SINT_UN_COL = "c_sint_un"
BINT_UN_COL = "c_bint_un"
INT_UN_COL = "c_int_un"
TINT_UN_COL = "c_utint"
SINT_UN_COL = "c_usint"
BINT_UN_COL = "c_ubint"
INT_UN_COL = "c_uint"
BINARY_COL = "c_binary"
NCHAR_COL = "c_nchar"
TS_COL = "c_ts"
NUM_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
CHAR_COL = [BINARY_COL, NCHAR_COL, ]
BOOLEAN_COL = [BOOL_COL, ]
TS_TYPE_COL = [TS_COL, ]
INT_TAG = "t_int"
ALL_COL = [PRIMARY_COL, INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BINARY_COL, NCHAR_COL, BOOL_COL, TS_COL]
TAG_COL = [INT_TAG]
# insert data args：
TIME_STEP = 10000
NOW = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
# init db/table
DBNAME = "db"
STBNAME = "stb1"
CTBNAME = "ct1"
NTBNAME = "nt1"
@dataclass
class DataSet:
@ -73,19 +82,25 @@ class DataSet:
@dataclass
class SMAschema:
creation : str = "CREATE"
index_name : str = "sma_index_1"
index_flag : str = "SMA INDEX"
operator : str = "ON"
tbname : str = None
watermark : str = None
maxdelay : str = None
func : Tuple[str] = None
interval : Tuple[str] = None
sliding : str = None
other : Any = None
drop : str = "DROP"
drop_flag : str = "INDEX"
creation : str = "CREATE"
index_name : str = "sma_index_1"
index_flag : str = "SMA INDEX"
operator : str = "ON"
tbname : str = None
watermark : str = "5s"
max_delay : str = "6m"
func : Tuple[str] = None
interval : Tuple[str] = ("6m", "10s")
sliding : str = "6m"
other : Any = None
drop : str = "DROP"
drop_flag : str = "INDEX"
querySmaOptimize : int = 1
show : str = "SHOW"
show_msg : str = "INDEXES"
show_oper : str = "FROM"
dbname : str = None
rollup_db : bool = False
def __post_init__(self):
if isinstance(self.other, dict):
@ -111,8 +126,8 @@ class SMAschema:
self.watermark = v
del self.other[k]
if k.lower() == "maxdelay" and isinstance(v, str) and not self.maxdelay:
self.maxdelay = v
if k.lower() == "max_delay" and isinstance(v, str) and not self.max_delay:
self.max_delay = v
del self.other[k]
if k.lower() == "functions" and isinstance(v, tuple) and not self.func:
@ -131,12 +146,36 @@ class SMAschema:
self.drop_flag = v
del self.other[k]
if k.lower() == "show_msg" and isinstance(v, str) and not self.show_msg:
self.show_msg = v
del self.other[k]
if k.lower() == "dbname" and isinstance(v, str) and not self.dbname:
self.dbname = v
del self.other[k]
if k.lower() == "show_oper" and isinstance(v, str) and not self.show_oper:
self.show_oper = v
del self.other[k]
if k.lower() == "rollup_db" and isinstance(v, bool) and not self.rollup_db:
self.rollup_db = v
del self.other[k]
# from ...pytest.util.sql import *
# from ...pytest.util.constant import *
class TDTestCase:
updatecfgDict = {"querySmaOptimize": 1}
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), False)
self.precision = "ms"
self.sma_count = 0
self.sma_created_index = []
"""
create sma index :
@ -155,13 +194,17 @@ class TDTestCase:
if sma.func:
sql += f" function({', '.join(sma.func)})"
if sma.interval:
sql += f" interval({', '.join(sma.interval)})"
interval, offset = self.__get_interval_offset(sma.interval)
if offset:
sql += f" interval({interval}, {offset})"
else:
sql += f" interval({interval})"
if sma.sliding:
sql += f" sliding({sma.sliding})"
if sma.watermark:
sql += f" watermark {sma.watermark}"
if sma.maxdelay:
sql += f" maxdelay {sma.maxdelay}"
if sma.max_delay:
sql += f" max_delay {sma.max_delay}"
if isinstance(sma.other, dict):
for k,v in sma.other.items():
if isinstance(v,tuple) or isinstance(v, list):
@ -171,53 +214,305 @@ class TDTestCase:
if isinstance(sma.other, tuple) or isinstance(sma.other, list):
sql += " ".join(sma.other)
if isinstance(sma.other, int) or isinstance(sma.other, float) or isinstance(sma.other, str):
sql += sma.other
sql += f" {sma.other}"
return sql
def sma_create_check(self, sma:SMAschema):
def __get_sma_func_col(self, func):
cols = []
if isinstance(func, str):
cols.append( func.split("(")[-1].split(")")[0] )
elif isinstance(func, tuple) or isinstance(func, list):
for func_col in func:
cols.append(func_col.split("(")[-1].split(")")[0])
else:
cols = []
return cols
def __check_sma_func(self, func:tuple):
if not isinstance(func, str) and not isinstance(func, tuple) and not isinstance(func, list):
return False
if isinstance(func, str) :
if "(" not in func or ")" not in func:
return False
if func.split("(")[0].upper() not in SMA_INDEX_FUNCTIONS:
return False
if func.split("(")[1].split(")")[0] not in ALL_COL and func.split("(")[1].split(")")[0] not in TAG_COL :
return False
if isinstance(func, tuple) or isinstance(func, list):
for arg in func:
if not isinstance(arg, str):
return False
if "(" not in arg or ")" not in arg:
return False
if arg.split("(")[0].upper() not in SMA_INDEX_FUNCTIONS:
return False
if arg.split("(")[1].split(")")[0] not in ALL_COL and arg.split("(")[1].split(")")[0] not in TAG_COL :
return False
return True
def __check_sma_watermark(self, arg):
if not arg:
return False
if not isinstance(arg, str):
return False
if arg[-1] not in SMA_WATMARK_MAXDELAY_INIT:
return False
if len(arg) == 1:
return False
if not arg[:-1].isdecimal():
return False
if tdSql.get_times(arg) > WATERMARK_MAX:
return False
if tdSql.get_times(arg) < WATERMARK_MIN:
return False
return True
def __check_sma_max_delay(self, arg):
if not self.__check_sma_watermark(arg):
return False
if tdSql.get_times(arg) < MAX_DELAY_MIN:
return False
return True
def __check_sma_sliding(self, arg):
if not isinstance(arg, str):
return False
if arg[-1] not in TAOS_TIME_INIT:
return False
if len(arg) == 1:
return False
if not arg[:-1].isdecimal():
return False
return True
def __get_interval_offset(self, args):
if isinstance(args, str):
interval, offset = args, None
elif isinstance(args,tuple) or isinstance(args, list):
if len(args) == 1:
interval, offset = args[0], None
elif len(args) == 2:
interval, offset = args
else:
interval, offset = False, False
else:
interval, offset = False, False
return interval, offset
def __check_sma_interval(self, args):
if not isinstance(args, tuple) and not isinstance(args,str):
return False
interval, offset = self.__get_interval_offset(args)
if not interval:
return False
if not self.__check_sma_sliding(interval):
return False
if tdSql.get_times(interval) < INTERVAL_MIN:
return False
if offset:
if not self.__check_sma_sliding(offset):
return False
if tdSql.get_times(interval) <= tdSql.get_times(offset) :
return False
return True
def __sma_create_check(self, sma:SMAschema):
if self.updatecfgDict["querySmaOptimize"] == 0:
return False
# # TODO: if database is a rollup-db, can not create sma index
# tdSql.query("select database()")
# if sma.rollup_db :
# return False
tdSql.query("show stables")
if not sma.tbname:
return False
stb_in_list = False
for row in tdSql.queryResult:
if sma.tbname == row[0]:
stb_in_list = True
break
if not stb_in_list:
tdSql.error(self.__create_sma_index(sma))
if not sma.creation:
tdSql.error(self.__create_sma_index(sma))
if not sma.index_flag:
tdSql.error(self.__create_sma_index(sma))
if not sma.index_name:
tdSql.error(self.__create_sma_index(sma))
if not sma.operator:
tdSql.error(self.__create_sma_index(sma))
if not sma.tbname:
tdSql.error(self.__create_sma_index(sma))
if not sma.func:
tdSql.error(self.__create_sma_index(sma))
if not sma.interval:
tdSql.error(self.__create_sma_index(sma))
if not sma.sliding:
tdSql.error(self.__create_sma_index(sma))
return False
if not sma.creation or not isinstance(sma.creation, str) or sma.creation.upper() != "CREATE":
return False
if not sma.index_flag or not isinstance(sma.index_flag, str) or sma.index_flag.upper() != "SMA INDEX" :
return False
if not sma.index_name or not isinstance(sma.index_name, str) or sma.index_name.upper() in TAOS_KEYWORDS:
return False
if not sma.operator or not isinstance(sma.operator, str) or sma.operator.upper() != "ON":
return False
if not sma.func or not self.__check_sma_func(sma.func):
return False
tdSql.query(f"desc {sma.tbname}")
_col_list = []
for col_row in tdSql.queryResult:
_col_list.append(col_row[0])
_sma_func_cols = self.__get_sma_func_col(sma.func)
for _sma_func_col in _sma_func_cols:
if _sma_func_col not in _col_list:
return False
if sma.sliding and not self.__check_sma_sliding(sma.sliding):
return False
interval, _ = self.__get_interval_offset(sma.interval)
if not sma.interval or not self.__check_sma_interval(sma.interval) :
return False
if sma.sliding and tdSql.get_times(interval) < tdSql.get_times(sma.sliding):
return False
if sma.watermark and not self.__check_sma_watermark(sma.watermark):
return False
if sma.max_delay and not self.__check_sma_max_delay(sma.max_delay):
return False
if sma.other:
return False
return True
def sma_create_check(self, sma:SMAschema):
if self.__sma_create_check(sma):
tdSql.query(self.__create_sma_index(sma))
self.sma_count += 1
self.sma_created_index.append(sma.index_name)
tdSql.query("show streams")
tdSql.checkRows(self.sma_count)
else:
tdSql.error(self.__create_sma_index(sma))
def __drop_sma_index(self, sma:SMAschema):
sql = f"{sma.drop} {sma.drop_flag} {sma.index_name}"
return sql
def __sma_drop_check(self, sma:SMAschema):
if not sma.drop:
return False
if not sma.drop_flag:
return False
if not sma.index_name:
return False
return True
def sma_drop_check(self, sma:SMAschema):
if self.__sma_drop_check(sma):
tdSql.query(self.__drop_sma_index(sma))
print(self.__drop_sma_index(sma))
self.sma_count -= 1
self.sma_created_index = list(filter(lambda x: x != sma.index_name, self.sma_created_index))
tdSql.query("show streams")
tdSql.checkRows(self.sma_count)
else:
tdSql.error(self.__drop_sma_index(sma))
def __show_sma_index(self, sma:SMAschema):
sql = f"{sma.show} {sma.show_msg} {sma.show_oper} {sma.tbname}"
return sql
def __sma_show_check(self, sma:SMAschema):
if not sma.show:
return False
if not sma.show_msg:
return False
if not sma.show_oper:
return False
if not sma.tbname:
return False
return True
def sma_show_check(self, sma:SMAschema):
if self.__sma_show_check(sma):
tdSql.query(self.__show_sma_index(sma))
tdSql.checkRows(self.sma_count)
else:
tdSql.error(self.__show_sma_index(sma))
@property
def __create_sma_sql(self):
err_sqls = []
cur_sqls = []
# err_set
# # case 1: required fields check
err_sqls.append( SMAschema(creation="", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
err_sqls.append( SMAschema(index_name="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
err_sqls.append( SMAschema(index_flag="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
err_sqls.append( SMAschema(operator="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
err_sqls.append( SMAschema(tbname="", func=(f"min({INT_COL})",f"max({INT_COL})") ) )
err_sqls.append( SMAschema(func=("",),tbname=STBNAME ) )
err_sqls.append( SMAschema(interval=(""),tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
# # case 2: err fields
err_sqls.append( SMAschema(creation="show",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
err_sqls.append( SMAschema(creation="alter",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
err_sqls.append( SMAschema(creation="select",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
err_sqls.append( SMAschema(index_flag="SMA INDEXES", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
err_sqls.append( SMAschema(index_flag="SMA INDEX ,", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
err_sqls.append( SMAschema(index_name="tbname", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
# current_set
cur_sqls.append( SMAschema(max_delay="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
cur_sqls.append( SMAschema(watermark="",index_name="sma_index_2",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
cur_sqls.append( SMAschema(sliding="",index_name='sma_index_3',tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) )
return err_sqls, cur_sqls
def test_create_sma(self):
err_sqls , cur_sqls = self.__create_sma_sql
for err_sql in err_sqls:
self.sma_create_check(err_sql)
for cur_sql in cur_sqls:
self.sma_create_check(cur_sql)
@property
def __drop_sma_sql(self):
err_sqls = []
cur_sqls = []
# err_set
## case 1: required fields check
err_sqls.append( SMAschema(drop="") )
err_sqls.append( SMAschema(drop_flag="") )
err_sqls.append( SMAschema(index_name="") )
for index in self.sma_created_index:
cur_sqls.append(SMAschema(index_name=index))
return err_sqls, cur_sqls
def test_drop_sma(self):
err_sqls , cur_sqls = self.__drop_sma_sql
for err_sql in err_sqls:
self.sma_drop_check(err_sql)
# for cur_sql in cur_sqls:
# self.sma_drop_check(cur_sql)
def all_test(self):
self.test_create_sma()
self.test_drop_sma()
pass
def __create_tb(self):
tdLog.printNoPrefix("==========step: create table")
create_stb_sql = f'''create table stb1(
create_stb_sql = f'''create table {STBNAME}(
ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
{FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
{BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp,
{TINT_UN_COL} tinyint unsigned, {SINT_UN_COL} smallint unsigned,
{INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned
) tags (tag1 int)
) tags ({INT_TAG} int)
'''
create_ntb_sql = f'''create table t1(
create_ntb_sql = f'''create table {NTBNAME}(
ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
{FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
{BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp,
@ -253,6 +548,7 @@ class TDTestCase:
return data_set
def __insert_data(self):
tdLog.printNoPrefix("==========step: start inser data into tables now.....")
data = self.__data_set(rows=self.rows)
# now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
@ -278,7 +574,7 @@ class TDTestCase:
tdSql.execute(
f"insert into ct4 values ( {NOW - i * int(TIME_STEP * 0.8) }, {row_data} )")
tdSql.execute(
f"insert into t1 values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )")
f"insert into {NTBNAME} values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )")
tdSql.execute(
f"insert into ct2 values ( {NOW + int(TIME_STEP * 0.6)}, {null_data} )")
@ -295,28 +591,31 @@ class TDTestCase:
f"insert into ct4 values ( {NOW - self.rows * int(TIME_STEP * 0.39)}, {null_data} )")
tdSql.execute(
f"insert into t1 values ( {NOW + int(TIME_STEP * 1.2)}, {null_data} )")
f"insert into {NTBNAME} values ( {NOW + int(TIME_STEP * 1.2)}, {null_data} )")
tdSql.execute(
f"insert into t1 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )")
f"insert into {NTBNAME} values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )")
tdSql.execute(
f"insert into t1 values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )")
f"insert into {NTBNAME} values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )")
def run(self):
sma1 = SMAschema(func=("min(c1)","max(c2)"))
sql1 = self.__create_sma_index(sma1)
print("================")
print(sql1)
# a = DataSet()
# return
self.rows = 10
tdLog.printNoPrefix("==========step0:all check")
# self.all_test()
tdLog.printNoPrefix("==========step1:create table in normal database")
tdSql.prepare()
self.__create_tb()
self.__insert_data()
# self.__insert_data()
self.all_test()
# drop databases, create same name db、stb and sma index
# tdSql.prepare()
# self.__create_tb()
# self.__insert_data()
# self.all_test()
return
tdLog.printNoPrefix("==========step2:create table in rollup database")

39
tests/system-test/2-query/abs.py
View File

@ -23,7 +23,6 @@ class TDTestCase:
self.time_step = 1000
def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ):
tdLog.info(" prepare datas for auto check abs function ")
tdSql.execute(" create database test ")
@ -36,7 +35,7 @@ class TDTestCase:
ts = self.ts
for row in range(rownums):
ts += time_step*row
ts = self.ts + time_step*row
c1 = random.randint(0,10000)
c2 = random.randint(0,100000)
c3 = random.randint(0,125)
@ -538,25 +537,41 @@ class TDTestCase:
# tdSql.query(" select sum(c1) from stb1 where t1+10 >1; ") #　taosd crash
tdSql.query("select c1 ,t1 from stb1 where t1 =0 ")
tdSql.checkRows(13)
# tdSql.query("select t1 from stb1 where t1 >0 ")
# tdSql.checkRows(3)
tdSql.query("select t1 from stb1 where t1 >0 ")
tdSql.checkRows(3)
tdSql.query("select t1 from stb1 where t1 =3 ")
tdSql.checkRows(1)
# tdSql.query("select sum(t1) from (select c1 ,t1 from stb1)")
# tdSql.checkData(0,0,61)
# tdSql.query("select distinct(c1) ,t1 from stb1")
# tdSql.checkRows(20)
# tdSql.query("select max(t2) , t1 ,c1, t2 from stb1")
# tdSql.checkData(0,3,33333)
tdSql.query("select max(t2) , t1 ,c1, t2 from stb1")
tdSql.checkData(0,3,33333)
# tag filter with abs function
# tdSql.query("select t1 from stb1 where abs(t1)=1")
# tdSql.checkRows(1)
tdSql.query("select t1 from stb1 where abs(t1)=1")
tdSql.checkRows(1)
tdSql.query("select t1 from stb1 where abs(c1+t1)=1")
tdSql.checkRows(1)
# tdSql.query("select t1 from stb1 where abs(t1+c1)=1")
# tdSql.checkRows(1)
tdSql.checkData(0,0,0)
tdSql.query(
"select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1")
def support_super_table_test(self):
tdSql.execute(" use testdb ")
self.check_result_auto( " select c1 from stb1 order by ts " , "select abs(c1) from stb1 order by ts" )
self.check_result_auto( " select c1 from stb1 order by tbname " , "select abs(c1) from stb1 order by tbname" )
self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto( " select t1,c1 from stb1 order by ts " , "select t1, abs(c1) from stb1 order by ts" )
self.check_result_auto( " select t2,c1 from stb1 order by tbname " , "select t2 ,abs(c1) from stb1 order by tbname" )
self.check_result_auto( " select t3,c1 from stb1 where c1 > 0 order by tbname " , "select t3 ,abs(c1) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto( " select t4,c1 from stb1 where c1 > 0 order by tbname " , "select t4 , abs(c1) from stb1 where c1 > 0 order by tbname" )
pass
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
@ -593,6 +608,10 @@ class TDTestCase:
self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step)
tdLog.printNoPrefix("==========step8: check abs result of stable query ============")
self.support_super_table_test()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")

22
tests/system-test/2-query/arccos.py
View File

@ -479,6 +479,20 @@ class TDTestCase:
tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2))
self.check_result_auto_acos("select num1,num2 from tb3;" , "select acos(num1),acos(num2) from tb3")
def support_super_table_test(self):
tdSql.execute(" use db ")
self.check_result_auto_acos( " select c5 from stb1 order by ts " , "select acos(c5) from stb1 order by ts" )
self.check_result_auto_acos( " select c5 from stb1 order by tbname " , "select acos(c5) from stb1 order by tbname" )
self.check_result_auto_acos( " select c5 from stb1 where c1 > 0 order by tbname " , "select acos(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_acos( " select c5 from stb1 where c1 > 0 order by tbname " , "select acos(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_acos( " select t1,c5 from stb1 order by ts " , "select acos(t1), acos(c5) from stb1 order by ts" )
self.check_result_auto_acos( " select t1,c5 from stb1 order by tbname " , "select acos(t1) ,acos(c5) from stb1 order by tbname" )
self.check_result_auto_acos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select acos(t1) ,acos(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_acos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select acos(t1) , acos(c5) from stb1 where c1 > 0 order by tbname" )
pass
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
@ -512,6 +526,14 @@ class TDTestCase:
self.abs_func_filter()
tdLog.printNoPrefix("==========step7: acos filter query ============")
self.abs_func_filter()
tdLog.printNoPrefix("==========step8: check acos result of stable query ============")
self.support_super_table_test()
def stop(self):
tdSql.close()

18
tests/system-test/2-query/arcsin.py
View File

@ -479,6 +479,20 @@ class TDTestCase:
tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2))
self.check_result_auto_asin("select num1,num2 from tb3;" , "select asin(num1),asin(num2) from tb3")
def support_super_table_test(self):
tdSql.execute(" use db ")
self.check_result_auto_asin( " select c5 from stb1 order by ts " , "select asin(c5) from stb1 order by ts" )
self.check_result_auto_asin( " select c5 from stb1 order by tbname " , "select asin(c5) from stb1 order by tbname" )
self.check_result_auto_asin( " select c5 from stb1 where c1 > 0 order by tbname " , "select asin(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_asin( " select c5 from stb1 where c1 > 0 order by tbname " , "select asin(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_asin( " select t1,c5 from stb1 order by ts " , "select asin(t1), asin(c5) from stb1 order by ts" )
self.check_result_auto_asin( " select t1,c5 from stb1 order by tbname " , "select asin(t1) ,asin(c5) from stb1 order by tbname" )
self.check_result_auto_asin( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select asin(t1) ,asin(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_asin( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select asin(t1) , asin(c5) from stb1 where c1 > 0 order by tbname" )
pass
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
@ -512,6 +526,10 @@ class TDTestCase:
self.abs_func_filter()
tdLog.printNoPrefix("==========step8: check asin result of stable query ============")
self.support_super_table_test()
def stop(self):
tdSql.close()

18
tests/system-test/2-query/arctan.py
View File

@ -476,6 +476,20 @@ class TDTestCase:
tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2))
self.check_result_auto_atan("select num1,num2 from tb3;" , "select atan(num1),atan(num2) from tb3")
def support_super_table_test(self):
tdSql.execute(" use db ")
self.check_result_auto_atan( " select c5 from stb1 order by ts " , "select atan(c5) from stb1 order by ts" )
self.check_result_auto_atan( " select c5 from stb1 order by tbname " , "select atan(c5) from stb1 order by tbname" )
self.check_result_auto_atan( " select c5 from stb1 where c1 > 0 order by tbname " , "select atan(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_atan( " select c5 from stb1 where c1 > 0 order by tbname " , "select atan(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_atan( " select t1,c5 from stb1 order by ts " , "select atan(t1), atan(c5) from stb1 order by ts" )
self.check_result_auto_atan( " select t1,c5 from stb1 order by tbname " , "select atan(t1) ,atan(c5) from stb1 order by tbname" )
self.check_result_auto_atan( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select atan(t1) ,atan(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_atan( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select atan(t1) , atan(c5) from stb1 where c1 > 0 order by tbname" )
pass
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
@ -509,6 +523,10 @@ class TDTestCase:
self.abs_func_filter()
tdLog.printNoPrefix("==========step8: check arctan result of stable query ============")
self.support_super_table_test()
def stop(self):

16
tests/system-test/2-query/ceil.py
View File

@ -427,6 +427,18 @@ class TDTestCase:
self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select ceil(c1+1) ,ceil(c2) , ceil(c3*1) , ceil(c4/2), ceil(c5)/2, ceil(c6) from sub1_bound ")
def support_super_table_test(self):
tdSql.execute(" use db ")
self.check_result_auto( " select c5 from stb1 order by ts " , "select ceil(c5) from stb1 order by ts" )
self.check_result_auto( " select c5 from stb1 order by tbname " , "select ceil(c5) from stb1 order by tbname" )
self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select ceil(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select ceil(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto( " select t1,c5 from stb1 order by ts " , "select ceil(t1), ceil(c5) from stb1 order by ts" )
self.check_result_auto( " select t1,c5 from stb1 order by tbname " , "select ceil(t1) ,ceil(c5) from stb1 order by tbname" )
self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select ceil(t1) ,ceil(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select ceil(t1) , ceil(c5) from stb1 where c1 > 0 order by tbname" )
pass
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
@ -455,6 +467,10 @@ class TDTestCase:
self.abs_func_filter()
tdLog.printNoPrefix("==========step7: check ceil result of stable query ============")
self.support_super_table_test()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")

17
tests/system-test/2-query/cos.py
View File

@ -476,6 +476,19 @@ class TDTestCase:
tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2))
self.check_result_auto_cos("select num1,num2 from tb3;" , "select cos(num1),cos(num2) from tb3")
def support_super_table_test(self):
tdSql.execute(" use db ")
self.check_result_auto_cos( " select c5 from stb1 order by ts " , "select cos(c5) from stb1 order by ts" )
self.check_result_auto_cos( " select c5 from stb1 order by tbname " , "select cos(c5) from stb1 order by tbname" )
self.check_result_auto_cos( " select c5 from stb1 where c1 > 0 order by tbname " , "select cos(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_cos( " select c5 from stb1 where c1 > 0 order by tbname " , "select cos(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_cos( " select t1,c5 from stb1 order by ts " , "select cos(t1), cos(c5) from stb1 order by ts" )
self.check_result_auto_cos( " select t1,c5 from stb1 order by tbname " , "select cos(t1) ,cos(c5) from stb1 order by tbname" )
self.check_result_auto_cos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select cos(t1) ,cos(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_cos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select cos(t1) , cos(c5) from stb1 where c1 > 0 order by tbname" )
pass
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
@ -509,6 +522,10 @@ class TDTestCase:
self.abs_func_filter()
tdLog.printNoPrefix("==========step8: check cos result of stable query ============")
self.support_super_table_test()
def stop(self):
tdSql.close()

17
tests/system-test/2-query/floor.py
View File

@ -427,6 +427,19 @@ class TDTestCase:
self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select floor(c1+1) ,floor(c2) , floor(c3*1) , floor(c4/2), floor(c5)/2, floor(c6) from sub1_bound ")
def support_super_table_test(self):
tdSql.execute(" use db ")
self.check_result_auto( " select c5 from stb1 order by ts " , "select floor(c5) from stb1 order by ts" )
self.check_result_auto( " select c5 from stb1 order by tbname " , "select floor(c5) from stb1 order by tbname" )
self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select floor(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select floor(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto( " select t1,c5 from stb1 order by ts " , "select floor(t1), floor(c5) from stb1 order by ts" )
self.check_result_auto( " select t1,c5 from stb1 order by tbname " , "select floor(t1) ,floor(c5) from stb1 order by tbname" )
self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select floor(t1) ,floor(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select floor(t1) , floor(c5) from stb1 where c1 > 0 order by tbname" )
pass
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
@ -454,6 +467,10 @@ class TDTestCase:
self.abs_func_filter()
tdLog.printNoPrefix("==========step7: check floor result of stable query ============")
self.support_super_table_test()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")

2
tests/system-test/2-query/join.py
View File

@ -28,7 +28,7 @@ class TDTestCase:
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
tdSql.init(conn.cursor(), False)
def __query_condition(self,tbname):
query_condition = []

2
tests/system-test/2-query/join2.py
View File

@ -28,7 +28,7 @@ class TDTestCase:
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
tdSql.init(conn.cursor(), False)
def __query_condition(self,tbname):
query_condition = []

18
tests/system-test/2-query/log.py
View File

@ -670,7 +670,19 @@ class TDTestCase:
tdSql.checkData(0,2,math.log(32767.000000000,2))
tdSql.checkData(0,3,math.log(63.500000000,2))
tdSql.checkData(0,4,63.999401166)
def support_super_table_test(self):
tdSql.execute(" use db ")
self.check_result_auto_log2( " select c5 from stb1 order by ts " , "select log(c5,2) from stb1 order by ts" )
self.check_result_auto_log2( " select c5 from stb1 order by tbname " , "select log(c5,2) from stb1 order by tbname" )
self.check_result_auto_log2( " select c5 from stb1 where c1 > 0 order by tbname " , "select log(c5,2) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_log2( " select c5 from stb1 where c1 > 0 order by tbname " , "select log(c5,2) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_log2( " select t1,c5 from stb1 order by ts " , "select log(t1,2), log(c5,2) from stb1 order by ts" )
self.check_result_auto_log2( " select t1,c5 from stb1 order by tbname " , "select log(t1,2) ,log(c5,2) from stb1 order by tbname" )
self.check_result_auto_log2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select log(t1,2) ,log(c5,2) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_log2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select log(t1,2) , log(c5,2) from stb1 where c1 > 0 order by tbname" )
pass
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
@ -707,7 +719,9 @@ class TDTestCase:
self.abs_func_filter()
tdLog.printNoPrefix("==========step9: check log result of stable query ============")
self.support_super_table_test()
def stop(self):
tdSql.close()

20
tests/system-test/2-query/pow.py
View File

@ -606,7 +606,23 @@ class TDTestCase:
tdSql.checkData(0,3,math.pow(63.500000000,2))
tdSql.checkData(0,5,None)
def support_super_table_test(self):
tdSql.execute(" use db ")
self.check_result_auto_pow2( " select c5 from stb1 order by ts " , "select pow(c5,2) from stb1 order by ts" )
self.check_result_auto_pow2( " select c5 from stb1 order by tbname " , "select pow(c5,2) from stb1 order by tbname" )
self.check_result_auto_pow2( " select c5 from stb1 where c1 > 0 order by tbname " , "select pow(c5,2) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_pow2( " select c5 from stb1 where c1 > 0 order by tbname " , "select pow(c5,2) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_pow2( " select t1,c5 from stb1 order by ts " , "select pow(t1,2), pow(c5,2) from stb1 order by ts" )
self.check_result_auto_pow2( " select t1,c5 from stb1 order by tbname " , "select pow(t1,2) ,pow(c5,2) from stb1 order by tbname" )
self.check_result_auto_pow2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select pow(t1,2) ,pow(c5,2) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto_pow2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select pow(t1,2) , pow(c5,2) from stb1 where c1 > 0 order by tbname" )
pass
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
@ -642,7 +658,9 @@ class TDTestCase:
self.abs_func_filter()
tdLog.printNoPrefix("==========step9: check pow result of stable query ============")
self.support_super_table_test()
def stop(self):
tdSql.close()

18
tests/system-test/2-query/round.py
View File

@ -432,6 +432,20 @@ class TDTestCase:
self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select round(c1+1) ,round(c2) , round(c3*1) , round(c4/2), round(c5)/2, round(c6) from sub1_bound ")
def support_super_table_test(self):
tdSql.execute(" use db ")
self.check_result_auto( " select c5 from stb1 order by ts " , "select round(c5) from stb1 order by ts" )
self.check_result_auto( " select c5 from stb1 order by tbname " , "select round(c5) from stb1 order by tbname" )
self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select round(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname " , "select round(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto( " select t1,c5 from stb1 order by ts " , "select round(t1), round(c5) from stb1 order by ts" )
self.check_result_auto( " select t1,c5 from stb1 order by tbname " , "select round(t1) ,round(c5) from stb1 order by tbname" )
self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select round(t1) ,round(c5) from stb1 where c1 > 0 order by tbname" )
self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select round(t1) , round(c5) from stb1 where c1 > 0 order by tbname" )
pass
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
@ -459,6 +473,10 @@ class TDTestCase:
self.abs_func_filter()
tdLog.printNoPrefix("==========step7: check round result of stable query ============")
self.support_super_table_test()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")

Some files were not shown because too many files have changed in this diff Show More