xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into fix/3_liaohj

This commit is contained in:
54liuyao 2023-12-26 18:36:18 +08:00
parent e2564a6189 130f57859f
commit ccf861dec8
40 changed files with 973 additions and 1871 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmake/curl_CMakeLists.txt.in
View File

@ -1,6 +1,7 @@
# curl # curl
ExternalProject_Add(curl2 ExternalProject_Add(curl2
URL https://curl.se/download/curl-8.2.1.tar.gz URL https://github.com/curl/curl/releases/download/curl-8_2_1/curl-8.2.1.tar.gz
#URL https://curl.se/download/curl-8.2.1.tar.gz
URL_HASH MD5=b25588a43556068be05e1624e0e74d41 URL_HASH MD5=b25588a43556068be05e1624e0e74d41
DOWNLOAD_NO_PROGRESS 1 DOWNLOAD_NO_PROGRESS 1
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"

1
include/util/taoserror.h
View File

@ -355,6 +355,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL TAOS_DEF_ERROR_CODE(0, 0x03D5) #define TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL TAOS_DEF_ERROR_CODE(0, 0x03D5)
#define TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x03D6) //internal #define TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x03D6) //internal
#define TSDB_CODE_MND_TRANS_SYNC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x03D7) #define TSDB_CODE_MND_TRANS_SYNC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x03D7)
#define TSDB_CODE_MND_TRANS_CTX_SWITCH TAOS_DEF_ERROR_CODE(0, 0x03D8)
#define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03DF) #define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03DF)
// mnode-mq // mnode-mq

15
source/common/src/tglobal.c
View File

@ -1432,6 +1432,13 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize,
static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
if (strcasecmp(name, "resetlog") == 0) {
// trigger, no item in cfg
taosResetLog();
cfgDumpCfg(tsCfg, 0, false);
return 0;
}
SConfigItem *pItem = cfgGetItem(pCfg, name); SConfigItem *pItem = cfgGetItem(pCfg, name);
if (!pItem || (pItem->dynScope & CFG_DYN_SERVER) == 0) { if (!pItem || (pItem->dynScope & CFG_DYN_SERVER) == 0) {
uError("failed to config:%s, not support", name); uError("failed to config:%s, not support", name);
@ -1445,12 +1452,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
return 0; return 0;
} }
if (strcasecmp(name, "resetlog") == 0) {
taosResetLog();
cfgDumpCfg(tsCfg, 0, false);
return 0;
}
{ // 'bool/int32_t/int64_t/float/double' variables with general modification function { // 'bool/int32_t/int64_t/float/double' variables with general modification function
static OptionNameAndVar debugOptions[] = { static OptionNameAndVar debugOptions[] = {
{"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, {"mDebugFlag", &mDebugFlag}, {"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, {"mDebugFlag", &mDebugFlag},
@ -1774,4 +1775,4 @@ void taosSetAllDebugFlag(int32_t flag, bool rewrite) {
uInfo("all debug flag are set to %d", flag); uInfo("all debug flag are set to %d", flag);
} }
int8_t taosGranted() { return atomic_load_8(&tsGrant); } int8_t taosGranted() { return atomic_load_8(&tsGrant); }

10
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -108,11 +108,11 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CONFIG_CHANGE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CONFIG_CHANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

2
source/dnode/mnode/impl/inc/mndTrans.h
View File

@ -97,7 +97,7 @@ SSdbRaw *mndTransEncode(STrans *pTrans);
SSdbRow *mndTransDecode(SSdbRaw *pRaw); SSdbRow *mndTransDecode(SSdbRaw *pRaw);
void mndTransDropData(STrans *pTrans); void mndTransDropData(STrans *pTrans);
bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans); bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans, bool topHalf);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

4
source/dnode/mnode/impl/src/mndCompactDetail.c
View File

@ -290,11 +290,11 @@ int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* p
SSdbRaw *pVgRaw = mndCompactDetailActionEncode(&compactDetail); SSdbRaw *pVgRaw = mndCompactDetailActionEncode(&compactDetail);
if (pVgRaw == NULL) return -1; if (pVgRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw); sdbFreeRaw(pVgRaw);
return -1; return -1;
} }
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
return 0; return 0;
} }

2
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -1215,7 +1215,7 @@ int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName)
return -1; return -1;
} }
if (mndSetDropSubRedoLogs(pMnode, pTrans, pSub) < 0) { if (mndSetDropSubCommitLogs(pMnode, pTrans, pSub) < 0) {
sdbRelease(pSdb, pSub); sdbRelease(pSdb, pSub);
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return -1; return -1;

17
source/dnode/mnode/impl/src/mndSync.c
View File

@ -180,7 +180,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
goto _OUT; goto _OUT;
} }
mInfo("trans:%d, is proposed, saved:%d code:0x%x, apply index:%" PRId64 " term:%" PRIu64 " config:%" PRId64 mInfo("trans:%d, process sync proposal, saved:%d code:0x%x, apply index:%" PRId64 " term:%" PRIu64 " config:%" PRId64
" role:%s raw:%p sec:%d seq:%" PRId64, " role:%s raw:%p sec:%d seq:%" PRId64,
transId, pMgmt->transId, pMeta->code, pMeta->index, pMeta->term, pMeta->lastConfigIndex, syncStr(pMeta->state), transId, pMgmt->transId, pMeta->code, pMeta->index, pMeta->term, pMeta->lastConfigIndex, syncStr(pMeta->state),
pRaw, pMgmt->transSec, pMgmt->transSeq); pRaw, pMgmt->transSec, pMgmt->transSeq);
@ -208,15 +208,11 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
} }
if (pTrans->stage == TRN_STAGE_PREPARE) { if (pTrans->stage == TRN_STAGE_PREPARE) {
bool continueExec = mndTransPerformPrepareStage(pMnode, pTrans); bool continueExec = mndTransPerformPrepareStage(pMnode, pTrans, false);
if (!continueExec) goto _OUT; if (!continueExec) goto _OUT;
} }
if (pTrans->id != pMgmt->transId) { mndTransRefresh(pMnode, pTrans);
mInfo("trans:%d, execute in mnode which not leader or sync timeout, createTime:%" PRId64 " saved trans:%d",
pTrans->id, pTrans->createdTime, pMgmt->transId);
mndTransRefresh(pMnode, pTrans);
}
sdbSetApplyInfo(pMnode->pSdb, pMeta->index, pMeta->term, pMeta->lastConfigIndex); sdbSetApplyInfo(pMnode->pSdb, pMeta->index, pMeta->term, pMeta->lastConfigIndex);
sdbWriteFile(pMnode->pSdb, tsMndSdbWriteDelta); sdbWriteFile(pMnode->pSdb, tsMndSdbWriteDelta);
@ -234,6 +230,7 @@ static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) {
goto _OUT; goto _OUT;
} }
int32_t transId = pMgmt->transId;
pMgmt->transId = 0; pMgmt->transId = 0;
pMgmt->transSec = 0; pMgmt->transSec = 0;
pMgmt->transSeq = 0; pMgmt->transSeq = 0;
@ -241,9 +238,9 @@ static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) {
tsem_post(&pMgmt->syncSem); tsem_post(&pMgmt->syncSem);
if (pMgmt->errCode != 0) { if (pMgmt->errCode != 0) {
mError("trans:%d, failed to propose since %s, post sem", pMgmt->transId, tstrerror(pMgmt->errCode)); mError("trans:%d, failed to propose since %s, post sem", transId, tstrerror(pMgmt->errCode));
} else { } else {
mInfo("trans:%d, is proposed and post sem, seq:%" PRId64, pMgmt->transId, pMgmt->transSeq); mInfo("trans:%d, is proposed and post sem, seq:%" PRId64, transId, pMgmt->transSeq);
} }
_OUT: _OUT:
@ -542,7 +539,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
taosThreadMutexLock(&pMgmt->lock); taosThreadMutexLock(&pMgmt->lock);
pMgmt->errCode = 0; pMgmt->errCode = 0;
if (pMgmt->transId != 0 /* && pMgmt->transId != transId*/) { if (pMgmt->transId != 0) {
mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId); mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId);
taosThreadMutexUnlock(&pMgmt->lock); taosThreadMutexUnlock(&pMgmt->lock);
rpcFreeCont(req.pCont); rpcFreeCont(req.pCont);

155
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -36,21 +36,25 @@ static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw);
static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction); static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction);
static void mndTransDropLogs(SArray *pArray); static void mndTransDropLogs(SArray *pArray);
static void mndTransDropActions(SArray *pArray); static void mndTransDropActions(SArray *pArray);
static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray);
static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray, bool topHalf);
static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans, bool topHalf);
static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans, bool topHalf);
static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans, bool topHalf);
static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans, bool topHalf);
static bool mndTransPerformRedoLogStage(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans, bool topHalf);
static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformRedoLogStage(SMnode *pMnode, STrans *pTrans, bool topHalf);
static bool mndTransPerformUndoLogStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf);
static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformUndoLogStage(SMnode *pMnode, STrans *pTrans, bool topHalf);
static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf);
static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf);
static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans, bool topHalf);
static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans); static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool topHalf);
static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsLeader(pMnode); } static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans, bool topHalf);
static bool mndCannotExecuteTransAction(SMnode *pMnode, bool topHalf) {
return (!pMnode->deploy && !mndIsLeader(pMnode)) || !topHalf;
}
static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans);
static int32_t mndProcessTransTimer(SRpcMsg *pReq); static int32_t mndProcessTransTimer(SRpcMsg *pReq);
@ -1090,8 +1094,9 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray)
} }
} }
static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) {
if (pAction->rawWritten) return 0; if (pAction->rawWritten) return 0;
if (topHalf) return TSDB_CODE_MND_TRANS_CTX_SWITCH;
int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pAction->pRaw); int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pAction->pRaw);
if (code == 0 || terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { if (code == 0 || terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
@ -1112,9 +1117,9 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi
return code; return code;
} }
static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) {
if (pAction->msgSent) return 0; if (pAction->msgSent) return 0;
if (mndCannotExecuteTransAction(pMnode)) return -1; if (mndCannotExecuteTransAction(pMnode, topHalf)) return TSDB_CODE_MND_TRANS_CTX_SWITCH;
int64_t signature = pTrans->id; int64_t signature = pTrans->id;
signature = (signature << 32); signature = (signature << 32);
@ -1159,7 +1164,8 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio
return code; return code;
} }
static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) {
if (!topHalf) return TSDB_CODE_MND_TRANS_CTX_SWITCH;
pAction->rawWritten = 0; pAction->rawWritten = 0;
pAction->errCode = 0; pAction->errCode = 0;
mInfo("trans:%d, %s:%d confirm action executed", pTrans->id, mndTransStr(pAction->stage), pAction->id); mInfo("trans:%d, %s:%d confirm action executed", pTrans->id, mndTransStr(pAction->stage), pAction->id);
@ -1168,34 +1174,39 @@ static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction
return 0; return 0;
} }
static int32_t mndTransExecSingleAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { static int32_t mndTransExecSingleAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) {
if (pAction->actionType == TRANS_ACTION_RAW) { if (pAction->actionType == TRANS_ACTION_RAW) {
return mndTransWriteSingleLog(pMnode, pTrans, pAction); return mndTransWriteSingleLog(pMnode, pTrans, pAction, topHalf);
} else if (pAction->actionType == TRANS_ACTION_MSG) { } else if (pAction->actionType == TRANS_ACTION_MSG) {
return mndTransSendSingleMsg(pMnode, pTrans, pAction); return mndTransSendSingleMsg(pMnode, pTrans, pAction, topHalf);
} else { } else {
return mndTransExecNullMsg(pMnode, pTrans, pAction); return mndTransExecNullMsg(pMnode, pTrans, pAction, topHalf);
} }
} }
static int32_t mndTransExecSingleActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { static int32_t mndTransExecSingleActions(SMnode *pMnode, STrans *pTrans, SArray *pArray, bool topHalf) {
int32_t numOfActions = taosArrayGetSize(pArray); int32_t numOfActions = taosArrayGetSize(pArray);
int32_t code = 0; int32_t code = 0;
for (int32_t action = 0; action < numOfActions; ++action) { for (int32_t action = 0; action < numOfActions; ++action) {
STransAction *pAction = taosArrayGet(pArray, action); STransAction *pAction = taosArrayGet(pArray, action);
code = mndTransExecSingleAction(pMnode, pTrans, pAction); code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf);
if (code != 0) break; if (code != 0) {
mInfo("trans:%d, action:%d not executed since %s. numOfActions:%d", pTrans->id, action, tstrerror(code),
numOfActions);
break;
}
} }
return code; return code;
} }
static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray, bool topHalf) {
int32_t numOfActions = taosArrayGetSize(pArray); int32_t numOfActions = taosArrayGetSize(pArray);
int32_t code = 0;
if (numOfActions == 0) return 0; if (numOfActions == 0) return 0;
if (mndTransExecSingleActions(pMnode, pTrans, pArray) != 0) { if ((code = mndTransExecSingleActions(pMnode, pTrans, pArray, topHalf)) != 0) {
return -1; return -1;
} }
@ -1248,31 +1259,31 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA
} }
} }
static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) {
int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions, topHalf);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("failed to execute redoActions since:%s, code:0x%x", terrstr(), terrno); mError("failed to execute redoActions since:%s, code:0x%x", terrstr(), terrno);
} }
return code; return code;
} }
static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) {
int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions, topHalf);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("failed to execute undoActions since %s", terrstr()); mError("failed to execute undoActions since %s", terrstr());
} }
return code; return code;
} }
static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans, bool topHalf) {
int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->commitActions); int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->commitActions, topHalf);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("failed to execute commitActions since %s", terrstr()); mError("failed to execute commitActions since %s", terrstr());
} }
return code; return code;
} }
static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans, bool topHalf) {
int32_t code = 0; int32_t code = 0;
int32_t numOfActions = taosArrayGetSize(pTrans->redoActions); int32_t numOfActions = taosArrayGetSize(pTrans->redoActions);
if (numOfActions == 0) return code; if (numOfActions == 0) return code;
@ -1289,7 +1300,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
for (int32_t action = pTrans->redoActionPos; action < numOfActions; ++action) { for (int32_t action = pTrans->redoActionPos; action < numOfActions; ++action) {
STransAction *pAction = taosArrayGet(pTrans->redoActions, pTrans->redoActionPos); STransAction *pAction = taosArrayGet(pTrans->redoActions, pTrans->redoActionPos);
code = mndTransExecSingleAction(pMnode, pTrans, pAction); code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf);
if (code == 0) { if (code == 0) {
if (pAction->msgSent) { if (pAction->msgSent) {
if (pAction->msgReceived) { if (pAction->msgReceived) {
@ -1317,14 +1328,16 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
} }
mndSetTransLastAction(pTrans, pAction); mndSetTransLastAction(pTrans, pAction);
if (mndCannotExecuteTransAction(pMnode)) break; if (mndCannotExecuteTransAction(pMnode, topHalf)) break;
if (code == 0) { if (code == 0) {
pTrans->code = 0; pTrans->code = 0;
pTrans->redoActionPos++; pTrans->redoActionPos++;
mInfo("trans:%d, %s:%d is executed and need sync to other mnodes", pTrans->id, mndTransStr(pAction->stage), mInfo("trans:%d, %s:%d is executed and need sync to other mnodes", pTrans->id, mndTransStr(pAction->stage),
pAction->id); pAction->id);
taosThreadMutexUnlock(&pTrans->mutex);
code = mndTransSync(pMnode, pTrans); code = mndTransSync(pMnode, pTrans);
taosThreadMutexLock(&pTrans->mutex);
if (code != 0) { if (code != 0) {
pTrans->redoActionPos--; pTrans->redoActionPos--;
pTrans->code = terrno; pTrans->code = terrno;
@ -1357,7 +1370,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
return code; return code;
} }
bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans, bool topHalf) {
bool continueExec = true; bool continueExec = true;
int32_t code = 0; int32_t code = 0;
@ -1368,7 +1381,7 @@ bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) {
for (int32_t action = 0; action < numOfActions; ++action) { for (int32_t action = 0; action < numOfActions; ++action) {
STransAction *pAction = taosArrayGet(pTrans->prepareActions, action); STransAction *pAction = taosArrayGet(pTrans->prepareActions, action);
code = mndTransExecSingleAction(pMnode, pTrans, pAction); code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf);
if (code != 0) { if (code != 0) {
mError("trans:%d, failed to execute prepare action:%d, numOfActions:%d", pTrans->id, action, numOfActions); mError("trans:%d, failed to execute prepare action:%d, numOfActions:%d", pTrans->id, action, numOfActions);
return false; return false;
@ -1381,17 +1394,17 @@ _OVER:
return continueExec; return continueExec;
} }
static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) { static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf) {
bool continueExec = true; bool continueExec = true;
int32_t code = 0; int32_t code = 0;
if (pTrans->exec == TRN_EXEC_SERIAL) { if (pTrans->exec == TRN_EXEC_SERIAL) {
code = mndTransExecuteRedoActionsSerial(pMnode, pTrans); code = mndTransExecuteRedoActionsSerial(pMnode, pTrans, topHalf);
} else { } else {
code = mndTransExecuteRedoActions(pMnode, pTrans); code = mndTransExecuteRedoActions(pMnode, pTrans, topHalf);
} }
if (mndCannotExecuteTransAction(pMnode)) return false; if (mndCannotExecuteTransAction(pMnode, topHalf)) return false;
terrno = code; terrno = code;
if (code == 0) { if (code == 0) {
@ -1431,8 +1444,8 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) {
return continueExec; return continueExec;
} }
static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans, bool topHalf) {
if (mndCannotExecuteTransAction(pMnode)) return false; if (mndCannotExecuteTransAction(pMnode, topHalf)) return false;
bool continueExec = true; bool continueExec = true;
int32_t code = mndTransCommit(pMnode, pTrans); int32_t code = mndTransCommit(pMnode, pTrans);
@ -1452,9 +1465,9 @@ static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) {
return continueExec; return continueExec;
} }
static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans) { static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf) {
bool continueExec = true; bool continueExec = true;
int32_t code = mndTransExecuteCommitActions(pMnode, pTrans); int32_t code = mndTransExecuteCommitActions(pMnode, pTrans, topHalf);
if (code == 0) { if (code == 0) {
pTrans->code = 0; pTrans->code = 0;
@ -1471,9 +1484,9 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans) {
return continueExec; return continueExec;
} }
static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf) {
bool continueExec = true; bool continueExec = true;
int32_t code = mndTransExecuteUndoActions(pMnode, pTrans); int32_t code = mndTransExecuteUndoActions(pMnode, pTrans, topHalf);
if (code == 0) { if (code == 0) {
pTrans->stage = TRN_STAGE_PRE_FINISH; pTrans->stage = TRN_STAGE_PRE_FINISH;
@ -1491,8 +1504,8 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) {
return continueExec; return continueExec;
} }
static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool topHalf) {
if (mndCannotExecuteTransAction(pMnode)) return false; if (mndCannotExecuteTransAction(pMnode, topHalf)) return false;
bool continueExec = true; bool continueExec = true;
int32_t code = mndTransRollback(pMnode, pTrans); int32_t code = mndTransRollback(pMnode, pTrans);
@ -1510,8 +1523,8 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) {
return continueExec; return continueExec;
} }
static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans) { static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans, bool topHalf) {
if (mndCannotExecuteTransAction(pMnode)) return false; if (mndCannotExecuteTransAction(pMnode, topHalf)) return false;
bool continueExec = true; bool continueExec = true;
int32_t code = mndTransPreFinish(pMnode, pTrans); int32_t code = mndTransPreFinish(pMnode, pTrans);
@ -1529,8 +1542,9 @@ static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans) {
return continueExec; return continueExec;
} }
static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans) { static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans, bool topHalf) {
bool continueExec = false; bool continueExec = false;
if (topHalf) return continueExec;
SSdbRaw *pRaw = mndTransEncode(pTrans); SSdbRaw *pRaw = mndTransEncode(pTrans);
if (pRaw == NULL) { if (pRaw == NULL) {
@ -1558,43 +1572,28 @@ void mndTransExecuteImp(SMnode *pMnode, STrans *pTrans, bool topHalf) {
pTrans->lastExecTime = taosGetTimestampMs(); pTrans->lastExecTime = taosGetTimestampMs();
switch (pTrans->stage) { switch (pTrans->stage) {
case TRN_STAGE_PREPARE: case TRN_STAGE_PREPARE:
continueExec = mndTransPerformPrepareStage(pMnode, pTrans); continueExec = mndTransPerformPrepareStage(pMnode, pTrans, topHalf);
break; break;
case TRN_STAGE_REDO_ACTION: case TRN_STAGE_REDO_ACTION:
continueExec = mndTransPerformRedoActionStage(pMnode, pTrans); continueExec = mndTransPerformRedoActionStage(pMnode, pTrans, topHalf);
break; break;
case TRN_STAGE_COMMIT: case TRN_STAGE_COMMIT:
if (topHalf) { continueExec = mndTransPerformCommitStage(pMnode, pTrans, topHalf);
continueExec = mndTransPerformCommitStage(pMnode, pTrans);
} else {
mInfo("trans:%d, can not commit since not leader", pTrans->id);
continueExec = false;
}
break; break;
case TRN_STAGE_COMMIT_ACTION: case TRN_STAGE_COMMIT_ACTION:
continueExec = mndTransPerformCommitActionStage(pMnode, pTrans); continueExec = mndTransPerformCommitActionStage(pMnode, pTrans, topHalf);
break; break;
case TRN_STAGE_ROLLBACK: case TRN_STAGE_ROLLBACK:
if (topHalf) { continueExec = mndTransPerformRollbackStage(pMnode, pTrans, topHalf);
continueExec = mndTransPerformRollbackStage(pMnode, pTrans);
} else {
mInfo("trans:%d, can not rollback since not leader", pTrans->id);
continueExec = false;
}
break; break;
case TRN_STAGE_UNDO_ACTION: case TRN_STAGE_UNDO_ACTION:
continueExec = mndTransPerformUndoActionStage(pMnode, pTrans); continueExec = mndTransPerformUndoActionStage(pMnode, pTrans, topHalf);
break; break;
case TRN_STAGE_PRE_FINISH: case TRN_STAGE_PRE_FINISH:
if (topHalf) { continueExec = mndTransPerformPreFinishStage(pMnode, pTrans, topHalf);
continueExec = mndTransPerformPreFinishStage(pMnode, pTrans);
} else {
mInfo("trans:%d, can not pre-finish since not leader", pTrans->id);
continueExec = false;
}
break; break;
case TRN_STAGE_FINISH: case TRN_STAGE_FINISH:
continueExec = mndTransPerformFinishStage(pMnode, pTrans); continueExec = mndTransPerformFinishStage(pMnode, pTrans, topHalf);
break; break;
default: default:
continueExec = false; continueExec = false;

22
source/dnode/vnode/src/meta/metaTable.c
View File

@ -36,7 +36,6 @@ static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME);
static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) {
pInfo->uid = pEntry->uid; pInfo->uid = pEntry->uid;
pInfo->version = pEntry->version; pInfo->version = pEntry->version;
@ -562,6 +561,7 @@ int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn);
metaULock(pMeta); metaULock(pMeta);
metaDestroyTagIdxKey(pTagIdxKey); metaDestroyTagIdxKey(pTagIdxKey);
pTagIdxKey = NULL;
} }
nStbEntry.version = version; nStbEntry.version = version;
@ -692,6 +692,7 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq)
tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn);
metaULock(pMeta); metaULock(pMeta);
metaDestroyTagIdxKey(pTagIdxKey); metaDestroyTagIdxKey(pTagIdxKey);
pTagIdxKey = NULL;
} }
// clear idx flag // clear idx flag
@ -1076,7 +1077,7 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) {
return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx);
} }
static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t* pSysTbl) { static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl) {
void *pData = NULL; void *pData = NULL;
int nData = 0; int nData = 0;
int rc = 0; int rc = 0;
@ -1146,6 +1147,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p
tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn);
} }
metaDestroyTagIdxKey(pTagIdxKey); metaDestroyTagIdxKey(pTagIdxKey);
pTagIdxKey = NULL;
} }
} }
tDecoderClear(&tdc); tDecoderClear(&tdc);
@ -1865,6 +1867,7 @@ static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTb
} }
tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn);
metaDestroyTagIdxKey(pTagIdxKey); metaDestroyTagIdxKey(pTagIdxKey);
pTagIdxKey = NULL;
} }
tdbTbcClose(pCtbIdxc); tdbTbcClose(pCtbIdxc);
return 0; return 0;
@ -2122,7 +2125,7 @@ int metaUpdateChangeTimeWithLock(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeM
if (!tsTtlChangeOnWrite) return 0; if (!tsTtlChangeOnWrite) return 0;
metaWLock(pMeta); metaWLock(pMeta);
int ret = metaUpdateChangeTime(pMeta, uid, changeTimeMs); int ret = metaUpdateChangeTime(pMeta, uid, changeTimeMs);
metaULock(pMeta); metaULock(pMeta);
return ret; return ret;
} }
@ -2228,15 +2231,14 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
nTagData = tDataTypes[pTagColumn->type].bytes; nTagData = tDataTypes[pTagColumn->type].bytes;
} }
if (pTagData != NULL) { if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type,
if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) {
pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { ret = -1;
ret = -1; goto end;
goto end;
}
tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn);
} }
tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn);
metaDestroyTagIdxKey(pTagIdxKey); metaDestroyTagIdxKey(pTagIdxKey);
pTagIdxKey = NULL;
} }
} }
end: end:

4
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -1623,6 +1623,7 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num
pDummy[i].functionId = pCtx[i].functionId; pDummy[i].functionId = pCtx[i].functionId;
pDummy[i].isNotNullFunc = pCtx[i].isNotNullFunc; pDummy[i].isNotNullFunc = pCtx[i].isNotNullFunc;
pDummy[i].isPseudoFunc = pCtx[i].isPseudoFunc; pDummy[i].isPseudoFunc = pCtx[i].isPseudoFunc;
pDummy[i].fpSet.init = pCtx[i].fpSet.init;
} }
} }
@ -2774,6 +2775,9 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) {
for (int32_t i = 0; i < num; i++) { for (int32_t i = 0; i < num; i++) {
SResultWindowInfo winInfo = {0}; SResultWindowInfo winInfo = {0};
getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo); getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo);
if (!IS_VALID_SESSION_WIN(winInfo)) {
continue;
}
compactSessionSemiWindow(pOperator, &winInfo); compactSessionSemiWindow(pOperator, &winInfo);
saveSessionOutputBuf(pAggSup, &winInfo); saveSessionOutputBuf(pAggSup, &winInfo);
} }

3
source/libs/parser/CMakeLists.txt
View File

@ -4,9 +4,6 @@ IF (TD_ENTERPRISE)
LIST(APPEND PARSER_SRC ${TD_ENTERPRISE_DIR}/src/plugins/view/src/parserView.c) LIST(APPEND PARSER_SRC ${TD_ENTERPRISE_DIR}/src/plugins/view/src/parserView.c)
ENDIF () ENDIF ()
IF (TD_BI_SUPPORT)
LIST(APPEND PARSER_SRC ${TD_ENTERPRISE_DIR}/src/plugins/bi/src/biRewriteQuery.c)
ENDIF ()
add_library(parser STATIC ${PARSER_SRC}) add_library(parser STATIC ${PARSER_SRC})
target_include_directories( target_include_directories(
parser parser

196
source/libs/parser/src/parTranslater.c
View File

@ -1104,11 +1104,192 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
#ifndef TD_ENTERPRISE static void biMakeAliasNameInMD5(char* pExprStr, int32_t len, char* pAlias) {
T_MD5_CTX ctx;
tMD5Init(&ctx);
tMD5Update(&ctx, pExprStr, len);
tMD5Final(&ctx);
char* p = pAlias;
for (uint8_t i = 0; i < tListLen(ctx.digest); ++i) {
sprintf(p, "%02x", ctx.digest[i]);
p += 2;
}
}
static SNode* biMakeTbnameProjectAstNode(char* funcName, char* tableAlias) {
SValueNode* valNode = NULL;
if (tableAlias != NULL) {
SValueNode* n = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
n->literal = tstrdup(tableAlias);
n->node.resType.type = TSDB_DATA_TYPE_BINARY;
n->node.resType.bytes = strlen(n->literal);
n->isDuration = false;
n->translate = false;
valNode = n;
}
SFunctionNode* tbNameFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
tstrncpy(tbNameFunc->functionName, "tbname", TSDB_FUNC_NAME_LEN);
if (valNode != NULL) {
nodesListMakeAppend(&tbNameFunc->pParameterList, (SNode*)valNode);
}
snprintf(tbNameFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias),
(tableAlias)? "%s.tbname" : "%stbname",
(tableAlias)? tableAlias : "");
strncpy(tbNameFunc->node.aliasName, tbNameFunc->functionName, TSDB_COL_NAME_LEN);
if (funcName == NULL) {
return (SNode*)tbNameFunc;
} else {
SFunctionNode* multiResFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
tstrncpy(multiResFunc->functionName, funcName, TSDB_FUNC_NAME_LEN);
nodesListMakeAppend(&multiResFunc->pParameterList, (SNode*)tbNameFunc);
if (tsKeepColumnName) {
snprintf(multiResFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias),
(tableAlias)? "%s.tbname" : "%stbname",
(tableAlias)? tableAlias : "");
strcpy(multiResFunc->node.aliasName, tbNameFunc->functionName);
} else {
snprintf(multiResFunc->node.userAlias, sizeof(multiResFunc->node.userAlias),
tableAlias? "%s(%s.tbname)" : "%s(%stbname)", funcName,
tableAlias? tableAlias: "");
biMakeAliasNameInMD5(multiResFunc->node.userAlias, strlen(multiResFunc->node.userAlias), multiResFunc->node.aliasName);
}
return (SNode*)multiResFunc;
}
}
static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt* pSelect, SNode* pNode, SListCell* pSelectListCell) {
SNodeList* pTbnameNodeList = nodesMakeList();
SFunctionNode* pFunc = (SFunctionNode*)pNode;
if (strcasecmp(pFunc->functionName, "last") == 0 ||
strcasecmp(pFunc->functionName, "last_row") == 0 ||
strcasecmp(pFunc->functionName, "first") == 0) {
SNodeList* pParams = pFunc->pParameterList;
SNode* pPara = NULL;
FOREACH(pPara, pParams) {
if (nodesIsStar(pPara)) {
SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel);
size_t n = taosArrayGetSize(pTables);
for (int32_t i = 0; i < n; ++i) {
STableNode* pTable = taosArrayGetP(pTables, i);
if (nodeType(pTable) == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)pTable)->pMeta != NULL &&
((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) {
SNode* pTbnameNode = biMakeTbnameProjectAstNode(pFunc->functionName, NULL);
nodesListAppend(pTbnameNodeList, pTbnameNode);
}
}
if (LIST_LENGTH(pTbnameNodeList) > 0) {
nodesListInsertListAfterPos(pSelect->pProjectionList, pSelectListCell, pTbnameNodeList);
}
} else if (nodesIsTableStar(pPara)) {
char* pTableAlias = ((SColumnNode*)pPara)->tableAlias;
STableNode* pTable = NULL;
int32_t code = findTable(pCxt, pTableAlias, &pTable);
if (TSDB_CODE_SUCCESS == code && nodeType(pTable) == QUERY_NODE_REAL_TABLE &&
((SRealTableNode*)pTable)->pMeta != NULL &&
((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) {
SNode* pTbnameNode = biMakeTbnameProjectAstNode(pFunc->functionName, pTableAlias);
nodesListAppend(pTbnameNodeList, pTbnameNode);
}
if (LIST_LENGTH(pTbnameNodeList) > 0) {
nodesListInsertListAfterPos(pSelect->pProjectionList, pSelectListCell, pTbnameNodeList);
}
}
}
}
return TSDB_CODE_SUCCESS;
}
// after translate from
// before translate select list
int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
SNode* pNode = NULL;
SNodeList* pTbnameNodeList = nodesMakeList();
WHERE_EACH(pNode, pSelect->pProjectionList) {
if (nodesIsStar(pNode)) {
SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel);
size_t n = taosArrayGetSize(pTables);
for (int32_t i = 0; i < n; ++i) {
STableNode* pTable = taosArrayGetP(pTables, i);
if (nodeType(pTable) == QUERY_NODE_REAL_TABLE &&
((SRealTableNode*)pTable)->pMeta != NULL &&
((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) {
SNode* pTbnameNode = biMakeTbnameProjectAstNode(NULL, NULL);
nodesListAppend(pTbnameNodeList, pTbnameNode);
}
}
if (LIST_LENGTH(pTbnameNodeList) > 0) {
nodesListInsertListAfterPos(pSelect->pProjectionList, cell, pTbnameNodeList);
}
} else if (nodesIsTableStar(pNode)) {
char* pTableAlias = ((SColumnNode*)pNode)->tableAlias;
STableNode* pTable = NULL;
int32_t code = findTable(pCxt, pTableAlias, &pTable);
if (TSDB_CODE_SUCCESS == code &&
nodeType(pTable) == QUERY_NODE_REAL_TABLE &&
((SRealTableNode*)pTable)->pMeta != NULL &&
((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) {
SNode* pTbnameNode = biMakeTbnameProjectAstNode(NULL, pTableAlias);
nodesListAppend(pTbnameNodeList, pTbnameNode);
}
if (LIST_LENGTH(pTbnameNodeList) > 0) {
nodesListInsertListAfterPos(pSelect->pProjectionList, cell, pTbnameNodeList);
}
} else if (nodeType(pNode) == QUERY_NODE_FUNCTION) {
biRewriteSelectFuncParamStar(pCxt, pSelect, pNode, cell);
}
WHERE_NEXT;
}
return TSDB_CODE_SUCCESS;
}
bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode) { bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode) {
SColumnNode* pCol = (SColumnNode*)(*ppNode);
if ((strcasecmp(pCol->colName, "tbname") == 0) &&
((SSelectStmt*)pCxt->pCurrStmt)->pFromTable &&
QUERY_NODE_REAL_TABLE == nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) {
SFunctionNode* tbnameFuncNode = NULL;
tbnameFuncNode = (SFunctionNode*)biMakeTbnameProjectAstNode(NULL, (pCol->tableAlias[0]!='\0') ? pCol->tableAlias : NULL);
tbnameFuncNode->node.resType = pCol->node.resType;
strcpy(tbnameFuncNode->node.aliasName, pCol->node.aliasName);
strcpy(tbnameFuncNode->node.userAlias, pCol->node.userAlias);
nodesDestroyNode(*ppNode);
*ppNode = (SNode*)tbnameFuncNode;
return true;
}
return false; return false;
} }
#endif
int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
if (pStmt->pTags) {
SNode* pNode = NULL;
FOREACH(pNode, pStmt->pTags) {
SColumnDefNode* pTag = (SColumnDefNode*)pNode;
if (strcasecmp(pTag->colName, "tbname") == 0) {
int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, "tbname can not used for tags in BI mode");
return code;
}
}
}
if (pStmt->pCols) {
SNode* pNode = NULL;
FOREACH(pNode, pStmt->pCols) {
SColumnDefNode* pCol = (SColumnDefNode*)pNode;
if (strcasecmp(pCol->colName, "tbname") == 0) {
int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, "tbname can not used for columns in BI mode");
return code;
}
}
}
return TSDB_CODE_SUCCESS;
}
static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) { static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
if (NULL == pCxt->pCurrStmt || if (NULL == pCxt->pCurrStmt ||
@ -3178,10 +3359,6 @@ static int32_t createTags(STranslateContext* pCxt, SNodeList** pOutput) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
#ifndef TD_ENTERPRISE
int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; }
#endif
static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) { static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
SNode* pNode = NULL; SNode* pNode = NULL;
WHERE_EACH(pNode, pSelect->pProjectionList) { WHERE_EACH(pNode, pSelect->pProjectionList) {
@ -5743,11 +5920,6 @@ static int32_t checkTableDeleteMarkOption(STranslateContext* pCxt, STableOptions
return code; return code;
} }
#ifndef TD_ENTERPRISE
int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
return TSDB_CODE_SUCCESS;
}
#endif
static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, bool createStable) { static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, bool createStable) {
if (NULL != strchr(pStmt->tableName, '.')) { if (NULL != strchr(pStmt->tableName, '.')) {
@ -8769,7 +8941,7 @@ static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pS
(*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY;
(*pSchema)[0].bytes = TSDB_CONFIG_OPTION_LEN; (*pSchema)[0].bytes = TSDB_CONFIG_OPTION_LEN;
strcpy((*pSchema)[0].name, "name"); strcpy((*pSchema)[0].name, "result");
(*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY;
(*pSchema)[1].bytes = TSDB_CONFIG_VALUE_LEN; (*pSchema)[1].bytes = TSDB_CONFIG_VALUE_LEN;

3
source/libs/sync/src/syncMain.c
View File

@ -1162,9 +1162,12 @@ int32_t syncNodeRestore(SSyncNode* pSyncNode) {
ASSERTS(pSyncNode->pLogStore != NULL, "log store not created"); ASSERTS(pSyncNode->pLogStore != NULL, "log store not created");
ASSERTS(pSyncNode->pLogBuf != NULL, "ring log buffer not created"); ASSERTS(pSyncNode->pLogBuf != NULL, "ring log buffer not created");
taosThreadMutexLock(&pSyncNode->pLogBuf->mutex);
SyncIndex lastVer = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); SyncIndex lastVer = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
SyncIndex commitIndex = pSyncNode->pLogStore->syncLogCommitIndex(pSyncNode->pLogStore); SyncIndex commitIndex = pSyncNode->pLogStore->syncLogCommitIndex(pSyncNode->pLogStore);
SyncIndex endIndex = pSyncNode->pLogBuf->endIndex; SyncIndex endIndex = pSyncNode->pLogBuf->endIndex;
taosThreadMutexUnlock(&pSyncNode->pLogBuf->mutex);
if (lastVer != -1 && endIndex != lastVer + 1) { if (lastVer != -1 && endIndex != lastVer + 1) {
terrno = TSDB_CODE_WAL_LOG_INCOMPLETE; terrno = TSDB_CODE_WAL_LOG_INCOMPLETE;
sError("vgId:%d, failed to restore sync node since %s. expected lastLogIndex:%" PRId64 ", lastVer:%" PRId64 "", sError("vgId:%d, failed to restore sync node since %s. expected lastLogIndex:%" PRId64 ", lastVer:%" PRId64 "",

15
source/libs/sync/src/syncSnapshot.c
View File

@ -1087,7 +1087,10 @@ static int32_t syncSnapSenderExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotSe
// sender // sender
static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) {
int32_t code = -1;
SSnapshot snapshot = {0}; SSnapshot snapshot = {0};
taosThreadMutexLock(&pSender->pSndBuf->mutex);
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
// prepare <begin, end> // prepare <begin, end>
@ -1103,20 +1106,24 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend
// start reader // start reader
if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
if (syncSnapSenderExchgSnapInfo(pSyncNode, pSender, pMsg) != 0) { if (syncSnapSenderExchgSnapInfo(pSyncNode, pSender, pMsg) != 0) {
return -1; goto _out;
} }
} }
int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader); code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader);
if (code != 0) { if (code != 0) {
sSError(pSender, "prepare snapshot failed since %s", terrstr()); sSError(pSender, "prepare snapshot failed since %s", terrstr());
return -1; goto _out;
} }
// update next index // update next index
syncIndexMgrSetIndex(pSyncNode->pNextIndex, &pMsg->srcId, snapshot.lastApplyIndex + 1); syncIndexMgrSetIndex(pSyncNode->pNextIndex, &pMsg->srcId, snapshot.lastApplyIndex + 1);
return snapshotSend(pSender); code = snapshotSend(pSender);
_out:
taosThreadMutexUnlock(&pSender->pSndBuf->mutex);
return code;
} }
static int32_t snapshotSenderSignatureCmp(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { static int32_t snapshotSenderSignatureCmp(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) {

1
source/util/src/terror.c
View File

@ -286,6 +286,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CLOG_IS_NULL, "Transaction commitlog
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL, "Unable to establish connection While execute transaction and will continue in the background") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL, "Unable to establish connection While execute transaction and will continue in the background")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED, "Last Transaction not finished") TAOS_DEFINE_ERROR(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED, "Last Transaction not finished")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_SYNC_TIMEOUT, "Sync timeout While execute transaction and will continue in the background") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_SYNC_TIMEOUT, "Sync timeout While execute transaction and will continue in the background")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CTX_SWITCH, "Transaction context switch")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error")
// mnode-mq // mnode-mq

4
source/util/src/tlog.c
View File

@ -337,14 +337,10 @@ static int32_t taosOpenNewLogFile() {
} }
void taosResetLog() { void taosResetLog() {
char lastName[LOG_FILE_NAME_LEN + 20];
sprintf(lastName, "%s.%d", tsLogObj.logName, tsLogObj.flag);
// force create a new log file // force create a new log file
tsLogObj.lines = tsNumOfLogLines + 10; tsLogObj.lines = tsNumOfLogLines + 10;
taosOpenNewLogFile(); taosOpenNewLogFile();
(void)taosRemoveFile(lastName);
uInfo("=================================="); uInfo("==================================");
uInfo(" reset log file "); uInfo(" reset log file ");

58
tests/army/enterprise/multi-level/mlevel_basic.json Normal file
View File

@ -0,0 +1,58 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"connection_pool_size": 8,
"num_of_records_per_req": 2000,
"thread_count": 2,
"create_table_thread_count": 1,
"confirm_parameter_prompt": "no",
"databases": [
{
"dbinfo": {
"name": "db",
"drop": "yes",
"vgroups": 2,
"replica": 1,
"duration":"1d",
"keep": "3d,6d,30d"
},
"super_tables": [
{
"name": "stb",
"child_table_exists": "no",
"childtable_count": 4,
"insert_rows": 1000000,
"childtable_prefix": "d",
"insert_mode": "taosc",
"timestamp_step": 1000,
"start_timestamp":"now-12d",
"columns": [
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc" },
{ "type": "double", "name": "dc"},
{ "type": "tinyint", "name": "ti", "values":["1"]},
{ "type": "smallint", "name": "si" },
{ "type": "int", "name": "ic" },
{ "type": "bigint", "name": "bi" },
{ "type": "utinyint", "name": "uti"},
{ "type": "usmallint", "name": "usi"},
{ "type": "uint", "name": "ui" },
{ "type": "ubigint", "name": "ubi"},
{ "type": "binary", "name": "bin", "len": 32},
{ "type": "nchar", "name": "nch", "len": 64}
],
"tags": [
{"type": "tinyint", "name": "groupid","max": 10,"min": 1},
{"name": "location","type": "binary", "len": 16, "values":
["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"]
}
]
}
]
}
]
}

60
tests/army/enterprise/multi-level/mlevel_basic.py
View File

@ -15,26 +15,62 @@ import sys
import time import time
import taos import taos
import frame
import frame.etool
from frame.log import * from frame.log import *
from frame.cases import * from frame.cases import *
from frame.sql import * from frame.sql import *
from frame.caseBase import *
from frame import *
class TDTestCase:
# init class TDTestCase(TBase):
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}") def insertData(self):
tdSql.init(conn.cursor(), True) tdLog.info(f"insert data.")
# taosBenchmark run
json = etool.curFile(__file__, "mlevel_basic.json")
etool.runBenchmark(json=json)
tdSql.execute(f"use {self.db}")
# set insert data information
self.childtable_count = 4
self.insert_rows = 1000000
self.timestamp_step = 1000
def doAction(self):
tdLog.info(f"do action.")
self.flushDb()
self.trimDb()
self.compactDb()
# run # run
def run(self): def run(self):
# check two db query result same tdLog.debug(f"start to excute {__file__}")
tdLog.info(f"hello world.")
# insert data
self.insertData()
# check insert data correct
self.checkInsertCorrect()
# save
self.snapshotAgg()
# do action
self.doAction()
# check save agg result correct
self.checkAggCorrect()
# check insert correct again
self.checkInsertCorrect()
# stop
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed") tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase()) tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase())

106
tests/army/frame/caseBase.py Normal file
View File

@ -0,0 +1,106 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import os
import time
import datetime
from frame.log import *
from frame.sql import *
# test case base
class TBase:
#
# frame call
#
# init
def init(self, conn, logSql, replicaVar=1):
# save param
self.replicaVar = int(replicaVar)
tdSql.init(conn.cursor(), True)
# record server information
self.dnodeNum = 0
self.mnodeNum = 0
self.mLevel = 0
self.mLevelDisk = 0
# test case information
self.db = "db"
self.stb = "stb"
# variant in taosBenchmark json
self.childtable_count = 2
self.insert_rows = 1000000
self.timestamp_step = 1000
# sql
self.sqlSum = f"select sum(ic) from {self.stb}"
self.sqlMax = f"select max(ic) from {self.stb}"
self.sqlMin = f"select min(ic) from {self.stb}"
self.sqlAvg = f"select avg(ic) from {self.stb}"
# stop
def stop(self):
tdSql.close()
#
# db action
#
def trimDb(self):
tdSql.execute(f"trim database {self.db}")
def compactDb(self):
tdSql.execute(f"compact database {self.db}")
def flushDb(self):
tdSql.execute(f"flush database {self.db}")
#
# check db correct
#
# basic
def checkInsertCorrect(self):
# check count
sql = f"select count(*) from {self.stb}"
tdSql.checkAgg(sql, self.childtable_count * self.insert_rows)
# check child table count
sql = f" select count(*) from (select count(*) as cnt , tbname from {self.stb} group by tbname) where cnt = {self.insert_rows} "
tdSql.checkAgg(sql, self.childtable_count)
# check step
sql = f"select count(*) from (select diff(ts) as dif from {self.stb} partition by tbname) where dif != {self.timestamp_step}"
tdSql.checkAgg(sql, 0)
# save agg result
def snapshotAgg(self):
self.sum = tdSql.getFirstValue(self.sqlSum)
self.avg = tdSql.getFirstValue(self.sqlAvg)
self.min = tdSql.getFirstValue(self.sqlMin)
self.max = tdSql.getFirstValue(self.sqlMax)
# check agg
def checkAggCorrect(self):
tdSql.checkAgg(self.sqlSum, self.sum)
tdSql.checkAgg(self.sqlAvg, self.avg)
tdSql.checkAgg(self.sqlMin, self.min)
tdSql.checkAgg(self.sqlMax, self.max)

2
tests/army/frame/common.py
View File

@ -23,7 +23,7 @@ import taos
from frame.log import * from frame.log import *
from frame.sql import * from frame.sql import *
from frame.cases import * from frame.cases import *
from frame.dnodes import * from frame.server.dnodes import *
from frame.common import * from frame.common import *
from frame.constant import * from frame.constant import *
from dataclasses import dataclass,field from dataclasses import dataclass,field

502
tests/army/frame/dnodes-default.py
View File

@ -1,502 +0,0 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import os
import os.path
import subprocess
from frame.log import *
class TDSimClient:
def __init__(self):
self.testCluster = False
self.cfgDict = {
"numOfLogLines": "100000000",
"numOfThreadsPerCore": "2.0",
"locale": "en_US.UTF-8",
"charset": "UTF-8",
"asyncLog": "0",
"minTablesPerVnode": "4",
"maxTablesPerVnode": "1000",
"tableIncStepPerVnode": "10000",
"maxVgroupsPerDb": "1000",
"sdbDebugFlag": "143",
"rpcDebugFlag": "135",
"tmrDebugFlag": "131",
"cDebugFlag": "135",
"udebugFlag": "135",
"jnidebugFlag": "135",
"qdebugFlag": "135",
"telemetryReporting": "0",
}
def init(self, path):
self.__init__()
self.path = path
def getLogDir(self):
self.logDir = os.path.join(self.path,"sim","psim","log")
return self.logDir
def getCfgDir(self):
self.cfgDir = os.path.join(self.path,"sim","psim","cfg")
return self.cfgDir
def setTestCluster(self, value):
self.testCluster = value
def addExtraCfg(self, option, value):
self.cfgDict.update({option: value})
def cfg(self, option, value):
cmd = "echo %s %s >> %s" % (option, value, self.cfgPath)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def deploy(self):
self.logDir = os.path.join(self.path,"sim","psim","log")
self.cfgDir = os.path.join(self.path,"sim","psim","cfg")
self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg")
cmd = "rm -rf " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "rm -rf " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "touch " + self.cfgPath
if os.system(cmd) != 0:
tdLog.exit(cmd)
if self.testCluster:
self.cfg("masterIp", "192.168.0.1")
self.cfg("secondIp", "192.168.0.2")
self.cfg("logDir", self.logDir)
for key, value in self.cfgDict.items():
self.cfg(key, value)
tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath))
class TDDnode:
def __init__(self, index):
self.index = index
self.running = 0
self.deployed = 0
self.testCluster = False
self.valgrind = 0
def init(self, path):
self.path = path
def setTestCluster(self, value):
self.testCluster = value
def setValgrind(self, value):
self.valgrind = value
def getDataSize(self):
totalSize = 0
if (self.deployed == 1):
for dirpath, dirnames, filenames in os.walk(self.dataDir):
for f in filenames:
fp = os.path.join(dirpath, f)
if not os.path.islink(fp):
totalSize = totalSize + os.path.getsize(fp)
return totalSize
def deploy(self):
self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log")
self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data")
self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg")
self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg")
cmd = "rm -rf " + self.dataDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "rm -rf " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "rm -rf " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.dataDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "touch " + self.cfgPath
if os.system(cmd) != 0:
tdLog.exit(cmd)
if self.testCluster:
self.startIP()
if self.testCluster:
self.cfg("masterIp", "192.168.0.1")
self.cfg("secondIp", "192.168.0.2")
self.cfg("publicIp", "192.168.0.%d" % (self.index))
self.cfg("internalIp", "192.168.0.%d" % (self.index))
self.cfg("privateIp", "192.168.0.%d" % (self.index))
self.cfg("dataDir", self.dataDir)
self.cfg("logDir", self.logDir)
self.cfg("numOfLogLines", "100000000")
self.cfg("mnodeEqualVnodeNum", "0")
self.cfg("walLevel", "2")
self.cfg("fsync", "1000")
self.cfg("statusInterval", "1")
self.cfg("numOfMnodes", "3")
self.cfg("numOfThreadsPerCore", "2.0")
self.cfg("monitor", "0")
self.cfg("maxVnodeConnections", "30000")
self.cfg("maxMgmtConnections", "30000")
self.cfg("maxMeterConnections", "30000")
self.cfg("maxShellConns", "30000")
self.cfg("locale", "en_US.UTF-8")
self.cfg("charset", "UTF-8")
self.cfg("asyncLog", "0")
self.cfg("anyIp", "0")
self.cfg("dDebugFlag", "135")
self.cfg("mDebugFlag", "135")
self.cfg("sdbDebugFlag", "135")
self.cfg("rpcDebugFlag", "135")
self.cfg("tmrDebugFlag", "131")
self.cfg("cDebugFlag", "135")
self.cfg("httpDebugFlag", "135")
self.cfg("monitorDebugFlag", "135")
self.cfg("udebugFlag", "135")
self.cfg("jnidebugFlag", "135")
self.cfg("qdebugFlag", "135")
self.deployed = 1
tdLog.debug(
"dnode:%d is deployed and configured by %s" %
(self.index, self.cfgPath))
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root)-len("/build/bin")]
break
return buildPath
def start(self):
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info("taosd found in %s" % buildPath)
binPath = buildPath + "/build/bin/taosd"
if self.deployed == 0:
tdLog.exit("dnode:%d is not deployed" % (self.index))
if self.valgrind == 0:
cmd = "nohup %s -c %s > /dev/null 2>&1 & " % (
binPath, self.cfgDir)
else:
valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"
cmd = "nohup %s %s -c %s 2>&1 & " % (
valgrindCmdline, binPath, self.cfgDir)
print(cmd)
if os.system(cmd) != 0:
tdLog.exit(cmd)
self.running = 1
tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index))
time.sleep(5)
def stop(self):
if self.valgrind == 0:
toBeKilled = "taosd"
else:
toBeKilled = "valgrind.bin"
if self.running != 0:
psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -INT %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
for port in range(6030, 6041):
fuserCmd = "fuser -k -n tcp %d" % port
os.system(fuserCmd)
if self.valgrind:
time.sleep(2)
self.running = 0
tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index))
def forcestop(self):
if self.valgrind == 0:
toBeKilled = "taosd"
else:
toBeKilled = "valgrind.bin"
if self.running != 0:
psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
for port in range(6030, 6041):
fuserCmd = "fuser -k -n tcp %d" % port
os.system(fuserCmd)
if self.valgrind:
time.sleep(2)
self.running = 0
tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index))
def startIP(self):
cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def stopIP(self):
cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % (
self.index, self.index)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def cfg(self, option, value):
cmd = "echo %s %s >> %s" % (option, value, self.cfgPath)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def getDnodeRootDir(self, index):
dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index)
return dnodeRootDir
def getDnodesRootDir(self):
dnodesRootDir = os.path.join(self.path,"sim","psim")
return dnodesRootDir
class TDDnodes:
def __init__(self):
self.dnodes = []
self.dnodes.append(TDDnode(1))
self.dnodes.append(TDDnode(2))
self.dnodes.append(TDDnode(3))
self.dnodes.append(TDDnode(4))
self.dnodes.append(TDDnode(5))
self.dnodes.append(TDDnode(6))
self.dnodes.append(TDDnode(7))
self.dnodes.append(TDDnode(8))
self.dnodes.append(TDDnode(9))
self.dnodes.append(TDDnode(10))
self.simDeployed = False
def init(self, path):
psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
binPath = os.path.dirname(os.path.realpath(__file__))
binPath = binPath + "/../../../debug/"
tdLog.debug("binPath %s" % (binPath))
binPath = os.path.realpath(binPath)
tdLog.debug("binPath real path %s" % (binPath))
# cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath)
# tdLog.debug(cmd)
# os.system(cmd)
# cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath)
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
# tdLog.debug("execute %s" % (cmd))
# cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath)
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
# tdLog.debug("execute %s" % (cmd))
if path == "":
# self.path = os.path.expanduser('~')
self.path = os.path.abspath(binPath + "../../")
else:
self.path = os.path.realpath(path)
for i in range(len(self.dnodes)):
self.dnodes[i].init(self.path)
self.sim = TDSimClient()
self.sim.init(self.path)
def setTestCluster(self, value):
self.testCluster = value
def setValgrind(self, value):
self.valgrind = value
def deploy(self, index):
self.sim.setTestCluster(self.testCluster)
if (self.simDeployed == False):
self.sim.deploy()
self.simDeployed = True
self.check(index)
self.dnodes[index - 1].setTestCluster(self.testCluster)
self.dnodes[index - 1].setValgrind(self.valgrind)
self.dnodes[index - 1].deploy()
def cfg(self, index, option, value):
self.check(index)
self.dnodes[index - 1].cfg(option, value)
def start(self, index):
self.check(index)
self.dnodes[index - 1].start()
def stop(self, index):
self.check(index)
self.dnodes[index - 1].stop()
def getDataSize(self, index):
self.check(index)
return self.dnodes[index - 1].getDataSize()
def forcestop(self, index):
self.check(index)
self.dnodes[index - 1].forcestop()
def startIP(self, index):
self.check(index)
if self.testCluster:
self.dnodes[index - 1].startIP()
def stopIP(self, index):
self.check(index)
if self.dnodes[index - 1].testCluster:
self.dnodes[index - 1].stopIP()
def check(self, index):
if index < 1 or index > 10:
tdLog.exit("index:%d should on a scale of [1, 10]" % (index))
def stopAll(self):
tdLog.info("stop all dnodes")
for i in range(len(self.dnodes)):
self.dnodes[i].stop()
psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
if processID:
cmd = "sudo systemctl stop taosd"
os.system(cmd)
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
def getDnodesRootDir(self):
dnodesRootDir = "%s/sim" % (self.path)
return dnodesRootDir
def getSimCfgPath(self):
return self.sim.getCfgDir()
def getSimLogPath(self):
return self.sim.getLogDir()
def addSimExtraCfg(self, option, value):
self.sim.addExtraCfg(option, value)
tdDnodes = TDDnodes()

500
tests/army/frame/dnodes-no-random-fail.py
View File

@ -1,500 +0,0 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import os
import os.path
import subprocess
from frame.log import *
class TDSimClient:
def __init__(self):
self.testCluster = False
self.cfgDict = {
"numOfLogLines": "100000000",
"numOfThreadsPerCore": "2.0",
"locale": "en_US.UTF-8",
"charset": "UTF-8",
"asyncLog": "0",
"anyIp": "0",
"sdbDebugFlag": "135",
"rpcDebugFlag": "135",
"tmrDebugFlag": "131",
"cDebugFlag": "135",
"udebugFlag": "135",
"jnidebugFlag": "135",
"qdebugFlag": "135",
"telemetryReporting": "0",
}
def init(self, path):
self.__init__()
self.path = path
def getLogDir(self):
self.logDir = os.path.join(self.path,"sim","psim","log")
return self.logDir
def getCfgDir(self):
self.cfgDir = os.path.join(self.path,"sim","psim","cfg")
return self.cfgDir
def setTestCluster(self, value):
self.testCluster = value
def addExtraCfg(self, option, value):
self.cfgDict.update({option: value})
def cfg(self, option, value):
cmd = "echo %s %s >> %s" % (option, value, self.cfgPath)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def deploy(self):
self.logDir = os.path.join(self.path,"sim","psim","log")
self.cfgDir = os.path.join(self.path,"sim","psim","cfg")
self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg")
cmd = "rm -rf " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "rm -rf " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "touch " + self.cfgPath
if os.system(cmd) != 0:
tdLog.exit(cmd)
if self.testCluster:
self.cfg("masterIp", "192.168.0.1")
self.cfg("secondIp", "192.168.0.2")
self.cfg("logDir", self.logDir)
for key, value in self.cfgDict.items():
self.cfg(key, value)
tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath))
class TDDnode:
def __init__(self, index):
self.index = index
self.running = 0
self.deployed = 0
self.testCluster = False
self.valgrind = 0
def init(self, path):
self.path = path
def setTestCluster(self, value):
self.testCluster = value
def setValgrind(self, value):
self.valgrind = value
def getDataSize(self):
totalSize = 0
if (self.deployed == 1):
for dirpath, dirnames, filenames in os.walk(self.dataDir):
for f in filenames:
fp = os.path.join(dirpath, f)
if not os.path.islink(fp):
totalSize = totalSize + os.path.getsize(fp)
return totalSize
def deploy(self):
self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log")
self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data")
self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg")
self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg")
cmd = "rm -rf " + self.dataDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "rm -rf " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "rm -rf " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.dataDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "touch " + self.cfgPath
if os.system(cmd) != 0:
tdLog.exit(cmd)
if self.testCluster:
self.startIP()
if self.testCluster:
self.cfg("masterIp", "192.168.0.1")
self.cfg("secondIp", "192.168.0.2")
self.cfg("publicIp", "192.168.0.%d" % (self.index))
self.cfg("internalIp", "192.168.0.%d" % (self.index))
self.cfg("privateIp", "192.168.0.%d" % (self.index))
self.cfg("dataDir", self.dataDir)
self.cfg("logDir", self.logDir)
self.cfg("numOfLogLines", "100000000")
self.cfg("mnodeEqualVnodeNum", "0")
self.cfg("walLevel", "2")
self.cfg("fsync", "1000")
self.cfg("statusInterval", "1")
self.cfg("numOfMnodes", "3")
self.cfg("numOfThreadsPerCore", "2.0")
self.cfg("monitor", "0")
self.cfg("maxVnodeConnections", "30000")
self.cfg("maxMgmtConnections", "30000")
self.cfg("maxMeterConnections", "30000")
self.cfg("maxShellConns", "30000")
self.cfg("locale", "en_US.UTF-8")
self.cfg("charset", "UTF-8")
self.cfg("asyncLog", "0")
self.cfg("anyIp", "0")
self.cfg("dDebugFlag", "135")
self.cfg("mDebugFlag", "135")
self.cfg("sdbDebugFlag", "135")
self.cfg("rpcDebugFlag", "135")
self.cfg("tmrDebugFlag", "131")
self.cfg("cDebugFlag", "135")
self.cfg("httpDebugFlag", "135")
self.cfg("monitorDebugFlag", "135")
self.cfg("udebugFlag", "135")
self.cfg("jnidebugFlag", "135")
self.cfg("qdebugFlag", "135")
self.deployed = 1
tdLog.debug(
"dnode:%d is deployed and configured by %s" %
(self.index, self.cfgPath))
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root)-len("/build/bin")]
break
return buildPath
def start(self):
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info("taosd found in %s" % buildPath)
binPath = buildPath + "/build/bin/taosd"
if self.deployed == 0:
tdLog.exit("dnode:%d is not deployed" % (self.index))
if self.valgrind == 0:
cmd = "nohup %s -c %s --random-file-fail-factor 0 > /dev/null 2>&1 & " % (
binPath, self.cfgDir)
else:
valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"
cmd = "nohup %s %s -c %s 2>&1 & " % (
valgrindCmdline, binPath, self.cfgDir)
print(cmd)
if os.system(cmd) != 0:
tdLog.exit(cmd)
self.running = 1
tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index))
time.sleep(5)
def stop(self):
if self.valgrind == 0:
toBeKilled = "taosd"
else:
toBeKilled = "valgrind.bin"
if self.running != 0:
psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -INT %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
for port in range(6030, 6041):
fuserCmd = "fuser -k -n tcp %d" % port
os.system(fuserCmd)
if self.valgrind:
time.sleep(2)
self.running = 0
tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index))
def forcestop(self):
if self.valgrind == 0:
toBeKilled = "taosd"
else:
toBeKilled = "valgrind.bin"
if self.running != 0:
psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
for port in range(6030, 6041):
fuserCmd = "fuser -k -n tcp %d" % port
os.system(fuserCmd)
if self.valgrind:
time.sleep(2)
self.running = 0
tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index))
def startIP(self):
cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def stopIP(self):
cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % (
self.index, self.index)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def cfg(self, option, value):
cmd = "echo %s %s >> %s" % (option, value, self.cfgPath)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def getDnodeRootDir(self, index):
dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index)
return dnodeRootDir
def getDnodesRootDir(self):
dnodesRootDir = os.path.join(self.path,"sim","psim")
return dnodesRootDir
class TDDnodes:
def __init__(self):
self.dnodes = []
self.dnodes.append(TDDnode(1))
self.dnodes.append(TDDnode(2))
self.dnodes.append(TDDnode(3))
self.dnodes.append(TDDnode(4))
self.dnodes.append(TDDnode(5))
self.dnodes.append(TDDnode(6))
self.dnodes.append(TDDnode(7))
self.dnodes.append(TDDnode(8))
self.dnodes.append(TDDnode(9))
self.dnodes.append(TDDnode(10))
self.simDeployed = False
def init(self, path):
psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
binPath = os.path.dirname(os.path.realpath(__file__))
binPath = binPath + "/../../../debug/"
tdLog.debug("binPath %s" % (binPath))
binPath = os.path.realpath(binPath)
tdLog.debug("binPath real path %s" % (binPath))
# cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath)
# tdLog.debug(cmd)
# os.system(cmd)
# cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath)
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
# tdLog.debug("execute %s" % (cmd))
# cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath)
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
# tdLog.debug("execute %s" % (cmd))
if path == "":
# self.path = os.path.expanduser('~')
self.path = os.path.abspath(binPath + "../../")
else:
self.path = os.path.realpath(path)
for i in range(len(self.dnodes)):
self.dnodes[i].init(self.path)
self.sim = TDSimClient()
self.sim.init(self.path)
def setTestCluster(self, value):
self.testCluster = value
def setValgrind(self, value):
self.valgrind = value
def deploy(self, index):
self.sim.setTestCluster(self.testCluster)
if (self.simDeployed == False):
self.sim.deploy()
self.simDeployed = True
self.check(index)
self.dnodes[index - 1].setTestCluster(self.testCluster)
self.dnodes[index - 1].setValgrind(self.valgrind)
self.dnodes[index - 1].deploy()
def cfg(self, index, option, value):
self.check(index)
self.dnodes[index - 1].cfg(option, value)
def start(self, index):
self.check(index)
self.dnodes[index - 1].start()
def stop(self, index):
self.check(index)
self.dnodes[index - 1].stop()
def getDataSize(self, index):
self.check(index)
return self.dnodes[index - 1].getDataSize()
def forcestop(self, index):
self.check(index)
self.dnodes[index - 1].forcestop()
def startIP(self, index):
self.check(index)
if self.testCluster:
self.dnodes[index - 1].startIP()
def stopIP(self, index):
self.check(index)
if self.dnodes[index - 1].testCluster:
self.dnodes[index - 1].stopIP()
def check(self, index):
if index < 1 or index > 10:
tdLog.exit("index:%d should on a scale of [1, 10]" % (index))
def stopAll(self):
tdLog.info("stop all dnodes")
for i in range(len(self.dnodes)):
self.dnodes[i].stop()
psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
if processID:
cmd = "sudo systemctl stop taosd"
os.system(cmd)
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
def getDnodesRootDir(self):
dnodesRootDir = "%s/sim" % (self.path)
return dnodesRootDir
def getSimCfgPath(self):
return self.sim.getCfgDir()
def getSimLogPath(self):
return self.sim.getLogDir()
def addSimExtraCfg(self, option, value):
self.sim.addExtraCfg(option, value)
tdDnodes = TDDnodes()

497
tests/army/frame/dnodes-random-fail.py
View File

@ -1,497 +0,0 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import os
import os.path
import subprocess
from frame.log import *
class TDSimClient:
def __init__(self):
self.testCluster = False
self.cfgDict = {
"numOfLogLines": "100000000",
"locale": "en_US.UTF-8",
"charset": "UTF-8",
"asyncLog": "0",
"rpcDebugFlag": "135",
"tmrDebugFlag": "131",
"cDebugFlag": "135",
"udebugFlag": "135",
"jnidebugFlag": "135",
"qdebugFlag": "135",
"telemetryReporting": "0",
}
def init(self, path):
self.__init__()
self.path = path
def getLogDir(self):
self.logDir = os.path.join(self.path,"sim","psim","log")
return self.logDir
def getCfgDir(self):
self.cfgDir = os.path.join(self.path,"sim","psim","cfg")
return self.cfgDir
def setTestCluster(self, value):
self.testCluster = value
def addExtraCfg(self, option, value):
self.cfgDict.update({option: value})
def cfg(self, option, value):
cmd = "echo %s %s >> %s" % (option, value, self.cfgPath)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def deploy(self):
self.logDir = os.path.join(self.path,"sim","psim","log")
self.cfgDir = os.path.join(self.path,"sim","psim","cfg")
self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg")
cmd = "rm -rf " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "rm -rf " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "touch " + self.cfgPath
if os.system(cmd) != 0:
tdLog.exit(cmd)
if self.testCluster:
self.cfg("masterIp", "192.168.0.1")
self.cfg("secondIp", "192.168.0.2")
self.cfg("logDir", self.logDir)
for key, value in self.cfgDict.items():
self.cfg(key, value)
tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath))
class TDDnode:
def __init__(self, index):
self.index = index
self.running = 0
self.deployed = 0
self.testCluster = False
self.valgrind = 0
def init(self, path):
self.path = path
def setTestCluster(self, value):
self.testCluster = value
def setValgrind(self, value):
self.valgrind = value
def getDataSize(self):
totalSize = 0
if (self.deployed == 1):
for dirpath, dirnames, filenames in os.walk(self.dataDir):
for f in filenames:
fp = os.path.join(dirpath, f)
if not os.path.islink(fp):
totalSize = totalSize + os.path.getsize(fp)
return totalSize
def deploy(self):
self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log")
self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data")
self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg")
self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg")
cmd = "rm -rf " + self.dataDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "rm -rf " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "rm -rf " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.dataDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.logDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "mkdir -p " + self.cfgDir
if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "touch " + self.cfgPath
if os.system(cmd) != 0:
tdLog.exit(cmd)
if self.testCluster:
self.startIP()
if self.testCluster:
self.cfg("masterIp", "192.168.0.1")
self.cfg("secondIp", "192.168.0.2")
self.cfg("publicIp", "192.168.0.%d" % (self.index))
self.cfg("internalIp", "192.168.0.%d" % (self.index))
self.cfg("privateIp", "192.168.0.%d" % (self.index))
self.cfg("dataDir", self.dataDir)
self.cfg("logDir", self.logDir)
self.cfg("numOfLogLines", "100000000")
self.cfg("mnodeEqualVnodeNum", "0")
self.cfg("walLevel", "2")
self.cfg("fsync", "1000")
self.cfg("statusInterval", "1")
self.cfg("numOfMnodes", "3")
self.cfg("numOfThreadsPerCore", "2.0")
self.cfg("monitor", "0")
self.cfg("maxVnodeConnections", "30000")
self.cfg("maxMgmtConnections", "30000")
self.cfg("maxMeterConnections", "30000")
self.cfg("maxShellConns", "30000")
self.cfg("locale", "en_US.UTF-8")
self.cfg("charset", "UTF-8")
self.cfg("asyncLog", "0")
self.cfg("anyIp", "0")
self.cfg("dDebugFlag", "135")
self.cfg("mDebugFlag", "135")
self.cfg("sdbDebugFlag", "135")
self.cfg("rpcDebugFlag", "135")
self.cfg("tmrDebugFlag", "131")
self.cfg("cDebugFlag", "135")
self.cfg("httpDebugFlag", "135")
self.cfg("monitorDebugFlag", "135")
self.cfg("udebugFlag", "135")
self.cfg("jnidebugFlag", "135")
self.cfg("qdebugFlag", "135")
self.deployed = 1
tdLog.debug(
"dnode:%d is deployed and configured by %s" %
(self.index, self.cfgPath))
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root)-len("/build/bin")]
break
return buildPath
def start(self):
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info("taosd found in %s" % buildPath)
binPath = buildPath + "/build/bin/taosd"
if self.deployed == 0:
tdLog.exit("dnode:%d is not deployed" % (self.index))
if self.valgrind == 0:
cmd = "nohup %s -c %s --alloc-random-fail --random-file-fail-factor 5 > /dev/null 2>&1 & " % (
binPath, self.cfgDir)
else:
valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"
cmd = "nohup %s %s -c %s 2>&1 & " % (
valgrindCmdline, binPath, self.cfgDir)
print(cmd)
if os.system(cmd) != 0:
tdLog.exit(cmd)
self.running = 1
tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index))
time.sleep(5)
def stop(self):
if self.valgrind == 0:
toBeKilled = "taosd"
else:
toBeKilled = "valgrind.bin"
if self.running != 0:
psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -INT %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
for port in range(6030, 6041):
fuserCmd = "fuser -k -n tcp %d" % port
os.system(fuserCmd)
if self.valgrind:
time.sleep(2)
self.running = 0
tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index))
def forcestop(self):
if self.valgrind == 0:
toBeKilled = "taosd"
else:
toBeKilled = "valgrind.bin"
if self.running != 0:
psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
for port in range(6030, 6041):
fuserCmd = "fuser -k -n tcp %d" % port
os.system(fuserCmd)
if self.valgrind:
time.sleep(2)
self.running = 0
tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index))
def startIP(self):
cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def stopIP(self):
cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % (
self.index, self.index)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def cfg(self, option, value):
cmd = "echo %s %s >> %s" % (option, value, self.cfgPath)
if os.system(cmd) != 0:
tdLog.exit(cmd)
def getDnodeRootDir(self, index):
dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index)
return dnodeRootDir
def getDnodesRootDir(self):
dnodesRootDir = os.path.join(self.path,"sim","psim")
return dnodesRootDir
class TDDnodes:
def __init__(self):
self.dnodes = []
self.dnodes.append(TDDnode(1))
self.dnodes.append(TDDnode(2))
self.dnodes.append(TDDnode(3))
self.dnodes.append(TDDnode(4))
self.dnodes.append(TDDnode(5))
self.dnodes.append(TDDnode(6))
self.dnodes.append(TDDnode(7))
self.dnodes.append(TDDnode(8))
self.dnodes.append(TDDnode(9))
self.dnodes.append(TDDnode(10))
self.simDeployed = False
def init(self, path):
psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
binPath = os.path.dirname(os.path.realpath(__file__))
binPath = binPath + "/../../../debug/"
tdLog.debug("binPath %s" % (binPath))
binPath = os.path.realpath(binPath)
tdLog.debug("binPath real path %s" % (binPath))
# cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath)
# tdLog.debug(cmd)
# os.system(cmd)
# cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath)
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
# tdLog.debug("execute %s" % (cmd))
# cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath)
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
# tdLog.debug("execute %s" % (cmd))
if path == "":
# self.path = os.path.expanduser('~')
self.path = os.path.abspath(binPath + "../../")
else:
self.path = os.path.realpath(path)
for i in range(len(self.dnodes)):
self.dnodes[i].init(self.path)
self.sim = TDSimClient()
self.sim.init(self.path)
def setTestCluster(self, value):
self.testCluster = value
def setValgrind(self, value):
self.valgrind = value
def deploy(self, index):
self.sim.setTestCluster(self.testCluster)
if (self.simDeployed == False):
self.sim.deploy()
self.simDeployed = True
self.check(index)
self.dnodes[index - 1].setTestCluster(self.testCluster)
self.dnodes[index - 1].setValgrind(self.valgrind)
self.dnodes[index - 1].deploy()
def cfg(self, index, option, value):
self.check(index)
self.dnodes[index - 1].cfg(option, value)
def start(self, index):
self.check(index)
self.dnodes[index - 1].start()
def stop(self, index):
self.check(index)
self.dnodes[index - 1].stop()
def getDataSize(self, index):
self.check(index)
return self.dnodes[index - 1].getDataSize()
def forcestop(self, index):
self.check(index)
self.dnodes[index - 1].forcestop()
def startIP(self, index):
self.check(index)
if self.testCluster:
self.dnodes[index - 1].startIP()
def stopIP(self, index):
self.check(index)
if self.dnodes[index - 1].testCluster:
self.dnodes[index - 1].stopIP()
def check(self, index):
if index < 1 or index > 10:
tdLog.exit("index:%d should on a scale of [1, 10]" % (index))
def stopAll(self):
tdLog.info("stop all dnodes")
for i in range(len(self.dnodes)):
self.dnodes[i].stop()
psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
if processID:
cmd = "sudo systemctl stop taosd"
os.system(cmd)
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
while(processID):
killCmd = "kill -TERM %s > /dev/null 2>&1" % processID
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(
psCmd, shell=True).decode("utf-8")
# if os.system(cmd) != 0 :
# tdLog.exit(cmd)
def getDnodesRootDir(self):
dnodesRootDir = "%s/sim" % (self.path)
return dnodesRootDir
def getSimCfgPath(self):
return self.sim.getCfgDir()
def getSimLogPath(self):
return self.sim.getLogDir()
def addSimExtraCfg(self, option, value):
self.sim.addExtraCfg(option, value)
tdDnodes = TDDnodes()

36
tests/army/frame/eos.py Normal file
View File

@ -0,0 +1,36 @@
###################################################################
# Copyright (c) 2023 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
#
# about system funciton extension
#
import sys
import os
import time
import datetime
import platform
# if windows platform return True
def isWin():
return platform.system().lower() == 'windows'
# wait util execute file finished
def exe(file):
return os.system(file)
# execute file and return immediately
def exeNoWait(file):
print("exe no wait")

53
tests/army/frame/epath.py Normal file
View File

@ -0,0 +1,53 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
#
# about path function extension
#
import os
from frame.log import *
# build/bin path
binDir = ""
def binPath():
global binDir
if binDir != "":
return binDir
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community/tests" in selfPath):
projPath = selfPath[:selfPath.find("community/tests")]
else:
projPath = selfPath[:selfPath.find("TDengine/tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root)-len("/build/bin")]
break
# check
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info(f"taosd found in {buildPath}")
# return
binDir = buildPath + "/build/bin/"
return binDir
def binFile(filename):
return binPath() + filename

23
tests/army/frame/etime.py Normal file
View File

@ -0,0 +1,23 @@
###################################################################
# Copyright (c) 2023 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
#
# about tools funciton extension
#
import sys
import os
import time
import datetime

46
tests/army/frame/etool.py Normal file
View File

@ -0,0 +1,46 @@
###################################################################
# Copyright (c) 2023 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
#
# about system funciton extension
#
import sys
import os
import time
import datetime
import frame.epath
import frame.eos
from frame.log import *
# run taosBenchmark with command or json file mode
def runBenchmark(command = "", json = "") :
# get taosBenchmark path
bmFile = frame.epath.binFile("taosBenchmark")
if frame.eos.isWin():
bmFile += ".exe"
# run
if command != "":
frame.eos.exe(bmFile + " " + command)
if json != "":
cmd = f"{bmFile} -f {json}"
print(cmd)
status = frame.eos.exe(cmd)
if status !=0:
tdLog.exit(f"run failed {cmd} status={status}")
# get current directory file name
def curFile(fullPath, filename):
return os.path.dirname(fullPath) + "/" + filename

83
tests/army/frame/pathFinding.py
View File

@ -1,83 +0,0 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import os
from frame.log import *
class TDFindPath:
"""This class is for finding path within TDengine
"""
def __init__(self):
self.file = ""
def init(self, file):
"""[summary]
Args:
file (str): the file location you want to start the query. Generally using __file__
"""
self.file = file
def getTaosdemoPath(self):
"""for finding the path of directory containing taosdemo
Returns:
str: the path to directory containing taosdemo
"""
selfPath = os.path.dirname(os.path.realpath(self.file))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root)-len("/build/bin")]
break
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info(f"taosd found in {buildPath}")
return buildPath + "/build/bin/"
def getTDenginePath(self):
"""for finding the root path of TDengine
Returns:
str: the root path of TDengine
"""
selfPath = os.path.dirname(os.path.realpath(self.file))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
print(projPath)
for root, dirs, files in os.walk(projPath):
if ("sim" in dirs):
print(root)
rootRealPath = os.path.realpath(root)
if (rootRealPath == ""):
tdLog.exit("TDengine not found!")
else:
tdLog.info(f"TDengine found in {rootRealPath}")
return rootRealPath
tdFindPath = TDFindPath()

10
tests/army/frame/cluster.py → tests/army/frame/server/cluster.py
View File

@ -8,7 +8,7 @@ import socket
from frame.log import * from frame.log import *
from frame.sql import * from frame.sql import *
from frame.cases import * from frame.cases import *
from frame.dnodes import * from frame.server.dnodes import *
from frame.common import * from frame.common import *
class ClusterDnodes(TDDnodes): class ClusterDnodes(TDDnodes):
@ -35,17 +35,21 @@ class ConfigureyCluster:
self.startPort = 6030 self.startPort = 6030
self.portStep = 100 self.portStep = 100
self.mnodeNums = 0 self.mnodeNums = 0
self.level = 0
self.disk = 0
def configure_cluster(self ,dnodeNums=5,mnodeNums=0,independentMnode=True,startPort=6030,portStep=100,hostname="%s"%hostname): def configure_cluster(self ,dnodeNums=5, mnodeNums=0, independentMnode=True, startPort=6030, portStep=100, hostname="%s"%hostname, level=1, disk=1):
self.startPort=int(startPort) self.startPort=int(startPort)
self.portStep=int(portStep) self.portStep=int(portStep)
self.hostname=hostname self.hostname=hostname
self.dnodeNums = int(dnodeNums) self.dnodeNums = int(dnodeNums)
self.mnodeNums = int(mnodeNums) self.mnodeNums = int(mnodeNums)
self.level = int(level)
self.disk = int(disk)
self.dnodes = [] self.dnodes = []
startPort_sec = int(startPort+portStep) startPort_sec = int(startPort+portStep)
for num in range(1, (self.dnodeNums+1)): for num in range(1, (self.dnodeNums+1)):
dnode = TDDnode(num) dnode = TDDnode(num, self.level, self.disk)
dnode.addExtraCfg("firstEp", f"{hostname}:{self.startPort}") dnode.addExtraCfg("firstEp", f"{hostname}:{self.startPort}")
dnode.addExtraCfg("fqdn", f"{hostname}") dnode.addExtraCfg("fqdn", f"{hostname}")
dnode.addExtraCfg("serverPort", f"{self.startPort + (num-1)*self.portStep}") dnode.addExtraCfg("serverPort", f"{self.startPort + (num-1)*self.portStep}")

48
tests/army/frame/dnodes.py → tests/army/frame/server/dnodes.py
View File

@ -115,8 +115,11 @@ class TDSimClient:
class TDDnode: class TDDnode:
def __init__(self, index): def __init__(self, index=1, level=1, disk=1):
self.index = index self.index = index
self.level = level
self.disk = disk
self.dataDir = []
self.running = 0 self.running = 0
self.deployed = 0 self.deployed = 0
self.testCluster = False self.testCluster = False
@ -209,14 +212,30 @@ class TDDnode:
self.remote_conn.run("python3 ./test.py %s -d %s -e %s"%(valgrindStr,remoteCfgDictStr,execCmdStr)) self.remote_conn.run("python3 ./test.py %s -d %s -e %s"%(valgrindStr,remoteCfgDictStr,execCmdStr))
def deploy(self, *updatecfgDict): def deploy(self, *updatecfgDict):
# logDir
self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log")
self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") # dataDir
simPath = os.path.join(self.path, "sim", "dnode%d" % self.index)
primary = 1
if self.level == 1 and self.disk == 1:
eDir = os.path.join(simPath, "data")
self.dataDir.append(eDir)
else:
for i in range(self.level):
for j in range(self.disk):
eDir = os.path.join(simPath, f"data{i}{j}")
self.dataDir.append(f"{eDir} {i} {primary}")
if primary == 1:
primary = 0
# taos.cfg
self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg")
self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg")
cmd = "rm -rf " + self.dataDir for eDir in self.dataDir:
if os.system(cmd) != 0: cmd = "rm -rf " + eDir
tdLog.exit(cmd) if os.system(cmd) != 0:
tdLog.exit(cmd)
cmd = "rm -rf " + self.logDir cmd = "rm -rf " + self.logDir
if os.system(cmd) != 0: if os.system(cmd) != 0:
@ -229,7 +248,8 @@ class TDDnode:
# cmd = "mkdir -p " + self.dataDir # cmd = "mkdir -p " + self.dataDir
# if os.system(cmd) != 0: # if os.system(cmd) != 0:
# tdLog.exit(cmd) # tdLog.exit(cmd)
os.makedirs(self.dataDir) for eDir in self.dataDir:
os.makedirs(eDir.split(' ')[0])
# cmd = "mkdir -p " + self.logDir # cmd = "mkdir -p " + self.logDir
# if os.system(cmd) != 0: # if os.system(cmd) != 0:
@ -275,7 +295,11 @@ class TDDnode:
self.addExtraCfg(key, value) self.addExtraCfg(key, value)
if (self.remoteIP == ""): if (self.remoteIP == ""):
for key, value in self.cfgDict.items(): for key, value in self.cfgDict.items():
self.cfg(key, value) if type(value) == list:
for v in value:
self.cfg(key, v)
else:
self.cfg(key, value)
else: else:
self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)"%self.index) self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)"%self.index)
@ -887,4 +911,10 @@ class TDDnodes:
def getAsan(self): def getAsan(self):
return self.asan return self.asan
tdDnodes = TDDnodes() def setLevelDisk(self, level, disk):
for i in range(len(self.dnodes)):
self.dnodes[i].level = int(level)
self.dnodes[i].disk = int(disk)
tdDnodes = TDDnodes()

14
tests/army/frame/sql.py
View File

@ -535,6 +535,16 @@ class TDSql:
tdLog.info("%s(%d) failed: sql:%s, elm:%s == expect_elm:%s" % args) tdLog.info("%s(%d) failed: sql:%s, elm:%s == expect_elm:%s" % args)
raise Exception raise Exception
# check like select count(*) ... sql
def checkAgg(self, sql, expectCnt):
self.query(sql)
self.checkData(0, 0, expectCnt)
# get first value
def getFirstValue(self, sql) :
self.query(sql)
return self.getData(0, 0)
def get_times(self, time_str, precision="ms"): def get_times(self, time_str, precision="ms"):
caller = inspect.getframeinfo(inspect.stack()[1][0]) caller = inspect.getframeinfo(inspect.stack()[1][0])
if time_str[-1] not in TAOS_TIME_INIT: if time_str[-1] not in TAOS_TIME_INIT:
@ -602,6 +612,7 @@ class TDSql:
if self.cursor.istype(col, "BIGINT UNSIGNED"): if self.cursor.istype(col, "BIGINT UNSIGNED"):
return "BIGINT UNSIGNED" return "BIGINT UNSIGNED"
'''
def taosdStatus(self, state): def taosdStatus(self, state):
tdLog.sleep(5) tdLog.sleep(5)
pstate = 0 pstate = 0
@ -630,6 +641,7 @@ class TDSql:
tdLog.exit("taosd state is %d != expect:%d" %args) tdLog.exit("taosd state is %d != expect:%d" %args)
pass pass
def haveFile(self, dir, state): def haveFile(self, dir, state):
if os.path.exists(dir) and os.path.isdir(dir): if os.path.exists(dir) and os.path.isdir(dir):
if not os.listdir(dir): if not os.listdir(dir):
@ -644,6 +656,7 @@ class TDSql:
tdLog.exit("dir: %s is not empty, expect: empty" %dir) tdLog.exit("dir: %s is not empty, expect: empty" %dir)
else: else:
tdLog.exit("dir: %s doesn't exist" %dir) tdLog.exit("dir: %s doesn't exist" %dir)
def createDir(self, dir): def createDir(self, dir):
if os.path.exists(dir): if os.path.exists(dir):
shrmtree(dir) shrmtree(dir)
@ -651,5 +664,6 @@ class TDSql:
os.makedirs( dir, 755 ) os.makedirs( dir, 755 )
tdLog.info("dir: %s is created" %dir) tdLog.info("dir: %s is created" %dir)
pass pass
'''
tdSql = TDSql() tdSql = TDSql()

27
tests/army/frame/srvCtl.py Normal file
View File

@ -0,0 +1,27 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import os
import time
import datetime
class srvCtl:
def __init__(self):
# record server information
self.dnodeNum = 0
self.mnodeNum = 0
self.mLevel = 0
self.mLevelDisk = 0
sc = srvCtl()

32
tests/army/test.py
View File

@ -28,9 +28,9 @@ import importlib
import toml import toml
from frame.log import * from frame.log import *
from frame.dnodes import * from frame.server.dnodes import *
from frame.server.cluster import *
from frame.cases import * from frame.cases import *
from frame.cluster import *
from frame.taosadapter import * from frame.taosadapter import *
import taos import taos
@ -111,8 +111,12 @@ if __name__ == "__main__":
asan = False asan = False
independentMnode = False independentMnode = False
previousCluster = False previousCluster = False
opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWD:n:i:aP', [ level = 1
'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode','previous']) disk = 1
opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWU:n:i:aP:L:D:', [
'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums',
'queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode',"asan",'previous','level','disk'])
for key, value in opts: for key, value in opts:
if key in ['-h', '--help']: if key in ['-h', '--help']:
tdLog.printNoPrefix( tdLog.printNoPrefix(
@ -134,11 +138,13 @@ if __name__ == "__main__":
tdLog.printNoPrefix('-C create Dnode Numbers in one cluster') tdLog.printNoPrefix('-C create Dnode Numbers in one cluster')
tdLog.printNoPrefix('-R restful realization form') tdLog.printNoPrefix('-R restful realization form')
tdLog.printNoPrefix('-W websocket connection') tdLog.printNoPrefix('-W websocket connection')
tdLog.printNoPrefix('-D taosadapter update cfg dict ') tdLog.printNoPrefix('-U taosadapter update cfg dict ')
tdLog.printNoPrefix('-n the number of replicas') tdLog.printNoPrefix('-n the number of replicas')
tdLog.printNoPrefix('-i independentMnode Mnode') tdLog.printNoPrefix('-i independentMnode Mnode')
tdLog.printNoPrefix('-a address sanitizer mode') tdLog.printNoPrefix('-a address sanitizer mode')
tdLog.printNoPrefix('-P run case with [P]revious cluster, do not create new cluster to run case.') tdLog.printNoPrefix('-P run case with [P]revious cluster, do not create new cluster to run case.')
tdLog.printNoPrefix('-L set multiple level number. range 1 ~ 3')
tdLog.printNoPrefix('-D set disk number on each level. range 1 ~ 10')
sys.exit(0) sys.exit(0)
@ -213,7 +219,7 @@ if __name__ == "__main__":
if key in ['-a', '--asan']: if key in ['-a', '--asan']:
asan = True asan = True
if key in ['-D', '--adaptercfgupdate']: if key in ['-U', '--adaptercfgupdate']:
try: try:
adaptercfgupdate = eval(base64.b64decode(value.encode()).decode()) adaptercfgupdate = eval(base64.b64decode(value.encode()).decode())
except: except:
@ -226,6 +232,12 @@ if __name__ == "__main__":
if key in ['-P', '--previous']: if key in ['-P', '--previous']:
previousCluster = True previousCluster = True
if key in ['-L', '--level']:
level = value
if key in ['-D', '--disk']:
disk = value
# #
# do exeCmd command # do exeCmd command
# #
@ -361,6 +373,7 @@ if __name__ == "__main__":
tAdapter.stop(force_kill=True) tAdapter.stop(force_kill=True)
if dnodeNums == 1 : if dnodeNums == 1 :
tdDnodes.setLevelDisk(level, disk)
tdDnodes.deploy(1,updateCfgDict) tdDnodes.deploy(1,updateCfgDict)
tdDnodes.start(1) tdDnodes.start(1)
tdCases.logSql(logSql) tdCases.logSql(logSql)
@ -391,7 +404,7 @@ if __name__ == "__main__":
tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") tdLog.exit(f"alter queryPolicy to {queryPolicy} failed")
else : else :
tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums))
dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode) dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk)
tdDnodes = ClusterDnodes(dnodeslist) tdDnodes = ClusterDnodes(dnodeslist)
tdDnodes.init(deployPath, masterIp) tdDnodes.init(deployPath, masterIp)
tdDnodes.setTestCluster(testCluster) tdDnodes.setTestCluster(testCluster)
@ -498,6 +511,7 @@ if __name__ == "__main__":
else: else:
tdLog.info("not need to query") tdLog.info("not need to query")
else: else:
# except windows
tdDnodes.setKillValgrind(killValgrind) tdDnodes.setKillValgrind(killValgrind)
tdDnodes.init(deployPath, masterIp) tdDnodes.init(deployPath, masterIp)
tdDnodes.setTestCluster(testCluster) tdDnodes.setTestCluster(testCluster)
@ -529,6 +543,7 @@ if __name__ == "__main__":
if dnodeNums == 1 : if dnodeNums == 1 :
# dnode is one # dnode is one
tdDnodes.setLevelDisk(level, disk)
tdDnodes.deploy(1,updateCfgDict) tdDnodes.deploy(1,updateCfgDict)
tdDnodes.start(1) tdDnodes.start(1)
tdCases.logSql(logSql) tdCases.logSql(logSql)
@ -574,7 +589,8 @@ if __name__ == "__main__":
# dnode > 1 cluster # dnode > 1 cluster
tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums))
print(independentMnode,"independentMnode valuse") print(independentMnode,"independentMnode valuse")
dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode) # create dnode list
dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk)
tdDnodes = ClusterDnodes(dnodeslist) tdDnodes = ClusterDnodes(dnodeslist)
tdDnodes.init(deployPath, masterIp) tdDnodes.init(deployPath, masterIp)
tdDnodes.setTestCluster(testCluster) tdDnodes.setTestCluster(testCluster)

2
tests/parallel_test/cases.task
View File

@ -6,7 +6,7 @@
,,y,unit-test,bash test.sh ,,y,unit-test,bash test.sh
#army-test #army-test
,,y,army,./pytest.sh python3 ./test.py -f empty.py ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2
#system test #system test
,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_basic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_basic.py

101
tests/pytest/util/sql.py
View File

@ -78,6 +78,26 @@ class TDSql:
self.cursor.execute(s) self.cursor.execute(s)
time.sleep(2) time.sleep(2)
def execute(self, sql, queryTimes=30, show=False):
self.sql = sql
if show:
tdLog.info(sql)
i=1
while i <= queryTimes:
try:
self.affectedRows = self.cursor.execute(sql)
return self.affectedRows
except Exception as e:
tdLog.notice("Try to execute sql again, query times: %d "%i)
if i == queryTimes:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, sql, repr(e))
tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
raise Exception(repr(e))
i+=1
time.sleep(1)
pass
def error(self, sql, expectedErrno = None, expectErrInfo = None, fullMatched = True): def error(self, sql, expectedErrno = None, expectErrInfo = None, fullMatched = True):
caller = inspect.getframeinfo(inspect.stack()[1][0]) caller = inspect.getframeinfo(inspect.stack()[1][0])
expectErrNotOccured = True expectErrNotOccured = True
@ -108,7 +128,7 @@ class TDSql:
if expectErrInfo == self.error_info: if expectErrInfo == self.error_info:
tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo))
else: else:
tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo '%s' occured, but not expected errno '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo '%s' occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo))
else: else:
if expectedErrno != None: if expectedErrno != None:
if expectedErrno in self.errno: if expectedErrno in self.errno:
@ -120,7 +140,7 @@ class TDSql:
if expectErrInfo in self.error_info: if expectErrInfo in self.error_info:
tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo))
else: else:
tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected errno '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo))
return self.error_info return self.error_info
@ -158,6 +178,63 @@ class TDSql:
time.sleep(1) time.sleep(1)
pass pass
def query_success_failed(self, sql, row_tag=None, queryTimes=10, count_expected_res=None, expectErrInfo = None, fullMatched = True):
self.sql = sql
i=1
while i <= queryTimes:
try:
self.cursor.execute(sql)
self.queryResult = self.cursor.fetchall()
self.queryRows = len(self.queryResult)
self.queryCols = len(self.cursor.description)
if count_expected_res is not None:
counter = 0
while count_expected_res != self.queryResult[0][0]:
self.cursor.execute(sql)
self.queryResult = self.cursor.fetchall()
if counter < queryTimes:
counter += 0.5
time.sleep(0.5)
else:
return False
tdLog.info("query is success")
time.sleep(1)
continue
except Exception as e:
tdLog.notice("Try to query again, query times: %d "%i)
caller = inspect.getframeinfo(inspect.stack()[1][0])
if i < queryTimes:
error_info = repr(e)
print(error_info)
self.error_info = ','.join(error_info[error_info.index('(')+1:-1].split(",")[:-1]).replace("'","")
self.queryRows = 0
self.queryCols = 0
self.queryResult = None
if fullMatched:
if expectErrInfo != None:
if expectErrInfo == self.error_info:
tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo))
else:
tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo '%s' occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo))
else:
if expectErrInfo != None:
if expectErrInfo in self.error_info:
tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo))
else:
tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo))
return self.error_info
elif i == queryTimes:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, sql, repr(e))
tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
raise Exception(repr(e))
i+=1
time.sleep(1)
pass
def is_err_sql(self, sql): def is_err_sql(self, sql):
err_flag = True err_flag = True
@ -471,25 +548,7 @@ class TDSql:
time.sleep(1) time.sleep(1)
continue continue
def execute(self, sql, queryTimes=30, show=False):
self.sql = sql
if show:
tdLog.info(sql)
i=1
while i <= queryTimes:
try:
self.affectedRows = self.cursor.execute(sql)
return self.affectedRows
except Exception as e:
tdLog.notice("Try to execute sql again, query times: %d "%i)
if i == queryTimes:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, sql, repr(e))
tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
raise Exception(repr(e))
i+=1
time.sleep(1)
pass
def checkAffectedRows(self, expectAffectedRows): def checkAffectedRows(self, expectAffectedRows):
if self.affectedRows != expectAffectedRows: if self.affectedRows != expectAffectedRows:

36
tests/script/tsim/stream/session1.sim
View File

@ -327,4 +327,40 @@ if $rows != 1 then
goto loop17 goto loop17
endi endi
sql create database test2 vgroups 4;
sql use test2;
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create stream streams4 trigger at_once ignore update 0 ignore expired 0 into streamt4 as select _wstart, count(*) c1, count(a) c2 from st session(ts, 2s) ;
sql insert into t1 values(1648791255100,1,2,3);
sql insert into t1 values(1648791255300,1,2,3);
sleep 1000
sql insert into t1 values(1648791253000,1,2,3) (1648791254000,1,2,3);
$loop_count = 0
loop18:
sleep 1000
sql select * from streamt4;
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 1 then
print =====rows=$rows
goto loop18
endi
if $data01 != 4 then
print =====data01=$data01
goto loop18
endi
print =====over
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT

6
tests/system-test/0-others/test_hot_refresh_configurations.py
View File

@ -215,6 +215,12 @@ class TDTestCase:
self.svr_check(item["name"], item["alias"], item["except_values"], True) self.svr_check(item["name"], item["alias"], item["except_values"], True)
else: else:
raise Exception(f"unknown key: {key}") raise Exception(f"unknown key: {key}")
# reset log
path = os.sep.join([tdDnodes.getDnodesRootDir(), "dnode1", "log", "taosdlog.*"])
tdSql.execute("alter all dnodes 'resetlog';")
r = subprocess.Popen("cat {} | grep 'reset log file'".format(path), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = r.communicate()
assert('reset log file' in stdout.decode())
def stop(self): def stop(self):
tdSql.close() tdSql.close()

54
tests/system-test/1-insert/ts-4272.py
View File

@ -26,12 +26,10 @@ class TDTestCase:
self.file1 = f"{self.testcasePath}/b.csv" self.file1 = f"{self.testcasePath}/b.csv"
self.file2 = f"{self.testcasePath}/c.csv" self.file2 = f"{self.testcasePath}/c.csv"
#os.system("rm -rf %s/b.csv" %self.testcasePath)
tdLog.debug(f"start to excute {__file__}") tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), logSql) tdSql.init(conn.cursor(), logSql)
def check_count(self, rows, records): def check_count(self, rows, records):
tdSql.execute(f"use {self.db};")
tdSql.query(f"select tbname,count(*) from {self.stable0} group by tbname order by tbname;") tdSql.query(f"select tbname,count(*) from {self.stable0} group by tbname order by tbname;")
tdSql.checkRows(rows) tdSql.checkRows(rows)
for i in range(rows): for i in range(rows):
@ -39,13 +37,6 @@ class TDTestCase:
def reset_tb(self): def reset_tb(self):
# create database and tables # create database and tables
# os.system("taos -s 'drop database if exists d1;'")
# os.system("taos -s 'create database d1;use d1;create stable meters (ts timestamp, current float, voltage int, phase float) tags (location binary(64), groupId int);'")
# os.system(f"taos -s 'use d1;create table d2001 using meters(groupId) tags(5);'")
# res = os.system(f"taos -s 'use d1;create table d2002 using meters(groupId) tags(6);'")
# if (0 != res):
# tdLog.exit(f"create tb error")
tdSql.execute(f"drop database if exists {self.db};") tdSql.execute(f"drop database if exists {self.db};")
tdSql.execute(f"create database {self.db};") tdSql.execute(f"create database {self.db};")
tdSql.execute(f"use {self.db};") tdSql.execute(f"use {self.db};")
@ -54,13 +45,14 @@ class TDTestCase:
tdSql.execute(f"create table {self.tb2} {self.tag2};") tdSql.execute(f"create table {self.tb2} {self.tag2};")
tdSql.execute(f"create stable {self.stable1} (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);") tdSql.execute(f"create stable {self.stable1} (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);")
tdSql.execute(f"create stable {self.stable2} (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);") tdSql.execute(f"create stable {self.stable2} (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);")
tdSql.execute(f"create table {self.stable1}_1 using {self.stable1}(t_int) tags(1);")
tdSql.execute(f"create table {self.stable2}_1 using {self.stable2}(t_int) tags(2);")
def test(self, sql): def test(self, sql):
sql = f"use {self.db};" + sql # sql = f"use {self.db};" + sql
res = os.system(f'taos -s "{sql}"') # os.system(f'taos -s "{sql}"')
# if (0 != res): print(f'{sql}\n')
# tdLog.exit(f"taos sql error") tdSql.execute(sql, 1)
def check(self): def check(self):
# same table, auto create + create # same table, auto create + create
@ -95,13 +87,8 @@ class TDTestCase:
sql = f"insert into {self.tb1} file '{self.file1}' {self.tb2} file '{self.file2}';" sql = f"insert into {self.tb1} file '{self.file1}' {self.tb2} file '{self.file2}';"
self.test(sql) self.test(sql)
# bigNum = 1010000 self.check_count(2, [2010000, 1000000])
# self.check_count(5, [2100, 2100, bigNum, bigNum, bigNum])
result = os.popen("taos -s 'select count(*) from %s.%s'" %(self.db, self.tb1))
res = result.read()
if (f"OK" in res):
tdLog.info(f"check count success")
def make_csv(self, filepath, once, qtime, startts): def make_csv(self, filepath, once, qtime, startts):
f = open(filepath, 'w') f = open(filepath, 'w')
@ -118,10 +105,8 @@ class TDTestCase:
def test_mix(self): def test_mix(self):
#forbid use both value and file in one insert #forbid use both value and file in one insert
result = os.popen(f"insert into {self.tb1} file '{self.file2}' {self.tb2} values('2021-07-13 14:06:34.630', 10.2, 219, 0.32);") self.make_csv(self.file2, 10, 10, self.ts)
res = result.read() tdSql.error(f"insert into {self.tb1} file '{self.file2}' {self.tb2} values('2021-07-13 14:06:34.630', 10.2, 219, 0.32);")
if (f"error" in res):
tdLog.info(f"forbid success")
def test_bigcsv(self): def test_bigcsv(self):
# prepare csv # prepare csv
@ -144,7 +129,6 @@ class TDTestCase:
self.test(sql) self.test(sql)
print("end insert to table") print("end insert to table")
#tdSql.execute(f"use d1;")
tdSql.query(f"select tbname,count(*) from {self.stable0} group by tbname order by tbname;") tdSql.query(f"select tbname,count(*) from {self.stable0} group by tbname order by tbname;")
tdSql.checkRows(2) tdSql.checkRows(2)
tdSql.checkData(0, 1, rowNum1) tdSql.checkData(0, 1, rowNum1)
@ -160,7 +144,7 @@ class TDTestCase:
ts = startts + offset ts = startts + offset
rows = [] rows = []
for i in range(once): for i in range(once):
rows.append([table_name, ts + i, offset + i, 'NULL']) rows.append([f"\'{table_name}\'", ts + i, offset + i, 'NULL'])
writer.writerows(rows) writer.writerows(rows)
f.close() f.close()
print(datetime.now(), filepath, " ready!") print(datetime.now(), filepath, " ready!")
@ -171,22 +155,22 @@ class TDTestCase:
once = 10000 once = 10000
qtime1 = 101 qtime1 = 101
qtime2 = 100 qtime2 = 100
# rowNum1 = qtime1 * once
# rowNum2 = qtime2 * once
child_1 = f"{self.stable1}_1" child_1 = f"{self.stable1}_1"
child_2 = f"{self.stable2}_1" child_2 = f"{self.stable2}_1"
self.make_stable_csv(self.file1, once, qtime1, self.ts - 86400000, child_1) self.make_stable_csv(self.file1, once, qtime1, self.ts - 86400000, child_1)
self.make_stable_csv(self.file2, once, qtime2, self.ts, child_2) self.make_stable_csv(self.file2, once, qtime2, self.ts, child_2)
print("end stable_csv data prepare") print("end stable_csv data prepare")
# insert create child table of stable
sql = f"insert into {self.db}.{self.stable1}(tbname,ts,q_int,q_binary) file '{self.file1}' {self.db}.{self.stable2}(tbname,ts,q_int,q_binary) file '{self.file2}';" sql = f"insert into {self.db}.{self.stable1}(tbname,ts,q_int,q_binary) file '{self.file1}' {self.db}.{self.stable2}(tbname,ts,q_int,q_binary) file '{self.file2}';"
self.test(sql) self.test(sql)
print("end insert to stable") print("end insert to stable")
#tdSql.execute(f"insert into {self.db}.{child_1}(ts, q_int) values(now, 1);") tdSql.query(f"select tbname,count(*) from {self.stable1} group by tbname;")
tdSql.query(f"select tbname,count(*) from {self.stable1} group by tbname order by tbname;") tdSql.checkRows(1)
tdSql.checkRows(0) tdSql.checkData(0, 1, qtime1 * once)
tdSql.query(f"select tbname,count(*) from {self.stable2} group by tbname;")
tdSql.checkRows(1)
tdSql.checkData(0, 1, qtime2 * once)
print("check stable success") print("check stable success")
def run(self): def run(self):
@ -194,8 +178,10 @@ class TDTestCase:
self.reset_tb() self.reset_tb()
self.test_stable_csv() self.test_stable_csv()
self.test_bigcsv() self.test_bigcsv()
self.test_mix()
self.check() self.check()
self.test_mix()
os.system(f"rm -rf {self.file1}")
os.system(f"rm -rf {self.file2}")
tdSql.close() tdSql.close()
def stop(self): def stop(self):

91
tests/system-test/2-query/last_cache_scan.py
View File

@ -28,7 +28,7 @@ class TDTestCase:
def init(self, conn, logSql, replicaVar=1): def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar) self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}") tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), False) tdSql.init(conn.cursor(), True)
def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'):
if dropFlag == 1: if dropFlag == 1:
@ -341,44 +341,12 @@ class TDTestCase:
tdSql.checkData(0, 0, '999') tdSql.checkData(0, 0, '999')
p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c12 int"]) p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c12 int"])
p.check_returncode() p.check_returncode()
tdSql.query("select last(c1) from meters", queryTimes=1) tdSql.query_success_failed("select ts, last(c1), c1, ts, c1 from meters", queryTimes=10, expectErrInfo="Invalid column name: c1")
tdSql.checkData(0, 0, None) tdSql.query('select last(c12), c12, ts from meters', queryTimes=1)
tdSql.query('select last(*) from meters', queryTimes=1)
print(str(tdSql.queryResult))
tdSql.checkData(0, 1, None)
tdSql.query('select last(c1), c1, ts from meters', queryTimes=1)
tdSql.checkRows(1)
tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, None)
tdSql.checkData(0, 2, None)
try:
tdSql.query('select ts, last(c1), c1, ts, c1 from meters', queryTimes=1)
except Exception as e:
if str(e).count('Invalid column name') == 1:
print('column has been dropped, the cache has been updated: %s' % (str(e)))
return
else:
raise
tdSql.checkRows(1)
tdSql.checkCols(5)
tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, None)
tdSql.checkData(0, 2, None)
tdSql.checkData(0, 3, None)
tdSql.checkData(0, 4, None)
try:
tdSql.query('select last(c1), last(c2), last(c3) from meters', queryTimes=1)
except Exception as e:
if str(e).count('Invalid column name') == 1:
print('column has been dropped, the cache has been updated: %s' % (str(e)))
return
else:
raise
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.checkCols(3) tdSql.checkCols(3)
tdSql.checkData(0, 0, None) tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, None)
def test_cache_scan_with_drop_column(self): def test_cache_scan_with_drop_column(self):
tdSql.query('select last(*) from meters') tdSql.query('select last(*) from meters')
@ -403,49 +371,48 @@ class TDTestCase:
tdSql.checkData(0, 10, None) tdSql.checkData(0, 10, None)
def test_cache_scan_last_row_with_drop_column2(self): def test_cache_scan_last_row_with_drop_column2(self):
tdSql.query('select last_row(c1) from meters') tdSql.query('select last_row(c2) from meters')
print(str(tdSql.queryResult)) print(str(tdSql.queryResult))
tdSql.checkCols(1) tdSql.checkCols(1)
p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) p = subprocess.run(["taos", '-s', "alter table test.meters drop column c2; alter table test.meters add column c1 int"])
p.check_returncode() p.check_returncode()
tdSql.query('select last_row(c1) from meters', queryTimes=1) tdSql.query_success_failed("select ts, last_row(c2), c12, ts, c12 from meters", queryTimes=10, expectErrInfo="Invalid column name: c2")
print(str(tdSql.queryResult)) tdSql.query('select last(c1), c1, ts from meters', queryTimes=1)
tdSql.checkCols(1) tdSql.checkRows(1)
tdSql.checkCols(3)
tdSql.checkData(0, 0, None) tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, None)
def test_cache_scan_last_row_with_partition_by(self): def test_cache_scan_last_row_with_partition_by(self):
tdSql.query('select last(c1) from meters partition by t1') tdSql.query('select last(c1) from meters partition by t1')
print(str(tdSql.queryResult)) print(str(tdSql.queryResult))
tdSql.checkCols(1) tdSql.checkCols(1)
tdSql.checkRows(5) tdSql.checkRows(2)
p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c2 int"])
p.check_returncode() p.check_returncode()
tdSql.query('select last_row(c1) from meters partition by t1', queryTimes=1) tdSql.query_success_failed('select last(c1) from meters partition by t1', queryTimes=10, expectErrInfo="Invalid column name: c1")
tdSql.query('select last(c2), c2, ts from meters', queryTimes=1)
print(str(tdSql.queryResult)) print(str(tdSql.queryResult))
tdSql.checkCols(1) tdSql.checkRows(1)
tdSql.checkRows(5) tdSql.checkCols(3)
tdSql.checkData(0, 0, None) tdSql.checkData(0, 0, None)
tdSql.checkData(1, 0, None) tdSql.checkData(0, 1, None)
tdSql.checkData(2, 0, None)
tdSql.checkData(3, 0, None)
tdSql.checkData(4, 0, None)
def test_cache_scan_last_row_with_partition_by_tbname(self): def test_cache_scan_last_row_with_partition_by_tbname(self):
tdSql.query('select last(c1) from meters partition by tbname', queryTimes=1) tdSql.query('select last(c2) from meters partition by tbname')
print(str(tdSql.queryResult)) print(str(tdSql.queryResult))
tdSql.checkCols(1) tdSql.checkCols(1)
tdSql.checkRows(10) tdSql.checkRows(10)
p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) p = subprocess.run(["taos", '-s', "alter table test.meters drop column c2; alter table test.meters add column c1 int"])
p.check_returncode() p.check_returncode()
tdSql.query('select last_row(c1) from meters partition by tbname', queryTimes=1) tdSql.query_success_failed('select last_row(c2) from meters partition by tbname', queryTimes=10, expectErrInfo="Invalid column name: c2")
tdSql.query('select last(c1), c1, ts from meters', queryTimes=1)
print(str(tdSql.queryResult)) print(str(tdSql.queryResult))
tdSql.checkCols(1) tdSql.checkRows(1)
tdSql.checkRows(10) tdSql.checkCols(3)
tdSql.checkData(0, 0, None) tdSql.checkData(0, 0, None)
tdSql.checkData(1, 0, None) tdSql.checkData(0, 1, None)
tdSql.checkData(2, 0, None)
tdSql.checkData(3, 0, None)
tdSql.checkData(4, 0, None)
@ -457,9 +424,9 @@ class TDTestCase:
self.test_cache_scan_with_drop_and_add_column2() self.test_cache_scan_with_drop_and_add_column2()
#self.test_cache_scan_with_drop_column() #self.test_cache_scan_with_drop_column()
#self.test_cache_scan_last_row_with_drop_column() #self.test_cache_scan_last_row_with_drop_column()
#self.test_cache_scan_last_row_with_drop_column2() self.test_cache_scan_last_row_with_drop_column2()
#self.test_cache_scan_last_row_with_partition_by() self.test_cache_scan_last_row_with_partition_by()
#self.test_cache_scan_last_row_with_partition_by_tbname() self.test_cache_scan_last_row_with_partition_by_tbname()
def stop(self): def stop(self):
tdSql.close() tdSql.close()