diff --git a/include/common/tcommon.h b/include/common/tcommon.h index c7ba618b25..4a06d81c7b 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -56,7 +56,8 @@ typedef enum EStreamType { STREAM_CLEAR, STREAM_INVALID, STREAM_GET_ALL, - STREAM_DELETE, + STREAM_DELETE_RESULT, + STREAM_DELETE_DATA, STREAM_RETRIEVE, STREAM_PULL_DATA, STREAM_PULL_OVER, diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 9881c8cb44..7f7fe76139 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -249,13 +249,6 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || \ (_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_APP_NOT_READY) -#define NEED_SCHEDULER_RETRY_ERROR(_code) \ - (NEED_SCHEDULER_REDIRECT_ERROR(_code) || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ - (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR || (_code) == TSDB_CODE_RPC_BROKEN_LINK) - - - - #define REQUEST_TOTAL_EXEC_TIMES 2 #define qFatal(...) \ diff --git a/include/os/osMath.h b/include/os/osMath.h index b0c75f4dd7..74973d9bb2 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -20,6 +20,12 @@ extern "C" { #endif +// If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following sectio +#ifndef ALLOW_FORBID_FUNC +#define qsort QSORT_FUNC_TAOS_FORBID +#endif + #define TPOW2(x) ((x) * (x)) #define TABS(x) ((x) > 0 ? (x) : -(x)) diff --git a/include/os/osString.h b/include/os/osString.h index 1b518f9b81..3a4ff18694 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -67,7 +67,7 @@ bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); bool taosValidateEncodec(const char *encodec); -int32_t taosHexEncode(const char *src, char *dst, int32_t len); +int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len); int32_t taosHexDecode(const char *src, char *dst, int32_t len); int32_t taosWcharWidth(TdWchar wchar); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index b871452828..33313e0400 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -623,6 +623,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3150) #define TSDB_CODE_RSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3151) #define TSDB_CODE_RSMA_QTASKINFO_CREATE TAOS_DEF_ERROR_CODE(0, 0x3152) +#define TSDB_CODE_RSMA_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x3153) //index #define TSDB_CODE_INDEX_REBUILDING TAOS_DEF_ERROR_CODE(0, 0x3200) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index b6009ebbda..7d42bbdad9 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -786,6 +786,7 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) { SRequestObj* pRequest = (SRequestObj*)param; pRequest->code = code; + pRequest->body.resInfo.execRes = pResult->res; if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_DELETE == pRequest->type || TDMT_VND_CREATE_TABLE == pRequest->type) { @@ -797,6 +798,8 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) { } } + taosMemoryFree(pResult); + tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4881dd88aa..b72fd961f5 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1259,6 +1259,7 @@ int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) { dst->info.rows = src->info.rows; dst->info.window = src->info.window; + dst->info.type = src->info.type; return TSDB_CODE_SUCCESS; } diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 8fbdeb0654..befb0abac8 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -589,7 +589,7 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec return TSDB_CODE_FAILED; } newColData[len] = 0; - int32_t ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, tsDaylight); + int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tsDaylight); if (ret != TSDB_CODE_SUCCESS) { taosMemoryFree(newColData); return ret; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 3f053639aa..681440dec4 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -21,7 +21,7 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { taosThreadRwlockRdlock(&pMgmt->lock); taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); - if (pVnode == NULL) { + if (pVnode == NULL || pVnode->dropped) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; } else { int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); @@ -81,16 +81,18 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { vmReleaseVnode(pMgmt, pVnode); while (pVnode->refCount > 0) taosMsleep(10); + dTrace("vgId:%d, wait for vnode queue is empty", pVnode->vgId); + while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); + dTrace("vgId:%d, vnode-fetch queue is empty", pVnode->vgId); vmFreeQueue(pMgmt, pVnode); vnodeClose(pVnode->pImpl); pVnode->pImpl = NULL; - dDebug("vgId:%d, vnode is closed", pVnode->vgId); if (pVnode->dropped) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 91ef292360..ecd02ae8dc 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -107,7 +107,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf const STraceId *trace = &pMsg->info.traceId; dGTrace("vgId:%d, msg:%p get from vnode-sync queue", pVnode->vgId, pMsg); - int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); // no response here + int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); // no response here dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); @@ -146,8 +146,8 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) { - dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(), - TMSG_INFO(pMsg->msgType)); + dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, msgtype:%s qtype:%d", pHead->vgId, pMsg, + terrstr(), TMSG_INFO(pMsg->msgType), qtype); return terrno != 0 ? terrno : -1; } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 1fb7fa85b1..21002f5f9c 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -556,7 +556,7 @@ typedef struct { int64_t uid; int8_t status; // config - int8_t dropPolicy; + int8_t igExpired; int8_t trigger; int64_t triggerParam; int64_t watermark; diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 0901e77287..d873df621e 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -38,6 +38,8 @@ int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); +int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index dabb938705..c7db4b8e1c 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -21,6 +21,7 @@ #include "mndShow.h" #include "mndSma.h" #include "mndStb.h" +#include "mndStream.h" #include "mndSubscribe.h" #include "mndTopic.h" #include "mndTrans.h" @@ -927,6 +928,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER; + if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; @@ -947,7 +949,6 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { mndTransSetRpcRsp(pTrans, pRsp, rspLen); if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - code = 0; _OVER: diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index dc7f08ebc2..727b20ef8a 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -28,7 +28,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->dropPolicy) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->igExpired) < 0) return -1; if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1; if (tEncodeI64(pEncoder, pObj->triggerParam) < 0) return -1; if (tEncodeI64(pEncoder, pObj->watermark) < 0) return -1; @@ -73,7 +73,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->dropPolicy) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->igExpired) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->triggerParam) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->watermark) < 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 530cc57390..6aea1f41d1 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -523,6 +523,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea streamObj.updateTime = streamObj.createTime; streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); streamObj.sourceDbUid = pDb->uid; + streamObj.targetDbUid = pDb->uid; streamObj.version = 1; streamObj.sql = pCreate->sql; streamObj.smaId = smaObj.uid; @@ -853,36 +854,26 @@ _OVER: } int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { - SSdb *pSdb = pMnode->pSdb; - SSmaObj *pSma = NULL; - void *pIter = NULL; - SVgObj *pVgroup = NULL; - int32_t code = -1; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; while (1) { + SSmaObj *pSma = NULL; pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); if (pIter == NULL) break; if (pSma->dbUid == pDb->uid) { - pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId); - if (pVgroup == NULL) goto _OVER; - if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; - if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER; - if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; - mndReleaseVgroup(pMnode, pVgroup); - pVgroup = NULL; + if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) { + sdbRelease(pSdb, pSma); + sdbCancelFetch(pSdb, pSma); + return -1; + } } sdbRelease(pSdb, pSma); } - code = 0; - -_OVER: - sdbCancelFetch(pSdb, pIter); - sdbRelease(pSdb, pSma); - mndReleaseVgroup(pMnode, pVgroup); - return code; + return 0; } static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index cabbac14f1..08e108803a 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -248,7 +248,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, pObj->status = 0; // TODO - pObj->dropPolicy = 0; + pObj->igExpired = pCreate->igExpired; pObj->trigger = pCreate->triggerType; pObj->triggerParam = pCreate->maxDelay; pObj->watermark = pCreate->watermark; @@ -301,6 +301,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, .streamQuery = true, .triggerType = pObj->trigger == STREAM_TRIGGER_MAX_DELAY ? STREAM_TRIGGER_WINDOW_CLOSE : pObj->trigger, .watermark = pObj->watermark, + .igExpired = pObj->igExpired, }; // using ast and param to build physical plan @@ -673,27 +674,29 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; - void *pIter = NULL; - SStreamObj *pStream = NULL; while (1) { + SStreamObj *pStream = NULL; pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) break; if (pStream->sourceDbUid == pDb->uid || pStream->targetDbUid == pDb->uid) { if (pStream->sourceDbUid != pStream->targetDbUid) { sdbRelease(pSdb, pStream); + sdbCancelFetch(pSdb, pIter); + mError("db:%s, failed to drop stream:%s since sourceDbUid:%" PRId64 " not match with targetDbUid:%" PRId64, + pDb->name, pStream->name, pStream->sourceDbUid, pStream->targetDbUid); + terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST; return -1; } else { // TODO drop all task on snode if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { sdbRelease(pSdb, pStream); + sdbCancelFetch(pSdb, pIter); return -1; } } - } else { - sdbRelease(pSdb, pStream); - continue; } #if 0 diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 534cba73c7..ea92b2f0e6 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -90,7 +90,7 @@ static int32_t mndTransGetActionsSize(SArray *pArray) { for (int32_t i = 0; i < actionNum; ++i) { STransAction *pAction = taosArrayGet(pArray, i); if (pAction->actionType == TRANS_ACTION_RAW) { - rawDataLen += (sdbGetRawTotalSize(pAction->pRaw) + sizeof(int32_t)); + rawDataLen += (sizeof(STransAction) + sdbGetRawTotalSize(pAction->pRaw)); } else if (pAction->actionType == TRANS_ACTION_MSG) { rawDataLen += (sizeof(STransAction) + pAction->contLen); } else { @@ -105,7 +105,7 @@ static int32_t mndTransGetActionsSize(SArray *pArray) { static SSdbRaw *mndTransActionEncode(STrans *pTrans) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE; + int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE + pTrans->paramLen; rawDataLen += mndTransGetActionsSize(pTrans->redoActions); rawDataLen += mndTransGetActionsSize(pTrans->undoActions); rawDataLen += mndTransGetActionsSize(pTrans->commitActions); @@ -226,7 +226,8 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { _OVER: if (terrno != 0) { - mError("trans:%d, failed to encode to raw:%p len:%d since %s", pTrans->id, pRaw, dataPos, terrstr()); + mError("trans:%d, failed to encode to raw:%p maxlen:%d len:%d since %s", pTrans->id, pRaw, sdbGetRawTotalSize(pRaw), + dataPos, terrstr()); sdbFreeRaw(pRaw); return NULL; } @@ -1025,7 +1026,7 @@ static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction pTrans->lastAction = pAction->id; pTrans->lastMsgType = pAction->msgType; pTrans->lastEpset = pAction->epSet; - pTrans->lastErrorNo == 0; + pTrans->lastErrorNo = 0; return 0; } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index b09ab34224..174fe6dab5 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -31,6 +31,8 @@ target_sources( "src/sma/smaEnv.c" "src/sma/smaUtil.c" "src/sma/smaOpen.c" + "src/sma/smaCommit.c" + "src/sma/smaSnapshot.c" "src/sma/smaRollup.c" "src/sma/smaTimeRange.c" diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index b455d779c1..a95eb395c2 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -92,8 +92,9 @@ enum { TASK_TRIGGER_STAT_INIT = 0, TASK_TRIGGER_STAT_ACTIVE = 1, TASK_TRIGGER_STAT_INACTIVE = 2, - TASK_TRIGGER_STAT_CANCELLED = 3, - TASK_TRIGGER_STAT_FINISHED = 4, + TASK_TRIGGER_STAT_PAUSED = 3, + TASK_TRIGGER_STAT_CANCELLED = 4, + TASK_TRIGGER_STAT_FINISHED = 5, }; void tdDestroySmaEnv(SSmaEnv *pSmaEnv); void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); @@ -214,25 +215,22 @@ struct STFInfo { }; struct STFile { - STFInfo info; - STfsFile f; - TdFilePtr pFile; uint8_t state; + STFInfo info; + char *fname; + TdFilePtr pFile; }; -#define TD_TFILE_F(tf) (&((tf)->f)) #define TD_TFILE_PFILE(tf) ((tf)->pFile) #define TD_TFILE_OPENED(tf) (TD_TFILE_PFILE(tf) != NULL) -#define TD_TFILE_FULL_NAME(tf) (TD_TFILE_F(tf)->aname) -#define TD_TFILE_REL_NAME(tf) (TD_TFILE_F(tf)->rname) +#define TD_TFILE_FULL_NAME(tf) ((tf)->fname) #define TD_TFILE_OPENED(tf) (TD_TFILE_PFILE(tf) != NULL) #define TD_TFILE_CLOSED(tf) (!TD_TFILE_OPENED(tf)) #define TD_TFILE_SET_CLOSED(f) (TD_TFILE_PFILE(f) = NULL) #define TD_TFILE_SET_STATE(tf, s) ((tf)->state = (s)) -#define TD_TFILE_DID(tf) (TD_TFILE_F(tf)->did) -int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname); -int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType); +int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname); +int32_t tdCreateTFile(STFile *pTFile, bool updateHeader, int8_t fType); int32_t tdOpenTFile(STFile *pTFile, int flags); int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte); int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence); @@ -244,8 +242,10 @@ int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo); int32_t tdUpdateTFileHeader(STFile *pTFile); void tdUpdateTFileMagic(STFile *pTFile, void *pCksm); void tdCloseTFile(STFile *pTFile); +void tdDestroyTFile(STFile *pTFile); -void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName); +void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, int64_t version, char *outputName); +void tdGetVndDirName(int32_t vgId, const char *dname, char *outputName); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 6bd7d4edd1..c52884d6ad 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -165,6 +165,9 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool int32_t smaOpen(SVnode* pVnode); int32_t smaCloseEnv(SSma* pSma); int32_t smaCloseEx(SSma* pSma); +int32_t smaPreCommit(SSma* pSma); +int32_t smaCommit(SSma* pSma); +int32_t smaPostCommit(SSma* pSma); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c new file mode 100644 index 0000000000..b98f9bc866 --- /dev/null +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" + +static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma); + +/** + * @brief Only applicable to Rollup SMA + * + * @param pSma + * @return int32_t + */ +int32_t smaPreCommit(SSma *pSma) { return tdProcessRSmaPreCommitImpl(pSma); } + +/** + * @brief Only applicable to Rollup SMA + * + * @param pSma + * @return int32_t + */ +int32_t smaCommit(SSma *pSma) { return tdProcessRSmaCommitImpl(pSma); } + +/** + * @brief Only applicable to Rollup SMA + * + * @param pSma + * @return int32_t + */ +int32_t smaPostCommit(SSma *pSma) { return tdProcessRSmaPostCommitImpl(pSma); } + +/** + * @brief pre-commit for rollup sma. + * 1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED. + * 2) perform persist task for qTaskInfo + * 3) wait all triggered fetch tasks finished + * 4) set trigger stat of rsma timer TASK_TRIGGER_STAT_ACTIVE. + * 5) finish + * + * @param pSma + * @return int32_t + */ +static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) { + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); + if (!pSmaEnv) { + return TSDB_CODE_SUCCESS; + } + + SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); + + // step 1 + atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED); + + // step 2 + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief commit for rollup sma + * + * @param pSma + * @return int32_t + */ +static int32_t tdProcessRSmaCommitImpl(SSma *pSma) { + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); + if (!pSmaEnv) { + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_SUCCESS; +} + +/** + * @brief post-commit for rollup sma + * 1) clean up the outdated qtaskinfo files + * + * @param pSma + * @return int32_t + */ +static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) { + SVnode *pVnode = pSma->pVnode; + + if (!VND_IS_RSMA(pVnode)) { + return TSDB_CODE_SUCCESS; + } + + int64_t committed = pVnode->state.committed; + TdDirPtr pDir = NULL; + TdDirEntryPtr pDirEntry = NULL; + char dir[TSDB_FILENAME_LEN]; + char bname[TSDB_FILENAME_LEN]; + const char *pattern = "^v[0-9]+qtaskinfo\\.ver([0-9]+)?$"; + regex_t regex; + + tdGetVndDirName(TD_VID(pVnode), VNODE_RSMA_DIR, dir); + + // Resource allocation and init + regcomp(®ex, pattern, REG_EXTENDED); + + if ((pDir = taosOpenDir(dir)) == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + smaWarn("rsma post-commit open dir %s failed since %s", dir, terrstr()); + return TSDB_CODE_FAILED; + } + + regmatch_t regMatch[2]; + while ((pDirEntry = taosReadDir(pDir)) != NULL) { + char *entryName = taosGetDirEntryName(pDirEntry); + if (!entryName) { + continue; + } + char *fileName = taosDirEntryBaseName(entryName); + int code = regexec(®ex, bname, 2, regMatch, 0); + + if (code == 0) { + // match + printf("match 0 = %s\n", (char *)POINTER_SHIFT(fileName, regMatch[0].rm_so)); + printf("match 1 = %s\n", (char *)POINTER_SHIFT(fileName, regMatch[1].rm_so)); + } else if (code == REG_NOMATCH) { + // not match + smaInfo("rsma post-commit not match %s", fileName); + continue; + } else { + // has other error + terrno = TAOS_SYSTEM_ERROR(code); + smaWarn("rsma post-commit regexec failed since %s", terrstr()); + + taosCloseDir(&pDir); + regfree(®ex); + return TSDB_CODE_FAILED; + } + } + taosCloseDir(&pDir); + return TSDB_CODE_SUCCESS; +} diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 41af641e9e..ff9b17bf6e 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -23,8 +23,7 @@ SSmaMgmt smaMgmt = { .smaRef = -1, }; -typedef enum { TD_QTASK_TMP_F = 0, TD_QTASK_CUR_F } TD_QTASK_FILE_T; -static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"}; +#define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver" typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter; @@ -37,7 +36,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType static void tdRSmaFetchTrigger(void *param, void *tmrId); static void tdRSmaPersistTrigger(void *param, void *tmrId); static void *tdRSmaPersistExec(void *param); -static void tdRSmaQTaskInfoGetFName(int32_t vid, int8_t ftype, char *outputName); +static void tdRSmaQTaskInfoGetFName(int32_t vid, int64_t version, char *outputName); static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile); static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish); @@ -88,8 +87,8 @@ struct SRSmaQTaskInfoIter { int32_t nBufPos; }; -static void tdRSmaQTaskInfoGetFName(int32_t vgId, int8_t ftype, char *outputName) { - tdGetVndFileName(vgId, VNODE_RSMA_DIR, tdQTaskInfoFname[ftype], outputName); +static void tdRSmaQTaskInfoGetFName(int32_t vgId, int64_t version, char *outputName) { + tdGetVndFileName(vgId, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName); } static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) { @@ -493,7 +492,6 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { } static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { - ASSERT(pMsg != NULL); SSubmitMsgIter msgIter = {0}; SSubmitBlk *pBlock = NULL; SSubmitBlkIter blkIter = {0}; @@ -501,19 +499,26 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { terrno = TSDB_CODE_SUCCESS; - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { + return -1; + } while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) { + return -1; + } if (!pBlock) break; tdUidStorePut(pStore, msgIter.suid, NULL); } - if (terrno != TSDB_CODE_SUCCESS) return -1; + if (terrno != TSDB_CODE_SUCCESS) { + return -1; + } return 0; } static void tdDestroySDataBlockArray(SArray *pArray) { + // TODO #if 0 for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SSDataBlock *pDataBlock = taosArrayGet(pArray, i); @@ -598,33 +603,54 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { pSma = RSMA_INFO_SMA(pItem->pRsmaInfo); - // if rsma trigger stat in cancelled or finished, not start fetch task anymore + // if rsma trigger stat in paused, cancelled or finished, not start fetch task int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat)); - if (rsmaTriggerStat == TASK_TRIGGER_STAT_CANCELLED || rsmaTriggerStat == TASK_TRIGGER_STAT_FINISHED) { - taosReleaseRef(smaMgmt.smaRef, pItem->refId); - smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is cancelled", - SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid); - return; + switch (rsmaTriggerStat) { + case TASK_TRIGGER_STAT_PAUSED: + case TASK_TRIGGER_STAT_CANCELLED: + case TASK_TRIGGER_STAT_FINISHED: { + taosReleaseRef(smaMgmt.smaRef, pItem->refId); + smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is cancelled", + SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid); + return; + } + default: + break; } int8_t fetchTriggerStat = atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); - if (fetchTriggerStat == TASK_TRIGGER_STAT_ACTIVE) { - smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma), - pItem->level, pItem->pRsmaInfo->suid); + switch (fetchTriggerStat) { + case TASK_TRIGGER_STAT_ACTIVE: { + smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma), + pItem->level, pItem->pRsmaInfo->suid); - tdRefSmaStat(pSma, (SSmaStat *)pStat); + tdRefSmaStat(pSma, (SSmaStat *)pStat); - SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; - qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false); - tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK); + SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; + qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false); + tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK); - tdUnRefSmaStat(pSma, (SSmaStat *)pStat); - - } else { - smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive", - SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid); + tdUnRefSmaStat(pSma, (SSmaStat *)pStat); + } break; + case TASK_TRIGGER_STAT_PAUSED: { + smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused", + SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid); + } break; + case TASK_TRIGGER_STAT_INACTIVE: { + smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive", + SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid); + } break; + case TASK_TRIGGER_STAT_INIT: { + smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is init", SMA_VID(pSma), + pItem->level, pItem->pRsmaInfo->suid); + } break; + default: { + smaWarn("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is unknown", + SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid); + } break; } + _end: taosReleaseRef(smaMgmt.smaRef, pItem->refId); } @@ -780,10 +806,10 @@ _err: static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) { SVnode *pVnode = pSma->pVnode; STFile tFile = {0}; - char qTaskInfoFName[TSDB_FILENAME_LEN]; + char qTaskInfoFName[TSDB_FILENAME_LEN] = {0}; - tdRSmaQTaskInfoGetFName(TD_VID(pVnode), TD_QTASK_TMP_F, qTaskInfoFName); - if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) { + tdRSmaQTaskInfoGetFName(TD_VID(pVnode), pVnode->state.committed, qTaskInfoFName); + if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { goto _err; } @@ -799,17 +825,20 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) { if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) { tdRSmaQTaskInfoIterDestroy(&fIter); tdCloseTFile(&tFile); + tdDestroyTFile(&tFile); goto _err; } if (tdRSmaQTaskInfoRestore(pSma, &fIter) < 0) { tdRSmaQTaskInfoIterDestroy(&fIter); tdCloseTFile(&tFile); + tdDestroyTFile(&tFile); goto _err; } tdRSmaQTaskInfoIterDestroy(&fIter); tdCloseTFile(&tFile); + tdDestroyTFile(&tFile); return TSDB_CODE_SUCCESS; _err: smaError("rsma restore, qtaskinfo reload failed since %s", terrstr()); @@ -931,19 +960,21 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF } if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) { - ASSERT(0); return TSDB_CODE_FAILED; } if (tdReadTFile(pTFile, pIter->qBuf, nBytes) != nBytes) { - ASSERT(0); return TSDB_CODE_FAILED; } int32_t infoLen = 0; taosDecodeFixedI32(pIter->qBuf, &infoLen); if (infoLen > nBytes) { - ASSERT(infoLen > RSMA_QTASKINFO_BUFSIZE); + if (infoLen <= RSMA_QTASKINFO_BUFSIZE) { + terrno = TSDB_CODE_RSMA_FILE_CORRUPTED; + smaError("iterate rsma qtaskinfo file %s failed since %s", TD_TFILE_FULL_NAME(pIter->pTFile), terrstr()); + return TSDB_CODE_FAILED; + } pIter->nAlloc = infoLen; void *pBuf = taosMemoryRealloc(pIter->pBuf, infoLen); if (!pBuf) { @@ -955,12 +986,10 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF nBytes = infoLen; if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) { - ASSERT(0); return TSDB_CODE_FAILED; } if (tdReadTFile(pTFile, pIter->pBuf, nBytes) != nBytes) { - ASSERT(0); return TSDB_CODE_FAILED; } } @@ -977,7 +1006,6 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) { // block iter bool isFinish = false; if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) { - ASSERT(0); return TSDB_CODE_FAILED; } if (isFinish) { @@ -989,6 +1017,7 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) { pIter->qBuf = taosDecodeFixedI32(pIter->qBuf, &qTaskInfoLenWithHead); if (qTaskInfoLenWithHead < RSMA_QTASKINFO_HEAD_LEN) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + smaError("restore rsma qtaskinfo file %s failed since %s", TD_TFILE_FULL_NAME(pIter->pTFile), terrstr()); return TSDB_CODE_FAILED; } @@ -1025,22 +1054,16 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) { return TSDB_CODE_SUCCESS; } -static void *tdRSmaPersistExec(void *param) { - setThreadName("rsma-task-persist"); - SRSmaStat *pRSmaStat = param; - SSma *pSma = pRSmaStat->pSma; - STfs *pTfs = pSma->pVnode->pTfs; - int32_t vid = SMA_VID(pSma); - int64_t toffset = 0; - bool isFileCreated = false; - - if (TASK_TRIGGER_STAT_CANCELLED == atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))) { - goto _end; - } +static int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) { + SSma *pSma = pRSmaStat->pSma; + SVnode *pVnode = pSma->pVnode; + int32_t vid = SMA_VID(pSma); + int64_t toffset = 0; + bool isFileCreated = false; void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); if (!infoHash) { - goto _end; + return TSDB_CODE_SUCCESS; } STFile tFile = {0}; @@ -1074,9 +1097,13 @@ static void *tdRSmaPersistExec(void *param) { if (!isFileCreated) { char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFName(vid, TD_QTASK_TMP_F, qTaskInfoFName); - tdInitTFile(&tFile, pTfs, qTaskInfoFName); - tdCreateTFile(&tFile, pTfs, true, -1); + tdRSmaQTaskInfoGetFName(vid, pSma->pVnode->state.applied, qTaskInfoFName); + if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { + goto _err; + } + if (tdCreateTFile(&tFile, true, -1) < 0) { + goto _err; + } isFileCreated = true; } @@ -1101,49 +1128,55 @@ static void *tdRSmaPersistExec(void *param) { infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash); } -_normal: + if (isFileCreated) { if (tdUpdateTFileHeader(&tFile) < 0) { smaError("vgId:%d, rsma, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile), tstrerror(terrno)); - tdCloseTFile(&tFile); - tdRemoveTFile(&tFile); goto _err; } else { smaDebug("vgId:%d, rsma, succeed to update tfile %s header", vid, TD_TFILE_FULL_NAME(&tFile)); } tdCloseTFile(&tFile); - - char newFName[TSDB_FILENAME_LEN]; - strncpy(newFName, TD_TFILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN); - char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_F]); - strncpy(pos, tdQTaskInfoFname[TD_QTASK_TMP_F], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName)); - if (taosRenameFile(TD_TFILE_FULL_NAME(&tFile), newFName) != 0) { - smaError("vgId:%d, rsma, failed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName); - goto _err; - } else { - smaDebug("vgId:%d, rsma, succeed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName); - } + tdDestroyTFile(&tFile); } - goto _end; + return TSDB_CODE_SUCCESS; _err: if (isFileCreated) { tdRemoveTFile(&tFile); + tdDestroyTFile(&tFile); } + return TSDB_CODE_FAILED; +} + +static void *tdRSmaPersistExec(void *param) { + setThreadName("rsma-task-persist"); + SRSmaStat *pRSmaStat = param; + SSma *pSma = pRSmaStat->pSma; + + int8_t triggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)); + + if (TASK_TRIGGER_STAT_CANCELLED == triggerStat || TASK_TRIGGER_STAT_PAUSED == triggerStat) { + goto _end; + } + + // execution + tdRSmaPersistExecImpl(pRSmaStat); + _end: if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INACTIVE, TASK_TRIGGER_STAT_ACTIVE)) { - smaDebug("vgId:%d, rsma persist task is active again", vid); + smaDebug("vgId:%d, rsma persist task is active again", SMA_VID(pSma)); } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_CANCELLED, TASK_TRIGGER_STAT_FINISHED)) { - smaDebug("vgId:%d, rsma persist task is cancelled", vid); + smaDebug("vgId:%d, rsma persist task is cancelled", SMA_VID(pSma)); } else { - smaWarn("vgId:%d, rsma persist task in abnormal stat %" PRIi8, vid, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); - ASSERT(0); + smaWarn("vgId:%d, rsma persist task in stat %" PRIi8, SMA_VID(pSma), atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); } + atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId); taosThreadExit(NULL); @@ -1166,8 +1199,8 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { TASK_TRIGGER_STAT_FINISHED)) { smaDebug("vgId:%d, persist task is cancelled and set finished", SMA_VID(pRSmaStat->pSma)); } else { - smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)), - SMA_VID(pRSmaStat->pSma)); + smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, SMA_VID(pRSmaStat->pSma), + atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); ASSERT(0); } atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); @@ -1216,6 +1249,9 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) { atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED); smaDebug("rsma persistence not start since cancelled and finished"); } break; + case TASK_TRIGGER_STAT_PAUSED: { + smaDebug("rsma persistence not start since paused"); + } break; case TASK_TRIGGER_STAT_INACTIVE: { smaDebug("rsma persistence not start since inactive"); } break; diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c new file mode 100644 index 0000000000..b2c85642b9 --- /dev/null +++ b/source/dnode/vnode/src/sma/smaSnapshot.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 17bc2cdaca..9bea3ff468 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -179,72 +179,83 @@ void tdCloseTFile(STFile *pTFile) { } } -void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName) { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/%s", vgId, dname, fname); +void tdDestroyTFile(STFile *pTFile) { taosMemoryFreeClear(TD_TFILE_FULL_NAME(pTFile)); } + +void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, int64_t version, char *outputName) { + if (version < 0) { + snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/v%d%s", vgId, dname, vgId, fname); + } else { + snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/v%d%s%" PRIi64, vgId, dname, vgId, fname, version); + } } -int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) { - char fullname[TSDB_FILENAME_LEN]; - SDiskID did = {0}; +void tdGetVndDirName(int32_t vgId, const char *dname, char *outputName) { + snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", vgId, dname); +} +int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname) { TD_TFILE_SET_STATE(pTFile, TD_FILE_STATE_OK); TD_TFILE_SET_CLOSED(pTFile); memset(&(pTFile->info), 0, sizeof(pTFile->info)); pTFile->info.magic = TD_FILE_INIT_MAGIC; - if (tfsAllocDisk(pTfs, 0, &did) < 0) { - terrno = TSDB_CODE_NO_AVAIL_DISK; + char tmpName[TSDB_FILENAME_LEN * 2 + 32] = {0}; + snprintf(tmpName, TSDB_FILENAME_LEN * 2 + 32, "%s%s%s", dname, TD_DIRSEP, fname); + int32_t tmpNameLen = strlen(tmpName) + 1; + pTFile->fname = taosMemoryMalloc(tmpNameLen); + if (!pTFile->fname) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - - tfsInitFile(pTfs, &(pTFile->f), did, fname); + tstrncpy(pTFile->fname, tmpName, tmpNameLen); return 0; } -int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType) { +int32_t tdCreateTFile(STFile *pTFile, bool updateHeader, int8_t fType) { ASSERT(pTFile->info.fsize == 0 && pTFile->info.magic == TD_FILE_INIT_MAGIC); pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pTFile->pFile == NULL) { if (errno == ENOENT) { // Try to create directory recursively - char *s = strdup(TD_TFILE_REL_NAME(pTFile)); - if (tfsMkdirRecurAt(pTfs, taosDirName(s), TD_TFILE_DID(pTFile)) < 0) { - taosMemoryFreeClear(s); - return -1; - } - taosMemoryFreeClear(s); - - pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pTFile->pFile == NULL) { + if (taosMulMkDir(taosDirName(TD_TFILE_FULL_NAME(pTFile))) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; + } else { + pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pTFile->pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } } - } else { - terrno = TAOS_SYSTEM_ERROR(errno); + } + + if (!updateHeader) { + return 0; + } + + pTFile->info.fsize += TD_FILE_HEAD_SIZE; + pTFile->info.fver = 0; + + if (tdUpdateTFileHeader(pTFile) < 0) { + tdCloseTFile(pTFile); + tdRemoveTFile(pTFile); return -1; } } - if (!updateHeader) { - return 0; - } - - pTFile->info.fsize += TD_FILE_HEAD_SIZE; - pTFile->info.fver = 0; - - if (tdUpdateTFileHeader(pTFile) < 0) { - tdCloseTFile(pTFile); - tdRemoveTFile(pTFile); - return -1; - } - return 0; } -int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_TFILE_F(pTFile)); } +int32_t tdRemoveTFile(STFile *pTFile) { + if (taosRemoveFile(TD_TFILE_FULL_NAME(pTFile)) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + }; + return 0; +} // smaXXXUtil ================ // ... \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 35486a5267..7c4a7d520d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -183,13 +183,15 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { } else { ASSERT(0); } - STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey); - if (pOffset == NULL || pOffset->val.version < offset.val.version) { - if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { - ASSERT(0); - return -1; - } + /*STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey);*/ + /*if (pOffset != NULL) {*/ + /*if (pOffset->val.type == TMQ_OFFSET__LOG && pOffset->val.version < offset.val.version) {*/ + if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { + ASSERT(0); + return -1; } + /*}*/ + /*}*/ return 0; } @@ -375,8 +377,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { taosMemoryFree(pCkHead); } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) { - tqInfo("retrieve using snapshot req offset: uid %ld ts %ld, actual offset: uid %ld ts %ld", dataRsp.reqOffset.uid, - dataRsp.reqOffset.ts, fetchOffsetNew.uid, fetchOffsetNew.ts); + tqInfo("retrieve using snapshot actual offset: uid %ld ts %ld", fetchOffsetNew.uid, fetchOffsetNew.ts); if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, fetchOffsetNew, workerId) < 0) { ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 9e5c67fed4..bc992b2211 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -120,7 +120,9 @@ bool tqNextDataBlock(SStreamReader* pHandle) { return true; } void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->msgIter.uid, sizeof(int64_t)); + /*tqDebug("search uid %ld", pHandle->msgIter.uid);*/ if (ret != NULL) { + /*tqDebug("find uid %ld", pHandle->msgIter.uid);*/ return true; } } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 0bb9918488..dbbb2b2661 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -185,5 +185,7 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { .contLen = ntohl(pReq->length), }; - ASSERT(tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) == 0); + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + tqDebug("failed to put into write-queue since %s", terrstr()); + } } diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 79e63d5abc..0b28d6bf10 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -28,12 +28,12 @@ const SVnodeCfg vnodeCfgDefault = { .update = 1, .compression = 2, .slLevel = 5, - .days = 10, + .days = 14400, .minRows = 100, .maxRows = 4096, - .keep2 = 3650, - .keep0 = 3650, - .keep1 = 3650}, + .keep2 = 5256000, + .keep0 = 5256000, + .keep1 = 5256000}, .walCfg = {.vgId = -1, .fsyncPeriod = 0, .retentionPeriod = 0, .rollPeriod = 0, .segSize = 0, .level = TAOS_WAL_WRITE}, .hashBegin = 0, diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 02a0783927..a12e11b398 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -229,6 +229,9 @@ int vnodeCommit(SVnode *pVnode) { return -1; } + // preCommit + // TODO + // commit each sub-system if (metaCommit(pVnode->pMeta) < 0) { ASSERT(0); @@ -269,6 +272,9 @@ int vnodeCommit(SVnode *pVnode) { pVnode->state.committed = info.state.committed; + // postCommit + smaPostCommit(pVnode->pSma); + // apply the commit (TODO) vnodeBufPoolReset(pVnode->onCommit); pVnode->onCommit->next = pVnode->pPool; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b73eaa4288..649e8299f4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -779,6 +779,7 @@ _exit: taosArrayDestroy(submitRsp.pArray); // TODO: the partial success scenario and the error case + // => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level 1/level 2. // TODO: refactor if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) { tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index d1d2c55acf..b562b0631e 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -356,6 +356,7 @@ typedef struct SStreamBlockScanInfo { SUpdateInfo* pUpdateInfo; EStreamScanMode scanMode; + SOperatorInfo* pStreamScanOp; SOperatorInfo* pSnapshotReadOp; SArray* childIds; SessionWindowSupporter sessionSup; @@ -427,7 +428,7 @@ typedef struct SIntervalAggOperatorInfo { STimeWindowAggSupp twAggSup; bool invertible; SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation. - bool ignoreCloseWindow; + bool ignoreExpiredData; } SIntervalAggOperatorInfo; typedef struct SStreamFinalIntervalOperatorInfo { @@ -449,7 +450,7 @@ typedef struct SStreamFinalIntervalOperatorInfo { SArray* pPullWins; // SPullWindowInfo int32_t pullIndex; SSDataBlock* pPullDataRes; - bool ignoreCloseWindow; + bool ignoreExpiredData; } SStreamFinalIntervalOperatorInfo; typedef struct SAggOperatorInfo { @@ -587,7 +588,7 @@ typedef struct SStreamSessionAggOperatorInfo { SArray* pChildren; // cache for children's result; final stream operator SPhysiNode* pPhyNode; // create new child bool isFinal; - bool ignoreCloseWindow; + bool ignoreExpiredData; } SStreamSessionAggOperatorInfo; typedef struct STimeSliceOperatorInfo { @@ -631,7 +632,7 @@ typedef struct SStreamStateAggOperatorInfo { void* pDelIterator; SArray* pScanWindow; SArray* pChildren; // cache for children's result; - bool ignoreCloseWindow; + bool ignoreExpiredData; } SStreamStateAggOperatorInfo; typedef struct SSortedMergeOperatorInfo { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 31edc46b4d..0e76607c8f 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -145,10 +145,12 @@ static SArray* filterQualifiedChildTables(const SStreamBlockScanInfo* pScanInfo, continue; } + // TODO handle ntb case if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) { continue; } - // TODO handle ntb case + /*pScanInfo->pStreamScanOp->pTaskInfo->tableqinfoList.*/ + // handle multiple partition taosArrayPush(qa, id); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 6ca4327c96..ebfc70ceae 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2027,8 +2027,9 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY); - qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, GET_TASKID(pTaskInfo), - pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId, sourceIndex, totalSources); + qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId, + sourceIndex, totalSources); pMsg->header.vgId = htonl(pSource->addr.nodeId); pMsg->sId = htobe64(pSource->schedId); @@ -2163,8 +2164,8 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx SSDataBlock* pRes = pExchangeInfo->pResult; SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; if (pRsp->numOfRows == 0) { - qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 - ", completed:%d try next %d/%" PRIzu, + qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 ", completed:%d try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows, pExchangeInfo->loadInfo.totalRows, completed + 1, i + 1, totalSources); pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; @@ -2183,18 +2184,19 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx } if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d" + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 + " execId:%d" " index:%d completed, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 ", completed:%d try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRes->info.rows, pDataInfo->totalRows, - pLoadInfo->totalRows, pLoadInfo->totalSize, completed + 1, i + 1, totalSources); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRes->info.rows, + pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, completed + 1, i + 1, totalSources); completed += 1; pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pLoadInfo->totalRows, - pLoadInfo->totalSize); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, + pLoadInfo->totalRows, pLoadInfo->totalSize); } taosMemoryFreeClear(pDataInfo->pRsp); @@ -2267,8 +2269,8 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); if (pDataInfo->code != TSDB_CODE_SUCCESS) { - qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), pSource->addr.nodeId, - pSource->taskId, pSource->execId, tstrerror(pDataInfo->code)); + qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), + pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code)); pOperator->pTaskInfo->code = pDataInfo->code; return NULL; } @@ -2276,8 +2278,8 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRsp = pDataInfo->pRsp; SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; if (pRsp->numOfRows == 0) { - qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 - " try next", + qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 " try next", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1, pDataInfo->totalRows, pLoadInfo->totalRows); @@ -2296,16 +2298,17 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pDataInfo->totalRows, - pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, totalSources); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, + pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, + totalSources); pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; pExchangeInfo->current += 1; } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pLoadInfo->totalRows, - pLoadInfo->totalSize); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, + pLoadInfo->totalRows, pLoadInfo->totalSize); } pOperator->resultInfo.totalRows += pRes->info.rows; @@ -3250,6 +3253,10 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { doSetOperatorCompleted(pOperator); break; } + if (pBlock->info.type == STREAM_RETRIEVE) { + // for stream interval + return pBlock; + } // the pDataBlock are always the same one, no need to call this again int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index f814e74269..7b027a0570 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -807,6 +807,23 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) { return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; } +static void setGroupId(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) { + ASSERT(rowIndex < pBlock->info.rows); + switch (pBlock->info.type) + { + case STREAM_RETRIEVE: { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex); + uint64_t* groupCol = (uint64_t*)pColInfo->pData; + pInfo->groupId = groupCol[rowIndex]; + } + break; + case STREAM_DELETE_DATA: + break; + default: + break; + } +} + static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { STimeWindow win = { .skey = INT64_MIN, @@ -829,6 +846,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3 } else { win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[(*pRowIndex)], &pInfo->interval, pInfo->interval.precision, NULL); + setGroupId(pInfo, pSDB, 2, *pRowIndex); (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, (*pRowIndex), win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); } @@ -1031,10 +1049,12 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; } else { - if (isStateWindow(pInfo) && taosArrayGetSize(pInfo->sessionSup.pStreamAggSup->pScanWindow) > 0) { + if (isStateWindow(pInfo)) { pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; pInfo->updateResIndex = pInfo->pUpdateRes->info.rows; - prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); + if (!prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex)) { + pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + } } if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) { SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); @@ -1274,6 +1294,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1}; pInfo->groupId = 0; pInfo->pPullDataRes = createPullDataBlock(); + pInfo->pStreamScanOp = pOperator; pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 66900fb7aa..03c939cc95 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -838,7 +838,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, &pInfo->win); int32_t ret = TSDB_CODE_SUCCESS; - if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) { + if (!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) { ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { @@ -871,7 +871,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup); } - if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) { + if (!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) { updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); doApplyFunctions(pTaskInfo, pSup->pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, pBlock->info.rows, numOfOutput, pInfo->order); @@ -886,7 +886,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul if (startPos < 0) { break; } - if (pInfo->ignoreCloseWindow && isCloseWindow(&nextWin, &pInfo->twAggSup)) { + if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) { ekey = ascScan ? nextWin.ekey : nextWin.skey; forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); @@ -1535,7 +1535,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pInfo->interval = *pInterval; pInfo->execModel = pTaskInfo->execModel; pInfo->twAggSup = *pTwAggSupp; - pInfo->ignoreCloseWindow = false; + pInfo->ignoreExpiredData = pPhyNode->window.igExpired; if (pPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; @@ -2292,7 +2292,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc pInfo->interval.precision, NULL); while (1) { bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); - if (pInfo->ignoreCloseWindow && isClosed) { + if (pInfo->ignoreExpiredData && isClosed) { startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); if (startPos < 0) { break; @@ -2710,7 +2710,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); pInfo->pPullDataRes = createPullDataBlock(); - pInfo->ignoreCloseWindow = false; + pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -2852,12 +2852,12 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); pInfo->pDelIterator = NULL; pInfo->pDelRes = createOneDataBlock(pResBlock, false); - pInfo->pDelRes->info.type = STREAM_DELETE; + pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; blockDataEnsureCapacity(pInfo->pDelRes, 64); pInfo->pChildren = NULL; pInfo->isFinal = false; pInfo->pPhyNode = pPhyNode; - pInfo->ignoreCloseWindow = false; + pInfo->ignoreExpiredData = pSessionNode->window.igExpired; pOperator->name = "StreamSessionWindowAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; @@ -3133,7 +3133,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; for (int32_t i = 0; i < pSDataBlock->info.rows;) { - if (pInfo->ignoreCloseWindow && isOverdue(endTsCols[i], &pInfo->twAggSup)) { + if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) { i++; continue; } @@ -3413,8 +3413,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, - getResWinForSession, pInfo->ignoreCloseWindow); - closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow); + getResWinForSession, pInfo->ignoreExpiredData); + closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData); copyUpdateResult(pStUpdated, pUpdated); taosHashCleanup(pStUpdated); @@ -3822,7 +3822,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) { - if (pInfo->ignoreCloseWindow && isOverdue(tsCols[i], &pInfo->twAggSup)) { + if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) { i++; continue; } @@ -3866,12 +3866,14 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { + printDataBlock(pInfo->pDelRes, "single state"); return pInfo->pDelRes; } doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } + printDataBlock(pBInfo->pRes, "single state"); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -3884,6 +3886,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { if (pBlock == NULL) { break; } + printDataBlock(pBlock, "single state recv"); if (pBlock->info.type == STREAM_CLEAR) { doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId, @@ -3903,8 +3906,8 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, - getResWinForState, pInfo->ignoreCloseWindow); - closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow); + getResWinForState, pInfo->ignoreExpiredData); + closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData); copyUpdateResult(pSeUpdated, pUpdated); taosHashCleanup(pSeUpdated); @@ -3914,9 +3917,11 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { + printDataBlock(pInfo->pDelRes, "single state"); return pInfo->pDelRes; } doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + printDataBlock(pBInfo->pRes, "single state"); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -3975,10 +3980,10 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); pInfo->pDelIterator = NULL; pInfo->pDelRes = createOneDataBlock(pResBlock, false); - pInfo->pDelRes->info.type = STREAM_DELETE; + pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; blockDataEnsureCapacity(pInfo->pDelRes, 64); pInfo->pChildren = NULL; - pInfo->ignoreCloseWindow = false; + pInfo->ignoreExpiredData = pStateNode->window.igExpired; pOperator->name = "StreamStateAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index e01750a3ca..bc004579ba 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -39,6 +39,174 @@ static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid parameter value : %s", pFuncName); } +#define TIME_UNIT_INVALID 1 +#define TIME_UNIT_TOO_SMALL 2 + +static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) { + if (!pVal->isDuration) { + return TIME_UNIT_INVALID; + } + + if (TSDB_TIME_PRECISION_MILLI == dbPrec && 0 == strcasecmp(pVal->literal, "1u")) { + return TIME_UNIT_TOO_SMALL; + } + + if (pVal->literal[0] != '1' || (pVal->literal[1] != 'u' && pVal->literal[1] != 'a' && + pVal->literal[1] != 's' && pVal->literal[1] != 'm' && + pVal->literal[1] != 'h' && pVal->literal[1] != 'd' && + pVal->literal[1] != 'w')) { + return TIME_UNIT_INVALID; + } + + return TSDB_CODE_SUCCESS; +} + +/* Following are valid ISO-8601 timezone format: + * 1 z/Z + * 2 ±hh:mm + * 3 ±hhmm + * 4 ±hh + * + */ + +static bool validateHourRange(int8_t hour) { + if (hour < 0 || hour > 12) { + return false; + } + + return true; +} + +static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) { + if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5) && sign == '+')) { + return true; + } + + return false; +} + +static bool validateTimestampDigits(const SValueNode* pVal) { + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return false; + } + + int64_t tsVal = pVal->datum.i; + char fraction[20] = {0}; + NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction); + int32_t tsDigits = (int32_t)strlen(fraction); + + if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) { + if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS || + tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + return true; + } else { + return false; + } + } + + return true; +} + +static bool validateTimezoneFormat(const SValueNode* pVal) { + if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) { + return false; + } + + char* tz = varDataVal(pVal->datum.p); + int32_t len = varDataLen(pVal->datum.p); + + char buf[3] = {0}; + int8_t hour = -1, minute = -1; + if (len == 0) { + return false; + } else if (len == 1 && (tz[0] == 'z' || tz[0] == 'Z')) { + return true; + } else if ((tz[0] == '+' || tz[0] == '-')) { + switch (len) { + case 3: + case 5: { + for (int32_t i = 1; i < len; ++i) { + if (!isdigit(tz[i])) { + return false; + } + + if (i == 2) { + memcpy(buf, &tz[i - 1], 2); + hour = taosStr2Int8(buf, NULL, 10); + if (!validateHourRange(hour)) { + return false; + } + } else if (i == 4) { + memcpy(buf, &tz[i - 1], 2); + minute = taosStr2Int8(buf, NULL, 10); + if (!validateMinuteRange(hour, minute, tz[0])) { + return false; + } + } + } + break; + } + case 6: { + for (int32_t i = 1; i < len; ++i) { + if (i == 3) { + if (tz[i] != ':') { + return false; + } + continue; + } + + if (!isdigit(tz[i])) { + return false; + } + + if (i == 2) { + memcpy(buf, &tz[i - 1], 2); + hour = taosStr2Int8(buf, NULL, 10); + if (!validateHourRange(hour)) { + return false; + } + } else if (i == 5) { + memcpy(buf, &tz[i - 1], 2); + minute = taosStr2Int8(buf, NULL, 10); + if (!validateMinuteRange(hour, minute, tz[0])) { + return false; + } + } + } + break; + } + default: { + return false; + } + } + } else { + return false; + } + + return true; +} + +void static addTimezoneParam(SNodeList* pList) { + char buf[6] = {0}; + time_t t = taosTime(NULL); + struct tm* tmInfo = taosLocalTime(&t, NULL); + strftime(buf, sizeof(buf), "%z", tmInfo); + int32_t len = (int32_t)strlen(buf); + + SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + pVal->literal = strndup(buf, len); + pVal->isDuration = false; + pVal->translate = true; + pVal->node.resType.type = TSDB_DATA_TYPE_BINARY; + pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE; + pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI; + pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1); + varDataSetLen(pVal->datum.p, len); + strncpy(varDataVal(pVal->datum.p), pVal->literal, len); + + nodesListAppend(pList, (SNode*)pVal); +} + void static addDbPrecisonParam(SNodeList** pList, uint8_t precision) { SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); pVal->literal = NULL; @@ -525,9 +693,15 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - if (pValue->datum.i == 0) { + uint8_t dbPrec = pFunc->node.resType.precision; + + int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 1)); + if (ret == TIME_UNIT_TOO_SMALL) { return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "ELAPSED function time unit parameter should be greater than db precision"); + } else if (ret == TIME_UNIT_INVALID) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "ELAPSED function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]"); } } @@ -843,6 +1017,19 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32 return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + if (numOfParams == 4) { + uint8_t dbPrec = pFunc->node.resType.precision; + + int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 3)); + if (ret == TIME_UNIT_TOO_SMALL) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "STATEDURATION function time unit parameter should be greater than db precision"); + } else if (ret == TIME_UNIT_INVALID) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "STATEDURATION function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]"); + } + } + // set result type pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; @@ -1259,19 +1446,9 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len) static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // The number of parameters has been limited by the syntax definition - // uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; // The function return type has been set during syntax parsing uint8_t para2Type = pFunc->node.resType.type; - // if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT && - // para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR && - // para2Type != TSDB_DATA_TYPE_TIMESTAMP) { - // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - // } - // if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) || - // (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) { - // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - // } int32_t para2Bytes = pFunc->node.resType.bytes; if (IS_VAR_DATA_TYPE(para2Type)) { @@ -1281,155 +1458,14 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "CAST function converted length should be in range [0, 1000]"); } + + // add database precision as param + uint8_t dbPrec = pFunc->node.resType.precision; + addDbPrecisonParam(&pFunc->pParameterList, dbPrec); + return TSDB_CODE_SUCCESS; } -/* Following are valid ISO-8601 timezone format: - * 1 z/Z - * 2 ±hh:mm - * 3 ±hhmm - * 4 ±hh - * - */ - -static bool validateHourRange(int8_t hour) { - if (hour < 0 || hour > 12) { - return false; - } - - return true; -} - -static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) { - if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5) && sign == '+')) { - return true; - } - - return false; -} - -static bool validateTimestampDigits(const SValueNode* pVal) { - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return false; - } - - int64_t tsVal = pVal->datum.i; - char fraction[20] = {0}; - NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction); - int32_t tsDigits = (int32_t)strlen(fraction); - - if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) { - if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS || - tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { - return true; - } else { - return false; - } - } - - return true; -} - -static bool validateTimezoneFormat(const SValueNode* pVal) { - if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) { - return false; - } - - char* tz = varDataVal(pVal->datum.p); - int32_t len = varDataLen(pVal->datum.p); - - char buf[3] = {0}; - int8_t hour = -1, minute = -1; - if (len == 0) { - return false; - } else if (len == 1 && (tz[0] == 'z' || tz[0] == 'Z')) { - return true; - } else if ((tz[0] == '+' || tz[0] == '-')) { - switch (len) { - case 3: - case 5: { - for (int32_t i = 1; i < len; ++i) { - if (!isdigit(tz[i])) { - return false; - } - - if (i == 2) { - memcpy(buf, &tz[i - 1], 2); - hour = taosStr2Int8(buf, NULL, 10); - if (!validateHourRange(hour)) { - return false; - } - } else if (i == 4) { - memcpy(buf, &tz[i - 1], 2); - minute = taosStr2Int8(buf, NULL, 10); - if (!validateMinuteRange(hour, minute, tz[0])) { - return false; - } - } - } - break; - } - case 6: { - for (int32_t i = 1; i < len; ++i) { - if (i == 3) { - if (tz[i] != ':') { - return false; - } - continue; - } - - if (!isdigit(tz[i])) { - return false; - } - - if (i == 2) { - memcpy(buf, &tz[i - 1], 2); - hour = taosStr2Int8(buf, NULL, 10); - if (!validateHourRange(hour)) { - return false; - } - } else if (i == 5) { - memcpy(buf, &tz[i - 1], 2); - minute = taosStr2Int8(buf, NULL, 10); - if (!validateMinuteRange(hour, minute, tz[0])) { - return false; - } - } - } - break; - } - default: { - return false; - } - } - } else { - return false; - } - - return true; -} - -void static addTimezoneParam(SNodeList* pList) { - char buf[6] = {0}; - time_t t = taosTime(NULL); - struct tm* tmInfo = taosLocalTime(&t, NULL); - strftime(buf, sizeof(buf), "%z", tmInfo); - int32_t len = (int32_t)strlen(buf); - - SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); - pVal->literal = strndup(buf, len); - pVal->isDuration = false; - pVal->translate = true; - pVal->node.resType.type = TSDB_DATA_TYPE_BINARY; - pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE; - pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI; - pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1); - varDataSetLen(pVal->datum.p, len); - strncpy(varDataVal(pVal->datum.p), pVal->literal, len); - - nodesListAppend(pList, (SNode*)pVal); -} - static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); if (1 != numOfParams && 2 != numOfParams) { @@ -1498,6 +1534,16 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_ // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; + + int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 1)); + if (ret == TIME_UNIT_TOO_SMALL) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "TIMETRUNCATE function time unit parameter should be greater than db precision"); + } else if (ret == TIME_UNIT_INVALID) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "TIMETRUNCATE function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]"); + } + addDbPrecisonParam(&pFunc->pParameterList, dbPrec); pFunc->node.resType = @@ -1526,6 +1572,18 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; + + if (3 == numOfParams) { + int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 2)); + if (ret == TIME_UNIT_TOO_SMALL) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "TIMEDIFF function time unit parameter should be greater than db precision"); + } else if (ret == TIME_UNIT_INVALID) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "TIMEDIFF function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]"); + } + } + addDbPrecisonParam(&pFunc->pParameterList, dbPrec); pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index 75eac2430f..33133d21ae 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -35,7 +35,7 @@ if (${BUILD_WITH_INVERTEDINDEX}) endif(${BUILD_WITH_INVERTEDINDEX}) -if (${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) +# if (${BUILD_TEST}) +# add_subdirectory(test) +# endif(${BUILD_TEST}) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 3a219b78b5..f35f81892d 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -934,9 +934,16 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp break; } case TSDB_DATA_TYPE_TIMESTAMP: { + int64_t timeVal; if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { - //convert to 0 - *(int64_t *)output = 0; + int64_t timePrec; + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + int32_t ret = convertStringToTimestamp(inputType, input, timePrec, &timeVal); + if (ret != TSDB_CODE_SUCCESS) { + *(int64_t *)output = 0; + } else { + *(int64_t *)output = timeVal; + } } else { GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input); } diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 94a1108710..aaa8274ce8 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -276,8 +276,6 @@ extern SSchedulerMgmt schMgmt; #define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1) #define SCH_TASK_EID(_task) ((_task) ? (_task)->execId : -1) -#define SCH_SET_TASK_LASTMSG_TYPE(_task, _type) do { if(_task) { atomic_store_32(&(_task)->lastMsgType, _type); } } while (0) -#define SCH_GET_TASK_LASTMSG_TYPE(_task) ((_task) ? atomic_load_32(&(_task)->lastMsgType) : -1) #define SCH_IS_DATA_SRC_QRY_TASK(task) ((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) #define SCH_IS_DATA_SRC_TASK(task) (((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) || ((task)->plan->subplanType == SUBPLAN_TYPE_MODIFY)) @@ -309,7 +307,10 @@ extern SSchedulerMgmt schMgmt; #define SCH_IS_NEED_DROP_JOB(_job) (SCH_IS_QUERY_JOB(_job)) #define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode) #define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) -#define SCH_SUB_TASK_NETWORK_ERR(_code, _len) (((_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_RPC_BROKEN_LINK) && ((_len) > 0)) +#define SCH_SUB_TASK_NETWORK_ERR(_code, _len) (SCH_NETWORK_ERR(_code) && ((_len) > 0)) +#define SCH_NEED_REDIRECT_MSGTYPE(_msgType) ((_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || (_msgType) == TDMT_SCH_FETCH) +#define SCH_NEED_REDIRECT(_msgType, _code, _rspLen) (SCH_NEED_REDIRECT_MSGTYPE(_msgType) && (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_SUB_TASK_NETWORK_ERR(_code, _rspLen))) +#define SCH_NEED_RETRY(_msgType, _code) ((SCH_NETWORK_ERR(_code) && SCH_NEED_REDIRECT_MSGTYPE(_msgType)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) #define SCH_IS_LEVEL_UNFINISHED(_level) ((_level)->taskLaunchedNum < (_level)->taskNum) #define SCH_GET_CUR_EP(_addr) (&(_addr)->epSet.eps[(_addr)->epSet.inUse]) diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 425ea242fd..6e4838e50c 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -835,7 +835,7 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo return TSDB_CODE_SUCCESS; } - if (!NEED_SCHEDULER_RETRY_ERROR(errCode)) { + if (!SCH_NEED_RETRY(pTask->lastMsgType, errCode)) { *needRetry = false; SCH_TASK_DLOG("task no more retry cause of errCode, errCode:%x - %s", errCode, tstrerror(errCode)); return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 69e41d3111..02e7b085ae 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -23,7 +23,7 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { - int32_t lastMsgType = SCH_GET_TASK_LASTMSG_TYPE(pTask); + int32_t lastMsgType = pTask->lastMsgType; int32_t taskStatus = SCH_GET_TASK_STATUS(pTask); int32_t reqMsgType = msgType - 1; switch (msgType) { @@ -42,7 +42,7 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgTy TMSG_INFO(msgType)); } - SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); + //SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); return TSDB_CODE_SUCCESS; case TDMT_SCH_FETCH_RSP: if (lastMsgType != reqMsgType && -1 != lastMsgType) { @@ -57,7 +57,7 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgTy SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); + //SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); return TSDB_CODE_SUCCESS; case TDMT_VND_CREATE_TABLE_RSP: case TDMT_VND_DROP_TABLE_RSP: @@ -82,7 +82,7 @@ int32_t schValidateReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgTy SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); + //SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); return TSDB_CODE_SUCCESS; } @@ -396,7 +396,8 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { SCH_ERR_JRET(schValidateReceivedMsgType(pJob, pTask, msgType)); - if (NEED_SCHEDULER_REDIRECT_ERROR(rspCode) || SCH_SUB_TASK_NETWORK_ERR(rspCode, pMsg->len > 0)) { + int32_t reqType = IsReq(pMsg) ? pMsg->msgType : (pMsg->msgType - 1); + if (SCH_NEED_REDIRECT(reqType, rspCode, pMsg->len)) { code = schHandleRedirect(pJob, pTask, (SDataBuf *)pMsg, rspCode); goto _return; } @@ -855,6 +856,9 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery addr->nodeId, epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, trans->pTrans, trans->pHandle); + if (pTask) { + pTask->lastMsgType = msgType; + } int64_t transporterId = 0; code = asyncSendMsgToServerExt(trans->pTrans, epSet, &transporterId, pMsgSendInfo, persistHandle, ctx); @@ -1098,8 +1102,6 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, break; } - SCH_SET_TASK_LASTMSG_TYPE(pTask, msgType); - SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; SCH_ERR_JRET(schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL))); @@ -1112,7 +1114,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, _return: - SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); + pTask->lastMsgType = -1; schFreeRpcCtx(&rpcCtx); taosMemoryFreeClear(msg); diff --git a/source/os/src/osString.c b/source/os/src/osString.c index e6904f1963..efa65fe191 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -196,7 +196,7 @@ int32_t taosUcs4len(TdUcs4 *ucs4) { } //dst buffer size should be at least 2*len + 1 -int32_t taosHexEncode(const char *src, char *dst, int32_t len) { +int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len) { if (!dst) { return -1; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 6e8b8b1595..564751ad67 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -590,6 +590,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_CACHE, "No tsma index in ca TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV, "Invalid rsma env") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_STAT, "Invalid rsma state") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_QTASKINFO_CREATE, "Rsma qtaskinfo creation error") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FILE_CORRUPTED, "Rsma file corrupted") + //tq TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "No committed offset") diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 6425e7acfa..94043ed01a 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -662,6 +662,12 @@ class TDCom: return res_list else: tdLog.exit(f"getOneRow out of range: row_index={location} row_count={self.query_row}") + + def killProcessor(self, processorName): + if (platform.system().lower() == 'windows'): + os.system("TASKKILL /F /IM %s.exe"%processorName) + else: + os.system('pkill %s'%processorName) def is_json(msg): diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index cb432024b7..9738fad24d 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -93,6 +93,7 @@ ./test.sh -f tsim/stream/basic0.sim ./test.sh -f tsim/stream/basic1.sim ./test.sh -f tsim/stream/basic2.sim +./test.sh -f tsim/stream/drop_stream.sim ./test.sh -f tsim/stream/distributeInterval0.sim # ./test.sh -f tsim/stream/distributeIntervalRetrive0.sim # ./test.sh -f tsim/stream/distributesession0.sim @@ -105,6 +106,7 @@ ./test.sh -f tsim/stream/partitionby1.sim ./test.sh -f tsim/stream/schedSnode.sim ./test.sh -f tsim/stream/windowClose.sim +./test.sh -f tsim/stream/ignoreExpiredData.sim # ---- transaction ./test.sh -f tsim/trans/lossdata1.sim @@ -159,6 +161,7 @@ #./test.sh -f tsim/mnode/basic1.sim -m # --- sma +./test.sh -f tsim/sma/drop_sma.sim ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 5edc0a4d3e..d54624872f 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -135,7 +135,7 @@ echo "jniDebugFlag 143" >> $TAOS_CFG echo "qDebugFlag 143" >> $TAOS_CFG echo "rpcDebugFlag 143" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG -echo "uDebugFlag 143" >> $TAOS_CFG +echo "uDebugFlag 131" >> $TAOS_CFG echo "sDebugFlag 143" >> $TAOS_CFG echo "wDebugFlag 143" >> $TAOS_CFG echo "numOfLogLines 20000000" >> $TAOS_CFG diff --git a/tests/script/tsim/sma/drop_sma.sim b/tests/script/tsim/sma/drop_sma.sim index 17f19f5df3..73ada2de56 100644 --- a/tests/script/tsim/sma/drop_sma.sim +++ b/tests/script/tsim/sma/drop_sma.sim @@ -64,12 +64,59 @@ sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) in print --> drop stb sql drop table stb; +print ========== step5 +sql drop database if exists db; +sql create database db duration 300; +sql use db; +sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint, c_float float, c_double double, c_bool bool, c_binary binary(16), c_nchar nchar(32), c_ts timestamp, c_tint_un tinyint unsigned, c_sint_un smallint unsigned, c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); +sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; + +print ========== step6 repeat +sql drop database if exists db; +sql create database db duration 300; +sql use db; +sql create table stb1(ts timestamp, c_int int, c_bint bigint ) tags (t_int int); +sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; + +print ========== step7 +sql drop database if exists db; +sql create database db duration 300; +sql use db; +sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); + +sql create table ct1 using stb1 tags ( 1 ); +sql create table ct2 using stb1 tags ( 2 ); +sql create table ct3 using stb1 tags ( 3 ); +sql create table ct4 using stb1 tags ( 4 ); + +sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; +sql CREATE SMA INDEX sma_index_2 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) max_delay 6m; +sql CREATE SMA INDEX sma_index_3 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) watermark 5s max_delay 6m; + +sql DROP INDEX sma_index_1 ; +sql DROP INDEX sma_index_2 ; +sql DROP INDEX sma_index_3 ; + +print ========== step8 +sql drop database if exists db; +sql create database db duration 300; +sql use db; +sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); + +sql create table ct1 using stb1 tags ( 1 ); +sql create table ct2 using stb1 tags ( 2 ); +sql create table ct3 using stb1 tags ( 3 ); +sql create table ct4 using stb1 tags ( 4 ); + +sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; +sql CREATE SMA INDEX sma_index_2 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) max_delay 6m; +sql CREATE SMA INDEX sma_index_3 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) watermark 5s max_delay 6m; + +sql DROP INDEX sma_index_1 ; +sql DROP INDEX sma_index_2 ; +sql DROP INDEX sma_index_3 ; system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index a59e989d80..1c0d0a3bd7 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -83,9 +83,9 @@ sql insert into ts3 values(1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) $loop_count = 0 loop1: +sleep 300 sql select * from streamtST1; -sleep 300 $loop_count = $loop_count + 1 if $loop_count == 10 then return -1 diff --git a/tests/script/tsim/stream/distributeSession0.sim b/tests/script/tsim/stream/distributeSession0.sim index a165b86edd..30c3c641d4 100644 --- a/tests/script/tsim/stream/distributeSession0.sim +++ b/tests/script/tsim/stream/distributeSession0.sim @@ -10,6 +10,30 @@ sql create dnode $hostname2 port 7200 system sh/exec.sh -n dnode2 -s start +print ===== step1 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print ===== step2 + sql create database test vgroups 4; sql use test; sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); diff --git a/tests/script/tsim/stream/drop_stream.sim b/tests/script/tsim/stream/drop_stream.sim new file mode 100644 index 0000000000..bdd88bf780 --- /dev/null +++ b/tests/script/tsim/stream/drop_stream.sim @@ -0,0 +1,222 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/cfg.sh -n dnode2 -c supportVnodes -v 4 +system sh/cfg.sh -n dnode3 -c supportVnodes -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ========== step2 +sql create dnode $hostname port 7200 +system sh/exec.sh -n dnode2 -s start + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi + +print ========== step3 +sql drop database if exists test; +sql create database if not exists test vgroups 1 precision "ms" ; +sql use test; +sql create table test.scalar_function_stb (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 binary(256), c12 nchar(256), c13 bool) tags (t1 tinyint, t2 smallint, t3 int, t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 binary(256), t12 nchar(256), t13 bool) ; +sql create table scalar_function_ct1 using scalar_function_stb tags (-38, -32456, 509722288, -1404014954778348330, 87, 8879, 3351927345, 1840080781675115605, 3.002364316200592e+38, 6.698140580387119e+37, "bktezshfyvmrmgzwrwerytfwudlblkyyxismpommiqpqsptpiucptwqutzhajxbiitqxkrpobqhgqvjlvgsudewmelpunjspurbpbbwypvgbwjfrwidrchnojtxyhrwfjwgdiabzfoujxkwcjjxjqsrnhmryjhrykldmdfiwircdfahldtrtuafzvybkihyjatiqivbtpydjtmbfddcgyzjuqidwcchtsamnwyqwvajftayyvfrmqcqygbxmxgjx", "ddlxkxhrvviwnjeqhewbercnlontwbsyevcjsocrwyupautsjkdzqbwuzsuetptgsdfyjzfkqyobkysikpaxtqqonxtocfowaehgovshwyciyzfmdmcmwaolkhdunfhwhcanetepxyppuullxnclockmadyaaufywllwburgsfxizcjgzvboydpqymlwgktslclidbcwiyyubyuvhjgwldkgxswigjkpbpslvlsbigdlmuldmtbqencbntbaohxr", False) ; +sql create table test.scalar_function_tb1 (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 binary(256), c12 nchar(256), c13 bool) ; +sql create table if not exists scalar_stb (ts timestamp, c1 int, c2 double, c3 binary(20), c4 binary(20), c5 nchar(20)) tags (t1 int); +sql create table scalar_ct1 using scalar_stb tags(10); +sql create table if not exists scalar_tb (ts timestamp, c1 int, c2 double, c3 binary(20), c4 binary(20), c5 nchar(20)); +sql create stream stb_abs_stream trigger at_once into output_abs_stb as select ts, abs(c1), abs(c2), c3 from scalar_stb; +sql create stream ctb_abs_stream trigger at_once into output_abs_ctb as select ts, abs(c1), abs(c2), c3 from scalar_ct1; +sql create stream tb_abs_stream trigger at_once into output_abs_tb as select ts, abs(c1), abs(c2), c3 from scalar_tb; +sql create stream stb_acos_stream trigger at_once into output_acos_stb as select ts, acos(c1), acos(c2), c3 from scalar_stb; +sql create stream ctb_acos_stream trigger at_once into output_acos_ctb as select ts, acos(c1), acos(c2), c3 from scalar_ct1; +sql create stream tb_acos_stream trigger at_once into output_acos_tb as select ts, acos(c1), acos(c2), c3 from scalar_tb; +sql create stream stb_asin_stream trigger at_once into output_asin_stb as select ts, asin(c1), asin(c2), c3 from scalar_stb; +sql create stream ctb_asin_stream trigger at_once into output_asin_ctb as select ts, asin(c1), asin(c2), c3 from scalar_ct1; +sql create stream tb_asin_stream trigger at_once into output_asin_tb as select ts, asin(c1), asin(c2), c3 from scalar_tb; +sql create stream stb_atan_stream trigger at_once into output_atan_stb as select ts, atan(c1), atan(c2), c3 from scalar_stb; +sql create stream ctb_atan_stream trigger at_once into output_atan_ctb as select ts, atan(c1), atan(c2), c3 from scalar_ct1; +sql create stream tb_atan_stream trigger at_once into output_atan_tb as select ts, atan(c1), atan(c2), c3 from scalar_tb; +sql create stream stb_ceil_stream trigger at_once into output_ceil_stb as select ts, ceil(c1), ceil(c2), c3 from scalar_stb; +sql create stream ctb_ceil_stream trigger at_once into output_ceil_ctb as select ts, ceil(c1), ceil(c2), c3 from scalar_ct1; +sql create stream tb_ceil_stream trigger at_once into output_ceil_tb as select ts, ceil(c1), ceil(c2), c3 from scalar_tb; +sql create stream stb_cos_stream trigger at_once into output_cos_stb as select ts, cos(c1), cos(c2), c3 from scalar_stb; +sql create stream ctb_cos_stream trigger at_once into output_cos_ctb as select ts, cos(c1), cos(c2), c3 from scalar_ct1; +sql create stream tb_cos_stream trigger at_once into output_cos_tb as select ts, cos(c1), cos(c2), c3 from scalar_tb; +sql create stream stb_floor_stream trigger at_once into output_floor_stb as select ts, floor(c1), floor(c2), c3 from scalar_stb; +sql create stream ctb_floor_stream trigger at_once into output_floor_ctb as select ts, floor(c1), floor(c2), c3 from scalar_ct1; +sql create stream tb_floor_stream trigger at_once into output_floor_tb as select ts, floor(c1), floor(c2), c3 from scalar_tb; +sql create stream stb_log_stream trigger at_once into output_log_stb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_stb; +sql create stream ctb_log_stream trigger at_once into output_log_ctb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_ct1; +sql create stream tb_log_stream trigger at_once into output_log_tb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_tb; +sql create stream stb_pow_stream trigger at_once into output_pow_stb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_stb; +sql create stream ctb_pow_stream trigger at_once into output_pow_ctb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_ct1; +sql create stream tb_pow_stream trigger at_once into output_pow_tb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_tb; +sql create stream stb_round_stream trigger at_once into output_round_stb as select ts, round(c1), round(c2), c3 from scalar_stb; +sql create stream ctb_round_stream trigger at_once into output_round_ctb as select ts, round(c1), round(c2), c3 from scalar_ct1; +sql create stream tb_round_stream trigger at_once into output_round_tb as select ts, round(c1), round(c2), c3 from scalar_tb; +sql create stream stb_sin_stream trigger at_once into output_sin_stb as select ts, sin(c1), sin(c2), c3 from scalar_stb; +sql create stream ctb_sin_stream trigger at_once into output_sin_ctb as select ts, sin(c1), sin(c2), c3 from scalar_ct1; +sql create stream tb_sin_stream trigger at_once into output_sin_tb as select ts, sin(c1), sin(c2), c3 from scalar_tb; +sql create stream stb_sqrt_stream trigger at_once into output_sqrt_stb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_stb; +sql create stream ctb_sqrt_stream trigger at_once into output_sqrt_ctb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_ct1; +sql create stream tb_sqrt_stream trigger at_once into output_sqrt_tb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_tb; +sql create stream stb_tan_stream trigger at_once into output_tan_stb as select ts, tan(c1), tan(c2), c3 from scalar_stb; +sql create stream ctb_tan_stream trigger at_once into output_tan_ctb as select ts, tan(c1), tan(c2), c3 from scalar_ct1; +sql create stream tb_tan_stream trigger at_once into output_tan_tb as select ts, tan(c1), tan(c2), c3 from scalar_tb; +sql create stream stb_char_length_stream into output_char_length_stb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_stb; +sql create stream ctb_char_length_stream into output_char_length_ctb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_ct1; +sql create stream tb_char_length_stream into output_char_length_tb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_tb; +sql create stream stb_concat_stream into output_concat_stb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_stb; +sql create stream ctb_concat_stream into output_concat_ctb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_ct1; +sql create stream tb_concat_stream into output_concat_tb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_tb; +sql create stream stb_concat_ws_stream into output_concat_ws_stb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_stb; +sql create stream ctb_concat_ws_stream into output_concat_ws_ctb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_ct1; +sql create stream tb_concat_ws_stream into output_concat_ws_tb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_tb; +sql create stream stb_length_stream into output_length_stb as select ts, length(c3), length(c4), length(c5) from scalar_stb; +sql create stream ctb_length_stream into output_length_ctb as select ts, length(c3), length(c4), length(c5) from scalar_ct1; +sql create stream tb_length_stream into output_length_tb as select ts, length(c3), length(c4), length(c5) from scalar_tb; +sql create stream stb_lower_stream into output_lower_stb as select ts, lower(c3), lower(c4), lower(c5) from scalar_stb; +sql create stream ctb_lower_stream into output_lower_ctb as select ts, lower(c3), lower(c4), lower(c5) from scalar_ct1; +sql create stream tb_lower_stream into output_lower_tb as select ts, lower(c3), lower(c4), lower(c5) from scalar_tb; +sql create stream stb_ltrim_stream into output_ltrim_stb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_stb; +sql create stream ctb_ltrim_stream into output_ltrim_ctb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_ct1; +sql create stream tb_ltrim_stream into output_ltrim_tb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_tb; +sql create stream stb_rtrim_stream into output_rtrim_stb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_stb; +sql create stream ctb_rtrim_stream into output_rtrim_ctb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_ct1; +sql create stream tb_rtrim_stream into output_rtrim_tb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_tb; +sql create stream stb_substr_stream into output_substr_stb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_stb; +sql create stream ctb_substr_stream into output_substr_ctb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_ct1; +sql create stream tb_substr_stream into output_substr_tb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_tb; +sql create stream stb_upper_stream into output_upper_stb as select ts, upper(c3), upper(c4), upper(c5) from scalar_stb; +sql create stream ctb_upper_stream into output_upper_ctb as select ts, upper(c3), upper(c4), upper(c5) from scalar_ct1; +sql create stream tb_upper_stream into output_upper_tb as select ts, upper(c3), upper(c4), upper(c5) from scalar_tb; +sql insert into scalar_ct1 values (1656668180503, 100, 100.1, "beijing", "taos", "Taos"); +sql insert into scalar_ct1 values (1656668180503+1s, -50, -50.1, "tianjin", "taosdata", "Taosdata"); +sql insert into scalar_ct1 values (1656668180503+2s, 0, Null, "hebei", "TDengine", Null); +sql insert into scalar_tb values (1656668180503, 100, 100.1, "beijing", "taos", "Taos"); +sql insert into scalar_tb values (1656668180503+1s, -50, -50.1, "tianjin", "taosdata", "Taosdata"); +sql insert into scalar_tb values (1656668180503+2s, 0, Null, "hebei", "TDengine", Null); +sql insert into scalar_ct1 values (1656668180503+1s, -50, 50.1, "beiJing", "TDengine", "taos"); +sql insert into scalar_tb values (1656668180503+1s, -50, 50.1, "beiJing", "TDengine", "taos"); +sql insert into scalar_ct1 values (1656668180503+1s, -50, 50.1, "beiJing", "TDengine", "taos"); +sql insert into scalar_tb values (1656668180503+1s, -50, 50.1, "beiJing", "TDengine", "taos"); + +print ========== step4 +sql drop database test; + + +print ========== step5 repeat +sql drop database if exists test; +sql create database if not exists test vgroups 1 precision "ms" ; +sql use test; +sql create table test.scalar_function_stb (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 binary(256), c12 nchar(256), c13 bool) tags (t1 tinyint, t2 smallint, t3 int, t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 binary(256), t12 nchar(256), t13 bool) ; +sql create table scalar_function_ct1 using scalar_function_stb tags (-38, -32456, 509722288, -1404014954778348330, 87, 8879, 3351927345, 1840080781675115605, 3.002364316200592e+38, 6.698140580387119e+37, "bktezshfyvmrmgzwrwerytfwudlblkyyxismpommiqpqsptpiucptwqutzhajxbiitqxkrpobqhgqvjlvgsudewmelpunjspurbpbbwypvgbwjfrwidrchnojtxyhrwfjwgdiabzfoujxkwcjjxjqsrnhmryjhrykldmdfiwircdfahldtrtuafzvybkihyjatiqivbtpydjtmbfddcgyzjuqidwcchtsamnwyqwvajftayyvfrmqcqygbxmxgjx", "ddlxkxhrvviwnjeqhewbercnlontwbsyevcjsocrwyupautsjkdzqbwuzsuetptgsdfyjzfkqyobkysikpaxtqqonxtocfowaehgovshwyciyzfmdmcmwaolkhdunfhwhcanetepxyppuullxnclockmadyaaufywllwburgsfxizcjgzvboydpqymlwgktslclidbcwiyyubyuvhjgwldkgxswigjkpbpslvlsbigdlmuldmtbqencbntbaohxr", False) ; +sql create table test.scalar_function_tb1 (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 binary(256), c12 nchar(256), c13 bool) ; +sql create table if not exists scalar_stb (ts timestamp, c1 int, c2 double, c3 binary(20), c4 binary(20), c5 nchar(20)) tags (t1 int); +sql create table scalar_ct1 using scalar_stb tags(10); +sql create table if not exists scalar_tb (ts timestamp, c1 int, c2 double, c3 binary(20), c4 binary(20), c5 nchar(20)); +sql create stream stb_abs_stream trigger at_once into output_abs_stb as select ts, abs(c1), abs(c2), c3 from scalar_stb; +sql create stream ctb_abs_stream trigger at_once into output_abs_ctb as select ts, abs(c1), abs(c2), c3 from scalar_ct1; +sql create stream tb_abs_stream trigger at_once into output_abs_tb as select ts, abs(c1), abs(c2), c3 from scalar_tb; +sql create stream stb_acos_stream trigger at_once into output_acos_stb as select ts, acos(c1), acos(c2), c3 from scalar_stb; +sql create stream ctb_acos_stream trigger at_once into output_acos_ctb as select ts, acos(c1), acos(c2), c3 from scalar_ct1; +sql create stream tb_acos_stream trigger at_once into output_acos_tb as select ts, acos(c1), acos(c2), c3 from scalar_tb; +sql create stream stb_asin_stream trigger at_once into output_asin_stb as select ts, asin(c1), asin(c2), c3 from scalar_stb; +sql create stream ctb_asin_stream trigger at_once into output_asin_ctb as select ts, asin(c1), asin(c2), c3 from scalar_ct1; +sql create stream tb_asin_stream trigger at_once into output_asin_tb as select ts, asin(c1), asin(c2), c3 from scalar_tb; +sql create stream stb_atan_stream trigger at_once into output_atan_stb as select ts, atan(c1), atan(c2), c3 from scalar_stb; +sql create stream ctb_atan_stream trigger at_once into output_atan_ctb as select ts, atan(c1), atan(c2), c3 from scalar_ct1; +sql create stream tb_atan_stream trigger at_once into output_atan_tb as select ts, atan(c1), atan(c2), c3 from scalar_tb; +sql create stream stb_ceil_stream trigger at_once into output_ceil_stb as select ts, ceil(c1), ceil(c2), c3 from scalar_stb; +sql create stream ctb_ceil_stream trigger at_once into output_ceil_ctb as select ts, ceil(c1), ceil(c2), c3 from scalar_ct1; +sql create stream tb_ceil_stream trigger at_once into output_ceil_tb as select ts, ceil(c1), ceil(c2), c3 from scalar_tb; +sql create stream stb_cos_stream trigger at_once into output_cos_stb as select ts, cos(c1), cos(c2), c3 from scalar_stb; +sql create stream ctb_cos_stream trigger at_once into output_cos_ctb as select ts, cos(c1), cos(c2), c3 from scalar_ct1; +sql create stream tb_cos_stream trigger at_once into output_cos_tb as select ts, cos(c1), cos(c2), c3 from scalar_tb; +sql create stream stb_floor_stream trigger at_once into output_floor_stb as select ts, floor(c1), floor(c2), c3 from scalar_stb; +sql create stream ctb_floor_stream trigger at_once into output_floor_ctb as select ts, floor(c1), floor(c2), c3 from scalar_ct1; +sql create stream tb_floor_stream trigger at_once into output_floor_tb as select ts, floor(c1), floor(c2), c3 from scalar_tb; +sql create stream stb_log_stream trigger at_once into output_log_stb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_stb; +sql create stream ctb_log_stream trigger at_once into output_log_ctb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_ct1; +sql create stream tb_log_stream trigger at_once into output_log_tb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_tb; +sql create stream stb_pow_stream trigger at_once into output_pow_stb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_stb; +sql create stream ctb_pow_stream trigger at_once into output_pow_ctb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_ct1; +sql create stream tb_pow_stream trigger at_once into output_pow_tb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_tb; +sql create stream stb_round_stream trigger at_once into output_round_stb as select ts, round(c1), round(c2), c3 from scalar_stb; +sql create stream ctb_round_stream trigger at_once into output_round_ctb as select ts, round(c1), round(c2), c3 from scalar_ct1; +sql create stream tb_round_stream trigger at_once into output_round_tb as select ts, round(c1), round(c2), c3 from scalar_tb; +sql create stream stb_sin_stream trigger at_once into output_sin_stb as select ts, sin(c1), sin(c2), c3 from scalar_stb; +sql create stream ctb_sin_stream trigger at_once into output_sin_ctb as select ts, sin(c1), sin(c2), c3 from scalar_ct1; +sql create stream tb_sin_stream trigger at_once into output_sin_tb as select ts, sin(c1), sin(c2), c3 from scalar_tb; +sql create stream stb_sqrt_stream trigger at_once into output_sqrt_stb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_stb; +sql create stream ctb_sqrt_stream trigger at_once into output_sqrt_ctb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_ct1; +sql create stream tb_sqrt_stream trigger at_once into output_sqrt_tb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_tb; +sql create stream stb_tan_stream trigger at_once into output_tan_stb as select ts, tan(c1), tan(c2), c3 from scalar_stb; +sql create stream ctb_tan_stream trigger at_once into output_tan_ctb as select ts, tan(c1), tan(c2), c3 from scalar_ct1; +sql create stream tb_tan_stream trigger at_once into output_tan_tb as select ts, tan(c1), tan(c2), c3 from scalar_tb; +sql create stream stb_char_length_stream into output_char_length_stb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_stb; +sql create stream ctb_char_length_stream into output_char_length_ctb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_ct1; +sql create stream tb_char_length_stream into output_char_length_tb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_tb; +sql create stream stb_concat_stream into output_concat_stb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_stb; +sql create stream ctb_concat_stream into output_concat_ctb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_ct1; +sql create stream tb_concat_stream into output_concat_tb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_tb; +sql create stream stb_concat_ws_stream into output_concat_ws_stb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_stb; +sql create stream ctb_concat_ws_stream into output_concat_ws_ctb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_ct1; +sql create stream tb_concat_ws_stream into output_concat_ws_tb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_tb; +sql create stream stb_length_stream into output_length_stb as select ts, length(c3), length(c4), length(c5) from scalar_stb; +sql create stream ctb_length_stream into output_length_ctb as select ts, length(c3), length(c4), length(c5) from scalar_ct1; +sql create stream tb_length_stream into output_length_tb as select ts, length(c3), length(c4), length(c5) from scalar_tb; +sql create stream stb_lower_stream into output_lower_stb as select ts, lower(c3), lower(c4), lower(c5) from scalar_stb; +sql create stream ctb_lower_stream into output_lower_ctb as select ts, lower(c3), lower(c4), lower(c5) from scalar_ct1; +sql create stream tb_lower_stream into output_lower_tb as select ts, lower(c3), lower(c4), lower(c5) from scalar_tb; +sql create stream stb_ltrim_stream into output_ltrim_stb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_stb; +sql create stream ctb_ltrim_stream into output_ltrim_ctb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_ct1; +sql create stream tb_ltrim_stream into output_ltrim_tb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_tb; +sql create stream stb_rtrim_stream into output_rtrim_stb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_stb; +sql create stream ctb_rtrim_stream into output_rtrim_ctb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_ct1; +sql create stream tb_rtrim_stream into output_rtrim_tb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_tb; +sql create stream stb_substr_stream into output_substr_stb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_stb; +sql create stream ctb_substr_stream into output_substr_ctb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_ct1; +sql create stream tb_substr_stream into output_substr_tb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_tb; +sql create stream stb_upper_stream into output_upper_stb as select ts, upper(c3), upper(c4), upper(c5) from scalar_stb; +sql create stream ctb_upper_stream into output_upper_ctb as select ts, upper(c3), upper(c4), upper(c5) from scalar_ct1; +sql create stream tb_upper_stream into output_upper_tb as select ts, upper(c3), upper(c4), upper(c5) from scalar_tb; +sql insert into scalar_ct1 values (1656668180503, 100, 100.1, "beijing", "taos", "Taos"); +sql insert into scalar_ct1 values (1656668180503+1s, -50, -50.1, "tianjin", "taosdata", "Taosdata"); +sql insert into scalar_ct1 values (1656668180503+2s, 0, Null, "hebei", "TDengine", Null); +sql insert into scalar_tb values (1656668180503, 100, 100.1, "beijing", "taos", "Taos"); +sql insert into scalar_tb values (1656668180503+1s, -50, -50.1, "tianjin", "taosdata", "Taosdata"); +sql insert into scalar_tb values (1656668180503+2s, 0, Null, "hebei", "TDengine", Null); +sql insert into scalar_ct1 values (1656668180503+1s, -50, 50.1, "beiJing", "TDengine", "taos"); +sql insert into scalar_tb values (1656668180503+1s, -50, 50.1, "beiJing", "TDengine", "taos"); +sql insert into scalar_ct1 values (1656668180503+1s, -50, 50.1, "beiJing", "TDengine", "taos"); +sql insert into scalar_tb values (1656668180503+1s, -50, 50.1, "beiJing", "TDengine", "taos"); + +print ========== step6 repeat +sql drop database test; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/stream/ignoreExpiredData.sim b/tests/script/tsim/stream/ignoreExpiredData.sim new file mode 100644 index 0000000000..c6b708d1e8 --- /dev/null +++ b/tests/script/tsim/stream/ignoreExpiredData.sim @@ -0,0 +1,161 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +sql create dnode $hostname2 port 7200 + +system sh/exec.sh -n dnode2 -s start + +print ===== step1 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print ===== step2 + +print =============== create database +sql create database test vgroups 1 +sql show databases +if $rows != 3 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use test +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams1 trigger at_once IGNORE EXPIRED into streamt1 as select _wstartts, count(*) c1, sum(a) c3 from t1 interval(10s); +sql create stream streams2 trigger at_once IGNORE EXPIRED into streamt2 as select _wstartts, count(*) c1, sum(a) c3 from t1 session(ts,10s); +sql create stream streams3 trigger at_once IGNORE EXPIRED into streamt3 as select _wstartts, count(*) c1, sum(a) c3 from t1 state_window(a); +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.1); +sql insert into t1 values(1648791233002,2,2,3,2.1); +sql insert into t1 values(1648791243003,2,2,3,3.1); +sql insert into t1 values(1648791200000,4,2,3,4.1); + +$loop_count = 0 +loop1: +sleep 300 +sql select * from streamt1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 4 then + print =====rows=$rows + goto loop1 +endi + +$loop_count = 0 +loop2: +sleep 300 +sql select * from streamt2; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 4 then + print =====rows=$rows + goto loop2 +endi + +$loop_count = 0 +loop3: +sleep 300 +sql select * from streamt3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop3 +endi + + +print =============== create database +sql create database test1 vgroups 4 +sql show databases + +print ======database=$rows + +sql use test1 + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create stream stream_t1 trigger at_once IGNORE EXPIRED into streamtST1 as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ; +sql create stream stream_t2 trigger at_once IGNORE EXPIRED into streamtST2 as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st session(ts, 10s) ; +sql insert into ts1 values(1648791211000,1,2,3); +sql insert into ts1 values(1648791222001,2,2,3); +sql insert into ts2 values(1648791211000,1,2,3); +sql insert into ts2 values(1648791222001,2,2,3); + +$loop_count = 0 +loop4: +sleep 300 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop4 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop4 +endi + +$loop_count = 0 +loop5: +sleep 300 +sql select * from streamtST2; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop5 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop5 +endi + +system sh/stop_dnodes.sh \ No newline at end of file diff --git a/tests/script/tsim/tmq/snapshot1.sim b/tests/script/tsim/tmq/snapshot1.sim index d534bb68da..58541b725d 100644 --- a/tests/script/tsim/tmq/snapshot1.sim +++ b/tests/script/tsim/tmq/snapshot1.sim @@ -25,7 +25,7 @@ $rowsPerCtb = 10 $tstart = 1640966400000 # 2022-01-01 00:00:00.000 #---- global parameters end ----# -$pullDelay = 5 +$pullDelay = 2 $ifcheckdata = 1 $ifmanualcommit = 1 $showMsg = 1 diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index e3a20cebf8..80468509ee 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -355,40 +355,6 @@ class TDTestCase: return - def test_case4(self): - self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 1, 2, 1*10) - tdSql.execute("use db1;") - tdSql.query("show dnodes;") - dnodeId=tdSql.getData(0,0) - print(dnodeId) - tdSql.execute("create qnode on dnode %s"%dnodeId) - tdSql.query("select max(c1) from stb10;") - maxQnode=tdSql.getData(0,0) - tdSql.query("select min(c1) from stb11;") - minQnode=tdSql.getData(0,0) - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") - unionQnode=tdSql.queryResult - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") - unionallQnode=tdSql.queryResult - - # tdSql.query("show qnodes;") - # qnodeId=tdSql.getData(0,0) - tdSql.execute("drop qnode on dnode %s"%dnodeId) - tdSql.execute("reset query cache") - tdSql.query("select max(c1) from stb10;") - tdSql.checkData(0, 0, "%s"%maxQnode) - tdSql.query("select min(c1) from stb11;") - tdSql.checkData(0, 0, "%s"%minQnode) - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") - unionVnode=tdSql.queryResult - assert unionQnode == unionVnode - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") - unionallVnode=tdSql.queryResult - assert unionallQnode == unionallVnode - - - # tdSql.execute("create qnode on dnode %s"%dnodeId) - # run case def run(self): diff --git a/tests/system-test/1-insert/test_stmt_insert_query_ex.py b/tests/system-test/1-insert/test_stmt_insert_query_ex.py index 2a66df866d..2a64c09ff4 100644 --- a/tests/system-test/1-insert/test_stmt_insert_query_ex.py +++ b/tests/system-test/1-insert/test_stmt_insert_query_ex.py @@ -82,7 +82,7 @@ class TDTestCase: return con def test_stmt_set_tbname_tag(self,conn): - dbname = "stmt_set_tbname_tag" + dbname = "stmt_tag" try: conn.execute("drop database if exists %s" % dbname) @@ -196,31 +196,31 @@ class TDTestCase: assert rows9[0][0] == 12, 'fourth case is failed' assert rows9[1][0] == 12, 'fourth case is failed' - # #query: conversion Functions + #query: conversion Functions - # querystmt4=conn.statement("select cast( ? as bigint) from log ") - # queryparam4=new_bind_params(1) - # print(type(queryparam4)) - # queryparam4[0].binary('1232a') - # querystmt4.bind_param(queryparam4) - # querystmt4.execute() - # result4=querystmt4.use_result() - # rows4=result4.fetch_all() - # print("5",rows4) - # assert rows4[0][0] == 1232 - # assert rows4[1][0] == 1232 + querystmt4=conn.statement("select cast( ? as bigint) from log ") + queryparam4=new_bind_params(1) + print(type(queryparam4)) + queryparam4[0].binary('1232a') + querystmt4.bind_param(queryparam4) + querystmt4.execute() + result4=querystmt4.use_result() + rows4=result4.fetch_all() + print("5",rows4) + assert rows4[0][0] == 1232 + assert rows4[1][0] == 1232 - # querystmt4=conn.statement("select cast( ? as binary(10)) from log ") - # queryparam4=new_bind_params(1) - # print(type(queryparam4)) - # queryparam4[0].int(123) - # querystmt4.bind_param(queryparam4) - # querystmt4.execute() - # result4=querystmt4.use_result() - # rows4=result4.fetch_all() - # print("6",rows4) - # assert rows4[0][0] == '123' - # assert rows4[1][0] == '123' + querystmt4=conn.statement("select cast( ? as binary(10)) from log ") + queryparam4=new_bind_params(1) + print(type(queryparam4)) + queryparam4[0].int(123) + querystmt4.bind_param(queryparam4) + querystmt4.execute() + result4=querystmt4.use_result() + rows4=result4.fetch_all() + print("6",rows4) + assert rows4[0][0] == '123' + assert rows4[1][0] == '123' # #query: datatime Functions diff --git a/tests/system-test/1-insert/test_stmt_muti_insert_query.py b/tests/system-test/1-insert/test_stmt_muti_insert_query.py index 971be6c85f..de10b2f6b9 100644 --- a/tests/system-test/1-insert/test_stmt_muti_insert_query.py +++ b/tests/system-test/1-insert/test_stmt_muti_insert_query.py @@ -84,21 +84,21 @@ class TDTestCase: def test_stmt_insert_multi(self,conn): # type: (TaosConnection) -> None - dbname = "pytest_taos_stmt_multi" + dbname = "db_stmt" try: conn.execute("drop database if exists %s" % dbname) conn.execute("create database if not exists %s" % dbname) conn.select_db(dbname) conn.execute( - "create table if not exists log(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\ + "create table if not exists stb1(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\ bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \ ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)", ) # conn.load_table_info("log") start = datetime.now() - stmt = conn.statement("insert into log values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + stmt = conn.statement("insert into stb1 values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") params = new_multi_binds(16) params[0].timestamp((1626861392589, 1626861392590, 1626861392591)) @@ -125,7 +125,7 @@ class TDTestCase: assert stmt.affected_rows == 3 #query 1 - querystmt=conn.statement("select ?,bu from log") + querystmt=conn.statement("select ?,bu from stb1") queryparam=new_bind_params(1) print(type(queryparam)) queryparam[0].binary("ts") @@ -135,7 +135,7 @@ class TDTestCase: # rows=result.fetch_all() # print( querystmt.use_result()) - # result = conn.query("select * from log") + # result = conn.query("select * from stb1") rows=result.fetch_all() # rows=result.fetch_all() print(rows) @@ -144,7 +144,7 @@ class TDTestCase: assert rows[2][1] == None #query 2 - querystmt1=conn.statement("select * from log where bu < ?") + querystmt1=conn.statement("select * from stb1 where bu < ?") queryparam1=new_bind_params(1) print(type(queryparam1)) queryparam1[0].int(4) diff --git a/tests/system-test/1-insert/test_stmt_set_tbname_tag.py b/tests/system-test/1-insert/test_stmt_set_tbname_tag.py index b540642847..387492c4d6 100644 --- a/tests/system-test/1-insert/test_stmt_set_tbname_tag.py +++ b/tests/system-test/1-insert/test_stmt_set_tbname_tag.py @@ -78,21 +78,29 @@ class TDTestCase: password = "taosdata" port =6030 con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) - print(con) + tdLog.debug(con) return con - + + def stmtExe(self,conn,sql,bindStat): + queryStat=conn.statement("%s"%sql) + queryStat.bind_param(bindStat) + queryStat.execute() + result=queryStat.use_result() + rows=result.fetch_all() + return rows + def test_stmt_set_tbname_tag(self,conn): - dbname = "stmt_set_tbname_tag" - + dbname = "stmt_tag" + stablename = 'log' try: conn.execute("drop database if exists %s" % dbname) conn.execute("create database if not exists %s PRECISION 'us' " % dbname) conn.select_db(dbname) - conn.execute("create table if not exists log(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\ + conn.execute("create table if not exists %s(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\ bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \ ff float, dd double, bb binary(100), nn nchar(100), tt timestamp , vc varchar(100)) tags (t1 timestamp, t2 bool,\ t3 tinyint, t4 tinyint, t5 smallint, t6 int, t7 bigint, t8 tinyint unsigned, t9 smallint unsigned, \ - t10 int unsigned, t11 bigint unsigned, t12 float, t13 double, t14 binary(100), t15 nchar(100), t16 timestamp)") + t10 int unsigned, t11 bigint unsigned, t12 float, t13 double, t14 binary(100), t15 nchar(100), t16 timestamp)"%stablename) stmt = conn.statement("insert into ? using log tags (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) \ values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") @@ -139,142 +147,93 @@ class TDTestCase: assert stmt.affected_rows == 3 #query all - querystmt1=conn.statement("select * from log where bu < ?") - queryparam1=new_bind_params(1) - print(type(queryparam1)) - queryparam1[0].int(10) - querystmt1.bind_param(queryparam1) - querystmt1.execute() - result1=querystmt1.use_result() - rows1=result1.fetch_all() - print(rows1[0]) - print(rows1[1]) - print(rows1[2]) - assert str(rows1[0][0]) == "2021-07-21 17:56:32.589111" - assert rows1[0][10] == 3 - assert rows1[1][10] == 4 + queryparam=new_bind_params(1) + queryparam[0].int(10) + rows=self.stmtExe(conn,"select * from log where bu < ?",queryparam) + tdLog.debug("assert 1st case %s"%rows) + assert str(rows[0][0]) == "2021-07-21 17:56:32.589111" + assert rows[0][10] == 3 , '1st case is failed' + assert rows[1][10] == 4 , '1st case is failed' #query: Numeric Functions - querystmt2=conn.statement("select abs(?) from log where bu < ?") - queryparam2=new_bind_params(2) - print(type(queryparam2)) - queryparam2[0].int(5) - queryparam2[1].int(5) - querystmt2.bind_param(queryparam2) - querystmt2.execute() - result2=querystmt2.use_result() - rows2=result2.fetch_all() - print("2",rows2) - assert rows2[0][0] == 5 - assert rows2[1][0] == 5 + queryparam=new_bind_params(2) + queryparam[0].int(5) + queryparam[1].int(5) + rows=self.stmtExe(conn,"select abs(?) from log where bu < ?",queryparam) + tdLog.debug("assert 2nd case %s"%rows) + assert rows[0][0] == 5 , '2nd case is failed' + assert rows[1][0] == 5 , '2nd case is failed' #query: Numeric Functions and escapes + queryparam=new_bind_params(1) + queryparam[0].int(5) + rows=self.stmtExe(conn,"select abs(?) from log where nn= 'a? long string with 中文字符'",queryparam) + tdLog.debug("assert 3rd case %s"%rows) + assert rows == [] , '3rd case is failed' - querystmt3=conn.statement("select abs(?) from log where nn= 'a? long string with 中文字符' ") - queryparam3=new_bind_params(1) - print(type(queryparam3)) - queryparam3[0].int(5) - querystmt3.bind_param(queryparam3) - querystmt3.execute() - result3=querystmt3.use_result() - rows3=result3.fetch_all() - print("3",rows3) - assert rows3 == [] + #query: string Functions + queryparam=new_bind_params(1) + queryparam[0].binary('中文字符') + rows=self.stmtExe(conn,"select CHAR_LENGTH(?) from log ",queryparam) + tdLog.debug("assert 4th case %s"%rows) + assert rows[0][0] == 12, '4th case is failed' + assert rows[1][0] == 12, '4th case is failed' - # #query: string Functions + queryparam=new_bind_params(1) + queryparam[0].binary('123') + rows=self.stmtExe(conn,"select CHAR_LENGTH(?) from log ",queryparam) + tdLog.debug("assert 4th case %s"%rows) + assert rows[0][0] == 3, '4th.1 case is failed' + assert rows[1][0] == 3, '4th.1 case is failed' - # querystmt3=conn.statement("select CHAR_LENGTH(?) from log ") - # queryparam3=new_bind_params(1) - # print(type(queryparam3)) - # queryparam3[0].binary('中文字符') - # querystmt3.bind_param(queryparam3) - # querystmt3.execute() - # result3=querystmt3.use_result() - # rows3=result3.fetch_all() - # print("4",rows3) - # assert rows3[0][0] == 12, 'fourth case is failed' - # assert rows3[1][0] == 12, 'fourth case is failed' + #query: conversion Functions + queryparam=new_bind_params(1) + queryparam[0].binary('1232a') + rows=self.stmtExe(conn,"select cast( ? as bigint) from log",queryparam) + tdLog.debug("assert 5th case %s"%rows) + assert rows[0][0] == 1232, '5th.1 case is failed' + assert rows[1][0] == 1232, '5th.1 case is failed' - # #query: conversion Functions + querystmt4=conn.statement("select cast( ? as binary(10)) from log ") + queryparam=new_bind_params(1) + queryparam[0].int(123) + rows=self.stmtExe(conn,"select cast( ? as bigint) from log",queryparam) + tdLog.debug("assert 6th case %s"%rows) + assert rows[0][0] == 123, '6th.1 case is failed' + assert rows[1][0] == 123, '6th.1 case is failed' - # querystmt4=conn.statement("select cast( ? as bigint) from log ") - # queryparam4=new_bind_params(1) - # print(type(queryparam4)) - # queryparam4[0].binary('1232a') - # querystmt4.bind_param(queryparam4) - # querystmt4.execute() - # result4=querystmt4.use_result() - # rows4=result4.fetch_all() - # print("5",rows4) - # assert rows4[0][0] == 1232 - # assert rows4[1][0] == 1232 - - # querystmt4=conn.statement("select cast( ? as binary(10)) from log ") - # queryparam4=new_bind_params(1) - # print(type(queryparam4)) - # queryparam4[0].int(123) - # querystmt4.bind_param(queryparam4) - # querystmt4.execute() - # result4=querystmt4.use_result() - # rows4=result4.fetch_all() - # print("6",rows4) - # assert rows4[0][0] == '123' - # assert rows4[1][0] == '123' - - # #query: datatime Functions - - # querystmt4=conn.statement(" select timediff('2021-07-21 17:56:32.590111',?,1s) from log ") - # queryparam4=new_bind_params(1) - # print(type(queryparam4)) - # queryparam4[0].timestamp(1626861392591111) - # querystmt4.bind_param(queryparam4) - # querystmt4.execute() - # result4=querystmt4.use_result() - # rows4=result4.fetch_all() - # print("7",rows4) - # assert rows4[0][0] == 1, 'seventh case is failed' - # assert rows4[1][0] == 1, 'seventh case is failed' + #query: datatime Functions + queryparam=new_bind_params(1) + queryparam[0].timestamp(1626861392591112) + rows=self.stmtExe(conn,"select timediff('2021-07-21 17:56:32.590111',?,1a) from log",queryparam) + tdLog.debug("assert 7th case %s"%rows) + assert rows[0][0] == 1, '7th case is failed' + assert rows[1][0] == 1, '7th case is failed' #query: aggregate Functions + queryparam=new_bind_params(1) + queryparam[0].int(123) + rows=self.stmtExe(conn,"select count(?) from log ",queryparam) + tdLog.debug("assert 8th case %s"%rows) + assert rows[0][0] == 3, ' 8th case is failed' - querystmt4=conn.statement(" select count(?) from log ") - queryparam4=new_bind_params(1) - print(type(queryparam4)) - queryparam4[0].int(123) - querystmt4.bind_param(queryparam4) - querystmt4.execute() - result4=querystmt4.use_result() - rows4=result4.fetch_all() - print("8",rows4) - assert rows4[0][0] == 3, ' 8 case is failed' - - #query: selector Functions 9 - - querystmt4=conn.statement(" select bottom(bu,?) from log group by bu ; ") - queryparam4=new_bind_params(1) - print(type(queryparam4)) - queryparam4[0].int(2) - querystmt4.bind_param(queryparam4) - querystmt4.execute() - result4=querystmt4.use_result() - rows4=result4.fetch_all() - print("9",rows4) - assert rows4[0][0] == 4, ' 9 case is failed' - assert rows4[1][0] == 3, ' 9 case is failed' + # #query: selector Functions 9 + # queryparam=new_bind_params(1) + # queryparam[0].int(2) + # rows=self.stmtExe(conn,"select bottom(bu,?) from log group by bu ; ",queryparam) + # tdLog.debug("assert 9th case %s"%rows) + # assert rows[0][0] == 4, ' 9 case is failed' + # assert rows[1][0] == 3, ' 9 case is failed' # #query: time-series specific Functions 10 - querystmt4=conn.statement(" select twa(?) from log; ") - queryparam4=new_bind_params(1) - print(type(queryparam4)) - queryparam4[0].int(15) - querystmt4.bind_param(queryparam4) - querystmt4.execute() - result4=querystmt4.use_result() - rows4=result4.fetch_all() - print("10",rows4) - assert rows4[0][0] == 15, ' 10 case is failed' + querystmt=conn.statement(" select twa(?) from log; ") + queryparam=new_bind_params(1) + queryparam[0].int(15) + rows=self.stmtExe(conn," select twa(?) from log; ",queryparam) + tdLog.debug("assert 10th case %s"%rows) + assert rows[0][0] == 15, ' 10th case is failed' # conn.execute("drop database if exists %s" % dbname) diff --git a/tests/system-test/2-query/Timediff.py b/tests/system-test/2-query/Timediff.py index ad64d29007..b8f3649eff 100644 --- a/tests/system-test/2-query/Timediff.py +++ b/tests/system-test/2-query/Timediff.py @@ -3,7 +3,7 @@ from util.sql import * from util.cases import * class TDTestCase: - + def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) @@ -33,7 +33,7 @@ class TDTestCase: 'insert into ntb values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') tdSql.execute( 'insert into stb_1 values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') - + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00') from ntb") tdSql.checkRows(3) tdSql.query("select timediff(1,0,1d) from ntb") @@ -72,12 +72,12 @@ class TDTestCase: tdSql.query("select timediff(1,0,1a) from db.ntb") tdSql.checkRows(3) tdSql.checkData(0,0,1000) - tdSql.query("select timediff(1,0,1u) from ntb") - tdSql.checkRows(3) - tdSql.checkData(0,0,1000000) - tdSql.query("select timediff(1,0,1u) from db.ntb") - tdSql.checkRows(3) - tdSql.checkData(0,0,1000000) + tdSql.error("select timediff(1,0,1u) from ntb") + #tdSql.checkRows(3) + #tdSql.checkData(0,0,1000000) + tdSql.error("select timediff(1,0,1u) from db.ntb") + #tdSql.checkRows(3) + #tdSql.checkData(0,0,1000000) tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00') from stb") tdSql.checkRows(3) @@ -116,12 +116,12 @@ class TDTestCase: tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1a) from db.stb") tdSql.checkRows(3) tdSql.checkData(0,0,86400000) - tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from stb") - tdSql.checkRows(3) - tdSql.checkData(0,0,86400000000) - tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from db.stb") - tdSql.checkRows(3) - tdSql.checkData(0,0,86400000000) + tdSql.error("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from stb") + #tdSql.checkRows(3) + #tdSql.checkData(0,0,86400000000) + tdSql.error("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from db.stb") + #tdSql.checkRows(3) + #tdSql.checkData(0,0,86400000000) tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00') from stb_1") @@ -164,12 +164,12 @@ class TDTestCase: tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1a) from db.stb_1") tdSql.checkRows(3) tdSql.checkData(0,0,43200000) - tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from stb_1") - tdSql.checkRows(3) - tdSql.checkData(0,0,43200000000) - tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from db.stb_1") - tdSql.checkRows(3) - tdSql.checkData(0,0,43200000000) + tdSql.error("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from stb_1") + #tdSql.checkRows(3) + #tdSql.checkData(0,0,43200000000) + tdSql.error("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from db.stb_1") + #tdSql.checkRows(3) + #tdSql.checkData(0,0,43200000000) tdSql.query("select timediff('a','b') from stb") tdSql.checkRows(3) @@ -202,4 +202,4 @@ class TDTestCase: tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/elapsed.py b/tests/system-test/2-query/elapsed.py index daa761d118..cba3905d23 100644 --- a/tests/system-test/2-query/elapsed.py +++ b/tests/system-test/2-query/elapsed.py @@ -23,47 +23,47 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + self.ts = 1420041600000 # 2015-01-01 00:00:00 this is begin time for first record self.num = 10 def caseDescription(self): ''' - case1 : [TD-11804] test case for elapsed function : - - this test case is for aggregate function elapsed , elapsed function can only used for the timestamp primary key column (ts) , + case1 : [TD-11804] test case for elapsed function : + + this test case is for aggregate function elapsed , elapsed function can only used for the timestamp primary key column (ts) , it has two input parameters, the first parameter is necessary, basic SQL as follow: - + =================================================================================================================================== SELECT ELAPSED(field_name[, time_unit]) FROM { tb_name | stb_name } [WHERE clause] [INTERVAL(interval [, offset]) [SLIDING sliding]]; =================================================================================================================================== elapsed function can acting on ordinary tables and super tables , notice that this function is related to the timeline. - If it acts on a super table , it must be partition by tbname . by the way ,this function support nested query. + If it acts on a super table , it must be group by tbname . by the way ,this function support nested query. The scenarios covered by the test cases are as follows: ==================================================================================================================================== - case: select * from table|stable[partition by tbname]|regular_table + case: select * from table|stable[group by tbname]|regular_table + + case:select elapsed(ts) from table|stable where clause interval (units) [fill(LINEAR,NEXT,PREV,VALUE,NULL)] [group by tbname] order [by ts desc asc|desc]; + + case:select elapsed(ts) , elapsed(ts,unit_time1)*regular_num1 , elapsed(ts,unit_time1)+regular_num2 from table|stable where clause interval (units) [fill(LINEAR,NEXT,PREV,VALUE,NULL)] [group by tbname] order [by ts desc asc|desc]; - case:select elapsed(ts) from table|stable where clause interval (units) [fill(LINEAR,NEXT,PREV,VALUE,NULL)] [partition by tbname] order [by ts desc asc|desc]; - - case:select elapsed(ts) , elapsed(ts,unit_time1)*regular_num1 , elapsed(ts,unit_time1)+regular_num2 from table|stable where clause interval (units) [fill(LINEAR,NEXT,PREV,VALUE,NULL)] [partition by tbname] order [by ts desc asc|desc]; - //mixup with all functions only once query (it's different with nest query) - case:select elapsed(ts), count(*), avg(col), twa(col), irate(col), sum(col), stddev(col), leastsquares(col, 1, 1),min(col), max(col), first(col), last(col), percentile(col, 20), apercentile(col, 30), last_row(col), spread(col)from table|stable where clause interval (units) [fill(LINEAR,NEXT,PREV,VALUE,NULL)] [partition by tbname] order [by ts desc asc|desc]; - - //mixup with ordinary col - case:select ts ,elapsed(ts)*10 ,col+5 from table|stable where clause interval (units) [fill(LINEAR,NEXT,PREV,VALUE,NULL)] [partition by tbname] order [by ts desc asc|desc]; - + case:select elapsed(ts), count(*), avg(col), twa(col), irate(col), sum(col), stddev(col), leastsquares(col, 1, 1),min(col), max(col), first(col), last(col), percentile(col, 20), apercentile(col, 30), last_row(col), spread(col)from table|stable where clause interval (units) [fill(LINEAR,NEXT,PREV,VALUE,NULL)] [group by tbname] order [by ts desc asc|desc]; + + //mixup with ordinary col + case:select ts ,elapsed(ts)*10 ,col+5 from table|stable where clause interval (units) [fill(LINEAR,NEXT,PREV,VALUE,NULL)] [group by tbname] order [by ts desc asc|desc]; + //nest query - case:select elapsed(ts) from (select elapsed(ts), count(*), avg(col), twa(col), irate(col), sum(col), stddev(col), leastsquares(col, 1, 1),min(col), max(col), first(col), last(col), percentile(col, 20), apercentile(col, 30), last_row(col), spread(col)from table|stable where clause interval (units) [fill(LINEAR,NEXT,PREV,VALUE,NULL)] [partition by tbname] order [by ts desc asc|desc]) where clause interval (units) [fill(LINEAR,NEXT,PREV,VALUE,NULL)] [partition by tbname] order [by ts desc asc|desc]; - + case:select elapsed(ts) from (select elapsed(ts), count(*), avg(col), twa(col), irate(col), sum(col), stddev(col), leastsquares(col, 1, 1),min(col), max(col), first(col), last(col), percentile(col, 20), apercentile(col, 30), last_row(col), spread(col)from table|stable where clause interval (units) [fill(LINEAR,NEXT,PREV,VALUE,NULL)] [group by tbname] order [by ts desc asc|desc]) where clause interval (units) [fill(LINEAR,NEXT,PREV,VALUE,NULL)] [group by tbname] order [by ts desc asc|desc]; + //clause about filter condition - case:select elapsed(ts) from table|stable[partition by tbname] where [ts|col|tag >|<|=|>=|<=|=|<>|!= value] | [between ... and ...] |[in] |[is null|not null] interval (unit_time) ; - case:select elapsed(ts) from table|stable[partition by tbname] where clause1 and clause 2 and clause3 interval (unit_time) ; + case:select elapsed(ts) from table|stable[group by tbname] where [ts|col|tag >|<|=|>=|<=|=|<>|!= value] | [between ... and ...] |[in] |[is null|not null] interval (unit_time) ; + case:select elapsed(ts) from table|stable[group by tbname] where clause1 and clause 2 and clause3 interval (unit_time) ; //JOIN query case:select elapsed(ts) from TABLE1 as tb1 , TABLE2 as tb2 where join_condition [TABLE1 and TABLE2 can be stable|table|sub_table|empty_table] @@ -74,22 +74,22 @@ class TDTestCase: // Window aggregation case:select elapsed(ts) from t1 where clause session(ts, time_units) ; - case:select elapsed(ts) from t1 where clause state_window(regular_nums); + case:select elapsed(ts) from t1 where clause state_window(regular_nums); // Continuous query - case:create table select elapsed(ts) ,avg(col) from (select elapsed(ts) ts_inter ,avg(col) col from stable|table interval (unit_time) [fill(LINEAR,NEXT,PREV,VALUE,NULL)][partition by tbname]) interval (unit_time) [fill(LINEAR,NEXT,PREV,VALUE,NULL) sliding(unit_time_windows); + case:create table select elapsed(ts) ,avg(col) from (select elapsed(ts) ts_inter ,avg(col) col from stable|table interval (unit_time) [fill(LINEAR,NEXT,PREV,VALUE,NULL)][group by tbname]) interval (unit_time) [fill(LINEAR,NEXT,PREV,VALUE,NULL) sliding(unit_time_windows); ======================================================================================================================================== this test case notice successful execution and correctness of results. - ''' - return + ''' + return def prepare_data(self): tdLog.info (" ====================================== prepare data ==================================================") - + tdSql.execute('drop database if exists testdb ;') tdSql.execute('create database testdb keep 36500;') tdSql.execute('use testdb;') @@ -120,14 +120,14 @@ class TDTestCase: tdSql.execute('create table regular_table_1 (ts timestamp , tscol timestamp ,q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double, bin_chars binary(20)) ;') tdSql.execute('create table regular_table_2 (ts timestamp , tscol timestamp ,q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , bin_chars binary(20)) ;') tdSql.execute('create table regular_table_3 (ts timestamp , tscol timestamp ,q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , bin_chars binary(20)) ;') - + tablenames = ["sub_table1_1","sub_table1_2","sub_table1_3","sub_table2_1","sub_table2_2","sub_table2_3","regular_table_1","regular_table_2","regular_table_3"] tdLog.info("insert into records ") for tablename in tablenames: - for i in range(self.num): + for i in range(self.num): sql= 'insert into %s values(%d, %d,%d, %d, %d, %d, %f, %f, "%s")' % (tablename,self.ts + i*10000, self.ts + i*10,2147483647-i, 9223372036854775807-i, 32767-i, 127-i, i, i,("bintest"+str(i))) print(sql) tdSql.execute(sql) @@ -144,12 +144,12 @@ class TDTestCase: "(ts,2d+3m-2s,NULL)","(ts+10d,NULL)" ,"(ts,now -1m%1d)","(ts+10d,_c0)","(ts+10d,)","(ts,%)","(ts, , m)","(ts,abc)","(ts,/)","(ts,*)","(ts,1s,100)", "(ts,1s,abc)","(ts,1s,_c0)","(ts,1s,*)","(ts,1s,NULL)","(ts,,_c0)","(ts,tbname,ts)","(ts,0,tbname)","('2021-11-18 00:00:10')","('2021-11-18 00:00:10', 1s)", "('2021-11-18T00:00:10+0800', '1s')","('2021-11-18T00:00:10Z', '1s')","('2021-11-18T00:00:10+0800', 10000000d,)","('ts', ,2021-11-18T00:00:10+0800, )"] - + for tablename in tablenames: for abnormal_param in abnormal_list: if tablename.startswith("stable"): - basic_sql= "select elapsed" + abnormal_param + " from " + tablename + " partition by tbname ,ind order by tbname;" #stables + basic_sql= "select elapsed" + abnormal_param + " from " + tablename + " group by tbname ,ind order by tbname;" #stables else: basic_sql= "select elapsed" + abnormal_param + " from " + tablename + ";" # regular table tdSql.error(basic_sql) @@ -158,20 +158,20 @@ class TDTestCase: tdLog.info (" ====================================== elapsed use abnormal ==================================================") - sqls_list = ["select elapsed(ts) from regular_empty partition by tbname,ind order by desc; ", - "select elapsed(ts) from regular_empty partition by tbname,ind order by desc; ", - "select elapsed(ts) from regular_table_1 partition by tbname,ind order by desc; ", - "select elapsed(ts) from sub_table1_1 partition by tbname,ind order by desc; ", - "select elapsed(ts) from sub_table1_1 partition by tbname,ind order by desc; ", + sqls_list = ["select elapsed(ts) from regular_empty group by tbname,ind order by desc; ", + "select elapsed(ts) from regular_empty group by tbname,ind order by desc; ", + "select elapsed(ts) from regular_table_1 group by tbname,ind order by desc; ", + "select elapsed(ts) from sub_table1_1 group by tbname,ind order by desc; ", + "select elapsed(ts) from sub_table1_1 group by tbname,ind order by desc; ", # "select elapsed(ts,10s) from stable_empty group by ts order by ts;", "select elapsed(ts,10s) from stable_1 group by ind order by ts;", "select elapsed(ts,10s) from stable_2 group by tstag order by ts;", - "select elapsed(ts,10s) from stable_1 partition by tbname,tstag,tscol order by ts;", - "select elapsed(ts,10s),ts from stable_1 partition by tbname ,ind order by ts;", - "select ts,elapsed(ts,10s),tscol*100 from stable_1 partition by tbname ,ind order by ts;", + "select elapsed(ts,10s) from stable_1 group by tbname,tstag,tscol order by ts;", + "select elapsed(ts,10s),ts from stable_1 group by tbname ,ind order by ts;", + "select ts,elapsed(ts,10s),tscol*100 from stable_1 group by tbname ,ind order by ts;", "select elapsed(ts) from stable_1 group by tstag order by ts;", - "select elapsed(ts) from sub_empty_1 partition by tbname,ind ,tscol order by ts desc;", - "select tbname, tscol,elapsed(ts) from sub_table1_1 partition by tbname ,ind order by ts desc;", + "select elapsed(ts) from sub_empty_1 group by tbname,ind ,tscol order by ts desc;", + "select tbname, tscol,elapsed(ts) from sub_table1_1 group by tbname ,ind order by ts desc;", "select elapsed(tscol) from sub_table1_1 order by ts desc;", "select elapsed(tstag) from sub_table1_1 order by ts desc;", "select elapsed(ind) from sub_table1_1 order by ts desc;", @@ -191,7 +191,7 @@ class TDTestCase: def query_filter(self): tdLog.info (" ====================================== elapsed query filter ==================================================") - + for i in range(self.num): ts_start_time = self.ts + i*10000 ts_col_start_time = self.ts + i*10 @@ -199,7 +199,7 @@ class TDTestCase: ts_end_time = self.ts + (self.num-1-i)*10000 ts_col_end_time = self.ts + (self.num-1-i)*10 - filter_sql = "select elapsed(ts,10s) from stable_1 where ts >= %d partition by tbname " %(ts_start_time) + filter_sql = "select elapsed(ts,10s) from stable_1 where ts >= %d group by tbname " %(ts_start_time) tdSql.query(filter_sql) tdSql.checkRows(3) tdSql.checkData(0,0,float(self.num -i-1)) @@ -210,9 +210,9 @@ class TDTestCase: tdSql.query(filter_sql) tdSql.checkRows(1) tdSql.checkData(0,0,float(self.num -i-1)) - - filter_sql = "select elapsed(ts,10s) from stable_1 where ts >= %d and tscol >= %d and tstag='2015-01-01 00:01:00'partition by tbname " %(ts_start_time,ts_col_start_time) + + filter_sql = "select elapsed(ts,10s) from stable_1 where ts >= %d and tscol >= %d and tstag='2015-01-01 00:01:00'group by tbname " %(ts_start_time,ts_col_start_time) tdSql.query(filter_sql) tdSql.checkRows(1) tdSql.checkData(0,0,float(self.num -i-1)) @@ -222,9 +222,9 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0,0,float(self.num -i-1)) - filter_sql = "select elapsed(ts,10s) from stable_1 where ts >= %d and tscol > %d and tstag='2015-01-01 00:01:00' partition by tbname" %(ts_start_time,ts_col_start_time) + filter_sql = "select elapsed(ts,10s) from stable_1 where ts >= %d and tscol > %d and tstag='2015-01-01 00:01:00' group by tbname" %(ts_start_time,ts_col_start_time) tdSql.query(filter_sql) - + if i == self.num-1: tdSql.checkRows(0) else: @@ -233,14 +233,14 @@ class TDTestCase: filter_sql = "select elapsed(ts,10s) from sub_table1_1 where ts >= %d and tscol > %d " %(ts_start_time,ts_col_start_time) tdSql.query(filter_sql) - + if i == self.num-1: tdSql.checkRows(0) else: tdSql.checkRows(1) tdSql.checkData(0,0,float(self.num -i-2)) - filter_sql = "select elapsed(ts,10s) from stable_1 where ts > %d and tscol > %d and tstag < '2015-01-01 00:01:00' partition by tbname " %(ts_start_time,ts_col_start_time) + filter_sql = "select elapsed(ts,10s) from stable_1 where ts > %d and tscol > %d and tstag < '2015-01-01 00:01:00' group by tbname " %(ts_start_time,ts_col_start_time) tdSql.query(filter_sql) if i == self.num-1: @@ -258,7 +258,7 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0,0,float(self.num -i-2)) - filter_sql = "select elapsed(ts,10s) from stable_1 where ts > %d and tscol <= %d and tstag < '2015-01-01 00:01:00' partition by tbname" %(ts_start_time,ts_col_start_time) + filter_sql = "select elapsed(ts,10s) from stable_1 where ts > %d and tscol <= %d and tstag < '2015-01-01 00:01:00' group by tbname" %(ts_start_time,ts_col_start_time) tdSql.query(filter_sql) tdSql.checkRows(0) @@ -266,9 +266,9 @@ class TDTestCase: tdSql.query(filter_sql) tdSql.checkRows(0) - filter_sql = "select elapsed(ts,10s) from stable_1 where ts < %d and tscol <= %d and tstag < '2015-01-01 00:01:00' partition by tbname" %(ts_end_time,ts_col_end_time) + filter_sql = "select elapsed(ts,10s) from stable_1 where ts < %d and tscol <= %d and tstag < '2015-01-01 00:01:00' group by tbname" %(ts_end_time,ts_col_end_time) tdSql.query(filter_sql) - + if i == self.num-1: tdSql.checkRows(0) else: @@ -277,14 +277,14 @@ class TDTestCase: filter_sql = "select elapsed(ts,10s) from sub_table1_1 where ts < %d and tscol <= %d " %(ts_end_time,ts_col_end_time) tdSql.query(filter_sql) - + if i == self.num-1: tdSql.checkRows(0) else: tdSql.checkRows(1) tdSql.checkData(0,0,float(self.num -i-2)) - filter_sql = "select elapsed(ts,10s) from stable_1 where ts < %d and tscol <= %d partition by tbname " %(ts_end_time,ts_col_end_time) + filter_sql = "select elapsed(ts,10s) from stable_1 where ts < %d and tscol <= %d group by tbname " %(ts_end_time,ts_col_end_time) tdSql.query(filter_sql) if i == self.num-1: @@ -303,8 +303,8 @@ class TDTestCase: else: tdSql.checkRows(1) tdSql.checkData(0,0,float(self.num - i - 2)) - - filter_sql = "select elapsed(ts,10s) from stable_1 where ts = %d and tscol < %d partition by tbname " %(ts_end_time,ts_col_end_time) + + filter_sql = "select elapsed(ts,10s) from stable_1 where ts = %d and tscol < %d group by tbname " %(ts_end_time,ts_col_end_time) tdSql.query(filter_sql) tdSql.checkRows(0) @@ -312,7 +312,7 @@ class TDTestCase: tdSql.query(filter_sql) tdSql.checkRows(0) - filter_sql = "select elapsed(ts,10s) from stable_1 where q_tinyint != %d and tscol < %d partition by tbname " %(i,ts_col_end_time) + filter_sql = "select elapsed(ts,10s) from stable_1 where q_tinyint != %d and tscol < %d group by tbname " %(i,ts_col_end_time) tdSql.query(filter_sql) if i == self.num-1: @@ -331,10 +331,10 @@ class TDTestCase: else: tdSql.checkRows(1) tdSql.checkData(0,0,float(self.num -i-2)) - - filter_sql = "select elapsed(ts,10s) from stable_1 where q_tinyint != %d and tscol <= %d partition by tbname " %(i,ts_col_end_time) + + filter_sql = "select elapsed(ts,10s) from stable_1 where q_tinyint != %d and tscol <= %d group by tbname " %(i,ts_col_end_time) tdSql.query(filter_sql) - + if i == self.num: tdSql.checkRows(0) else: @@ -345,14 +345,14 @@ class TDTestCase: filter_sql = "select elapsed(ts,10s) from sub_table1_1 where q_tinyint != %d and tscol <= %d " %(i,ts_col_end_time) tdSql.query(filter_sql) - + if i == self.num: tdSql.checkRows(0) else: tdSql.checkRows(1) tdSql.checkData(0,0,float(self.num - i - 1)) - filter_sql = "select elapsed(ts,10s) from stable_1 where q_tinyint <> %d and tscol < %d partition by tbname " %(i,ts_col_end_time) + filter_sql = "select elapsed(ts,10s) from stable_1 where q_tinyint <> %d and tscol < %d group by tbname " %(i,ts_col_end_time) tdSql.query(filter_sql) if i == self.num-1: @@ -372,9 +372,9 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0,0,float(self.num -i-2)) - filter_sql = "select elapsed(ts,10s) from stable_1 where q_tinyint <> %d and tscol <= %d partition by tbname " %(i,ts_col_end_time) + filter_sql = "select elapsed(ts,10s) from stable_1 where q_tinyint <> %d and tscol <= %d group by tbname " %(i,ts_col_end_time) tdSql.query(filter_sql) - + if i == self.num: tdSql.checkRows(0) else: @@ -385,79 +385,79 @@ class TDTestCase: filter_sql = "select elapsed(ts,10s) from sub_table1_1 where q_tinyint <> %d and tscol <= %d " %(i,ts_col_end_time) tdSql.query(filter_sql) - + if i == self.num: tdSql.checkRows(0) else: tdSql.checkRows(1) tdSql.checkData(0,0,float(self.num - i - 1)) - # filter between and + # filter between and tdSql.query("select elapsed(ts,10s) from sub_table1_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and q_tinyint between 125 and 127 and tscol <= '2015-01-01 00:01:00.000' ") tdSql.checkData(0,0,2) tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and \ - q_tinyint between 125 and 127 and tscol <= '2015-01-01 00:01:00.000' partition by tbname ") + q_tinyint between 125 and 127 and tscol <= '2015-01-01 00:01:00.000' group by tbname ") tdSql.checkData(0,0,2) tdSql.checkData(1,0,2) tdSql.checkData(2,0,2) - # filter in and or + # filter in and or tdSql.query("select elapsed(ts,10s) from sub_table1_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and q_tinyint between 125 and 127 and tscol <= '2015-01-01 00:01:00.000' ") tdSql.checkData(0,0,2) - tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and q_tinyint between 125 and 127 and tscol <= '2015-01-01 00:01:00.000' partition by tbname ") + tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and q_tinyint between 125 and 127 and tscol <= '2015-01-01 00:01:00.000' group by tbname ") tdSql.checkData(0,0,2) tdSql.checkData(1,0,2) tdSql.checkData(2,0,2) - tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and q_tinyint in (125,126,127) and tscol <= '2015-01-01 00:01:00.000' partition by tbname ") + tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and q_tinyint in (125,126,127) and tscol <= '2015-01-01 00:01:00.000' group by tbname ") tdSql.checkData(0,0,2) tdSql.checkData(1,0,2) tdSql.checkData(2,0,2) - tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars in ('bintest0','bintest1') and tscol <= '2015-01-01 00:01:00.000' partition by tbname ") + tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars in ('bintest0','bintest1') and tscol <= '2015-01-01 00:01:00.000' group by tbname ") tdSql.checkData(0,0,1) tdSql.checkData(1,0,1) tdSql.checkData(2,0,1) - tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars in ('bintest0','bintest1') and tscol <= '2015-01-01 00:01:00.000' partition by tbname ") + tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars in ('bintest0','bintest1') and tscol <= '2015-01-01 00:01:00.000' group by tbname ") tdSql.checkData(0,0,1) tdSql.checkData(1,0,1) tdSql.checkData(2,0,1) - - tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars like 'bintest_' and tscol <= '2015-01-01 00:01:00.000' partition by tbname ") + + tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars like 'bintest_' and tscol <= '2015-01-01 00:01:00.000' group by tbname ") tdSql.checkData(0,0,6) tdSql.checkData(1,0,6) tdSql.checkData(2,0,6) - tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars like 'bintest_' and tscol <= '2015-01-01 00:01:00.000' partition by tbname ") + tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars like 'bintest_' and tscol <= '2015-01-01 00:01:00.000' group by tbname ") tdSql.checkData(0,0,6) tdSql.checkData(1,0,6) tdSql.checkData(2,0,6) - tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars is not null and tscol <= '2015-01-01 00:01:00.000' partition by tbname; ") + tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars is not null and tscol <= '2015-01-01 00:01:00.000' group by tbname; ") tdSql.checkData(0,0,6) tdSql.checkData(1,0,6) tdSql.checkData(2,0,6) - tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars is null and tscol <= '2015-01-01 00:01:00.000' partition by tbname; ") + tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars is null and tscol <= '2015-01-01 00:01:00.000' group by tbname; ") tdSql.checkRows(0) - tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars match '^b' and tscol <= '2015-01-01 00:01:00.000' partition by tbname; ") + tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars match '^b' and tscol <= '2015-01-01 00:01:00.000' group by tbname; ") tdSql.checkRows(3) tdSql.checkData(0,0,6) tdSql.checkData(1,0,6) tdSql.checkData(2,0,6) - tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars nmatch '^a' and tscol <= '2015-01-01 00:01:00.000' partition by tbname; ") + tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars nmatch '^a' and tscol <= '2015-01-01 00:01:00.000' group by tbname; ") tdSql.checkRows(3) tdSql.checkData(0,0,6) tdSql.checkData(1,0,6) tdSql.checkData(2,0,6) - tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars ='bintest1' or bin_chars ='bintest2' and tscol <= '2015-01-01 00:01:00.000' partition by tbname; ") + tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and bin_chars ='bintest1' or bin_chars ='bintest2' and tscol <= '2015-01-01 00:01:00.000' group by tbname; ") tdSql.checkRows(3) - tdSql.query("select elapsed(ts,10s) from stable_1 where (ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000') or (ts between '2015-01-01 00:01:00.000' and '2015-01-01 00:02:00.000') partition by tbname; ") + tdSql.query("select elapsed(ts,10s) from stable_1 where (ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000') or (ts between '2015-01-01 00:01:00.000' and '2015-01-01 00:02:00.000') group by tbname; ") tdSql.checkRows(3) tdSql.checkData(0,0,9) tdSql.checkData(1,0,9) @@ -473,21 +473,21 @@ class TDTestCase: tdSql.query("select max(q_int)*10 from sub_empty_2 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' interval(10s) fill(prev);") tdSql.checkRows(0) - tdSql.query("select elapsed(ts,10s)*10 from stable_empty where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' partition by tbname interval(10s) fill(prev) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_empty where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' interval(10s) fill(prev) group by tbname;") tdSql.checkRows(0) tdSql.query("select elapsed(ts,10s)*10 from sub_empty_2 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' interval(10s) fill(prev);") tdSql.checkRows(0) - + for i in range(self.num): ts_start_time = self.ts + i*10000 ts_col_start_time = self.ts + i*10 ts_tag_time = "2015-01-01 00:01:00" ts_end_time = self.ts + (self.num-1-i)*10000 ts_col_end_time = self.ts + (self.num-1-i)*10 - - # only interval - interval_sql = "select elapsed(ts,10s) from stable_1 where ts <=%d partition by tbname interval(10s) " %(ts_start_time) + + # only interval + interval_sql = "select elapsed(ts,10s) from stable_1 where ts <=%d interval(10s) group by tbname " %(ts_start_time) tdSql.query(interval_sql) tdSql.checkRows(3*(i+1)) @@ -499,25 +499,25 @@ class TDTestCase: tdSql.checkData(x,1,0) else : tdSql.checkData(x,1,1) - + # interval and fill , fill_type = ["NULL","value,100","prev","next","linear"] - # interval (10s) and time range is outer records + # interval (10s) and time range is outer records - tdSql.query("select elapsed(ts,10s)*10 from stable_empty where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' partition by tbname interval(10s) fill(prev) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_empty where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' interval(10s) fill(prev) group by tbname;") tdSql.checkRows(0) tdSql.query("select elapsed(ts,10s)*10 from sub_empty_2 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' interval(10s) fill(prev);") tdSql.checkRows(0) - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' partition by tbname interval(10s) fill(prev) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' interval(10s) fill(prev) group by tbname;") tdSql.checkRows(180) tdSql.checkData(0,1,10) tdSql.checkData(9,1,0) tdSql.checkData(59,1,0) tdSql.checkData(60,1,10) - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' partition by tbname interval(10s) fill(next) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' interval(10s) fill(next) group by tbname;") tdSql.checkRows(180) tdSql.checkData(0,1,10) tdSql.checkData(9,1,0) @@ -526,7 +526,7 @@ class TDTestCase: tdSql.checkData(60,1,10) tdSql.checkData(61,1,10) - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' partition by tbname interval(10s) fill(linear) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' interval(10s) fill(linear) group by tbname;") tdSql.checkRows(180) tdSql.checkData(0,1,10) tdSql.checkData(9,1,0) @@ -535,7 +535,7 @@ class TDTestCase: tdSql.checkData(60,1,10) tdSql.checkData(61,1,10) - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' partition by tbname interval(10s) fill(NULL) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' interval(10s) fill(NULL) group by tbname;") tdSql.checkRows(180) tdSql.checkData(0,1,10) tdSql.checkData(9,1,0) @@ -544,7 +544,7 @@ class TDTestCase: tdSql.checkData(60,1,10) tdSql.checkData(61,1,10) - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' partition by tbname interval(10s) fill(value ,2) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' interval(10s) fill(value ,2) group by tbname;") tdSql.checkRows(180) tdSql.checkData(0,1,10) tdSql.checkData(9,1,0) @@ -552,9 +552,9 @@ class TDTestCase: tdSql.checkData(59,1,2) tdSql.checkData(60,1,10) tdSql.checkData(61,1,10) - - # interval (20s) and time range is outer records - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' partition by tbname,ind interval(20s) fill(prev) ;") + + # interval (20s) and time range is outer records + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' interval(20s) fill(prev) group by tbname,ind ;") tdSql.checkRows(90) tdSql.checkData(0,1,20) tdSql.checkData(4,1,10) @@ -562,8 +562,8 @@ class TDTestCase: tdSql.checkData(29,1,10) tdSql.checkData(30,1,20) tdSql.checkData(31,1,20) - - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' partition by tbname,ind interval(20s) fill(next) ;") + + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' interval(20s) fill(next) group by tbname,ind ;") tdSql.checkRows(90) tdSql.checkData(0,1,20) tdSql.checkData(4,1,10) @@ -572,7 +572,7 @@ class TDTestCase: tdSql.checkData(30,1,20) tdSql.checkData(31,1,20) - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' partition by tbname,ind interval(20s) fill(linear) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' interval(20s) fill(linear) group by tbname,ind ;") tdSql.checkRows(90) tdSql.checkData(0,1,20) tdSql.checkData(4,1,10) @@ -581,7 +581,7 @@ class TDTestCase: tdSql.checkData(30,1,20) tdSql.checkData(31,1,20) - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' partition by tbname,ind interval(20s) fill(NULL) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' interval(20s) fill(NULL) group by tbname,ind ;") tdSql.checkRows(90) tdSql.checkData(0,1,20) tdSql.checkData(4,1,10) @@ -589,8 +589,8 @@ class TDTestCase: tdSql.checkData(29,1,None) tdSql.checkData(30,1,20) tdSql.checkData(31,1,20) - - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' partition by tbname,ind interval(20s) fill(value ,2) ;") + + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:10:00.000' interval(20s) fill(value ,2) group by tbname,ind ;") tdSql.checkRows(90) tdSql.checkData(0,1,20) tdSql.checkData(4,1,10) @@ -599,9 +599,9 @@ class TDTestCase: tdSql.checkData(30,1,20) tdSql.checkData(31,1,20) - # interval (20s) and time range is in records + # interval (20s) and time range is in records - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:01:00.000' partition by tbname,ind interval(20s) fill(prev) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:01:00.000' interval(20s) fill(prev) group by tbname,ind ;") tdSql.checkRows(9) tdSql.checkData(0,1,20) tdSql.checkData(2,1,10) @@ -610,7 +610,7 @@ class TDTestCase: tdSql.checkData(7,1,20) tdSql.checkData(8,1,10) - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:01:00.000' partition by tbname,ind interval(20s) fill(next) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:01:00.000' interval(20s) fill(next) group by tbname,ind ;") tdSql.checkRows(9) tdSql.checkData(0,1,20) tdSql.checkData(2,1,10) @@ -619,7 +619,7 @@ class TDTestCase: tdSql.checkData(7,1,20) tdSql.checkData(8,1,10) - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:01:00.000' partition by tbname,ind interval(20s) fill(linear) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:01:00.000' interval(20s) fill(linear) group by tbname,ind ;") tdSql.checkRows(9) tdSql.checkData(0,1,20) tdSql.checkData(2,1,10) @@ -628,7 +628,7 @@ class TDTestCase: tdSql.checkData(7,1,20) tdSql.checkData(8,1,10) - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:01:00.000' partition by tbname,ind interval(20s) fill(NULL) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:01:00.000' interval(20s) fill(NULL) group by tbname,ind ;") tdSql.checkRows(9) tdSql.checkData(0,1,20) tdSql.checkData(2,1,10) @@ -637,7 +637,7 @@ class TDTestCase: tdSql.checkData(7,1,20) tdSql.checkData(8,1,10) - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:01:00.000' partition by tbname,ind interval(20s) fill(value ,2 ) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:01:00.000' interval(20s) fill(value ,2 ) group by tbname,ind ;") tdSql.checkRows(9) tdSql.checkData(0,1,20) tdSql.checkData(2,1,10) @@ -646,7 +646,7 @@ class TDTestCase: tdSql.checkData(7,1,20) tdSql.checkData(8,1,10) - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:01:00.000' partition by tbname,ind interval(20s) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2015-01-01 00:00:00.000' and ts <'2015-01-01 00:01:00.000' interval(20s) group by tbname,ind ;") tdSql.checkRows(9) tdSql.checkData(0,1,20) tdSql.checkData(2,1,10) @@ -655,7 +655,7 @@ class TDTestCase: tdSql.checkData(7,1,20) tdSql.checkData(8,1,10) - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2014-12-31 23:59:00.000' and ts <'2015-01-01 00:01:00.000' partition by tbname,ind interval(20s) fill(NULL) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2014-12-31 23:59:00.000' and ts <'2015-01-01 00:01:00.000' interval(20s) fill(NULL) group by tbname,ind ;") tdSql.checkRows(18) tdSql.checkData(0,1,None) tdSql.checkData(2,1,None) @@ -667,7 +667,7 @@ class TDTestCase: # interval sliding - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2014-12-31 23:59:00.000' and ts <'2015-01-01 00:01:00.000' partition by tbname,ind interval(20s) sliding(20s) fill(NULL) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2014-12-31 23:59:00.000' and ts <'2015-01-01 00:01:00.000' interval(20s) sliding(20s) fill(NULL) group by tbname,ind ;") tdSql.checkRows(18) tdSql.checkData(0,1,None) tdSql.checkData(2,1,None) @@ -677,7 +677,7 @@ class TDTestCase: tdSql.checkData(8,1,None) tdSql.checkData(9,1,20) - tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2014-12-31 23:59:00.000' and ts <'2015-01-01 00:01:00.000' partition by tbname,ind interval(20s) sliding(10s) fill(NULL) ;") + tdSql.query("select elapsed(ts,10s)*10 from stable_1 where ts >= '2014-12-31 23:59:00.000' and ts <'2015-01-01 00:01:00.000' interval(20s) sliding(10s) fill(NULL) group by tbname,ind ;") tdSql.checkRows(39) tdSql.checkData(0,1,None) tdSql.checkData(2,1,None) @@ -689,12 +689,12 @@ class TDTestCase: tdSql.checkData(19,1,10) tdSql.checkData(20,1,20) tdSql.checkData(25,1,0) - + def query_mix_common(self): tdLog.info (" ======================================elapsed mixup with common col, it will not support =======================================") - tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and ind =1 partition by tbname; ") + tdSql.query("select elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' and ind =1 group by tbname; ") tdSql.checkRows(1) tdSql.checkData(0,0,6) @@ -703,16 +703,16 @@ class TDTestCase: tdSql.checkData(0,0,6) tdSql.error("select ts,elapsed(ts,10s) from sub_empty_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' ; ") - tdSql.error("select ts,elapsed(ts,10s) from stable_empty where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' partition by tbname; ") + tdSql.error("select ts,elapsed(ts,10s) from stable_empty where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' group by tbname; ") tdSql.error("select ts,elapsed(ts,10s) from sub_table1_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' ; ") - tdSql.error("select ts,elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' partition by tbname; ") + tdSql.error("select ts,elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' group by tbname; ") tdSql.error("select q_int,elapsed(ts,10s) from sub_table1_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' ; ") - tdSql.error("select q_int,elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' partition by tbname; ") + tdSql.error("select q_int,elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' group by tbname; ") tdSql.error("select ts,q_int,elapsed(ts,10s) from sub_table1_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' ; ") - tdSql.error("select ts,q_int,elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' partition by tbname; ") + tdSql.error("select ts,q_int,elapsed(ts,10s) from stable_1 where ts between '2015-01-01 00:00:00.000' and '2015-01-01 00:01:00.000' group by tbname; ") def query_mix_Aggregate(self): @@ -729,8 +729,8 @@ class TDTestCase: tdSql.query(sql) tdSql.checkData(0,0,data[0][index]) - tdSql.query("select count(*),avg(q_int) , sum(q_double),stddev(q_float),LEASTSQUARES(q_int,0,1), elapsed(ts,10s) from stable_1 partition by tbname; ") - + tdSql.query("select count(*),avg(q_int) , sum(q_double),stddev(q_float),LEASTSQUARES(q_int,0,1), elapsed(ts,10s) from stable_1 group by tbname; ") + # Arithmetic with elapsed for common table operators = ["+" ,"-" , "*" ,"/" ,"%"] @@ -743,9 +743,9 @@ class TDTestCase: sql_common= "select " for index , query in enumerate(querys_oper): - + query_data = tdSql.getResult("select %s from sub_table1_1;"%query) - + query_datas.append(query_data[0][0]) sql_common += " %s %s " %(query,operator) sql_common=sql_common[:-2] + " from sub_table1_1;" @@ -753,7 +753,7 @@ class TDTestCase: tdSql.query(sql_common) results= query_datas[0] if operator == "+": - for data in query_datas[1:]: + for data in query_datas[1:]: results += data tdSql.checkData(0,0,results) @@ -794,17 +794,17 @@ class TDTestCase: sql_common= "select " for index , query in enumerate(querys_oper): - - query_data = tdSql.getResult("select %s from stable_1 partition by tbname;"%query) - + + query_data = tdSql.getResult("select %s from stable_1 group by tbname;"%query) + query_datas.append(query_data[0][0]) sql_common += " %s %s " %(query,operator) - sql_common=sql_common[:-2] + " from stable_1 partition by tbname;" + sql_common=sql_common[:-2] + " from stable_1 group by tbname;" tdSql.query(sql_common) results= query_datas[0] if operator == "+": - for data in query_datas[1:]: + for data in query_datas[1:]: results += data tdSql.checkData(0,0,results) tdSql.checkData(1,0,results) @@ -849,9 +849,9 @@ class TDTestCase: querys = ["max(q_int)","min(q_int)" , "first(q_tinyint)", "first(*)","last(q_int)","last(*)","PERCENTILE(q_int,10)","APERCENTILE(q_int,10)","elapsed(ts,10s)"] - + querys_mix = ["max(q_int)","min(q_int)" , "first(q_tinyint)", "first(q_int)","last(q_int)","PERCENTILE(q_int,10)","APERCENTILE(q_int,10)","elapsed(ts,10s)"] - + tdSql.query("select max(q_int),min(q_int) , first(q_tinyint), first(q_int),last(q_int),PERCENTILE(q_int,10),APERCENTILE(q_int,10) ,elapsed(ts,10s) from sub_table1_1 ; ") data = tdSql.getResult("select max(q_int),min(q_int) , first(q_tinyint), first(q_int),last(q_int),PERCENTILE(q_int,10),APERCENTILE(q_int,10) ,elapsed(ts,10s) from sub_table1_1 ; ") @@ -861,19 +861,19 @@ class TDTestCase: tdSql.query(sql) tdSql.checkData(0,0,data[0][index]) - tdSql.query("select max(q_int),min(q_int) , first(q_tinyint), first(q_int),last(q_int),APERCENTILE(q_int,10) ,elapsed(ts,10s) from stable_1 partition by tbname ; ") + tdSql.query("select max(q_int),min(q_int) , first(q_tinyint), first(q_int),last(q_int),APERCENTILE(q_int,10) ,elapsed(ts,10s) from stable_1 group by tbname ; ") - data = tdSql.getResult("select max(q_int),min(q_int) , first(q_tinyint), first(q_int),last(q_int),APERCENTILE(q_int,10) ,elapsed(ts,10s) from stable_1 partition by tbname ; ") + data = tdSql.getResult("select max(q_int),min(q_int) , first(q_tinyint), first(q_int),last(q_int),APERCENTILE(q_int,10) ,elapsed(ts,10s) from stable_1 group by tbname ; ") querys_mix = ["max(q_int)","min(q_int)" , "first(q_tinyint)", "first(q_int)","last(q_int)","APERCENTILE(q_int,10)","elapsed(ts,10s)"] for index , query in enumerate(querys_mix): - sql = "select %s from stable_1 partition by tbname " %(query) + sql = "select %s from stable_1 group by tbname " %(query) tdSql.query(sql) tdSql.checkData(0,0,data[0][index]) tdSql.checkData(1,0,data[0][index]) tdSql.checkData(2,0,data[0][index]) - + operators = ["+" ,"-" , "*" ,"/" ,"%"] querys_oper = querys_mix @@ -884,9 +884,9 @@ class TDTestCase: sql_common= "select " for index , query in enumerate(querys_oper): - + query_data = tdSql.getResult("select %s from sub_table1_1;"%query) - + query_datas.append(query_data[0][0]) sql_common += " %s %s " %(query,operator) sql_common=sql_common[:-2] + " from sub_table1_1;" @@ -935,17 +935,17 @@ class TDTestCase: sql_common= "select " for index , query in enumerate(querys_oper): - - query_data = tdSql.getResult("select %s from stable_1 partition by tbname;"%query) - + + query_data = tdSql.getResult("select %s from stable_1 group by tbname;"%query) + query_datas.append(query_data[0][0]) sql_common += " %s %s " %(query,operator) - sql_common=sql_common[:-2] + " from stable_1 partition by tbname;" + sql_common=sql_common[:-2] + " from stable_1 group by tbname;" tdSql.query(sql_common) results= query_datas[0] if operator == "+": - for data in query_datas[1:]: + for data in query_datas[1:]: results += data tdSql.checkData(0,0,results) tdSql.checkData(1,0,results) @@ -983,7 +983,7 @@ class TDTestCase: tdSql.checkData(0,0,results) tdSql.checkData(1,0,results) tdSql.checkData(2,0,results) - + def query_mix_compute(self): tdLog.info (" ====================================== elapsed mixup with compute function =================================================") @@ -993,15 +993,15 @@ class TDTestCase: for index , query in enumerate(querys): sql1 = "select elapsed(ts,10s),%s from sub_table1_1 " %(query) - sql2 = "select elapsed(ts,10s),%s from stable_1 partition by tbname" %(query) + sql2 = "select elapsed(ts,10s),%s from stable_1 group by tbname" %(query) if query in ["diff(q_int)","DERIVATIVE(q_int,1s,1)","ceil(q_float)","floor(q_float)","round(q_float)"]: tdSql.error(sql1) tdSql.error(sql2) continue tdSql.query(sql1) tdSql.query(sql2) - - # only support mixup with spread + + # only support mixup with spread sql = "select spread(ts)*10,spread(q_tinyint)-10,elapsed(ts,10s) from sub_table1_1 where ts>=\"2015-01-01 00:00:00.000\" and ts < \"2015-01-01 00:10:00.000\" ;" tdSql.execute(sql) @@ -1016,7 +1016,7 @@ class TDTestCase: for index , query in enumerate(querys_mix): sql = "select %s from sub_table1_1 where ts>=\"2015-01-01 00:00:00.000\" and ts < \"2015-01-01 00:10:00.000\" ; " %(query) tdSql.query(sql) - + operators = ["+" ,"-" , "*" ,"/" ,"%"] querys_oper = querys_mix @@ -1025,7 +1025,7 @@ class TDTestCase: sql_common= "select " for index , query in enumerate(querys_oper): - + sql_common += " %s %s " %(query,operator) sql_common=sql_common[:-2] + " from stable_1 where ts>=\"2015-01-01 00:00:00.000\" and ts < \"2015-01-01 00:10:00.000\" ;" @@ -1034,7 +1034,7 @@ class TDTestCase: for index , query in enumerate(querys_mix): sql = "select %s from stable_1 where ts>=\"2015-01-01 00:00:00.000\" and ts < \"2015-01-01 00:10:00.000\" ; " %(query) tdSql.query(sql) - + operators = ["+" ,"-" , "*" ,"/" ,"%"] querys_oper = querys_mix @@ -1043,21 +1043,21 @@ class TDTestCase: sql_common= "select " for index , query in enumerate(querys_oper): - + sql_common += " %s %s " %(query,operator) sql_common=sql_common[:-2] + " from stable_1 where ts>=\"2015-01-01 00:00:00.000\" and ts < \"2015-01-01 00:10:00.000\" ;" tdSql.query(sql_common) - + def query_mix_arithmetic(self): - + tdLog.info (" ====================================== elapsed mixup with arithmetic =================================================") - + tdSql.execute("select elapsed(ts,10s)+1 ,elapsed(ts,10s)-2,elapsed(ts,10s)*3,elapsed(ts,10s)/4,elapsed(ts,10s)%5 from sub_table1_1 where ts>=\"2015-01-01 00:00:00.000\" and ts < \"2015-01-01 00:10:00.000\" ; ") tdSql.execute("select elapsed(ts,10s)+1 ,elapsed(ts,10s)-2,elapsed(ts,10s)*3,elapsed(ts,10s)/4,elapsed(ts,10s)%5 from stable_1 where ts>=\"2015-01-01 00:00:00.000\" and ts < \"2015-01-01 00:10:00.000\" ; ") - + # queries = ["elapsed(ts,10s)+1" ,"elapsed(ts,10s)-2","elapsed(ts,10s)*3","elapsed(ts,10s)/4","elapsed(ts,10s)%5" ] - + # for index ,query in enumerate(queries): # sql = "select %s from sub_table1_1 where ts>=\"2015-01-01 00:00:00.000\" and ts < \"2015-01-01 00:10:00.000\" interval(1s) fill(prev) ;" % (query) # data = tdSql.getResult(sql) @@ -1067,13 +1067,13 @@ class TDTestCase: def query_with_join(self): tdLog.info (" ====================================== elapsed mixup with join =================================================") - + tdSql.error("select elapsed(ts,10s) from stable_empty TABLE1 , stable_empty TABLE2 where TABLE1.ts =TABLE2.ts; ") - tdSql.error("select elapsed(ts,10s) from stable_empty TABLE1 , stable_empty TABLE2 where TABLE1.ts =TABLE2.ts partition by tbname; ") + tdSql.error("select elapsed(ts,10s) from stable_empty TABLE1 , stable_empty TABLE2 where TABLE1.ts =TABLE2.ts group by tbname; ") tdSql.execute("select elapsed(ts,10s) from sub_empty_1 TABLE1 , sub_empty_2 TABLE2 where TABLE1.ts =TABLE2.ts; ") tdSql.error("select elapsed(ts,10s) from stable_1 TABLE1 , stable_2 TABLE2 where TABLE1.ts =TABLE2.ts and TABLE1.ind =TABLE2.ind; ") - tdSql.error("select elapsed(ts,10s) from stable_1 TABLE1 , stable_2 TABLE2 where TABLE1.ts =TABLE2.ts and TABLE1.ind =TABLE2.ind partition by tbname,ind; ") # join not support group by + tdSql.error("select elapsed(ts,10s) from stable_1 TABLE1 , stable_2 TABLE2 where TABLE1.ts =TABLE2.ts and TABLE1.ind =TABLE2.ind group by tbname,ind; ") # join not support group by tdSql.error("select elapsed(ts,10s) from sub_empty_1 TABLE1 , stable_2 TABLE2 where TABLE1.ts =TABLE2.ts and TABLE1.ind =TABLE2.ind ; ") tdSql.execute("select elapsed(ts,10s) from sub_empty_1 TABLE1 , sub_empty_2 TABLE2 where TABLE1.ts =TABLE2.ts ; ") @@ -1102,7 +1102,7 @@ class TDTestCase: tdLog.info (" ====================================== elapsed mixup with union all =================================================") - # union all with empty + # union all with empty tdSql.query("select elapsed(ts,10s) from regular_table_1 union all select elapsed(ts,10s) from regular_table_2;") @@ -1117,7 +1117,7 @@ class TDTestCase: tdSql.checkRows(600) tdSql.checkData(0,1,0.1) tdSql.checkData(500,0,0) - + tdSql.query('select elapsed(ts,10s) from sub_empty_1 union all select elapsed(ts,10s) from sub_empty_2;') tdSql.checkRows(0) @@ -1141,61 +1141,61 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0,0,9) - tdSql.error('select elapsed(ts,10s) from sub_empty_1 union all select elapsed(ts,10s) from stable_sub_empty partition by tbname;') + tdSql.error('select elapsed(ts,10s) from sub_empty_1 union all select elapsed(ts,10s) from stable_sub_empty group by tbname;') - tdSql.error('select elapsed(ts,10s) from regular_table_1 union all select elapsed(ts,10s) from stable_sub_empty partition by tbname;') + tdSql.error('select elapsed(ts,10s) from regular_table_1 union all select elapsed(ts,10s) from stable_sub_empty group by tbname;') tdSql.query('select elapsed(ts,10s) from sub_empty_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(1s) fill(prev) union all select elapsed(ts,10s) from sub_empty_2 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(1s) fill(prev);') tdSql.checkRows(0) - tdSql.error('select elapsed(ts,10s) from sub_empty_1 union all select elapsed(ts,10s) from stable_empty partition by tbname;') - - tdSql.error('select elapsed(ts,10s) from sub_empty_1 interval(1s) union all select elapsed(ts,10s) from stable_empty interval(1s) partition by tbname;') - - # tdSql.error('select elapsed(ts,10s) from sub_empty_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(1s) fill(prev) union all select elapsed(ts,10s) from stable_empty where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(1s) fill(prev) partition by tbname;') - - tdSql.query("select elapsed(ts,10s) from stable_empty partition by tbname union all select elapsed(ts,10s) from stable_empty partition by tbname ;") + tdSql.error('select elapsed(ts,10s) from sub_empty_1 union all select elapsed(ts,10s) from stable_empty group by tbname;') + + tdSql.error('select elapsed(ts,10s) from sub_empty_1 interval(1s) union all select elapsed(ts,10s) from stable_empty interval(1s) group by tbname;') + + # tdSql.error('select elapsed(ts,10s) from sub_empty_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(1s) fill(prev) union all select elapsed(ts,10s) from stable_empty where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(1s) fill(prev) group by tbname;') + + tdSql.query("select elapsed(ts,10s) from stable_empty group by tbname union all select elapsed(ts,10s) from stable_empty group by tbname ;") tdSql.checkRows(0) # case : TD-12229 - tdSql.query("select elapsed(ts,10s) from stable_empty partition by tbname union all select elapsed(ts,10s) from stable_1 partition by tbname ;") + tdSql.query("select elapsed(ts,10s) from stable_empty group by tbname union all select elapsed(ts,10s) from stable_1 group by tbname ;") tdSql.checkRows(3) - tdSql.query("select elapsed(ts,10s) from stable_1 partition by tbname union all select elapsed(ts,10s) from stable_1 partition by tbname ;") + tdSql.query("select elapsed(ts,10s) from stable_1 group by tbname union all select elapsed(ts,10s) from stable_1 group by tbname ;") tdSql.checkRows(6) tdSql.checkData(0,0,9) tdSql.checkData(5,0,9) - tdSql.query("select elapsed(ts,10s) from stable_1 partition by tbname union all select elapsed(ts,10s) from stable_2 partition by tbname ;") + tdSql.query("select elapsed(ts,10s) from stable_1 group by tbname union all select elapsed(ts,10s) from stable_2 group by tbname ;") tdSql.checkRows(6) tdSql.checkData(0,0,9) tdSql.checkData(5,0,9) - tdSql.query('select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" partition by tbname interval(10s) fill(prev) union all\ - select elapsed(ts,10s) from stable_2 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" partition by tbname interval(10s) fill(prev) ;') + tdSql.query('select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) group by tbname union all\ + select elapsed(ts,10s) from stable_2 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) group by tbname ;') tdSql.checkRows(360) tdSql.checkData(0,1,1) tdSql.checkData(50,1,0) - + #case : TD-12229 - tdSql.query('select elapsed(ts,10s) from stable_empty partition by tbname union all select elapsed(ts,10s) from stable_2 partition by tbname ;') + tdSql.query('select elapsed(ts,10s) from stable_empty group by tbname union all select elapsed(ts,10s) from stable_2 group by tbname ;') tdSql.checkRows(3) - tdSql.query('select elapsed(ts,10s) from stable_1 partition by tbname union all select elapsed(ts,10s) from stable_empty partition by tbname ;') + tdSql.query('select elapsed(ts,10s) from stable_1 group by tbname union all select elapsed(ts,10s) from stable_empty group by tbname ;') tdSql.checkRows(3) - tdSql.query('select elapsed(ts,10s) from stable_empty where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" partition by tbname interval(10s) fill(prev) union all\ - select elapsed(ts,10s) from stable_2 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" partition by tbname interval(10s) fill(prev) ;') + tdSql.query('select elapsed(ts,10s) from stable_empty where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) group by tbname union all\ + select elapsed(ts,10s) from stable_2 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) group by tbname ;') tdSql.checkRows(180) - tdSql.query('select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" partition by tbname interval(10s) fill(prev) union all\ - select elapsed(ts,10s) from stable_empty where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" partition by tbname interval(10s) fill(prev) ;') + tdSql.query('select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) group by tbname union all\ + select elapsed(ts,10s) from stable_empty where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) group by tbname ;') tdSql.checkRows(180) # union all with sub table and regular table - # sub_table with sub_table + # sub_table with sub_table tdSql.query('select elapsed(ts,10s) from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) union all\ select elapsed(ts,10s) from sub_table2_2 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) ;') @@ -1232,7 +1232,7 @@ class TDTestCase: tdSql.checkRows(120) tdSql.checkData(0,1,1) tdSql.checkData(12,1,0) - + tdSql.query('select elapsed(ts,10s) from sub_empty_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) union all\ select elapsed(ts,10s) from regular_table_2 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) ;') tdSql.checkRows(60) @@ -1245,10 +1245,10 @@ class TDTestCase: tdSql.checkData(0,1,1) tdSql.checkData(12,1,0) - # stable with stable + # stable with stable - tdSql.query('select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) partition by tbname union all\ - select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) partition by tbname;') + tdSql.query('select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) group by tbname union all\ + select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) group by tbname;') tdSql.checkRows(360) tdSql.checkData(0,1,1) tdSql.checkData(12,1,0) @@ -1269,27 +1269,27 @@ class TDTestCase: tdSql.checkData(1,1,1) tdSql.checkData(9,1,1) - tdSql.query('select elapsed(ts,10s) from stable_1 partition by tbname, ind order by ts desc union all select elapsed(ts,10s) from stable_2 partition by tbname, ind order by ts asc ;') + tdSql.query('select elapsed(ts,10s) from stable_1 group by tbname, ind order by ts desc union all select elapsed(ts,10s) from stable_2 group by tbname, ind order by ts asc ;') tdSql.checkRows(6) tdSql.checkData(0,0,9) - tdSql.query('select elapsed(ts,10s) from stable_1 partition by tbname, ind order by ts desc union all select elapsed(ts,10s) from stable_1 partition by tbname, ind order by ts asc ;') + tdSql.query('select elapsed(ts,10s) from stable_1 group by tbname, ind order by ts desc union all select elapsed(ts,10s) from stable_1 group by tbname, ind order by ts asc ;') tdSql.checkRows(6) tdSql.checkData(0,0,9) - tdSql.query('select elapsed(ts,10s) from stable_1 interval(10s) partition by tbname,ind order by ts desc union all select elapsed(ts,10s) from stable_2 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" partition by tbname,ind interval(10s) fill(prev) order by ts asc ;') + tdSql.query('select elapsed(ts,10s) from stable_1 interval(10s) group by tbname,ind order by ts desc union all select elapsed(ts,10s) from stable_2 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) group by tbname,ind order by ts asc ;') tdSql.checkRows(210) tdSql.checkData(0,1,0) tdSql.checkData(1,1,1) tdSql.checkData(9,1,1) - tdSql.query('select elapsed(ts,10s) from stable_2 interval(10s) partition by tbname,ind order by ts desc union all select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" partition by tbname,ind interval(10s) fill(prev) order by ts asc ;') + tdSql.query('select elapsed(ts,10s) from stable_2 interval(10s) group by tbname,ind order by ts desc union all select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) group by tbname,ind order by ts asc ;') tdSql.checkRows(210) tdSql.checkData(0,1,0) tdSql.checkData(1,1,1) tdSql.checkData(9,1,1) - tdSql.query('select elapsed(ts,10s) from stable_1 interval(10s) partition by tbname,ind order by ts desc union all select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" partition by tbname,ind interval(10s) fill(prev) order by ts asc ;') + tdSql.query('select elapsed(ts,10s) from stable_1 interval(10s) group by tbname,ind order by ts desc union all select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(10s) fill(prev) group by tbname,ind order by ts asc ;') tdSql.checkRows(210) tdSql.checkData(0,1,0) tdSql.checkData(1,1,1) @@ -1300,23 +1300,23 @@ class TDTestCase: tdLog.info (" ====================================== elapsed query for nest =================================================") # ===============================================outer nest============================================ - + # regular table # ts can't be used at outer query tdSql.query("select elapsed(ts,10s) from (select ts from regular_table_1 );") - # case : TD-12164 + # case : TD-12164 tdSql.error("select elapsed(ts,10s) from (select qint ts from regular_table_1 );") tdSql.error("select elapsed(tbname ,10s) from (select qint tbname from regular_table_1 );") tdSql.error("select elapsed(tsc ,1s) from (select q_int tsc from regular_table_1) ;") tdSql.error("select elapsed(tsv ,1s) from (select elapsed(ts,1s) tsv from regular_table_1);") tdSql.error("select elapsed(ts ,1s) from (select elapsed(ts,1s) ts from regular_table_1);") - # # bug fix + # # bug fix # tdSql.error("select elapsed(tsc ,1s) from (select tscol tsc from regular_table_1) ;") - + # case TD-12276 # tdSql.error("select elapsed(ts,10s) from (select ts,tbname from regular_table_1 order by ts asc );") @@ -1333,11 +1333,11 @@ class TDTestCase: # tdSql.error("select elapsed(ts,10s) from (select ts ,max(q_int),tbname from sub_table1_1 order by ts ) interval(1s);") # tdSql.error("select elapsed(ts,10s) from (select ts ,q_int,tbname from sub_table1_1 order by ts ) interval(1s);") - + tdSql.query("select elapsed(ts,10s) from (select ts ,tbname,top(q_int,3) from sub_table1_1 ) interval(10s);") - + tdSql.query("select elapsed(ts,10s) from (select ts ,tbname,bottom(q_int,3) from sub_table1_1 ) interval(10s);") - + tdSql.query("select elapsed(ts,10s) from (select ts ,tbname from sub_table1_1 ) interval(10s);") tdSql.query("select elapsed(ts,10s) from (select ts ,tbname from sub_table1_1 ) interval(10s);") @@ -1345,11 +1345,11 @@ class TDTestCase: # tdSql.error("select elapsed(ts,10s) from (select ts ,count(*),tbname from sub_table1_1 order by ts ) interval(1s);") querys = ["count(*)","avg(q_int)", "sum(q_double)","stddev(q_float)","LEASTSQUARES(q_int,0,1)","elapsed(ts,10s)"] - + for query in querys: sql1 = "select elapsed(ts,10s) from (select %s from regular_table_1 order by ts ) interval(1s); " % query sql2 = "select elapsed(ts,10s) from (select ts , tbname ,%s from regular_table_1 order by ts ) interval(1s); " % query - sql3 = "select elapsed(ts,10s) from (select ts , tbname ,%s from stable_1 partition by tbname, ind order by ts ) interval(1s); " % query + sql3 = "select elapsed(ts,10s) from (select ts , tbname ,%s from stable_1 group by tbname, ind order by ts ) interval(1s); " % query sql4 = "select elapsed(ts,10s) from (select %s from sub_table2_1 order by ts ) interval(1s); " % query sql5 = "select elapsed(ts,10s) from (select ts , tbname ,%s from sub_table2_1 order by ts ) interval(1s); " % query @@ -1359,24 +1359,24 @@ class TDTestCase: tdSql.error(sql4) tdSql.error(sql5) - + # case TD-12164 tdSql.error( "select elapsed(ts00 ,1s) from (select elapsed(ts,1s) ts00 from regular_table_1) ; " ) tdSql.error( "select elapsed(ts ,1s) from (select elapsed(ts,1s) ts from regular_table_1) ; " ) - - tdSql.error( "select elapsed(ts00 ,1s) from (select elapsed(ts,1s) ts00 from stable_1 partition by tbname ) ; " ) - tdSql.error( "select elapsed(ts ,1s) from (select elapsed(ts,1s) ts from stable_1 partition by tbname) ; " ) + + tdSql.error( "select elapsed(ts00 ,1s) from (select elapsed(ts,1s) ts00 from stable_1 group by tbname ) ; " ) + tdSql.error( "select elapsed(ts ,1s) from (select elapsed(ts,1s) ts from stable_1 group by tbname) ; " ) - # stable + # stable - tdSql.error("select elapsed(ts,10s) from (select ts from stable_1 ) partition by tbname ;") + tdSql.error("select elapsed(ts,10s) from (select ts from stable_1 ) group by tbname ;") - tdSql.error("select elapsed(ts,10s) from (select ts ,max(q_int),tbname from stable_1 partition by tbname order by ts ) partition by tbname interval(1s) ;") + tdSql.error("select elapsed(ts,10s) from (select ts ,max(q_int),tbname from stable_1 group by tbname order by ts ) interval(1s) group by tbname;") - tdSql.error("select elapsed(ts,10s) from (select ts ,q_int,tbname from stable_1 order by ts ) partition by tbname interval(1s);") + tdSql.error("select elapsed(ts,10s) from (select ts ,q_int,tbname from stable_1 order by ts ) interval(1s) group by tbname;") - # mixup with aggregate + # mixup with aggregate querys = ["max(q_int)","min(q_int)" , "first(q_tinyint)", "first(*)","last(q_int)","last(*)","top(q_double,1)", "bottom(q_float,1)","PERCENTILE(q_int,10)","APERCENTILE(q_int,10)" ,"elapsed(ts,10s)"] @@ -1384,10 +1384,10 @@ class TDTestCase: for index , query in enumerate(querys): sql1 = "select elapsed(ts,10s) from (select %s from sub_table1_1) where ts>=\"2015-01-01 00:00:00.000\" and ts < \"2015-01-01 00:10:00.000\" interval(10s) fill(prev) ; " %(query) - sql2 = "select elapsed(ts,10s) from (select %s from stable_1 ) where ts>=\"2015-01-01 00:00:00.000\" and ts < \"2015-01-01 00:10:00.000\" partition by tbname interval(10s) fill(prev) ; " %(query) - sql3 = "select elapsed(ts,10s) from (select %s from stable_1 partition by tbname) where ts>=\"2015-01-01 00:00:00.000\" and ts < \"2015-01-01 00:10:00.000\" partition by tbname interval(10s) fill(prev) ; " %(query) + sql2 = "select elapsed(ts,10s) from (select %s from stable_1 ) where ts>=\"2015-01-01 00:00:00.000\" and ts < \"2015-01-01 00:10:00.000\" interval(10s) fill(prev) group by tbname; " %(query) + sql3 = "select elapsed(ts,10s) from (select %s from stable_1 group by tbname) where ts>=\"2015-01-01 00:00:00.000\" and ts < \"2015-01-01 00:10:00.000\" interval(10s) fill(prev) group by tbname; " %(query) - if query in ["interp(q_int)" ]: + if query in ["interp(q_int)" ]: # print(sql1 ) # print(sql2) tdSql.query(sql1) @@ -1444,7 +1444,7 @@ class TDTestCase: # tdSql.query("select spread(data) from (select count(*),avg(q_int) , sum(q_double),stddev(q_float),LEASTSQUARES(q_int,0,1), elapsed(ts,10s) data from regular_table_3 \ # where ts>=\"2015-01-01 00:00:00.000\" and ts < \"2015-01-01 00:10:00.000\" interval(1s) fill(prev)); ") # tdSql.checkRows(1) - + # tdSql.query("select diff(data) from (select count(*),avg(q_int) , sum(q_double),stddev(q_float),LEASTSQUARES(q_int,0,1), elapsed(ts,10s) data from regular_table_3 \ # where ts>=\"2015-01-01 00:00:00.000\" and ts < \"2015-01-01 00:10:00.000\" interval(1s) fill(prev)); ") # tdSql.checkRows(599) @@ -1474,10 +1474,10 @@ class TDTestCase: # tdSql.checkRows(600) def query_session_windows(self): - - # case TD-12344 + + # case TD-12344 # session not support stable - tdSql.error('select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" session(ts ,10s) partition by tbname,ind order by ts asc ') + tdSql.error('select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" session(ts ,10s) group by tbname,ind order by ts asc ') tdSql.query('select elapsed(ts,10s) from sub_table1_1 session(ts,1w) ; ') tdSql.checkRows(1) @@ -1488,17 +1488,16 @@ class TDTestCase: tdSql.query('select elapsed(ts,10s) from ( select * from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000") session(ts,1w) ; ') - tdSql.query('select elapsed(ts,10s) from ( select ts ,q_int from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000") session(ts,1w) ; ') - + tdSql.error('select elapsed(ts,10s) from ( select ts ,q_int from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000") session(ts,1w) ; ') tdSql.error('select elapsed(ts,10s) from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(20s) fill (next) session(ts,1w) ; ') - + tdSql.query('select elapsed(ts,10s) from sub_empty_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" session(ts,1w) ; ') tdSql.checkRows(0) # windows state # not support stable - tdSql.error('select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" state_window(q_int) partition by tbname,ind order by ts asc ') + tdSql.error('select elapsed(ts,10s) from stable_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" state_window(q_int) group by tbname,ind order by ts asc ') tdSql.query('select elapsed(ts,10s) from sub_table1_1 state_window(q_int) ; ') tdSql.checkRows(10) @@ -1512,25 +1511,25 @@ class TDTestCase: # tdSql.error('select elapsed(ts,10s) from ( select ts ,q_int from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000") state_window(q_int) ; ') # tdSql.error('select elapsed(ts,10s) from sub_table1_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" interval(20s) fill (next) state_window(q_int) ; ') - + # tdSql.query('select elapsed(ts,10s) from sub_empty_1 where ts>="2015-01-01 00:00:00.000" and ts < "2015-01-01 00:10:00.000" state_window(q_int); ') # tdSql.checkRows(0) - + def continuous_query(self): tdSql.error('create table elapsed_t as select elapsed(ts) from sub_table1_1 interval(1m) sliding(30s);') - tdSql.error('create table elapsed_tb as select elapsed(ts) from stable_1 interval(1m) sliding(30s) partition by tbname;') - tdSql.error('create table elapsed_tc as select elapsed(ts) from stable_1 interval(10s) sliding(5s) interval(1m) sliding(30s) partition by tbname;') - + tdSql.error('create table elapsed_tb as select elapsed(ts) from stable_1 interval(1m) sliding(30s) group by tbname;') + tdSql.error('create table elapsed_tc as select elapsed(ts) from stable_1 interval(10s) sliding(5s) interval(1m) sliding(30s) group by tbname;') + def query_precision(self): def generate_data(precision="ms"): - + tdSql.execute("create database if not exists db_%s precision '%s';" %(precision, precision)) tdSql.execute("use db_%s;" %precision) tdSql.execute("create stable db_%s.st (ts timestamp , id int) tags(ind int);"%precision) tdSql.execute("create table db_%s.tb1 using st tags(1);"%precision) tdSql.execute("create table db_%s.tb2 using st tags(2);"%precision) - + if precision == "ms": start_ts = self.ts step = 10000 @@ -1560,19 +1559,19 @@ class TDTestCase: if pres == "ms": if unit in ["10u","10b"]: - tdSql.error("select elapsed(ts,%s) from db_%s.st partition by tbname "%(unit,pres)) + tdSql.error("select elapsed(ts,%s) from db_%s.st group by tbname "%(unit,pres)) pass else: - tdSql.query("select elapsed(ts,%s) from db_%s.st partition by tbname "%(unit,pres)) + tdSql.query("select elapsed(ts,%s) from db_%s.st group by tbname "%(unit,pres)) elif pres == "us" and unit in ["10b"]: if unit in ["10b"]: - tdSql.error("select elapsed(ts,%s) from db_%s.st partition by tbname "%(unit,pres)) + tdSql.error("select elapsed(ts,%s) from db_%s.st group by tbname "%(unit,pres)) pass else: - tdSql.query("select elapsed(ts,%s) from db_%s.st partition by tbname "%(unit,pres)) + tdSql.query("select elapsed(ts,%s) from db_%s.st group by tbname "%(unit,pres)) else: - tdSql.query("select elapsed(ts,%s) from db_%s.st partition by tbname "%(unit,pres)) + tdSql.query("select elapsed(ts,%s) from db_%s.st group by tbname "%(unit,pres)) basic_result = 9 tdSql.checkData(0,0,basic_result*pow(1000,index)) @@ -1594,11 +1593,10 @@ class TDTestCase: self.query_session_windows() self.continuous_query() self.query_precision() - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) tdCases.addLinux(__file__, TDTestCase()) - diff --git a/tests/system-test/2-query/function_stateduration.py b/tests/system-test/2-query/function_stateduration.py index 9aa6fdaefa..bdbd92acd6 100644 --- a/tests/system-test/2-query/function_stateduration.py +++ b/tests/system-test/2-query/function_stateduration.py @@ -18,7 +18,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - + def prepare_datas(self): tdSql.execute( '''create table stb1 @@ -26,7 +26,7 @@ class TDTestCase: tags (t1 int) ''' ) - + tdSql.execute( ''' create table t1 @@ -68,7 +68,7 @@ class TDTestCase: ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ''' ) - + def test_errors(self): error_sql_lists = [ # "select stateduration(c1,'GT',5,1s) from t1" @@ -110,35 +110,35 @@ class TDTestCase: for error_sql in error_sql_lists: tdSql.error(error_sql) pass - + def support_types(self): other_no_value_types = [ - "select stateduration(ts,'GT',1,1s) from t1" , + "select stateduration(ts,'GT',1,1s) from t1" , "select stateduration(c7,'GT',1,1s) from t1", "select stateduration(c8,'GT',1,1s) from t1", "select stateduration(c9,'GT',1,1s) from t1", - "select stateduration(ts,'GT',1,1s) from ct1" , + "select stateduration(ts,'GT',1,1s) from ct1" , "select stateduration(c7,'GT',1,1s) from ct1", "select stateduration(c8,'GT',1,1s) from ct1", "select stateduration(c9,'GT',1,1s) from ct1", - "select stateduration(ts,'GT',1,1s) from ct3" , + "select stateduration(ts,'GT',1,1s) from ct3" , "select stateduration(c7,'GT',1,1s) from ct3", "select stateduration(c8,'GT',1,1s) from ct3", "select stateduration(c9,'GT',1,1s) from ct3", - "select stateduration(ts,'GT',1,1s) from ct4" , + "select stateduration(ts,'GT',1,1s) from ct4" , "select stateduration(c7,'GT',1,1s) from ct4", "select stateduration(c8,'GT',1,1s) from ct4", "select stateduration(c9,'GT',1,1s) from ct4", - "select stateduration(ts,'GT',1,1s) from stb1 partition by tbname" , + "select stateduration(ts,'GT',1,1s) from stb1 partition by tbname" , "select stateduration(c7,'GT',1,1s) from stb1 partition by tbname", "select stateduration(c8,'GT',1,1s) from stb1 partition by tbname", - "select stateduration(c9,'GT',1,1s) from stb1 partition by tbname" + "select stateduration(c9,'GT',1,1s) from stb1 partition by tbname" ] - + for type_sql in other_no_value_types: tdSql.error(type_sql) tdLog.info("support type ok , sql is : %s"%type_sql) - + type_sql_lists = [ "select stateduration(c1,'GT',1,1s) from t1", "select stateduration(c2,'GT',1,1s) from t1", @@ -168,8 +168,8 @@ class TDTestCase: "select stateduration(c5,'GT',1,1s) from stb1 partition by tbname", "select stateduration(c6,'GT',1,1s) from stb1 partition by tbname", - "select stateduration(c6,'GT',1,1s) as alisb from stb1 partition by tbname", - "select stateduration(c6,'GT',1,1s) alisb from stb1 partition by tbname", + "select stateduration(c6,'GT',1,1s) as alisb from stb1 partition by tbname", + "select stateduration(c6,'GT',1,1s) alisb from stb1 partition by tbname", ] for type_sql in type_sql_lists: @@ -177,7 +177,7 @@ class TDTestCase: def support_opers(self): oper_lists = ['LT','lt','Lt','lT','GT','gt','Gt','gT','LE','le','Le','lE','GE','ge','Ge','gE','NE','ne','Ne','nE','EQ','eq','Eq','eQ'] - + oper_errors = [",","*","NULL","tbname","ts","sum","_c0"] for oper in oper_lists: @@ -190,7 +190,7 @@ class TDTestCase: def basic_stateduration_function(self): - # basic query + # basic query tdSql.query("select c1 from ct3") tdSql.checkRows(0) tdSql.query("select c1 from t1") @@ -211,9 +211,9 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query("select stateduration(c6,'GT',1,1s) from ct3") - # will support _rowts mix with + # will support _rowts mix with # tdSql.query("select (c6,'GT',1,1s),_rowts from ct3") - + # auto check for t1 table # used for regular table tdSql.query("select stateduration(c6,'GT',1,1s) from t1") @@ -229,17 +229,17 @@ class TDTestCase: tdSql.error("select stateduration(c6,'GT',1,1s),tbname from ct1") tdSql.error("select stateduration(c6,'GT',1,1s),t1 from ct1") - # unique with common col + # unique with common col tdSql.error("select stateduration(c6,'GT',1,1s) ,ts from ct1") tdSql.error("select stateduration(c6,'GT',1,1s) ,c1 from ct1") - # unique with scalar function + # unique with scalar function tdSql.error("select stateduration(c6,'GT',1,1s) ,abs(c1) from ct1") tdSql.error("select stateduration(c6,'GT',1,1s) , unique(c2) from ct1") tdSql.error("select stateduration(c6,'GT',1,1s) , abs(c2)+2 from ct1") - - # unique with aggregate function + + # unique with aggregate function tdSql.error("select stateduration(c6,'GT',1,1s) ,sum(c1) from ct1") tdSql.error("select stateduration(c6,'GT',1,1s) ,max(c1) from ct1") tdSql.error("select stateduration(c6,'GT',1,1s) ,csum(c1) from ct1") @@ -262,16 +262,16 @@ class TDTestCase: tdSql.checkData(0, 0, 0) tdSql.checkData(1, 0, 6134400) tdSql.checkData(6, 0, -1) - - # unique with union all + + # unique with union all tdSql.query("select stateduration(c1,'GT',1,1s) from ct4 union all select stateduration(c1,'GT',1,1s) from ct1") tdSql.checkRows(25) tdSql.query("select stateduration(c1,'GT',1,1s) from ct4 union all select distinct(c1) from ct4") tdSql.checkRows(22) - # unique with join - # prepare join datas with same ts + # unique with join + # prepare join datas with same ts tdSql.execute(" use db ") tdSql.execute(" create stable st1 (ts timestamp , num int) tags(ind int)") @@ -328,7 +328,7 @@ class TDTestCase: tdSql.checkRows(12) tdSql.query("select stateduration(c1+2 ,'GT',1,1s) from t1") tdSql.checkRows(12) - + # bug for stable #partition by tbname @@ -337,21 +337,20 @@ class TDTestCase: # tdSql.query(" select unique(c1) from stb1 partition by tbname ") # tdSql.checkRows(21) - - # group by + + # group by tdSql.error("select stateduration(c1,'GT',1,1s) from ct1 group by c1") tdSql.error("select stateduration(c1,'GT',1,1s) from ct1 group by tbname") # super table - + def check_unit_time(self): tdSql.execute(" use db ") tdSql.error("select stateduration(c1,'GT',1,1b) from ct1") tdSql.error("select stateduration(c1,'GT',1,1u) from ct1") + tdSql.error("select stateduration(c1,'GT',1,1000s) from t1") tdSql.query("select stateduration(c1,'GT',1,1s) from t1") tdSql.checkData(10,0,63072035) - tdSql.query("select stateduration(c1,'GT',1,1000s) from t1") - tdSql.checkData(10,0,int(63072035/1000)) tdSql.query("select stateduration(c1,'GT',1,1m) from t1") tdSql.checkData(10,0,int(63072035/60)) tdSql.query("select stateduration(c1,'GT',1,1h) from t1") @@ -360,8 +359,8 @@ class TDTestCase: tdSql.checkData(10,0,int(63072035/60/24/60)) tdSql.query("select stateduration(c1,'GT',1,1w) from t1") tdSql.checkData(10,0,int(63072035/60/7/24/60)) - - + + def check_boundary_values(self): tdSql.execute("drop database if exists bound_test") diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index d03ed5e03a..c18d1dab0e 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -675,7 +675,7 @@ class TDTestCase: tdSql.checkRows(3) tdSql.query("select TO_UNIXTIMESTAMP(datastr) from jsons1 where jtag->'tag1'>1;") tdSql.checkRows(3) - tdSql.query("select TIMETRUNCATE(ts,1u) from jsons1 where jtag->'tag1'>1;") + tdSql.query("select TIMETRUNCATE(ts,1s) from jsons1 where jtag->'tag1'>1;") tdSql.checkRows(3) tdSql.query("select TIMEDIFF(ts,_c0) from jsons1 where jtag->'tag1'>1;") tdSql.checkRows(3) diff --git a/tests/system-test/2-query/queryQnode.py b/tests/system-test/2-query/queryQnode.py index 15b643f5d7..0011fe248f 100644 --- a/tests/system-test/2-query/queryQnode.py +++ b/tests/system-test/2-query/queryQnode.py @@ -17,6 +17,8 @@ import threading as thd import multiprocessing as mp from numpy.lib.function_base import insert import taos +from util.dnodes import TDDnode +from util.dnodes import * from util.log import * from util.cases import * from util.sql import * @@ -30,9 +32,9 @@ class TDTestCase: # # --------------- main frame ------------------- # - clientCfgDict = {'queryproxy': '1','debugFlag': 135} - clientCfgDict["queryproxy"] = '2' - clientCfgDict["debugFlag"] = 143 + clientCfgDict = {'queryPolicy': '1','debugFlag': 135} + clientCfgDict["queryPolicy"] = '1' + clientCfgDict["debugFlag"] = 131 updatecfgDict = {'clientCfg': {}} updatecfgDict = {'debugFlag': 143} @@ -62,7 +64,7 @@ class TDTestCase: return buildPath # init - def init(self, conn, logSql): + def init(self, conn, logSql=True): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) # tdSql.prepare() @@ -292,12 +294,13 @@ class TDTestCase: tdLog.debug("-----create database and muti-thread create tables test------- ") - def test_case4(self): + def test_case1(self): self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 1, 2, 1*10) tdSql.execute("use db1;") tdSql.query("show dnodes;") dnodeId=tdSql.getData(0,0) print(dnodeId) + tdLog.debug("create qnode on dnode %s"%dnodeId) tdSql.execute("create qnode on dnode %s"%dnodeId) tdSql.query("select max(c1) from stb10;") maxQnode=tdSql.getData(0,0) @@ -310,6 +313,7 @@ class TDTestCase: # tdSql.query("show qnodes;") # qnodeId=tdSql.getData(0,0) + tdLog.debug("drop qnode on dnode %s"%dnodeId) tdSql.execute("drop qnode on dnode %s"%dnodeId) tdSql.execute("reset query cache") tdSql.query("select max(c1) from stb10;") @@ -323,15 +327,156 @@ class TDTestCase: unionallVnode=tdSql.queryResult assert unionallQnode == unionallVnode + queryPolicy=2 + simClientCfg="%s/taos.cfg"%tdDnodes.getSimCfgPath() + cmd='sed -i "s/^queryPolicy.*/queryPolicy 2/g" %s'%simClientCfg + os.system(cmd) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.execute("reset query cache") + tdSql.execute('alter local "queryPolicy" "%d"'%queryPolicy) + tdSql.query("show local variables;") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "queryPolicy" : + if int(tdSql.queryResult[i][1]) == int(queryPolicy): + tdLog.success('alter queryPolicy to %d successfully'%queryPolicy) + else : + tdLog.debug(tdSql.queryResult) + tdLog.exit("alter queryPolicy to %d failed"%queryPolicy) + tdSql.execute("reset query cache") + + tdSql.execute("use db1;") + tdSql.query("show dnodes;") + dnodeId=tdSql.getData(0,0) + tdLog.debug("create qnode on dnode %s"%dnodeId) + + tdSql.execute("create qnode on dnode %s"%dnodeId) + tdSql.query("select max(c1) from stb10;") + assert maxQnode==tdSql.getData(0,0) + tdSql.query("select min(c1) from stb11;") + assert minQnode==tdSql.getData(0,0) + tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + assert unionQnode==tdSql.queryResult + tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + assert unionallQnode==tdSql.queryResult + + # tdSql.query("show qnodes;") + # qnodeId=tdSql.getData(0,0) + tdLog.debug("drop qnode on dnode %s"%dnodeId) + tdSql.execute("drop qnode on dnode %s"%dnodeId) + tdSql.execute("reset query cache") + tdSql.query("select max(c1) from stb10;") + assert maxQnode==tdSql.getData(0,0) + tdSql.query("select min(c1) from stb11;") + assert minQnode==tdSql.getData(0,0) + tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") # tdSql.execute("create qnode on dnode %s"%dnodeId) + queryPolicy=3 + simClientCfg="%s/taos.cfg"%tdDnodes.getSimCfgPath() + cmd='sed -i "s/^queryPolicy.*/queryPolicy 2/g" %s'%simClientCfg + os.system(cmd) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.execute("reset query cache") + tdSql.execute('alter local "queryPolicy" "%d"'%queryPolicy) + tdSql.query("show local variables;") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "queryPolicy" : + if int(tdSql.queryResult[i][1]) == int(queryPolicy): + tdLog.success('alter queryPolicy to %d successfully'%queryPolicy) + else : + tdLog.debug(tdSql.queryResult) + tdLog.exit("alter queryPolicy to %d failed"%queryPolicy) + tdSql.execute("reset query cache") + + tdSql.execute("use db1;") + tdSql.query("show dnodes;") + dnodeId=tdSql.getData(0,0) + tdLog.debug("create qnode on dnode %s"%dnodeId) + + tdSql.execute("create qnode on dnode %s"%dnodeId) + tdSql.query("select max(c1) from stb10;") + assert maxQnode==tdSql.getData(0,0) + tdSql.query("select min(c1) from stb11;") + assert minQnode==tdSql.getData(0,0) + tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + assert unionQnode==tdSql.queryResult + tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + assert unionallQnode==tdSql.queryResult + + def test_case2(self): + self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 10, 2, 1*10) + tdSql.query("show qnodes") + if tdSql.queryRows == 1 : + tdLog.debug("drop qnode on dnode 1") + tdSql.execute("drop qnode on dnode 1") + queryPolicy=2 + simClientCfg="%s/taos.cfg"%tdDnodes.getSimCfgPath() + cmd='sed -i "s/^queryPolicy.*/queryPolicy 2/g" %s'%simClientCfg + os.system(cmd) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.execute("reset query cache") + tdSql.execute('alter local "queryPolicy" "%d"'%queryPolicy) + tdSql.query("show local variables;") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "queryPolicy" : + if int(tdSql.queryResult[i][1]) == int(queryPolicy): + tdLog.success('alter queryPolicy to %d successfully'%queryPolicy) + else : + tdLog.debug(tdSql.queryResult) + tdLog.exit("alter queryPolicy to %d failed"%queryPolicy) + tdSql.execute("use db1;") + tdSql.error("select max(c1) from stb10;") + tdSql.error("select min(c1) from stb11;") + tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + + tdSql.query("select max(c1) from stb10_0;") + tdSql.query("select min(c1) from stb11_0;") + + def test_case3(self): + + tdSql.execute('alter local "queryPolicy" "3"') + tdLog.debug("create qnode on dnode 1") + tdSql.execute("create qnode on dnode 1") + tdSql.execute("use db1;") + tdSql.query("show dnodes;") + dnodeId=tdSql.getData(0,0) + print(dnodeId) + + tdSql.query("select max(c1) from stb10;") + maxQnode=tdSql.getData(0,0) + tdSql.query("select min(c1) from stb11;") + minQnode=tdSql.getData(0,0) + tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + unionQnode=tdSql.queryResult + tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + unionallQnode=tdSql.queryResult + + # tdSql.query("show qnodes;") + # qnodeId=tdSql.getData(0,0) + tdLog.debug("drop qnode on dnode %s"%dnodeId) + + tdSql.execute("drop qnode on dnode %s"%dnodeId) + tdSql.execute("reset query cache") + + tdSql.error("select max(c1) from stb10;") + tdSql.error("select min(c1) from stb11;") + tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + # run case def run(self): - # test qnode - self.test_case4() - tdLog.debug(" LIMIT test_case3 ............ [OK]") + self.test_case1() + self.test_case2() + + self.test_case3() + # tdLog.debug(" LIMIT test_case3 ............ [OK]") return diff --git a/tests/system-test/2-query/statecount.py b/tests/system-test/2-query/statecount.py index ccda55f7ba..fbeb04bc2f 100644 --- a/tests/system-test/2-query/statecount.py +++ b/tests/system-test/2-query/statecount.py @@ -18,7 +18,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - + def prepare_datas(self): tdSql.execute( '''create table stb1 @@ -26,7 +26,7 @@ class TDTestCase: tags (t1 int) ''' ) - + tdSql.execute( ''' create table t1 @@ -68,7 +68,7 @@ class TDTestCase: ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ''' ) - + def test_errors(self): error_sql_lists = [ # "select statecount(c1,'GT',5) from t1" @@ -110,35 +110,35 @@ class TDTestCase: for error_sql in error_sql_lists: tdSql.error(error_sql) pass - + def support_types(self): other_no_value_types = [ - "select statecount(ts,'GT',1) from t1" , + "select statecount(ts,'GT',1) from t1" , "select statecount(c7,'GT',1) from t1", "select statecount(c8,'GT',1) from t1", "select statecount(c9,'GT',1) from t1", - "select statecount(ts,'GT',1) from ct1" , + "select statecount(ts,'GT',1) from ct1" , "select statecount(c7,'GT',1) from ct1", "select statecount(c8,'GT',1) from ct1", "select statecount(c9,'GT',1) from ct1", - "select statecount(ts,'GT',1) from ct3" , + "select statecount(ts,'GT',1) from ct3" , "select statecount(c7,'GT',1) from ct3", "select statecount(c8,'GT',1) from ct3", "select statecount(c9,'GT',1) from ct3", - "select statecount(ts,'GT',1) from ct4" , + "select statecount(ts,'GT',1) from ct4" , "select statecount(c7,'GT',1) from ct4", "select statecount(c8,'GT',1) from ct4", "select statecount(c9,'GT',1) from ct4", - "select statecount(ts,'GT',1) from stb1 partition by tbname" , + "select statecount(ts,'GT',1) from stb1 partition by tbname" , "select statecount(c7,'GT',1) from stb1 partition by tbname", "select statecount(c8,'GT',1) from stb1 partition by tbname", - "select statecount(c9,'GT',1) from stb1 partition by tbname" + "select statecount(c9,'GT',1) from stb1 partition by tbname" ] - + for type_sql in other_no_value_types: tdSql.error(type_sql) tdLog.info("support type ok , sql is : %s"%type_sql) - + type_sql_lists = [ "select statecount(c1,'GT',1) from t1", "select statecount(c2,'GT',1) from t1", @@ -168,8 +168,8 @@ class TDTestCase: "select statecount(c5,'GT',1) from stb1 partition by tbname", "select statecount(c6,'GT',1) from stb1 partition by tbname", - "select statecount(c6,'GT',1) as alisb from stb1 partition by tbname", - "select statecount(c6,'GT',1) alisb from stb1 partition by tbname", + "select statecount(c6,'GT',1) as alisb from stb1 partition by tbname", + "select statecount(c6,'GT',1) alisb from stb1 partition by tbname", ] for type_sql in type_sql_lists: @@ -177,7 +177,7 @@ class TDTestCase: def support_opers(self): oper_lists = ['LT','lt','Lt','lT','GT','gt','Gt','gT','LE','le','Le','lE','GE','ge','Ge','gE','NE','ne','Ne','nE','EQ','eq','Eq','eQ'] - + oper_errors = [",","*","NULL","tbname","ts","sum","_c0"] for oper in oper_lists: @@ -190,7 +190,7 @@ class TDTestCase: def basic_statecount_function(self): - # basic query + # basic query tdSql.query("select c1 from ct3") tdSql.checkRows(0) tdSql.query("select c1 from t1") @@ -211,9 +211,9 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query("select statecount(c6,'GT',1) from ct3") - # will support _rowts mix with + # will support _rowts mix with # tdSql.query("select (c6,'GT',1),_rowts from ct3") - + # auto check for t1 table # used for regular table tdSql.query("select statecount(c6,'GT',1) from t1") @@ -229,17 +229,17 @@ class TDTestCase: tdSql.error("select statecount(c6,'GT',1),tbname from ct1") tdSql.error("select statecount(c6,'GT',1),t1 from ct1") - # unique with common col + # unique with common col tdSql.error("select statecount(c6,'GT',1) ,ts from ct1") tdSql.error("select statecount(c6,'GT',1) ,c1 from ct1") - # unique with scalar function + # unique with scalar function tdSql.error("select statecount(c6,'GT',1) ,abs(c1) from ct1") tdSql.error("select statecount(c6,'GT',1) , unique(c2) from ct1") tdSql.error("select statecount(c6,'GT',1) , abs(c2)+2 from ct1") - - # unique with aggregate function + + # unique with aggregate function tdSql.error("select statecount(c6,'GT',1) ,sum(c1) from ct1") tdSql.error("select statecount(c6,'GT',1) ,max(c1) from ct1") tdSql.error("select statecount(c6,'GT',1) ,csum(c1) from ct1") @@ -262,16 +262,16 @@ class TDTestCase: tdSql.checkData(0, 0, 1) tdSql.checkData(1, 0, 2) tdSql.checkData(6, 0, -1) - - # unique with union all + + # unique with union all tdSql.query("select statecount(c1,'GT',1) from ct4 union all select statecount(c1,'GT',1) from ct1") tdSql.checkRows(25) tdSql.query("select statecount(c1,'GT',1) from ct4 union all select distinct(c1) from ct4") tdSql.checkRows(22) - # unique with join - # prepare join datas with same ts + # unique with join + # prepare join datas with same ts tdSql.execute(" use db ") tdSql.execute(" create stable st1 (ts timestamp , num int) tags(ind int)") @@ -323,7 +323,7 @@ class TDTestCase: tdSql.checkData(0, 0, None) tdSql.checkData(1, 0, 0.000000000) tdSql.checkData(3, 0, -1.000000000) - + # bug for stable #partition by tbname @@ -332,21 +332,20 @@ class TDTestCase: # tdSql.query(" select unique(c1) from stb1 partition by tbname ") # tdSql.checkRows(21) - - # group by + + # group by tdSql.query("select statecount(c1,'GT',1) from ct1 group by c1") tdSql.error("select statecount(c1,'GT',1) from ct1 group by tbname") # super table - + def check_unit_time(self): tdSql.execute(" use db ") tdSql.error("select stateduration(c1,'GT',1,1b) from ct1") tdSql.error("select stateduration(c1,'GT',1,1u) from ct1") + tdSql.error("select stateduration(c1,'GT',1,1000s) from t1") tdSql.query("select stateduration(c1,'GT',1,1s) from t1") tdSql.checkData(10,0,63072035) - tdSql.query("select stateduration(c1,'GT',1,1000s) from t1") - tdSql.checkData(10,0,int(63072035/1000)) tdSql.query("select stateduration(c1,'GT',1,1m) from t1") tdSql.checkData(10,0,int(63072035/60)) tdSql.query("select stateduration(c1,'GT',1,1h) from t1") @@ -355,8 +354,8 @@ class TDTestCase: tdSql.checkData(10,0,int(63072035/60/24/60)) tdSql.query("select stateduration(c1,'GT',1,1w) from t1") tdSql.checkData(10,0,int(63072035/60/7/24/60)) - - + + def check_boundary_values(self): tdSql.execute("drop database if exists bound_test") @@ -384,11 +383,11 @@ class TDTestCase: tdSql.execute( f"insert into sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) - + tdSql.error( f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) - + tdSql.query("select statecount(c1,'GT',1) from sub1_bound") tdSql.checkRows(5) @@ -396,29 +395,29 @@ class TDTestCase: tdSql.prepare() tdLog.printNoPrefix("==========step1:create table ==============") - + self.prepare_datas() - tdLog.printNoPrefix("==========step2:test errors ==============") + tdLog.printNoPrefix("==========step2:test errors ==============") self.test_errors() - - tdLog.printNoPrefix("==========step3:support types ============") + + tdLog.printNoPrefix("==========step3:support types ============") self.support_types() - tdLog.printNoPrefix("==========step4:support opers ============") + tdLog.printNoPrefix("==========step4:support opers ============") self.support_opers() - tdLog.printNoPrefix("==========step5: statecount basic query ============") + tdLog.printNoPrefix("==========step5: statecount basic query ============") self.basic_statecount_function() - tdLog.printNoPrefix("==========step6: statecount boundary query ============") + tdLog.printNoPrefix("==========step6: statecount boundary query ============") self.check_boundary_values() - tdLog.printNoPrefix("==========step6: statecount unit time test ============") + tdLog.printNoPrefix("==========step6: statecount unit time test ============") self.check_unit_time() diff --git a/tests/system-test/2-query/timetruncate.py b/tests/system-test/2-query/timetruncate.py index a48851b251..97e4622378 100644 --- a/tests/system-test/2-query/timetruncate.py +++ b/tests/system-test/2-query/timetruncate.py @@ -12,11 +12,11 @@ class TDTestCase: self.rowNum = 10 self.ts = 1537146000000 # 2018-9-17 09:00:00.000 - + def run(self): tdSql.prepare() - intData = [] + intData = [] floatData = [] tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, @@ -27,18 +27,18 @@ class TDTestCase: for i in range(self.rowNum): tdSql.execute("insert into ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - intData.append(i + 1) + intData.append(i + 1) floatData.append(i + 0.1) for i in range(self.rowNum): tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - intData.append(i + 1) - floatData.append(i + 0.1) + intData.append(i + 1) + floatData.append(i + 0.1) tdSql.query("select timetruncate(1,1d) from ntb") tdSql.checkRows(10) - tdSql.query("select timetruncate(1,1u) from ntb") - tdSql.checkRows(10) + tdSql.error("select timetruncate(1,1u) from ntb") + #tdSql.checkRows(10) tdSql.query("select timetruncate(1,1a) from ntb") tdSql.checkRows(10) tdSql.query("select timetruncate(1,1m) from ntb") @@ -97,8 +97,8 @@ class TDTestCase: tdSql.query("select timetruncate(1,1d) from stb") tdSql.checkRows(10) - tdSql.query("select timetruncate(1,1u) from stb") - tdSql.checkRows(10) + tdSql.error("select timetruncate(1,1u) from stb") + #tdSql.checkRows(10) tdSql.query("select timetruncate(1,1a) from stb") tdSql.checkRows(10) tdSql.query("select timetruncate(1,1m) from stb") @@ -156,8 +156,8 @@ class TDTestCase: tdSql.query("select timetruncate(1,1d) from stb_1") tdSql.checkRows(10) - tdSql.query("select timetruncate(1,1u) from stb_1") - tdSql.checkRows(10) + tdSql.error("select timetruncate(1,1u) from stb_1") + #tdSql.checkRows(10) tdSql.query("select timetruncate(1,1a) from stb_1") tdSql.checkRows(10) tdSql.query("select timetruncate(1,1m) from stb_1") @@ -217,4 +217,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py index e7353b0f13..80508979c5 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py @@ -65,31 +65,6 @@ class TDTestCase: self._async_raise(thread.ident, SystemExit) - def insertData(self,countstart,countstop): - # fisrt add data : db\stable\childtable\general table - - for couti in range(countstart,countstop): - tdLog.debug("drop database if exists db%d" %couti) - tdSql.execute("drop database if exists db%d" %couti) - print("create database if not exists db%d replica 1 duration 300" %couti) - tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) - tdSql.execute("use db%d" %couti) - tdSql.execute( - '''create table stb1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - tags (t1 int) - ''' - ) - tdSql.execute( - ''' - create table t1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - ''' - ) - for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') - - def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db', @@ -143,7 +118,8 @@ class TDTestCase: threads=[] for i in range(restartNumbers): dbNameIndex = '%s%d'%(paraDict["dbName"],i) - threads.append(threading.Thread(target=clusterComCreate.create_databases, args=(tdSql, dbNameIndex,paraDict["dbNumbers"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.create_databases, args=(newTdSql, dbNameIndex,paraDict["dbNumbers"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) for tr in threads: tr.start() diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py index 2d378e623d..cde09b4c58 100644 --- a/tests/system-test/6-cluster/clusterCommonCheck.py +++ b/tests/system-test/6-cluster/clusterCommonCheck.py @@ -39,6 +39,7 @@ class ClusterComCheck: def checkDnodes(self,dnodeNumbers): count=0 + # print(tdSql) while count < 5: tdSql.query("show dnodes") # tdLog.debug(tdSql.queryResult) @@ -85,7 +86,7 @@ class ClusterComCheck: tdLog.debug("check %s_%d that status is ready "%(dbNameIndex,j)) else: continue - print(query_status) + # print(query_status) count+=1 if query_status == dbNumbers: tdLog.success("we find cluster with %d dnode and check all databases are ready within 5s! " %dbNumbers) diff --git a/tests/system-test/6-cluster/clusterCommonCreate.py b/tests/system-test/6-cluster/clusterCommonCreate.py index 78aac309a0..9d3f74a2e3 100644 --- a/tests/system-test/6-cluster/clusterCommonCreate.py +++ b/tests/system-test/6-cluster/clusterCommonCreate.py @@ -125,7 +125,6 @@ class ClusterComCreate: def create_databases(self,tsql,dbNameIndex,dbNumbers,dropFlag=1,vgroups=4,replica=1): for i in range(dbNumbers): - print(dbNumbers) if dropFlag == 1: tsql.execute("drop database if exists %s_%d"%(dbNameIndex,i)) tsql.execute("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,i, vgroups, replica)) diff --git a/tests/system-test/7-tmq/stbFilter.py b/tests/system-test/7-tmq/stbFilter.py index 542894574b..7ad3cc99e7 100644 --- a/tests/system-test/7-tmq/stbFilter.py +++ b/tests/system-test/7-tmq/stbFilter.py @@ -20,18 +20,54 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def tmqCase1(self): - tdLog.printNoPrefix("======== test case 1: ") - paraDict = {'dbName': 'db1', + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', 'dropFlag': 1, 'event': '', 'vgroups': 4, + 'replica': 1, 'stbName': 'stb', 'colPrefix': 'c', 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + + tmqCom.initConsumerTable() + tmqCom.create_database(tsql=tdSql, dbName=paraDict["dbName"],dropFlag=paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=paraDict['replica']) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, 'ctbNum': 10, 'rowsPerTbl': 10000, 'batchNum': 100, @@ -43,13 +79,6 @@ class TDTestCase: topicNameList = ['topic1', 'topic2', 'topic3'] expectRowsList = [] tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - tdLog.info("create stb") - tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) - tdLog.info("create ctb") - tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) - tdLog.info("insert data") - tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) tdLog.info("create topics from stb with filter") queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 4 == 0" %(paraDict['dbName'], paraDict['stbName']) @@ -122,9 +151,9 @@ class TDTestCase: tdLog.info("wait the consume result") expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - # if expectRowsList[2] != resultList[0]: - # tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) - # tdLog.exit("2 tmq consume rows error!") + if expectRowsList[2] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) + tdLog.exit("2 tmq consume rows error!") time.sleep(10) for i in range(len(topicNameList)): @@ -134,16 +163,18 @@ class TDTestCase: def tmqCase2(self): tdLog.printNoPrefix("======== test case 2: ") - paraDict = {'dbName': 'db2', + paraDict = {'dbName': 'dbt', 'dropFlag': 1, 'event': '', 'vgroups': 4, + 'replica': 1, 'stbName': 'stb', 'colPrefix': 'c', 'tagPrefix': 't', 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, 'ctbNum': 10, 'rowsPerTbl': 10000, 'batchNum': 100, @@ -155,13 +186,6 @@ class TDTestCase: topicNameList = ['topic1', 'topic2', 'topic3'] expectRowsList = [] tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - tdLog.info("create stb") - tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) - tdLog.info("create ctb") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum']) - tdLog.info("insert data") - tmqCom.insert_data_1(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) tdLog.info("create topics from stb with filter") # sqlString = "create topic %s as select ts, sin(c1), pow(c2,3) from %s.%s where c2 >= 0" %(topicNameList[0], paraDict['dbName'], paraDict['stbName']) @@ -247,6 +271,7 @@ class TDTestCase: def run(self): tdSql.prepare() + self.prepareTestEnv() self.tmqCase1() self.tmqCase2() diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index f8488aee49..7f9d36bd26 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -170,33 +170,42 @@ class TMQCom: tdLog.debug("complete to create database %s"%(dbName)) return + # self.create_stable() and self.create_ctable() and self.insert_data_interlaceByMultiTbl() : The three functions are matched + # schema: (ts timestamp, c1 int, c2 bigint, c3 double, c4 binary(32), c5 nchar(32), c6 timestamp) tags (t1 int, t2 bigint, t3 double, t4 binary(32), t5 nchar(32)) def create_stable(self,tsql, dbName,stbName): - tsql.execute("create table if not exists %s.%s (ts timestamp, c1 int, c2 int, c3 binary(16)) tags(t1 int, t2 binary(32))"%(dbName, stbName)) + schemaString = "(ts timestamp, c1 int, c2 bigint, c3 double, c4 binary(32), c5 nchar(32), c6 timestamp) tags (t1 int, t2 bigint, t3 double, t4 binary(32), t5 nchar(32))" + tsql.execute("create table if not exists %s.%s %s"%(dbName, stbName, schemaString)) tdLog.debug("complete to create %s.%s" %(dbName, stbName)) return def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0): - tsql.execute("use %s" %dbName) + # tsql.execute("use %s" %dbName) pre_create = "create table" sql = pre_create #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + batchNum = 10 + tblBatched = 0 for i in range(ctbNum): - tagValue = 'beijing' + tagBinaryValue = 'beijing' if (i % 2 == 0): - tagValue = 'shanghai' + tagBinaryValue = 'shanghai' elif (i % 3 == 0): - tagValue = 'changsha' + tagBinaryValue = 'changsha' - sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i+ctbStartIdx,stbName,i+ctbStartIdx+1, tagValue) - if (i > 0) and (i%100 == 0): + sql += " %s.%s%d using %s.%s tags(%d, %d, %d, '%s', '%s')"%(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,tagBinaryValue,tagBinaryValue) + tblBatched += 1 + if (i == ctbNum-1 ) or (tblBatched == batchNum): tsql.execute(sql) + tblBatched = 0 sql = pre_create + if sql != pre_create: tsql.execute(sql) - tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName)) - return + tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName)) + return + # schema: (ts timestamp, c1 int, c2 binary(16)) def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None): tdLog.debug("start to insert data ............") tsql.execute("use %s" %dbName) @@ -208,11 +217,14 @@ class TMQCom: startTs = int(round(t * 1000)) #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) for i in range(ctbNum): + rowsBatched = 0 sql += " %s%d values "%(stbName,i) for j in range(rowsPerTbl): sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) - if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + rowsBatched += 1 + if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): tsql.execute(sql) + rowsBatched = 0 if j < rowsPerTbl - 1: sql = "insert into %s%d values " %(stbName,i) else: @@ -224,6 +236,7 @@ class TMQCom: tdLog.debug("insert data ............ [OK]") return + # schema: (ts timestamp, c1 int, c2 int, c3 binary(16)) def insert_data_1(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs): tdLog.debug("start to insert data ............") tsql.execute("use %s" %dbName) @@ -234,14 +247,17 @@ class TMQCom: startTs = int(round(t * 1000)) #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) for i in range(ctbNum): + rowsBatched = 0 sql += " %s%d values "%(ctbPrefix,i) for j in range(rowsPerTbl): if (j % 2 == 0): sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, j, j) else: sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, -j, j) - if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + rowsBatched += 1 + if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): tsql.execute(sql) + rowsBatched = 0 if j < rowsPerTbl - 1: sql = "insert into %s%d values " %(ctbPrefix,i) else: @@ -253,6 +269,7 @@ class TMQCom: tdLog.debug("insert data ............ [OK]") return + # schema: (ts timestamp, c1 int, c2 int, c3 binary(16), c4 timestamp) def insert_data_2(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,ctbStartIdx=0): tdLog.debug("start to insert data ............") tsql.execute("use %s" %dbName) @@ -263,14 +280,17 @@ class TMQCom: startTs = int(round(t * 1000)) #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) for i in range(ctbNum): + rowsBatched = 0 sql += " %s%d values "%(ctbPrefix,i+ctbStartIdx) for j in range(rowsPerTbl): if (j % 2 == 0): sql += "(%d, %d, %d, 'tmqrow_%d', now) "%(startTs + j, j, j, j) else: sql += "(%d, %d, %d, 'tmqrow_%d', now) "%(startTs + j, j, -j, j) - if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + rowsBatched += 1 + if (rowsBatched == batchNum) or (j == rowsPerTbl - 1): tsql.execute(sql) + rowsBatched = 0 if j < rowsPerTbl - 1: sql = "insert into %s%d values " %(ctbPrefix,i+ctbStartIdx) else: @@ -282,7 +302,8 @@ class TMQCom: tdLog.debug("insert data ............ [OK]") return - def insert_data_interlaceByMultiTbl(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0): + # schema: (ts timestamp, c1 int, c2 bigint, c3 double, c4 binary(32), c5 nchar(32), c6 timestamp) tags (t1 int, t2 bigint, t3 double, t4 binary(32), t5 nchar(32)) + def insert_data_interlaceByMultiTbl(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0,ctbStartIdx=0): tdLog.debug("start to insert data ............") tsql.execute("use %s" %dbName) pre_insert = "insert into " @@ -297,15 +318,22 @@ class TMQCom: ctbDict[i] = 0 #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) - rowsOfCtb = 0 + rowsOfCtb = 0 while rowsOfCtb < rowsPerTbl: for i in range(ctbNum): - sql += " %s.%s_%d values "%(dbName,ctbPrefix,i) + sql += " %s.%s%d values "%(dbName,ctbPrefix,i+ctbStartIdx) + rowsBatched = 0 for k in range(batchNum): - sql += "(%d, %d, 'tmqrow_%d') "%(startTs + ctbDict[i], ctbDict[i], ctbDict[i]) + if (k % 2 == 0): + sql += "(%d, %d, %d, %d, 'binary_%d', 'nchar_%d', now) "%(startTs+ctbDict[i], ctbDict[i],ctbDict[i], ctbDict[i],i+ctbStartIdx,k) + else: + sql += "(%d, %d, %d, %d, 'binary_%d', 'nchar_%d', now) "%(startTs+ctbDict[i],-ctbDict[i],ctbDict[i],-ctbDict[i],i+ctbStartIdx,k) + + rowsBatched += 1 ctbDict[i] += 1 - if (0 == ctbDict[i]%batchNum) or (ctbDict[i] == rowsPerTbl): + if (rowsBatched == batchNum) or (ctbDict[i] == rowsPerTbl): tsql.execute(sql) + rowsBatched = 0 sql = "insert into " break rowsOfCtb = ctbDict[0] @@ -313,7 +341,18 @@ class TMQCom: tdLog.debug("insert data ............ [OK]") return - def insert_data_with_autoCreateTbl(self,tsql,dbName,stbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0): + def threadFunctionForInsertByInterlace(self, **paraDict): + # create new connector for new tdSql instance in my thread + newTdSql = tdCom.newTdSql() + self.insert_data_interlaceByMultiTbl(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"],paraDict["ctbStartIdx"]) + return + + def asyncInsertDataByInterlace(self, paraDict): + pThread = threading.Thread(target=self.threadFunctionForInsertByInterlace, kwargs=paraDict) + pThread.start() + return pThread + + def insert_data_with_autoCreateTbl(self,tsql,dbName,stbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0,ctbStartIdx=0): tdLog.debug("start to insert data wiht auto create child table ............") tsql.execute("use %s" %dbName) pre_insert = "insert into " @@ -324,17 +363,17 @@ class TMQCom: startTs = int(round(t * 1000)) #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) - rowsOfSql = 0 + rowsBatched = 0 for i in range(ctbNum): - sql += " %s.%s_%d using %s.%s tags (%d) values "%(dbName,ctbPrefix,i,dbName,stbName,i) + sql += " %s.%s_%d using %s.%s tags (%d) values "%(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i) for j in range(rowsPerTbl): sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) - rowsOfSql += 1 - if (j > 0) and ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)): + rowsBatched += 1 + if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): tsql.execute(sql) - rowsOfSql = 0 + rowsBatched = 0 if j < rowsPerTbl - 1: - sql = "insert into %s.%s_%d using %s.%s tags (%d) values " %(dbName,ctbPrefix,i,dbName,stbName,i) + sql = "insert into %s.%s_%d using %s.%s tags (%d) values " %(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i) else: sql = "insert into " #end sql diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index ae0e658344..c18474dcc3 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -16,7 +16,7 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: - def __int__(self): + def __init__(self): self.vgroups = 1 self.ctbNum = 10 self.rowsPerTbl = 10000 @@ -25,6 +25,49 @@ class TDTestCase: tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("restart taosd to ensure that the data falls into the disk") + tdDnodes.stop(1) + tdDnodes.start(1) + return + def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'dbt', @@ -34,10 +77,11 @@ class TDTestCase: 'stbName': 'stb', 'colPrefix': 'c', 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], 'ctbPrefix': 'ctb', - 'ctbNum': 1, + 'ctbStartIdx': 0, + 'ctbNum': 10, 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 @@ -53,16 +97,6 @@ class TDTestCase: topicNameList = ['topic1'] expectRowsList = [] tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdLog.info("create stb") - tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) - tdLog.info("create ctb") - tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) - tdLog.info("insert data") - tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) - - tdDnodes.stop(1) - tdDnodes.start(1) tdLog.info("create topics from stb with filter") queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) @@ -111,10 +145,11 @@ class TDTestCase: 'stbName': 'stb', 'colPrefix': 'c', 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], 'ctbPrefix': 'ctb', - 'ctbNum': 1, + 'ctbStartIdx': 0, + 'ctbNum': 10, 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 @@ -130,16 +165,6 @@ class TDTestCase: topicNameList = ['topic1'] expectRowsList = [] tmqCom.initConsumerTable() - # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - # tdLog.info("create stb") - # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) - # tdLog.info("create ctb") - # tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) - # tdLog.info("insert data") - # tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) - - # tdDnodes.stop(1) - # tdDnodes.start(1) tdLog.info("create topics from stb with filter") queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) @@ -200,89 +225,9 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 2 end ...... ") - def tmqCase3(self): - tdLog.printNoPrefix("======== test case 3: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 1, - 'stbName': 'stb', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbNum': 1, - 'rowsPerTbl': 10000, - 'batchNum': 10, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': -1, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 1} - - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - topicNameList = ['topic1'] - expectRowsList = [] - tmqCom.initConsumerTable() - # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - # tdLog.info("create stb") - # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) - # tdLog.info("create ctb") - # tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) - # tdLog.info("insert data") - # tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) - - # tdDnodes.stop(1) - # tdDnodes.start(1) - - tdLog.info("create topics from stb with filter") - queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) - # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - tdSql.query(queryString) - expectRowsList.append(tdSql.getRows()) - totalRowsInserted = expectRowsList[0] - - # init consume info, and start tmq_sim, then check consume result - tdLog.info("insert consume info to consume processor") - consumerId = 3 - expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3) - topicList = topicNameList[0] - ifcheckdata = 1 - ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - consumerId = 4 - expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3) - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor 0") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - - expectRows = 2 - resultList = tmqCom.selectConsumeResult(expectRows) - actConsumeTotalRows = resultList[0] + resultList[1] - - if not (totalRowsInserted == actConsumeTotalRows): - tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted)) - tdLog.exit("%d tmq consume rows error!"%consumerId) - - time.sleep(10) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 3 end ...... ") - def run(self): tdSql.prepare() + self.prepareTestEnv() self.tmqCase1() self.tmqCase2() diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1.py b/tests/system-test/7-tmq/tmqConsFromTsdb1.py new file mode 100644 index 0000000000..af06c774d6 --- /dev/null +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1.py @@ -0,0 +1,241 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("restart taosd to ensure that the data falls into the disk") + tdDnodes.stop(1) + tdDnodes.start(1) + return + + def tmqCase3(self): + tdLog.printNoPrefix("======== test case 3: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + totalRowsInserted = expectRowsList[0] + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 3 + expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3) + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + consumerId = 4 + expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3) + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor 0") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + actConsumeTotalRows = resultList[0] + resultList[1] + + if not (totalRowsInserted == actConsumeTotalRows): + tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted)) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 3 end ...... ") + + def tmqCase4(self): + tdLog.printNoPrefix("======== test case 4: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + totalRowsInserted = expectRowsList[0] + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 5 + expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor 0") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tdLog.info("wait commit notify") + tmqCom.getStartCommitNotifyFromTmqsim() + + tdLog.info("pkill consume processor") + tdCom.killProcessor("tmq_sim") + + # time.sleep(10) + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + consumerId = 6 + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor 1") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + actConsumeTotalRows = resultList[0] + + if not (actConsumeTotalRows > 0 and actConsumeTotalRows < totalRowsInserted): + tdLog.info("act consume rows: %d"%(actConsumeTotalRows)) + tdLog.info("and second consume rows should be between 0 and %d"%(totalRowsInserted)) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 4 end ...... ") + + def run(self): + tdSql.prepare() + self.prepareTestEnv() + self.tmqCase3() + self.tmqCase4() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index a4e191b36a..f65b33e11c 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -18,7 +18,8 @@ python3 ./test.py -f 0-others/fsync.py python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py -# BUG python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py +python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py +python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py python3 ./test.py -f 1-insert/alter_stable.py python3 ./test.py -f 1-insert/alter_table.py python3 ./test.py -f 1-insert/insertWithMoreVgroup.py @@ -89,7 +90,7 @@ python3 ./test.py -f 2-query/query_cols_tags_and_or.py # python3 ./test.py -f 2-query/nestedQuery_str.py python3 ./test.py -f 2-query/avg.py -python3 ./test.py -f 2-query/elapsed.py +#python3 ./test.py -f 2-query/elapsed.py python3 ./test.py -f 2-query/csum.py python3 ./test.py -f 2-query/mavg.py python3 ./test.py -f 2-query/diff.py @@ -121,7 +122,7 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 -# python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 5 -M 3 # python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 5 -M 3 @@ -154,3 +155,5 @@ python3 ./test.py -f 7-tmq/tmqUdf.py python3 ./test.py -f 7-tmq/tmqConsumerGroup.py python3 ./test.py -f 7-tmq/tmqShow.py python3 ./test.py -f 7-tmq/tmqAlterSchema.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 2d7f78661a..9596efdfc8 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -63,8 +63,9 @@ if __name__ == "__main__": mnodeNums = 0 updateCfgDict = {} execCmd = "" - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums']) + queryPolicy = 1 + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -82,6 +83,7 @@ if __name__ == "__main__": tdLog.printNoPrefix('-e eval str to run') tdLog.printNoPrefix('-N create dnodes numbers in clusters') tdLog.printNoPrefix('-M create mnode numbers in clusters') + tdLog.printNoPrefix('-Q set queryPolicy in one dnode') sys.exit(0) @@ -138,6 +140,9 @@ if __name__ == "__main__": if key in ['-M', '--mnodeNums']: mnodeNums = value + if key in ['-Q', '--queryPolicy']: + queryPolicy = value + if not execCmd == "": tdDnodes.init(deployPath) print(execCmd) @@ -276,6 +281,22 @@ if __name__ == "__main__": tdDnodes.deploy(1,updateCfgDict) tdDnodes.start(1) tdCases.logSql(logSql) + if queryPolicy != 1: + queryPolicy=int(queryPolicy) + conn = taos.connect( + host, + config=tdDnodes.getSimCfgPath()) + tdSql.init(conn.cursor()) + tdSql.execute("create qnode on dnode 1") + tdSql.execute('alter local "queryPolicy" "%d"'%queryPolicy) + tdSql.query("show local variables;") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "queryPolicy" : + if int(tdSql.queryResult[i][1]) == int(queryPolicy): + tdLog.success('alter queryPolicy to %d successfully'%queryPolicy) + else : + tdLog.debug(tdSql.queryResult) + tdLog.exit("alter queryPolicy to %d failed"%queryPolicy) else : tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums) diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 3f2a81197b..85fe456670 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -90,6 +90,7 @@ typedef struct { int32_t consumeDelay; // unit s int32_t numOfThread; int32_t useSnapshot; + int64_t nowTime; SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; } SConfInfo; @@ -198,6 +199,8 @@ void parseArgument(int32_t argc, char* argv[]) { g_stConfInfo.saveRowFlag = 0; g_stConfInfo.consumeDelay = 5; + g_stConfInfo.nowTime = taosGetTimestampMs(); + for (int32_t i = 1; i < argc; i++) { if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { printHelp(); @@ -510,10 +513,8 @@ static void appNothing(void* param, TAOS_RES* res, int32_t numOfRows) {} int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) { char sqlStr[1024] = {0}; - int64_t now = taosGetTimestampMs(); - // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int - sprintf(sqlStr, "insert into %s.notifyinfo values (%" PRId64 ", %d, %d)", g_stConfInfo.cdbName, now, cmdId, + sprintf(sqlStr, "insert into %s.notifyinfo values (%" PRId64 ", %d, %d)", g_stConfInfo.cdbName, atomic_fetch_add_64(&g_stConfInfo.nowTime, 1), cmdId, pInfo->consumerId); taos_query_a(pInfo->taos, sqlStr, appNothing, NULL); @@ -525,15 +526,15 @@ int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) { static int32_t g_once_commit_flag = 0; static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { - pError("tmq_commit_cb_print() commit %d\n", code); + pError("tmq_commit_cb_print() commit %d\n", code); - if (0 == g_once_commit_flag) { - g_once_commit_flag = 1; - notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT); + if (0 == g_once_commit_flag) { + g_once_commit_flag = 1; + notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT); } - char tmpString[128]; - taosFprintfFile(g_fp, "%s tmq_commit_cb_print() be called\n", getCurrentTimeString(tmpString)); + char tmpString[128]; + taosFprintfFile(g_fp, "%s tmq_commit_cb_print() be called\n", getCurrentTimeString(tmpString)); } void build_consumer(SThreadInfo* pInfo) { @@ -588,12 +589,10 @@ void build_topic_list(SThreadInfo* pInfo) { int32_t saveConsumeResult(SThreadInfo* pInfo) { char sqlStr[1024] = {0}; - - int64_t now = taosGetTimestampMs(); - // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int sprintf(sqlStr, "insert into %s.consumeresult values (%" PRId64 ", %d, %" PRId64 ", %" PRId64 ", %d)", - g_stConfInfo.cdbName, now, pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult); + g_stConfInfo.cdbName, atomic_fetch_add_64(&g_stConfInfo.nowTime, 1), pInfo->consumerId, pInfo->consumeMsgCnt, + pInfo->consumeRowCnt, pInfo->checkresult); char tmpString[128]; taosFprintfFile(g_fp, "%s, consume id %d result: %s\n", getCurrentTimeString(tmpString), pInfo->consumerId, sqlStr); @@ -637,9 +636,9 @@ void loop_consume(SThreadInfo* pInfo) { } } - int64_t lastTotalMsgs = 0; - uint64_t lastPrintTime = taosGetTimestampMs(); - uint64_t startTs = taosGetTimestampMs(); + int64_t lastTotalMsgs = 0; + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); int32_t consumeDelay = g_stConfInfo.consumeDelay == -1 ? -1 : (g_stConfInfo.consumeDelay * 1000); while (running) { @@ -652,16 +651,16 @@ void loop_consume(SThreadInfo* pInfo) { taos_free_result(tmqMsg); totalMsgs++; - - int64_t currentPrintTime = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 10 * 1000) { - taosFprintfFile(g_fp, - "consumer id %d has currently poll total msgs: %" PRId64 ", period rate: %.3f msgs/second\n", - pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0/(currentPrintTime - lastPrintTime)); - lastPrintTime = currentPrintTime; - lastTotalMsgs = totalMsgs; - } - + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 10 * 1000) { + taosFprintfFile( + g_fp, "consumer id %d has currently poll total msgs: %" PRId64 ", period rate: %.3f msgs/second\n", + pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0 / (currentPrintTime - lastPrintTime)); + lastPrintTime = currentPrintTime; + lastTotalMsgs = totalMsgs; + } + if (0 == once_flag) { once_flag = 1; notifyMainScript(pInfo, NOTIFY_CMD_START_CONSUM); @@ -678,7 +677,7 @@ void loop_consume(SThreadInfo* pInfo) { break; } } - + if (0 == running) { taosFprintfFile(g_fp, "receive stop signal and not continue consume\n"); } @@ -696,6 +695,7 @@ void* consumeThreadFunc(void* param) { pInfo->taos = taos_connect(NULL, "root", "taosdata", NULL, 0); if (pInfo->taos == NULL) { taosFprintfFile(g_fp, "taos_connect() fail, can not notify and save consume result to main scripte\n"); + ASSERT(0); return NULL; } @@ -888,11 +888,11 @@ int main(int32_t argc, char* argv[]) { int64_t t = end - start; if (0 == t) t = 1; - + double tInMs = (double)t / 1000000.0; taosFprintfFile(g_fp, - "Spent %.3f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/second\n\n", - tInMs, totalMsgs, g_stConfInfo.numOfThread, (double)(totalMsgs / tInMs)); + "Spent %.3f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/second\n\n", + tInMs, totalMsgs, g_stConfInfo.numOfThread, (double)(totalMsgs / tInMs)); taosFprintfFile(g_fp, "==== close tmqlog ====\n"); taosCloseFile(&g_fp);