diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index a0e50a02d3..0ff1371618 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 9cfe416 + GIT_TAG 7920f98 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index db2ae92f6e..aecb8a1750 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 1e15545 + GIT_TAG 0111c66 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/zh/14-reference/07-tdinsight/index.mdx b/docs/zh/14-reference/07-tdinsight/index.mdx index 8ec11378ee..319fa8df53 100644 --- a/docs/zh/14-reference/07-tdinsight/index.mdx +++ b/docs/zh/14-reference/07-tdinsight/index.mdx @@ -140,10 +140,10 @@ Install and configure TDinsight dashboard in Grafana on Ubuntu 18.04/20.04 syste | -n | --tdengine-ds-name | TDENGINE_DS_NAME | TDengine 数据源名称，默认为 TDengine。 | | -a | --tdengine-api | TDENGINE_API | TDengine REST API 端点。默认为`http://127.0.0.1:6041`。 | | -u | --tdengine-user | TDENGINE_USER | TDengine 用户名。 [默认值：root] | -| -p | --tdengine-密码 | TDENGINE_PASSWORD | TDengine 密码。 [默认：taosdata] | +| -p | --tdengine-password | TDENGINE_PASSWORD | TDengine 密码。 [默认：taosdata] | | -i | --tdinsight-uid | TDINSIGHT_DASHBOARD_UID | TDinsight 仪表盘`uid`。 [默认值：tdinsight] | | -t | --tdinsight-title | TDINSIGHT_DASHBOARD_TITLE | TDinsight 仪表盘标题。 [默认：TDinsight] | -| -e | --tdinsight-可编辑 | TDINSIGHT_DASHBOARD_EDITABLE | 如果配置仪表盘可以编辑。 [默认值：false] | +| -e | --tdinsight-editable | TDINSIGHT_DASHBOARD_EDITABLE | 如果配置仪表盘可以编辑。 [默认值：false] | | -E | --external-notifier | EXTERNAL_NOTIFIER | 将外部通知程序 uid 应用于 TDinsight 仪表盘。 | 假设您在主机 `tdengine` 上启动 TDengine 数据库，HTTP API 端口为 `6041`，用户为 `root1`，密码为 `pass5ord`。执行脚本： diff --git a/include/common/tmsg.h b/include/common/tmsg.h index cd63f7d278..6ca36e7433 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1812,7 +1812,7 @@ typedef struct { #define STREAM_TRIGGER_AT_ONCE 1 #define STREAM_TRIGGER_WINDOW_CLOSE 2 #define STREAM_TRIGGER_MAX_DELAY 3 -#define STREAM_DEFAULT_IGNORE_EXPIRED 0 +#define STREAM_DEFAULT_IGNORE_EXPIRED 1 #define STREAM_FILL_HISTORY_ON 1 #define STREAM_FILL_HISTORY_OFF 0 #define STREAM_DEFAULT_FILL_HISTORY STREAM_FILL_HISTORY_OFF @@ -2684,7 +2684,7 @@ typedef struct { char subKey[TSDB_SUBSCRIBE_KEY_LEN]; int8_t subType; int8_t withMeta; - char* qmsg; + char* qmsg; //SubPlanToString int64_t suid; } SMqRebVgReq; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 75860a4b1e..5106196ccd 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -116,6 +116,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x012B) #define TSDB_CODE_TIMEOUT_ERROR TAOS_DEF_ERROR_CODE(0, 0x012C) #define TSDB_CODE_MSG_ENCODE_ERROR TAOS_DEF_ERROR_CODE(0, 0x012D) +#define TSDB_CODE_NO_ENOUGH_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x012E) #define TSDB_CODE_APP_IS_STARTING TAOS_DEF_ERROR_CODE(0, 0x0130) // #define TSDB_CODE_APP_IS_STOPPING TAOS_DEF_ERROR_CODE(0, 0x0131) // diff --git a/include/util/ttrace.h b/include/util/ttrace.h index 5cdb1eecaa..e672882f57 100644 --- a/include/util/ttrace.h +++ b/include/util/ttrace.h @@ -59,11 +59,13 @@ typedef struct STraceId { char* _t = _buf; \ _t[0] = '0'; \ _t[1] = 'x'; \ - _t += titoa(rootId, 16, &_t[2]); \ + _t += 2; \ + _t += titoa(rootId, 16, &_t[0]); \ _t[0] = ':'; \ _t[1] = '0'; \ _t[2] = 'x'; \ - _t += titoa(msgId, 16, &_t[3]); \ + _t += 3; \ + _t += titoa(msgId, 16, &_t[0]); \ } while (0) #ifdef __cplusplus diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 5aeff0e2fa..2495e177e1 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -34,6 +34,7 @@ benchmarkName="taosBenchmark" dumpName="taosdump" demoName="taosdemo" xname="taosx" +explorerName="${clientName}-explorer" clientName2="taos" serverName2="taosd" @@ -214,6 +215,7 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/${benchmarkName} || : ${csudo}rm -f ${bin_link_dir}/${dumpName} || : ${csudo}rm -f ${bin_link_dir}/${xname} || : + ${csudo}rm -f ${bin_link_dir}/${explorerName} || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : @@ -228,6 +230,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : [ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -s ${install_main_dir}/bin/${xname} ${bin_link_dir}/${xname} || : + [ -x ${install_main_dir}/bin/${explorerName} ] && ${csudo}ln -s ${install_main_dir}/bin/${explorerName} ${bin_link_dir}/${explorerName} || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat index f5ed1cdf66..6dc7c356cd 100644 --- a/packaging/tools/make_install.bat +++ b/packaging/tools/make_install.bat @@ -70,6 +70,9 @@ if %Enterprise% == TRUE ( if exist %binary_dir%\\build\\bin\\taosx.exe ( copy %binary_dir%\\build\\bin\\taosx.exe %target_dir% > nul ) + if exist %binary_dir%\\build\\bin\\taos-explorer.exe ( + copy %binary_dir%\\build\\bin\\taos-explorer.exe %target_dir% > nul + ) if exist %binary_dir%\\build\\bin\\tmq_sim.exe ( copy %binary_dir%\\build\\bin\\tmq_sim.exe %target_dir% > nul ) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 0a34d81b7f..7ad3cf7b0a 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -89,11 +89,13 @@ else ${build_dir}/bin/tdengine-datasource.zip \ ${build_dir}/bin/tdengine-datasource.zip.md5sum" [ -f ${build_dir}/bin/taosx ] && taosx_bin="${build_dir}/bin/taosx" + explorer_bin_files=$(find ${build_dir}/bin/ -name '*-explorer') bin_files="${build_dir}/bin/${serverName} \ ${build_dir}/bin/${clientName} \ ${taostools_bin_files} \ ${taosx_bin} \ + ${explorer_bin_files} \ ${build_dir}/bin/taosadapter \ ${build_dir}/bin/udfd \ ${script_dir}/remove.sh \ diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 9c50c4582d..2479e48670 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -119,6 +119,7 @@ function clean_bin() { ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : ${csudo}rm -f ${bin_link_dir}/taoskeeper || : ${csudo}rm -f ${bin_link_dir}/taosx || : + ${csudo}rm -f ${bin_link_dir}/taos-explorer || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then ${csudo}rm -f ${bin_link_dir}/${clientName2} || : diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 7cc7a1717a..b10daa9c21 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -66,7 +66,8 @@ enum { typedef struct SAppInstInfo SAppInstInfo; typedef struct { - char* key; + char* key; + int32_t idx; // statistics int32_t reportCnt; int32_t connKeyCnt; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index b5838386db..c9c2e7a5f8 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -303,8 +303,12 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { } static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { + if (0 == atomic_load_8(&clientHbMgr.inited)) { + goto _return; + } + static int32_t emptyRspNum = 0; - char *key = (char *)param; + int32_t idx = *(int32_t *)param; SClientHbBatchRsp pRsp = {0}; if (TSDB_CODE_SUCCESS == code) { tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp); @@ -319,22 +323,24 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { int32_t rspNum = taosArrayGetSize(pRsp.rsps); - taosThreadMutexLock(&appInfo.mutex); + taosThreadMutexLock(&clientHbMgr.lock); - SAppInstInfo **pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); - if (pInst == NULL || NULL == *pInst) { - taosThreadMutexUnlock(&appInfo.mutex); - tscError("cluster not exist, key:%s", key); + SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, idx); + if (pAppHbMgr == NULL) { + taosThreadMutexUnlock(&clientHbMgr.lock); + tscError("appHbMgr not exist, idx:%d", idx); taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); tFreeClientHbBatchRsp(&pRsp); return -1; } + SAppInstInfo *pInst = pAppHbMgr->pAppInstInfo; + if (code != 0) { - (*pInst)->onlineDnodes = ((*pInst)->totalDnodes ? 0 : -1); - tscDebug("hb rsp error %s, update server status %d/%d", tstrerror(code), (*pInst)->onlineDnodes, - (*pInst)->totalDnodes); + pInst->onlineDnodes = pInst->totalDnodes ? 0 : -1; + tscDebug("hb rsp error %s, update server status %d/%d", tstrerror(code), pInst->onlineDnodes, + pInst->totalDnodes); } if (rspNum) { @@ -346,15 +352,17 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { for (int32_t i = 0; i < rspNum; ++i) { SClientHbRsp *rsp = taosArrayGet(pRsp.rsps, i); - code = (*clientHbMgr.rspHandle[rsp->connKey.connType])((*pInst)->pAppHbMgr, rsp); + code = (*clientHbMgr.rspHandle[rsp->connKey.connType])(pAppHbMgr, rsp); if (code) { break; } } - taosThreadMutexUnlock(&appInfo.mutex); + taosThreadMutexUnlock(&clientHbMgr.lock); tFreeClientHbBatchRsp(&pRsp); + +_return: taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); return code; @@ -788,7 +796,8 @@ static void *hbThreadFunc(void *param) { pInfo->msgInfo.pData = buf; pInfo->msgInfo.len = tlen; pInfo->msgType = TDMT_MND_HEARTBEAT; - pInfo->param = taosStrdup(pAppHbMgr->key); + pInfo->param = taosMemoryMalloc(sizeof(int32_t)); + *(int32_t *)pInfo->param = i; pInfo->paramFreeFp = taosMemoryFree; pInfo->requestId = generateRequestId(); pInfo->requestObjRefId = 0; @@ -874,6 +883,7 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { taosThreadMutexLock(&clientHbMgr.lock); taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr); + pAppHbMgr->idx = taosArrayGetSize(clientHbMgr.appHbMgrs) - 1; taosThreadMutexUnlock(&clientHbMgr.lock); return pAppHbMgr; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index c1b3cde9ea..7aa1c9f56a 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -192,8 +192,8 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp taosWriteQitem(pVnode->pFetchQ, pMsg); break; case WRITE_QUEUE: - if (!osDataSpaceAvailable()) { - terrno = TSDB_CODE_NO_DISKSPACE; + if (!osDataSpaceSufficient()) { + terrno = TSDB_CODE_NO_ENOUGH_DISKSPACE; code = terrno; dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code)); break; diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 25a4397b7d..493ba48601 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -2,8 +2,9 @@ aux_source_directory(src MNODE_SRC) IF (TD_PRIVILEGE) ADD_DEFINITIONS(-D_PRIVILEGE) ENDIF () -IF (TD_PRIVILEGE) +IF (TD_ENTERPRISE) LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/privilege/src/privilege.c) + LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndDb.c) ENDIF () add_library(mnode STATIC ${MNODE_SRC}) diff --git a/source/dnode/mnode/impl/inc/mndDb.h b/source/dnode/mnode/impl/inc/mndDb.h index 9edfd9bf3b..97d047d7a3 100644 --- a/source/dnode/mnode/impl/inc/mndDb.h +++ b/source/dnode/mnode/impl/inc/mndDb.h @@ -33,6 +33,8 @@ bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb); SSdbRaw *mndDbActionEncode(SDbObj *pDb); const char *mndGetDbStr(const char *src); +int32_t mndProcessCompactDbReq(SRpcMsg *pReq); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index ac0c68f652..345418801b 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -542,7 +542,7 @@ void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer typedef struct { int32_t vgId; - char* qmsg; + char* qmsg; //SubPlanToString SEpSet epSet; } SMqVgEp; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 3efd8fb249..5de76a9d46 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -41,12 +41,15 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq); static int32_t mndProcessAlterDbReq(SRpcMsg *pReq); static int32_t mndProcessDropDbReq(SRpcMsg *pReq); static int32_t mndProcessUseDbReq(SRpcMsg *pReq); -static int32_t mndProcessCompactDbReq(SRpcMsg *pReq); static int32_t mndProcessTrimDbReq(SRpcMsg *pReq); static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq); +#ifndef TD_ENTERPRISE +int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { return TSDB_CODE_OPS_NOT_SUPPORT; } +#endif + int32_t mndInitDb(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_DB, @@ -1395,98 +1398,6 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, return 0; } -static int32_t mndSetCompactDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, int64_t compactTs) { - SDbObj dbObj = {0}; - memcpy(&dbObj, pDb, sizeof(SDbObj)); - dbObj.compactStartTime = compactTs; - - SSdbRaw *pCommitRaw = mndDbActionEncode(&dbObj); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - sdbFreeRaw(pCommitRaw); - return -1; - } - - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - return 0; -} - -static int32_t mndSetCompactDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, int64_t compactTs) { - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - - while (1) { - SVgObj *pVgroup = NULL; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - if (pIter == NULL) break; - - if (mndVgroupInDb(pVgroup, pDb->uid)) { - if (mndBuildCompactVgroupAction(pMnode, pTrans, pDb, pVgroup, compactTs) != 0) { - sdbCancelFetch(pSdb, pIter); - sdbRelease(pSdb, pVgroup); - return -1; - } - } - - sdbRelease(pSdb, pVgroup); - } - - return 0; -} - -static int32_t mndCompactDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { - int64_t compactTs = taosGetTimestampMs(); - int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "compact-db"); - if (pTrans == NULL) goto _OVER; - - mInfo("trans:%d, used to compact db:%s", pTrans->id, pDb->name); - mndTransSetDbName(pTrans, pDb->name, NULL); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; - if (mndSetCompactDbCommitLogs(pMnode, pTrans, pDb, compactTs) != 0) goto _OVER; - if (mndSetCompactDbRedoActions(pMnode, pTrans, pDb, compactTs) != 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - code = 0; - -_OVER: - mndTransDrop(pTrans); - return code; -} - -static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - int32_t code = -1; - SDbObj *pDb = NULL; - SCompactDbReq compactReq = {0}; - - if (tDeserializeSCompactDbReq(pReq->pCont, pReq->contLen, &compactReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } - - mInfo("db:%s, start to compact", compactReq.db); - - pDb = mndAcquireDb(pMnode, compactReq.db); - if (pDb == NULL) { - goto _OVER; - } - - if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB, pDb) != 0) { - goto _OVER; - } - - code = mndCompactDb(pMnode, pReq, pDb); - if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; - -_OVER: - if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("db:%s, failed to process compact db req since %s", compactReq.db, terrstr()); - } - - mndReleaseDb(pMnode, pDb); - return code; -} - static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 53a5548b2f..b09a4f63a7 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -344,8 +344,8 @@ static int32_t mndInitWal(SMnode *pMnode) { .fsyncPeriod = 0, .rollPeriod = -1, .segSize = -1, - .retentionPeriod = -1, - .retentionSize = -1, + .retentionPeriod = 0, + .retentionSize = 0, .level = TAOS_WAL_FSYNC, }; @@ -370,7 +370,6 @@ static int32_t mndInitSdb(SMnode *pMnode) { opt.path = pMnode->path; opt.pMnode = pMnode; opt.pWal = pMnode->pWal; - opt.sync = pMnode->syncMgmt.sync; pMnode->pSdb = sdbInit(&opt); if (pMnode->pSdb == NULL) { @@ -552,16 +551,7 @@ void mndPreClose(SMnode *pMnode) { if (pMnode != NULL) { syncLeaderTransfer(pMnode->syncMgmt.sync); syncPreStop(pMnode->syncMgmt.sync); -#if 0 - while (syncSnapshotRecving(pMnode->syncMgmt.sync)) { - mInfo("vgId:1, snapshot is recving"); - taosMsleep(300); - } - while (syncSnapshotSending(pMnode->syncMgmt.sync)) { - mInfo("vgId:1, snapshot is sending"); - taosMsleep(300); - } -#endif + sdbWriteFile(pMnode->pSdb, 0); } } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 2a05511134..b7f80f6b0e 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1223,7 +1223,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock // node id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - int32_t nodeId = TMAX(pTask->nodeId, 0); + int64_t nodeId = TMAX(pTask->nodeId, 0); colDataSetVal(pColInfo, numOfRows, (const char *)&nodeId, false); // level diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index f618b8afae..edd75c62b9 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -118,12 +118,12 @@ int32_t mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta transId, pTrans->createdTime, pMgmt->transId); mndTransExecute(pMnode, pTrans, false); mndReleaseTrans(pMnode, pTrans); - // sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA); } else { mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); } } + sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA); return 0; } @@ -319,6 +319,7 @@ int32_t mndInitSync(SMnode *pMnode) { mError("failed to open sync since %s", terrstr()); return -1; } + pMnode->pSdb->sync = pMgmt->sync; mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync); return 0; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index a34dfff4d6..55e9faf020 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1645,8 +1645,6 @@ void mndTransPullup(SMnode *pMnode) { } mndReleaseTrans(pMnode, pTrans); } - - sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA); taosArrayDestroy(pArray); } diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index 8d2cec478c..1b7b2f9672 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -37,7 +37,7 @@ extern "C" { #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }} // clang-format on -#define SDB_WRITE_DELTA 20 +#define SDB_WRITE_DELTA 2000 #define SDB_GET_VAL(pData, dataPos, val, pos, func, type) \ { \ diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index bb8040da07..9797dd8337 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -53,7 +53,6 @@ SSdb *sdbInit(SSdbOpt *pOption) { } pSdb->pWal = pOption->pWal; - pSdb->sync = pOption->sync; pSdb->applyIndex = -1; pSdb->applyTerm = -1; pSdb->applyConfig = -1; diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index c2d27ad713..2e182ec10b 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -472,10 +472,7 @@ int32_t sdbWriteFile(SSdb *pSdb, int32_t delta) { taosThreadMutexLock(&pSdb->filelock); if (pSdb->pWal != NULL) { - // code = walBeginSnapshot(pSdb->pWal, pSdb->applyIndex, 0); - if (pSdb->sync == 0) { - code = 0; - } else { + if (pSdb->sync > 0) { code = syncBeginSnapshot(pSdb->sync, pSdb->applyIndex); } } @@ -484,11 +481,7 @@ int32_t sdbWriteFile(SSdb *pSdb, int32_t delta) { } if (code == 0) { if (pSdb->pWal != NULL) { - // code = walEndSnapshot(pSdb->pWal); - - if (pSdb->sync == 0) { - code = 0; - } else { + if (pSdb->sync > 0) { code = syncEndSnapshot(pSdb->sync); } } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index ea7046886e..8dc3f46ae3 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -14,7 +14,7 @@ target_sources( "src/vnd/vnodeSvr.c" "src/vnd/vnodeSync.c" "src/vnd/vnodeSnapshot.c" - "src/vnd/vnodeCompact.c" + "src/vnd/vnodeRetention.c" # meta "src/meta/metaOpen.c" @@ -52,7 +52,6 @@ target_sources( "src/tsdb/tsdbCacheRead.c" "src/tsdb/tsdbRetention.c" "src/tsdb/tsdbDiskData.c" - "src/tsdb/tsdbCompact.c" "src/tsdb/tsdbMergeTree.c" "src/tsdb/tsdbDataIter.c" @@ -68,10 +67,20 @@ target_sources( "src/tq/tqSnapshot.c" "src/tq/tqOffsetSnapshot.c" ) + +IF (TD_VNODE_PLUGINS) + target_sources( + vnode + PRIVATE + ${TD_ENTERPRISE_DIR}/src/plugins/vnode/src/tsdbCompact.c + ${TD_ENTERPRISE_DIR}/src/plugins/vnode/src/vnodeCompact.c + ) +ENDIF () + target_include_directories( vnode PUBLIC "inc" - PRIVATE "src/inc" + PUBLIC "src/inc" PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar" ) target_link_libraries( diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 104e0945ba..792fed2309 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -67,7 +67,7 @@ typedef struct { // tqExec typedef struct { - char* qmsg; + char* qmsg; // SubPlanToString } STqExecCol; typedef struct { diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 88cd1d99e1..134909090f 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -106,10 +106,6 @@ int32_t vnodeSyncCommit(SVnode* pVnode); int32_t vnodeAsyncCommit(SVnode* pVnode); bool vnodeShouldRollback(SVnode* pVnode); -// vnodeCompact.c -int32_t vnodeAsyncCompact(SVnode* pVnode); -int32_t vnodeSyncCompact(SVnode* pVnode); - // vnodeSync.c int32_t vnodeSyncOpen(SVnode* pVnode, char* path); int32_t vnodeSyncStart(SVnode* pVnode); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 93e611e870..c0d017e350 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -180,7 +180,6 @@ int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo); int32_t tsdbFinishCommit(STsdb* pTsdb); int32_t tsdbRollbackCommit(STsdb* pTsdb); -int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 37ae7d895e..99e171dde1 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -595,6 +595,7 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq2 *pMsg, STbUidStore *pStore) { return 0; } +#if 0 /** * @brief retention of rsma1/rsma2 * @@ -618,6 +619,7 @@ int32_t smaDoRetention(SSma *pSma, int64_t now) { _end: return code; } +#endif static void tdBlockDataDestroy(SArray *pBlockArr) { for (int32_t i = 0; i < taosArrayGetSize(pBlockArr); ++i) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c465617975..d07933c9f1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -575,7 +575,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } -#if 1 // till now, all data has been rsp to consumer, new data needs to push client once arrived. if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && dataRsp.reqOffset.version == dataRsp.rspOffset.version) { @@ -597,7 +596,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return 0; } } -#endif taosWUnLockLatch(&pTq->pushLock); if (tqSendDataRsp(pTq, pMsg, &req, &dataRsp) < 0) { @@ -613,10 +611,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } // for taosx - /*A(pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN);*/ - SMqMetaRsp metaRsp = {0}; - STaosxRsp taosxRsp = {0}; tqInitTaosxRsp(&taosxRsp, &req); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 5352ebe6d4..ee00cdc205 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -307,10 +307,10 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { while (1) { if (!fromProcessedMsg) { - SWalReader* pWalReader = pReader->pWalReader; - - if (walNextValidMsg(pWalReader) < 0) { - pReader->ver = pWalReader->curVersion - (pWalReader->curInvalid | pWalReader->curStopped); + if (walNextValidMsg(pReader->pWalReader) < 0) { + pReader->ver = + pReader->pWalReader->curVersion - pReader->pWalReader->curStopped; +// pReader->pWalReader->curVersion - (pReader->pWalReader->curInvalid | pReader->pWalReader->curStopped); ret->offset.type = TMQ_OFFSET__LOG; ret->offset.version = pReader->ver; ret->fetchType = FETCH_TYPE__NONE; @@ -318,25 +318,11 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { return -1; } - void* body = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); - int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); - int64_t ver = pWalReader->pHead->head.version; + void* body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); + int32_t bodyLen = pReader->pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); + int64_t ver = pReader->pWalReader->pHead->head.version; - tqDebug("tmq poll: extract submit msg from wal, version:%"PRId64" len:%d", ver, bodyLen); - -#if 0 - if (pWalReader->pHead->head.msgType != TDMT_VND_SUBMIT) { - // TODO do filter - ret->fetchType = FETCH_TYPE__META; - ret->meta = pWalReader->pHead->head.body; - return 0; - } else { -#endif tqReaderSetSubmitReq2(pReader, body, bodyLen, ver); - /*tqReaderSetDataMsg(pReader, body, pWalReader->pHead->head.version);*/ -#if 0 - } -#endif } while (tqNextDataBlock2(pReader)) { diff --git a/source/dnode/vnode/src/tsdb/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c deleted file mode 100644 index fc7df98217..0000000000 --- a/source/dnode/vnode/src/tsdb/tsdbCompact.c +++ /dev/null @@ -1,667 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tsdb.h" - -extern int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo); -extern int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg); -extern int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg); - -typedef struct { - STsdb *pTsdb; - int64_t commitID; - int8_t cmprAlg; - int32_t maxRows; - int32_t minRows; - - STsdbFS fs; - - int32_t fid; - TABLEID tbid; - SSkmInfo tbSkm; - - // Tombstone - SDelFReader *pDelFReader; - SArray *aDelIdx; // SArray - SArray *aDelData; // SArray - SArray *aSkyLine; // SArray - int32_t iDelIdx; - int32_t iSkyLine; - TSDBKEY *pDKey; - TSDBKEY dKey; - - // Reader - SDataFReader *pReader; - STsdbDataIter2 *iterList; // list of iterators - STsdbDataIter2 *pIter; - SRBTree rbt; - - // Writer - SDataFWriter *pWriter; - SArray *aBlockIdx; // SArray - SMapData mDataBlk; // SMapData - SArray *aSttBlk; // SArray - SBlockData bData; - SBlockData sData; -} STsdbCompactor; - -static int32_t tsdbCommitCompact(STsdbCompactor *pCompactor) { - int32_t code = 0; - int32_t lino = 0; - - STsdb *pTsdb = pCompactor->pTsdb; - - code = tsdbFSPrepareCommit(pTsdb, &pCompactor->fs); - TSDB_CHECK_CODE(code, lino, _exit); - - taosThreadRwlockWrlock(&pTsdb->rwLock); - - code = tsdbFSCommit(pTsdb); - if (code) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - TSDB_CHECK_CODE(code, lino, _exit); - } - - taosThreadRwlockUnlock(&pTsdb->rwLock); - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tsdbAbortCompact(STsdbCompactor *pCompactor) { - int32_t code = 0; - int32_t lino = 0; - - STsdb *pTsdb = pCompactor->pTsdb; - code = tsdbFSRollback(pTsdb); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); - } - return code; -} - -static int32_t tsdbCompactWriteTableDataStart(STsdbCompactor *pCompactor, TABLEID *pId) { - int32_t code = 0; - int32_t lino = 0; - - pCompactor->tbid = *pId; - - // tombstone - for (;;) { - if (pCompactor->iDelIdx >= taosArrayGetSize(pCompactor->aDelIdx)) { - pCompactor->pDKey = NULL; - break; - } - - SDelIdx *pDelIdx = (SDelIdx *)taosArrayGet(pCompactor->aDelIdx, pCompactor->iDelIdx); - int32_t c = tTABLEIDCmprFn(pDelIdx, &pCompactor->tbid); - if (c < 0) { - pCompactor->iDelIdx++; - } else if (c == 0) { - pCompactor->iDelIdx++; - - code = tsdbReadDelData(pCompactor->pDelFReader, pDelIdx, pCompactor->aDelData); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbBuildDeleteSkyline(pCompactor->aDelData, 0, taosArrayGetSize(pCompactor->aDelData) - 1, - pCompactor->aSkyLine); - TSDB_CHECK_CODE(code, lino, _exit); - - pCompactor->iSkyLine = 0; - if (pCompactor->iSkyLine < taosArrayGetSize(pCompactor->aSkyLine)) { - TSDBKEY *pKey = (TSDBKEY *)taosArrayGet(pCompactor->aSkyLine, pCompactor->iSkyLine); - - pCompactor->dKey.version = 0; - pCompactor->dKey.ts = pKey->ts; - pCompactor->pDKey = &pCompactor->dKey; - } else { - pCompactor->pDKey = NULL; - } - break; - } else { - pCompactor->pDKey = NULL; - break; - } - } - - // writer - code = tsdbUpdateTableSchema(pCompactor->pTsdb->pVnode->pMeta, pId->suid, pId->uid, &pCompactor->tbSkm); - TSDB_CHECK_CODE(code, lino, _exit); - - tMapDataReset(&pCompactor->mDataBlk); - - code = tBlockDataInit(&pCompactor->bData, pId, pCompactor->tbSkm.pTSchema, NULL, 0); - TSDB_CHECK_CODE(code, lino, _exit); - - if (!TABLE_SAME_SCHEMA(pCompactor->sData.suid, pCompactor->sData.uid, pId->suid, pId->uid)) { - if (pCompactor->sData.nRow > 0) { - code = tsdbWriteSttBlock(pCompactor->pWriter, &pCompactor->sData, pCompactor->aSttBlk, pCompactor->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - - TABLEID tbid = {.suid = pId->suid, .uid = pId->suid ? 0 : pId->uid}; - code = tBlockDataInit(&pCompactor->sData, &tbid, pCompactor->tbSkm.pTSchema, NULL, 0); - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } else { - tsdbDebug("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64, TD_VID(pCompactor->pTsdb->pVnode), __func__, pId->suid, - pId->uid); - } - return code; -} - -static int32_t tsdbCompactWriteTableDataEnd(STsdbCompactor *pCompactor) { - int32_t code = 0; - int32_t lino = 0; - - if (pCompactor->bData.nRow > 0) { - if (pCompactor->bData.nRow < pCompactor->minRows) { - for (int32_t iRow = 0; iRow < pCompactor->bData.nRow; iRow++) { - code = tBlockDataAppendRow(&pCompactor->sData, &tsdbRowFromBlockData(&pCompactor->bData, iRow), NULL, - pCompactor->tbid.uid); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pCompactor->sData.nRow >= pCompactor->maxRows) { - code = tsdbWriteSttBlock(pCompactor->pWriter, &pCompactor->sData, pCompactor->aSttBlk, pCompactor->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - tBlockDataClear(&pCompactor->bData); - } else { - code = tsdbWriteDataBlock(pCompactor->pWriter, &pCompactor->bData, &pCompactor->mDataBlk, pCompactor->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - if (pCompactor->mDataBlk.nItem > 0) { - SBlockIdx *pBlockIdx = (SBlockIdx *)taosArrayReserve(pCompactor->aBlockIdx, 1); - if (pBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - pBlockIdx->suid = pCompactor->tbid.suid; - pBlockIdx->uid = pCompactor->tbid.uid; - - code = tsdbWriteDataBlk(pCompactor->pWriter, &pCompactor->mDataBlk, pBlockIdx); - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } else { - tsdbDebug("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64, TD_VID(pCompactor->pTsdb->pVnode), __func__, - pCompactor->tbid.suid, pCompactor->tbid.uid); - } - return code; -} - -static bool tsdbCompactRowIsDeleted(STsdbCompactor *pCompactor, TSDBROW *pRow) { - TSDBKEY tKey = TSDBROW_KEY(pRow); - TSDBKEY *aKey = (TSDBKEY *)TARRAY_DATA(pCompactor->aSkyLine); - int32_t nKey = TARRAY_SIZE(pCompactor->aSkyLine); - - if (tKey.ts > pCompactor->pDKey->ts) { - do { - pCompactor->pDKey->version = aKey[pCompactor->iSkyLine].version; - pCompactor->iSkyLine++; - if (pCompactor->iSkyLine < nKey) { - pCompactor->dKey.ts = aKey[pCompactor->iSkyLine].ts; - } else { - if (pCompactor->pDKey->version == 0) { - pCompactor->pDKey = NULL; - return false; - } else { - pCompactor->pDKey->ts = INT64_MAX; - } - } - } while (tKey.ts > pCompactor->pDKey->ts); - } - - if (tKey.ts < pCompactor->pDKey->ts) { - if (tKey.version > pCompactor->pDKey->version) { - return false; - } else { - return true; - } - } else if (tKey.ts == pCompactor->pDKey->ts) { - ASSERT(pCompactor->iSkyLine < nKey); - if (tKey.version > TMAX(pCompactor->pDKey->version, aKey[pCompactor->iSkyLine].version)) { - return false; - } else { - return true; - } - } - - return false; -} - -static int32_t tsdbCompactWriteTableData(STsdbCompactor *pCompactor, SRowInfo *pRowInfo) { - int32_t code = 0; - int32_t lino = 0; - - // start a new table data write if need - if (pRowInfo == NULL || pRowInfo->uid != pCompactor->tbid.uid) { - if (pCompactor->tbid.uid) { - code = tsdbCompactWriteTableDataEnd(pCompactor); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (pRowInfo == NULL) { - if (pCompactor->sData.nRow > 0) { - code = tsdbWriteSttBlock(pCompactor->pWriter, &pCompactor->sData, pCompactor->aSttBlk, pCompactor->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - return code; - } - - code = tsdbCompactWriteTableDataStart(pCompactor, (TABLEID *)pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // check if row is deleted - if (pCompactor->pDKey && tsdbCompactRowIsDeleted(pCompactor, &pRowInfo->row)) goto _exit; - - if (tBlockDataTryUpsertRow(&pCompactor->bData, &pRowInfo->row, pRowInfo->uid) > pCompactor->maxRows) { - code = tsdbWriteDataBlock(pCompactor->pWriter, &pCompactor->bData, &pCompactor->mDataBlk, pCompactor->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tBlockDataUpsertRow(&pCompactor->bData, &pRowInfo->row, NULL, pRowInfo->uid); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } else if (pRowInfo) { - tsdbTrace("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64 " ts:%" PRId64 " version:%" PRId64, - TD_VID(pCompactor->pTsdb->pVnode), __func__, pRowInfo->suid, pRowInfo->uid, TSDBROW_TS(&pRowInfo->row), - TSDBROW_VERSION(&pRowInfo->row)); - } - return code; -} - -static bool tsdbCompactTableIsDropped(STsdbCompactor *pCompactor) { - SMetaInfo info; - - if (pCompactor->pIter->rowInfo.uid == pCompactor->tbid.uid) return false; - if (metaGetInfo(pCompactor->pTsdb->pVnode->pMeta, pCompactor->pIter->rowInfo.uid, &info, NULL)) { - return true; - } - return false; -} -static int32_t tsdbCompactNextRow(STsdbCompactor *pCompactor, SRowInfo **ppRowInfo) { - int32_t code = 0; - int32_t lino = 0; - - for (;;) { - if (pCompactor->pIter) { - code = tsdbDataIterNext2(pCompactor->pIter, NULL); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pCompactor->pIter->rowInfo.suid == 0 && pCompactor->pIter->rowInfo.uid == 0) { - pCompactor->pIter = NULL; - } else { - SRBTreeNode *pNode = tRBTreeMin(&pCompactor->rbt); - if (pNode) { - int32_t c = tsdbDataIterCmprFn(&pCompactor->pIter->rbtn, pNode); - if (c > 0) { - tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn); - pCompactor->pIter = NULL; - } else if (c == 0) { - ASSERT(0); - } - } - } - } - - if (pCompactor->pIter == NULL) { - SRBTreeNode *pNode = tRBTreeDropMin(&pCompactor->rbt); - if (pNode) { - pCompactor->pIter = TSDB_RBTN_TO_DATA_ITER(pNode); - } - } - - if (pCompactor->pIter) { - if (tsdbCompactTableIsDropped(pCompactor)) { - TABLEID tbid = {.suid = pCompactor->pIter->rowInfo.suid, .uid = pCompactor->pIter->rowInfo.uid}; - tRBTreeClear(&pCompactor->rbt); - for (pCompactor->pIter = pCompactor->iterList; pCompactor->pIter; pCompactor->pIter = pCompactor->pIter->next) { - code = tsdbDataIterNext2(pCompactor->pIter, - &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_TABLEID, .tbid = tbid}); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pCompactor->pIter->rowInfo.suid || pCompactor->pIter->rowInfo.uid) { - tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn); - } - } - } else { - *ppRowInfo = &pCompactor->pIter->rowInfo; - break; - } - } else { - *ppRowInfo = NULL; - break; - } - } - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCompactFileSetStart(STsdbCompactor *pCompactor, SDFileSet *pSet) { - int32_t code = 0; - int32_t lino = 0; - - pCompactor->fid = pSet->fid; - pCompactor->tbid = (TABLEID){0}; - - /* tombstone */ - pCompactor->iDelIdx = 0; - - /* reader */ - code = tsdbDataFReaderOpen(&pCompactor->pReader, pCompactor->pTsdb, pSet); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbOpenDataFileDataIter(pCompactor->pReader, &pCompactor->pIter); - TSDB_CHECK_CODE(code, lino, _exit); - - tRBTreeCreate(&pCompactor->rbt, tsdbDataIterCmprFn); - if (pCompactor->pIter) { - pCompactor->pIter->next = pCompactor->iterList; - pCompactor->iterList = pCompactor->pIter; - - code = tsdbDataIterNext2(pCompactor->pIter, NULL); - TSDB_CHECK_CODE(code, lino, _exit); - - ASSERT(pCompactor->pIter->rowInfo.suid || pCompactor->pIter->rowInfo.uid); - tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn); - } - - for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { - code = tsdbOpenSttFileDataIter(pCompactor->pReader, iStt, &pCompactor->pIter); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pCompactor->pIter) { - pCompactor->pIter->next = pCompactor->iterList; - pCompactor->iterList = pCompactor->pIter; - - code = tsdbDataIterNext2(pCompactor->pIter, NULL); - TSDB_CHECK_CODE(code, lino, _exit); - - ASSERT(pCompactor->pIter->rowInfo.suid || pCompactor->pIter->rowInfo.uid); - tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn); - } - } - pCompactor->pIter = NULL; - - /* writer */ - code = tsdbDataFWriterOpen(&pCompactor->pWriter, pCompactor->pTsdb, - &(SDFileSet){.fid = pCompactor->fid, - .diskId = pSet->diskId, - .pHeadF = &(SHeadFile){.commitID = pCompactor->commitID}, - .pDataF = &(SDataFile){.commitID = pCompactor->commitID}, - .pSmaF = &(SSmaFile){.commitID = pCompactor->commitID}, - .nSttF = 1, - .aSttF = {&(SSttFile){.commitID = pCompactor->commitID}}}); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pCompactor->aBlockIdx) { - taosArrayClear(pCompactor->aBlockIdx); - } else if ((pCompactor->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - tMapDataReset(&pCompactor->mDataBlk); - - if (pCompactor->aSttBlk) { - taosArrayClear(pCompactor->aSttBlk); - } else if ((pCompactor->aSttBlk = taosArrayInit(0, sizeof(SSttBlk))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - tBlockDataReset(&pCompactor->bData); - tBlockDataReset(&pCompactor->sData); - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino, - tstrerror(code), pCompactor->fid); - } else { - tsdbInfo("vgId:%d %s done, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, pCompactor->fid); - } - return code; -} - -static int32_t tsdbCompactFileSetEnd(STsdbCompactor *pCompactor) { - int32_t code = 0; - int32_t lino = 0; - - ASSERT(pCompactor->bData.nRow == 0); - ASSERT(pCompactor->sData.nRow == 0); - - /* update files */ - code = tsdbWriteSttBlk(pCompactor->pWriter, pCompactor->aSttBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbWriteBlockIdx(pCompactor->pWriter, pCompactor->aBlockIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbUpdateDFileSetHeader(pCompactor->pWriter); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbFSUpsertFSet(&pCompactor->fs, &pCompactor->pWriter->wSet); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbDataFWriterClose(&pCompactor->pWriter, 1); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbDataFReaderClose(&pCompactor->pReader); - TSDB_CHECK_CODE(code, lino, _exit); - - /* do clear */ - while ((pCompactor->pIter = pCompactor->iterList) != NULL) { - pCompactor->iterList = pCompactor->pIter->next; - tsdbCloseDataIter2(pCompactor->pIter); - } - - tBlockDataReset(&pCompactor->bData); - tBlockDataReset(&pCompactor->sData); - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino, - tstrerror(code), pCompactor->fid); - } else { - tsdbInfo("vgId:%d %s done, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, pCompactor->fid); - } - return code; -} - -static int32_t tsdbCompactFileSet(STsdbCompactor *pCompactor, SDFileSet *pSet) { - int32_t code = 0; - int32_t lino = 0; - - // start compact - code = tsdbCompactFileSetStart(pCompactor, pSet); - TSDB_CHECK_CODE(code, lino, _exit); - - // do compact, end with a NULL row - SRowInfo *pRowInfo; - do { - code = tsdbCompactNextRow(pCompactor, &pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbCompactWriteTableData(pCompactor, pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); - } while (pRowInfo); - - // end compact - code = tsdbCompactFileSetEnd(pCompactor); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino, - tstrerror(code), pCompactor->fid); - if (pCompactor->pWriter) tsdbDataFWriterClose(&pCompactor->pWriter, 0); - while ((pCompactor->pIter = pCompactor->iterList)) { - pCompactor->iterList = pCompactor->pIter->next; - tsdbCloseDataIter2(pCompactor->pIter); - } - if (pCompactor->pReader) tsdbDataFReaderClose(&pCompactor->pReader); - } - return code; -} - -static void tsdbEndCompact(STsdbCompactor *pCompactor) { - // writer - tBlockDataDestroy(&pCompactor->sData); - tBlockDataDestroy(&pCompactor->bData); - taosArrayDestroy(pCompactor->aSttBlk); - tMapDataClear(&pCompactor->mDataBlk); - taosArrayDestroy(pCompactor->aBlockIdx); - - // reader - - // tombstone - taosArrayDestroy(pCompactor->aSkyLine); - taosArrayDestroy(pCompactor->aDelData); - taosArrayDestroy(pCompactor->aDelIdx); - - // others - tDestroyTSchema(pCompactor->tbSkm.pTSchema); - tsdbFSDestroy(&pCompactor->fs); - - tsdbInfo("vgId:%d %s done, commit ID:%" PRId64, TD_VID(pCompactor->pTsdb->pVnode), __func__, pCompactor->commitID); -} - -static int32_t tsdbBeginCompact(STsdb *pTsdb, SCompactInfo *pInfo, STsdbCompactor *pCompactor) { - int32_t code = 0; - int32_t lino = 0; - - pCompactor->pTsdb = pTsdb; - pCompactor->commitID = pInfo->commitID; - pCompactor->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; - pCompactor->maxRows = pTsdb->pVnode->config.tsdbCfg.maxRows; - pCompactor->minRows = pTsdb->pVnode->config.tsdbCfg.minRows; - pCompactor->fid = INT32_MIN; - - code = tsdbFSCopy(pTsdb, &pCompactor->fs); - TSDB_CHECK_CODE(code, lino, _exit); - - /* tombstone */ - if (pCompactor->fs.pDelFile) { - code = tsdbDelFReaderOpen(&pCompactor->pDelFReader, pCompactor->fs.pDelFile, pTsdb); - TSDB_CHECK_CODE(code, lino, _exit); - - if ((pCompactor->aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - if ((pCompactor->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - if ((pCompactor->aSkyLine = taosArrayInit(0, sizeof(TSDBKEY))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tsdbReadDelIdx(pCompactor->pDelFReader, pCompactor->aDelIdx); - TSDB_CHECK_CODE(code, lino, _exit); - } - - /* reader */ - - /* writer */ - code = tBlockDataCreate(&pCompactor->bData); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tBlockDataCreate(&pCompactor->sData); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s, commit ID:%" PRId64, TD_VID(pTsdb->pVnode), __func__, lino, - tstrerror(code), pCompactor->commitID); - tBlockDataDestroy(&pCompactor->sData); - tBlockDataDestroy(&pCompactor->bData); - if (pCompactor->fs.pDelFile) { - taosArrayDestroy(pCompactor->aSkyLine); - taosArrayDestroy(pCompactor->aDelData); - taosArrayDestroy(pCompactor->aDelIdx); - if (pCompactor->pDelFReader) tsdbDelFReaderClose(&pCompactor->pDelFReader); - } - tsdbFSDestroy(&pCompactor->fs); - } else { - tsdbInfo("vgId:%d %s done, commit ID:%" PRId64, TD_VID(pTsdb->pVnode), __func__, pCompactor->commitID); - } - return code; -} - -int32_t tsdbCompact(STsdb *pTsdb, SCompactInfo *pInfo) { - int32_t code = 0; - - STsdbCompactor *pCompactor = &(STsdbCompactor){0}; - - if ((code = tsdbBeginCompact(pTsdb, pInfo, pCompactor))) return code; - - for (;;) { - SDFileSet *pSet = (SDFileSet *)taosArraySearch(pCompactor->fs.aDFileSet, &(SDFileSet){.fid = pCompactor->fid}, - tDFileSetCmprFn, TD_GT); - if (pSet == NULL) { - pCompactor->fid = INT32_MAX; - break; - } - - if ((code = tsdbCompactFileSet(pCompactor, pSet))) goto _exit; - } - - if ((code = tsdbFSUpsertDelFile(&pCompactor->fs, NULL))) goto _exit; - -_exit: - if (code) { - tsdbAbortCompact(pCompactor); - } else { - tsdbCommitCompact(pCompactor); - } - tsdbEndCompact(pCompactor); - return code; -} diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index c6e1ed99f1..7c7e1bd0f7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -15,7 +15,7 @@ #include "tsdb.h" -static bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { +static bool tsdbShouldDoRetentionImpl(STsdb *pTsdb, int64_t now) { for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); @@ -38,19 +38,21 @@ static bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { return false; } +bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { + bool should; + taosThreadRwlockRdlock(&pTsdb->rwLock); + should = tsdbShouldDoRetentionImpl(pTsdb, now); + taosThreadRwlockUnlock(&pTsdb->rwLock); + return should; +} int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { int32_t code = 0; - - if (!tsdbShouldDoRetention(pTsdb, now)) { - return code; - } - - // do retention + int32_t lino = 0; STsdbFS fs = {0}; code = tsdbFSCopy(pTsdb, &fs); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); iSet++) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); @@ -60,8 +62,10 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { if (expLevel < 0) { taosMemoryFree(pSet->pHeadF); taosMemoryFree(pSet->pDataF); - taosMemoryFree(pSet->aSttF[0]); taosMemoryFree(pSet->pSmaF); + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + taosMemoryFree(pSet->aSttF[iStt]); + } taosArrayRemove(fs.aDFileSet, iSet); iSet--; } else { @@ -78,35 +82,33 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { fSet.diskId = did; code = tsdbDFileSetCopy(pTsdb, pSet, &fSet); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbFSUpsertFSet(&fs, &fSet); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } // do change fs code = tsdbFSPrepareCommit(pTsdb, &fs); - if (code) goto _err; - - taosThreadRwlockWrlock(&pTsdb->rwLock); - - code = tsdbFSCommit(pTsdb); - if (code) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; - } - - taosThreadRwlockUnlock(&pTsdb->rwLock); - - tsdbFSDestroy(&fs); + TSDB_CHECK_CODE(code, lino, _exit); _exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + } + tsdbFSDestroy(&fs); return code; +} -_err: - tsdbError("vgId:%d, tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - ASSERT(0); - // tsdbFSRollback(pTsdb->pFS); - return code; +static int32_t tsdbCommitRetentionImpl(STsdb *pTsdb) { return tsdbFSCommit(pTsdb); } + +int32_t tsdbCommitRetention(STsdb *pTsdb) { + taosThreadRwlockWrlock(&pTsdb->rwLock); + tsdbCommitRetentionImpl(pTsdb); + taosThreadRwlockUnlock(&pTsdb->rwLock); + tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index bd2d263804..2dbac956ed 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -35,9 +35,7 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 return -1; } - if (pMsg) { - arrSize = taosArrayGetSize(pMsg->aSubmitTbData); - } + arrSize = taosArrayGetSize(pMsg->aSubmitTbData); // scan and convert if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) { diff --git a/source/dnode/vnode/src/vnd/vnodeCompact.c b/source/dnode/vnode/src/vnd/vnodeRetention.c similarity index 54% rename from source/dnode/vnode/src/vnd/vnodeCompact.c rename to source/dnode/vnode/src/vnd/vnodeRetention.c index 16e39d75dc..170deb4286 100644 --- a/source/dnode/vnode/src/vnd/vnodeCompact.c +++ b/source/dnode/vnode/src/vnd/vnodeRetention.c @@ -15,43 +15,54 @@ #include "vnd.h" -static int32_t vnodeCompactTask(void *param) { +typedef struct { + SVnode *pVnode; + int64_t now; + int64_t commitID; + SVnodeInfo info; +} SRetentionInfo; + +extern bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now); +extern int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now); +extern int32_t tsdbCommitRetention(STsdb *pTsdb); + +static int32_t vnodePrepareRentention(SVnode *pVnode, SRetentionInfo *pInfo) { int32_t code = 0; int32_t lino = 0; - SCompactInfo *pInfo = (SCompactInfo *)param; - SVnode *pVnode = pInfo->pVnode; + tsem_wait(&pVnode->canCommit); - // do compact - code = tsdbCompact(pInfo->pVnode->pTsdb, pInfo); - TSDB_CHECK_CODE(code, lino, _exit); + pInfo->commitID = ++pVnode->state.commitID; - // end compact char dir[TSDB_FILENAME_LEN] = {0}; if (pVnode->pTfs) { snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); } else { snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); } - vnodeCommitInfo(dir); + + if (vnodeLoadInfo(dir, &pInfo->info) < 0) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + } _exit: - tsem_post(&pInfo->pVnode->canCommit); - taosMemoryFree(pInfo); + if (code) { + vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + tsem_post(&pVnode->canCommit); + } else { + vInfo("vgId:%d %s done", TD_VID(pVnode), __func__); + } return code; } -static int32_t vnodePrepareCompact(SVnode *pVnode, SCompactInfo *pInfo) { + +static int32_t vnodeRetentionTask(void *param) { int32_t code = 0; int32_t lino = 0; - tsem_wait(&pVnode->canCommit); - - pInfo->pVnode = pVnode; - pInfo->flag = 0; - pInfo->commitID = ++pVnode->state.commitID; - - char dir[TSDB_FILENAME_LEN] = {0}; - SVnodeInfo info = {0}; + SRetentionInfo *pInfo = (SRetentionInfo *)param; + SVnode *pVnode = pInfo->pVnode; + char dir[TSDB_FILENAME_LEN] = {0}; if (pVnode->pTfs) { snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); @@ -59,57 +70,61 @@ static int32_t vnodePrepareCompact(SVnode *pVnode, SCompactInfo *pInfo) { snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); } - if (vnodeLoadInfo(dir, &info) < 0) { + // save info + pInfo->info.state.commitID = pInfo->commitID; + + if (vnodeSaveInfo(dir, &pInfo->info) < 0) { code = terrno; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } - info.state.commitID = pInfo->commitID; + // do job + code = tsdbDoRetention(pInfo->pVnode->pTsdb, pInfo->now); + TSDB_CHECK_CODE(code, lino, _exit); - if (vnodeSaveInfo(dir, &info) < 0) { - code = terrno; - goto _exit; - } + // commit info + vnodeCommitInfo(dir); + + // commit sub-job + tsdbCommitRetention(pVnode->pTsdb); _exit: if (code) { - vError("vgId:%d %s failed at line %d since %s, commit ID:%" PRId64, TD_VID(pVnode), __func__, lino, tstrerror(code), - pVnode->state.commitID); + vError("vgId:%d %s failed at line %d since %s", TD_VID(pInfo->pVnode), __func__, lino, tstrerror(code)); } else { - vDebug("vgId:%d %s done, commit ID:%" PRId64, TD_VID(pVnode), __func__, pVnode->state.commitID); + vInfo("vgId:%d %s done", TD_VID(pInfo->pVnode), __func__); } + tsem_post(&pInfo->pVnode->canCommit); + taosMemoryFree(pInfo); return code; } -int32_t vnodeAsyncCompact(SVnode *pVnode) { + +int32_t vnodeAsyncRentention(SVnode *pVnode, int64_t now) { int32_t code = 0; int32_t lino = 0; - SCompactInfo *pInfo = taosMemoryCalloc(1, sizeof(*pInfo)); + if (!tsdbShouldDoRetention(pVnode->pTsdb, now)) return code; + + SRetentionInfo *pInfo = (SRetentionInfo *)taosMemoryCalloc(1, sizeof(*pInfo)); if (pInfo == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); } - vnodeAsyncCommit(pVnode); + pInfo->pVnode = pVnode; + pInfo->now = now; - code = vnodePrepareCompact(pVnode, pInfo); + code = vnodePrepareRentention(pVnode, pInfo); TSDB_CHECK_CODE(code, lino, _exit); - vnodeScheduleTask(vnodeCompactTask, pInfo); + vnodeScheduleTask(vnodeRetentionTask, pInfo); _exit: if (code) { - vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + vError("vgId:%d %s failed at line %d since %s", TD_VID(pInfo->pVnode), __func__, lino, tstrerror(code)); if (pInfo) taosMemoryFree(pInfo); } else { - vInfo("vgId:%d %s done", TD_VID(pVnode), __func__); + vInfo("vgId:%d %s done", TD_VID(pInfo->pVnode), __func__); } - return code; -} - -int32_t vnodeSyncCompact(SVnode *pVnode) { - vnodeAsyncCompact(pVnode); - tsem_wait(&pVnode->canCommit); - tsem_post(&pVnode->canCommit); return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index a3438b611e..3525e696c5 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -586,6 +586,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { pMetaRsp->precision = pVnode->config.tsdbCfg.precision; } +extern int32_t vnodeAsyncRentention(SVnode *pVnode, int64_t now); static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; SVTrimDbReq trimReq = {0}; @@ -598,12 +599,16 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, vInfo("vgId:%d, trim vnode request will be processed, time:%d", pVnode->config.vgId, trimReq.timestamp); - // process +// process +#if 0 code = tsdbDoRetention(pVnode->pTsdb, trimReq.timestamp); if (code) goto _exit; code = smaDoRetention(pVnode->pSma, trimReq.timestamp); if (code) goto _exit; +#else + vnodeAsyncRentention(pVnode, trimReq.timestamp); +#endif _exit: return code; @@ -635,6 +640,10 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p ret = smaDoRetention(pVnode->pSma, ttlReq.timestamp); if (ret) goto end; +#else + vnodeAsyncRentention(pVnode, ttlReq.timestamp); + tsem_wait(&pVnode->canCommit); + tsem_post(&pVnode->canCommit); #endif end: @@ -1632,17 +1641,13 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *p return TSDB_CODE_SUCCESS; } +extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); + static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - SCompactVnodeReq req = {0}; - if (tDeserializeSCompactVnodeReq(pReq, len, &req) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return TSDB_CODE_INVALID_MSG; - } - vInfo("vgId:%d, compact msg will be processed, db:%s dbUid:%" PRId64 " compactStartTime:%" PRId64, TD_VID(pVnode), - req.db, req.dbUid, req.compactStartTime); - - vnodeAsyncCompact(pVnode); - vnodeBegin(pVnode); - - return 0; + return vnodeProcessCompactVnodeReqImpl(pVnode, version, pReq, len, pRsp); } + + +#ifndef TD_ENTERPRISE +int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { return 0; } +#endif diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 5df3b14a5b..be79054c1b 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -732,7 +732,7 @@ void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int3 STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); -int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag); +int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder); int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); extern void doDestroyExchangeOperatorInfo(void* param); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 89e0dd363c..0dcefec93d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1197,16 +1197,11 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { - // expand the result datablock capacity - if (pRow->numOfRows > pBlock->info.capacity) { - blockDataEnsureCapacity(pBlock, pRow->numOfRows); - qDebug("datablock capacity not sufficient, expand to requried:%d, current capacity:%d, %s", pRow->numOfRows, - pBlock->info.capacity, GET_TASKID(pTaskInfo)); + blockDataEnsureCapacity(pBlock, pBlock->info.rows + pRow->numOfRows); + qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s", + (pRow->numOfRows+pBlock->info.rows), + pBlock->info.capacity, GET_TASKID(pTaskInfo)); // todo set the pOperator->resultInfo size - } else { - releaseBufPage(pBuf, page); - break; - } } pGroupResInfo->index += 1; @@ -1449,16 +1444,21 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t return TSDB_CODE_SUCCESS; } -int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag) { +int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder) { // todo add more information about exchange operation int32_t type = pOperator->operatorType; - if (type == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE || type == QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN || - type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN || - type == QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN || - type == QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN) { + if (type == QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN || + type == QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN || + type == QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN) { *order = TSDB_ORDER_ASC; *scanFlag = MAIN_SCAN; return TSDB_CODE_SUCCESS; + } else if (type == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE) { + if (!inheritUsOrder) { + *order = TSDB_ORDER_ASC; + } + *scanFlag = MAIN_SCAN; + return TSDB_CODE_SUCCESS; } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pTableScanInfo = pOperator->info; *order = pTableScanInfo->base.cond.order; @@ -1473,7 +1473,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) { return TSDB_CODE_INVALID_PARA; } else { - return getTableScanInfo(pOperator->pDownstream[0], order, scanFlag); + return getTableScanInfo(pOperator->pDownstream[0], order, scanFlag, inheritUsOrder); } } } @@ -1589,7 +1589,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { } hasValidBlock = true; - int32_t code = getTableScanInfo(pOperator, &order, &scanFlag); + int32_t code = getTableScanInfo(pOperator, &order, &scanFlag, false); if (code != TSDB_CODE_SUCCESS) { destroyDataBlockForEmptyInput(blockAllocated, &pBlock); T_LONG_JMP(pTaskInfo->env, code); diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index adb045a69f..2a33e3527a 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -69,7 +69,7 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; - getTableScanInfo(pOperator, &order, &scanFlag); + getTableScanInfo(pOperator, &order, &scanFlag, false); int64_t ekey = pInfo->existNewGroupBlock->info.window.ekey; taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); @@ -128,7 +128,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; - getTableScanInfo(pOperator, &order, &scanFlag); + getTableScanInfo(pOperator, &order, &scanFlag, false); doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo); if (pResBlock->info.rows > 0) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 8f5c3786e0..65146edfac 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -383,7 +383,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { break; } - int32_t code = getTableScanInfo(pOperator, &order, &scanFlag); + int32_t code = getTableScanInfo(pOperator, &order, &scanFlag, false); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 3ae114c656..49bc5af634 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -289,7 +289,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again - int32_t code = getTableScanInfo(downstream, &order, &scanFlag); + int32_t code = getTableScanInfo(downstream, &order, &scanFlag, false); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -441,7 +441,7 @@ static void doHandleDataBlock(SOperatorInfo* pOperator, SSDataBlock* pBlock, SOp SExprSupp* pSup = &pOperator->exprSupp; // the pDataBlock are always the same one, no need to call this again - int32_t code = getTableScanInfo(downstream, &order, &scanFlag); + int32_t code = getTableScanInfo(downstream, &order, &scanFlag, false); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e1594b13a8..7961d5518a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1651,8 +1651,6 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { } else if (ret.fetchType == FETCH_TYPE__NONE || (ret.fetchType == FETCH_TYPE__SEP && pOperator->status == OP_EXEC_RECV)) { pTaskInfo->streamInfo.lastStatus = ret.offset; - ASSERT(pTaskInfo->streamInfo.lastStatus.version >= pTaskInfo->streamInfo.prepareStatus.version); - ASSERT(pTaskInfo->streamInfo.lastStatus.version + 1 == pInfo->tqReader->pWalReader->curVersion); char formatBuf[80]; tFormatOffset(formatBuf, 80, &ret.offset); qDebug("queue scan log return null, offset %s", formatBuf); @@ -1660,16 +1658,6 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { return NULL; } } -#if 0 - } else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { - SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); - if (pResult && pResult->info.rows > 0) { - qDebug("stream scan tsdb return %d rows", pResult->info.rows); - return pResult; - } - qDebug("stream scan tsdb return null"); - return NULL; -#endif } else { qError("unexpected streamInfo prepare type: %d", pTaskInfo->streamInfo.prepareStatus.type); return NULL; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 20d4f46eaf..6411d862ae 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1072,7 +1072,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { break; } - getTableScanInfo(pOperator, &pInfo->inputOrder, &scanFlag); + getTableScanInfo(pOperator, &pInfo->inputOrder, &scanFlag, true); if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; @@ -4294,7 +4294,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { } } - getTableScanInfo(pOperator, &pIaInfo->inputOrder, &scanFlag); + getTableScanInfo(pOperator, &pIaInfo->inputOrder, &scanFlag, false); setInputDataBlock(pSup, pBlock, pIaInfo->inputOrder, scanFlag, true); doMergeAlignedIntervalAggImpl(pOperator, &pIaInfo->binfo.resultRowInfo, pBlock, pRes); @@ -4621,7 +4621,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { break; } - getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag); + getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag, false); setInputDataBlock(pExpSupp, pBlock, iaInfo->inputOrder, scanFlag, true); doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 9986af1691..9631aa76bb 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5438,8 +5438,6 @@ int32_t blockDistFunction(SqlFunctionCtx* pCtx) { if (pDistInfo->maxRows < p1.maxRows) { pDistInfo->maxRows = p1.maxRows; } - pDistInfo->numOfVgroups += (p1.numOfTables != 0 ? 1 : 0); - pDistInfo->numOfVgroups += (p1.numOfTables != 0 ? 1 : 0); for (int32_t i = 0; i < tListLen(pDistInfo->blockRowsHisto); ++i) { pDistInfo->blockRowsHisto[i] += p1.blockRowsHisto[i]; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index b4f7ea866a..d9a4c5178f 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -35,15 +35,15 @@ memcpy(&((pDst)->fldname), &((pSrc)->fldname), size); \ } while (0) -#define COPY_CHAR_POINT_FIELD(fldname) \ - do { \ - if (NULL == (pSrc)->fldname) { \ - break; \ - } \ +#define COPY_CHAR_POINT_FIELD(fldname) \ + do { \ + if (NULL == (pSrc)->fldname) { \ + break; \ + } \ (pDst)->fldname = taosStrdup((pSrc)->fldname); \ - if (NULL == (pDst)->fldname) { \ - return TSDB_CODE_OUT_OF_MEMORY; \ - } \ + if (NULL == (pDst)->fldname) { \ + return TSDB_CODE_OUT_OF_MEMORY; \ + } \ } while (0) #define CLONE_NODE_FIELD(fldname) \ @@ -158,7 +158,7 @@ static int32_t valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { - int32_t len = varDataTLen(pSrc->datum.p) + 1; + int32_t len = pSrc->node.resType.bytes + 1; pDst->datum.p = taosMemoryCalloc(1, len); if (NULL == pDst->datum.p) { return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 099cd0d3b3..e18de1c1d2 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -3269,7 +3269,7 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { - pNode->datum.p = taosMemoryCalloc(1, pNode->node.resType.bytes + VARSTR_HEADER_SIZE + 1); + pNode->datum.p = taosMemoryCalloc(1, pNode->node.resType.bytes + 1); if (NULL == pNode->datum.p) { code = TSDB_CODE_OUT_OF_MEMORY; break; diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index ad80508c64..6c6b6c0e81 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -928,7 +928,7 @@ static int32_t msgToDatum(STlv* pTlv, void* pObj) { code = TSDB_CODE_FAILED; break; } - pNode->datum.p = taosMemoryCalloc(1, pNode->node.resType.bytes + VARSTR_HEADER_SIZE + 1); + pNode->datum.p = taosMemoryCalloc(1, pNode->node.resType.bytes + 1); if (NULL == pNode->datum.p) { code = TSDB_CODE_OUT_OF_MEMORY; break; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 024130b5f8..22afc7ef55 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -963,8 +963,6 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_SHOW_USERS_STMT: case QUERY_NODE_SHOW_LICENCES_STMT: case QUERY_NODE_SHOW_VGROUPS_STMT: - case QUERY_NODE_SHOW_DB_ALIVE_STMT: - case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_TOPICS_STMT: case QUERY_NODE_SHOW_CONSUMERS_STMT: case QUERY_NODE_SHOW_CONNECTIONS_STMT: diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index f613b1bd3d..634a239399 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1810,7 +1810,7 @@ SNode* createStreamOptions(SAstCreateContext* pCxt) { CHECK_PARSER_STATUS(pCxt); SStreamOptions* pOptions = (SStreamOptions*)nodesMakeNode(QUERY_NODE_STREAM_OPTIONS); CHECK_OUT_OF_MEM(pOptions); - pOptions->triggerType = STREAM_TRIGGER_AT_ONCE; + pOptions->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; pOptions->fillHistory = STREAM_DEFAULT_FILL_HISTORY; pOptions->ignoreExpired = STREAM_DEFAULT_IGNORE_EXPIRED; pOptions->ignoreUpdate = STREAM_DEFAULT_IGNORE_UPDATE; diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index d4f4949df0..b2fc88add1 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -320,6 +320,79 @@ static int32_t calcConstInsert(SCalcConstContext* pCxt, SInsertStmt* pInsert) { return code; } +static SNodeList* getChildProjection(SNode* pStmt) { + switch (nodeType(pStmt)) { + case QUERY_NODE_SELECT_STMT: + return ((SSelectStmt*)pStmt)->pProjectionList; + case QUERY_NODE_SET_OPERATOR: + return ((SSetOperator*)pStmt)->pProjectionList; + default: + break; + } + return NULL; +} + +static void eraseSetOpChildProjection(SSetOperator* pSetOp, int32_t index) { + SNodeList* pLeftProjs = getChildProjection(pSetOp->pLeft); + nodesListErase(pLeftProjs, nodesListGetCell(pLeftProjs, index)); + SNodeList* pRightProjs = getChildProjection(pSetOp->pRight); + nodesListErase(pRightProjs, nodesListGetCell(pRightProjs, index)); +} + +typedef struct SNotRefByOrderByCxt { + SColumnNode* pCol; + bool hasThisCol; +} SNotRefByOrderByCxt; + +static EDealRes notRefByOrderByImpl(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + SNotRefByOrderByCxt* pCxt = (SNotRefByOrderByCxt*)pContext; + if (nodesEqualNode((SNode*)pCxt->pCol, pNode)) { + pCxt->hasThisCol = true; + return DEAL_RES_END; + } + } + return DEAL_RES_CONTINUE; +} + +static bool notRefByOrderBy(SColumnNode* pCol, SNodeList* pOrderByList) { + SNotRefByOrderByCxt cxt = {.pCol = pCol, .hasThisCol = false}; + nodesWalkExprs(pOrderByList, notRefByOrderByImpl, &cxt); + return !cxt.hasThisCol; +} + +static int32_t calcConstSetOpProjections(SCalcConstContext* pCxt, SSetOperator* pSetOp, bool subquery) { + int32_t index = 0; + SNode* pProj = NULL; + WHERE_EACH(pProj, pSetOp->pProjectionList) { + if (subquery && notRefByOrderBy((SColumnNode*)pProj, pSetOp->pOrderByList) && isUselessCol((SExprNode*)pProj)) { + ERASE_NODE(pSetOp->pProjectionList); + eraseSetOpChildProjection(pSetOp, index); + continue; + } + ++index; + WHERE_NEXT; + } + if (0 == LIST_LENGTH(pSetOp->pProjectionList)) { + return nodesListStrictAppend(pSetOp->pProjectionList, createConstantValue()); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t calcConstSetOperator(SCalcConstContext* pCxt, SSetOperator* pSetOp, bool subquery) { + int32_t code = calcConstSetOpProjections(pCxt, pSetOp, subquery); + if (TSDB_CODE_SUCCESS == code) { + code = calcConstQuery(pCxt, pSetOp->pLeft, false); + } + if (TSDB_CODE_SUCCESS == code) { + code = calcConstQuery(pCxt, pSetOp->pRight, false); + } + if (TSDB_CODE_SUCCESS == code) { + code = calcConstList(pSetOp->pOrderByList); + } + return code; +} + static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt, bool subquery) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pStmt)) { @@ -330,11 +403,7 @@ static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt, bool subque code = calcConstQuery(pCxt, ((SExplainStmt*)pStmt)->pQuery, subquery); break; case QUERY_NODE_SET_OPERATOR: { - SSetOperator* pSetOp = (SSetOperator*)pStmt; - code = calcConstQuery(pCxt, pSetOp->pLeft, false); - if (TSDB_CODE_SUCCESS == code) { - code = calcConstQuery(pCxt, pSetOp->pRight, false); - } + code = calcConstSetOperator(pCxt, (SSetOperator*)pStmt, subquery); break; } case QUERY_NODE_DELETE_STMT: diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 8ba5802ad6..2dac35590e 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -781,16 +781,10 @@ TEST_F(ParserInitialCTest, createStream) { snprintf(expect.targetStbFullName, sizeof(expect.targetStbFullName), "0.test.%s", pDstStb); expect.igExists = igExists; expect.sql = taosStrdup(pSql); - expect.createStb = STREAM_CREATE_STABLE_TRUE; - expect.triggerType = STREAM_TRIGGER_AT_ONCE; - expect.maxDelay = 0; - expect.watermark = 0; - expect.fillHistory = STREAM_DEFAULT_FILL_HISTORY; - expect.igExpired = STREAM_DEFAULT_IGNORE_EXPIRED; }; auto setStreamOptions = - [&](int8_t createStb = STREAM_CREATE_STABLE_TRUE, int8_t triggerType = STREAM_TRIGGER_AT_ONCE, + [&](int8_t createStb = STREAM_CREATE_STABLE_TRUE, int8_t triggerType = STREAM_TRIGGER_WINDOW_CLOSE, int64_t maxDelay = 0, int64_t watermark = 0, int8_t igExpired = STREAM_DEFAULT_IGNORE_EXPIRED, int8_t fillHistory = STREAM_DEFAULT_FILL_HISTORY, int8_t igUpdate = STREAM_DEFAULT_IGNORE_UPDATE) { expect.createStb = createStb; @@ -852,6 +846,7 @@ TEST_F(ParserInitialCTest, createStream) { }); setCreateStreamReq("s1", "test", "create stream s1 into st3 as select count(*) from t1 interval(10s)", "st3"); + setStreamOptions(); run("CREATE STREAM s1 INTO st3 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)"); clearCreateStreamReq(); @@ -872,6 +867,7 @@ TEST_F(ParserInitialCTest, createStream) { "st3"); addTag("tname", TSDB_DATA_TYPE_VARCHAR, 10 + VARSTR_HEADER_SIZE); addTag("id", TSDB_DATA_TYPE_INT); + setStreamOptions(); run("CREATE STREAM s1 INTO st3 TAGS(tname VARCHAR(10), id INT) SUBTABLE(CONCAT('new-', tname)) " "AS SELECT _WSTART wstart, COUNT(*) cnt FROM st1 PARTITION BY TBNAME tname, tag1 id INTERVAL(10S)"); clearCreateStreamReq(); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 7bfd3df9e0..3e1e36ccc1 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -200,6 +200,7 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { pReader->curVersion, pReader->curInvalid, ver); pReader->curVersion = ver; + pReader->curInvalid = 0; return 0; } @@ -210,8 +211,8 @@ int32_t walReadSeekVer(SWalReader *pReader, int64_t ver) { return 0; } - pReader->curInvalid = 1; - pReader->curVersion = ver; +// pReader->curInvalid = 1; +// pReader->curVersion = ver; if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { wDebug("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pReader->pWal->cfg.vgId, @@ -219,8 +220,8 @@ int32_t walReadSeekVer(SWalReader *pReader, int64_t ver) { terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; return -1; } - if (ver < pWal->vers.snapshotVer) { - } +// if (ver < pWal->vers.snapshotVer) { +// } if (walReadSeekVerImpl(pReader, ver) < 0) { return -1; @@ -239,8 +240,8 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { if (pRead->curInvalid || pRead->curVersion != fetchVer) { if (walReadSeekVer(pRead, fetchVer) < 0) { - pRead->curVersion = fetchVer; - pRead->curInvalid = 1; +// pRead->curVersion = fetchVer; +// pRead->curInvalid = 1; return -1; } seeked = true; @@ -259,11 +260,11 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - pRead->curInvalid = 1; +// pRead->curInvalid = 1; return -1; } } - pRead->curInvalid = 0; +// pRead->curInvalid = 0; return 0; } @@ -295,13 +296,14 @@ static int32_t walFetchBodyNew(SWalReader *pReader) { pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - pReader->curInvalid = 1; +// pRead->curInvalid = 1; return -1; } if (walValidBodyCksum(pReader->pHead) != 0) { wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId, ver); - pReader->curInvalid = 1; +// pRead->curInvalid = 1; + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } @@ -317,7 +319,7 @@ static int32_t walSkipFetchBodyNew(SWalReader *pRead) { code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - pRead->curInvalid = 1; +// pRead->curInvalid = 1; return -1; } @@ -345,8 +347,8 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { if (pRead->curInvalid || pRead->curVersion != ver) { code = walReadSeekVer(pRead, ver); if (code < 0) { - pRead->curVersion = ver; - pRead->curInvalid = 1; +// pRead->curVersion = ver; +// pRead->curInvalid = 1; return -1; } seeked = true; @@ -366,7 +368,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - pRead->curInvalid = 1; +// pRead->curInvalid = 1; return -1; } } @@ -379,7 +381,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { return -1; } - pRead->curInvalid = 0; +// pRead->curInvalid = 0; return 0; } @@ -394,7 +396,7 @@ int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead) { code = taosLSeekFile(pRead->pLogFile, pHead->head.bodyLen, SEEK_CUR); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - pRead->curInvalid = 1; +// pRead->curInvalid = 1; return -1; } @@ -433,14 +435,14 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) { pRead->pWal->cfg.vgId, pReadHead->version, ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - pRead->curInvalid = 1; +// pRead->curInvalid = 1; return -1; } if (pReadHead->version != ver) { wError("vgId:%d, wal fetch body error, index:%" PRId64 ", read request index:%" PRId64, pRead->pWal->cfg.vgId, pReadHead->version, ver); - pRead->curInvalid = 1; +// pRead->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } @@ -448,7 +450,7 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) { if (walValidBodyCksum(*ppHead) != 0) { wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver); - pRead->curInvalid = 1; +// pRead->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } @@ -544,7 +546,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { if (pReader->pHead->head.version != ver) { wError("vgId:%d, unexpected wal log, index:%" PRId64 ", read request index:%" PRId64, pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver); - pReader->curInvalid = 1; +// pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; taosThreadMutexUnlock(&pReader->mutex); return -1; @@ -557,7 +559,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { uint32_t readCkSum = walCalcBodyCksum(pReader->pHead->head.body, pReader->pHead->head.bodyLen); uint32_t logCkSum = pReader->pHead->cksumBody; wError("checksum written into log:%u, checksum calculated:%u", logCkSum, readCkSum); - pReader->curInvalid = 1; +// pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; taosThreadMutexUnlock(&pReader->mutex); return -1; @@ -575,5 +577,6 @@ void walReadReset(SWalReader *pReader) { taosCloseFile(&pReader->pLogFile); pReader->curInvalid = 1; pReader->curFileFirstVer = -1; + pReader->curVersion = -1; taosThreadMutexUnlock(&pReader->mutex); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 4bb082800d..b22e2e924f 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -94,6 +94,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NO_AVAIL_DISK, "No available disk") TAOS_DEFINE_ERROR(TSDB_CODE_NOT_FOUND, "Not found") TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISKSPACE, "Out of disk space") TAOS_DEFINE_ERROR(TSDB_CODE_TIMEOUT_ERROR, "Operation timeout") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DISKSPACE, "No enough disk space") TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STARTING, "Database is starting up") TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STOPPING, "Database is closing down") diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 90b71751f5..93e11eb1fb 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -680,8 +680,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 @@ -913,13 +913,13 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQueryInterval.py -Q 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 3 @@ -927,9 +927,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py -Q 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 @@ -945,7 +945,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 3 @@ -991,7 +991,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 4 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 4 diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index c61c7667f8..e69875d69f 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -17,7 +17,7 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream streams1 trigger at_once into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791223001,2,2,3,1.1); sql insert into t1 values(1648791233002,3,2,3,2.1); @@ -545,8 +545,8 @@ sql create table t2 using st tags(2,2,2); sql create table t3 using st tags(2,2,2); sql create table t4 using st tags(2,2,2); sql create table t5 using st tags(2,2,2); -sql create stream streams2 trigger at_once into streamt as select _wstart, count(*) c1, sum(a) c3,max(b) c4 from st partition by tbname interval(10s); -sql create stream streams3 trigger at_once into streamt3 as select _wstart, count(*) c1, sum(a) c3,max(b) c4, now c5 from st partition by tbname interval(10s); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart, count(*) c1, sum(a) c3,max(b) c4 from st partition by tbname interval(10s); +sql create stream streams3 trigger at_once IGNORE EXPIRED 0 into streamt3 as select _wstart, count(*) c1, sum(a) c3,max(b) c4, now c5 from st partition by tbname interval(10s); sql insert into t1 values(1648791213000,1,1,1,1.0) t2 values(1648791213000,2,2,2,2.0) t3 values(1648791213000,3,3,3,3.0) t4 values(1648791213000,4,4,4,4.0); @@ -667,7 +667,7 @@ sql create database test3 vgroups 1; sql use test3; sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); -sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from ts1 interval(10s) ; +sql create stream stream_t3 trigger at_once IGNORE EXPIRED 0 into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from ts1 interval(10s) ; sql insert into ts1 values(1648791211000,1,2,3); sleep 50 @@ -701,7 +701,7 @@ endi sql create database test4 vgroups 1; sql use test4; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream streams4 trigger at_once into streamt4 as select _wstart, count(*) c1 from t1 where a > 5 interval(10s); +sql create stream streams4 trigger at_once IGNORE EXPIRED 0 into streamt4 as select _wstart, count(*) c1 from t1 where a > 5 interval(10s); sql insert into t1 values(1648791213000,1,2,3,1.0); $loop_count = 0 @@ -797,8 +797,8 @@ sql create database test5 vgroups 1; sql use test5; sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); -sql create stream streams5 trigger at_once into streamt5 as select count(*), _wstart, _wend, max(a) from ts1 interval(10s) ; -sql create stream streams6 trigger at_once into streamt6 as select count(*), _wstart, _wend, max(a), _wstart as ts from ts1 interval(10s) ; +sql create stream streams5 trigger at_once IGNORE EXPIRED 0 into streamt5 as select count(*), _wstart, _wend, max(a) from ts1 interval(10s) ; +sql create stream streams6 trigger at_once IGNORE EXPIRED 0 into streamt6 as select count(*), _wstart, _wend, max(a), _wstart as ts from ts1 interval(10s) ; sql_error create stream streams7 trigger at_once into streamt7 as select _wstart, count(*), _wstart, _wend, max(a) from ts1 interval(10s) ; sql_error create stream streams8 trigger at_once into streamt8 as select count(*), _wstart, _wstart, _wend, max(a) from ts1 interval(10s) ; @@ -840,7 +840,7 @@ sql create database test7 vgroups 1; sql use test7; sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); -sql create stream streams7 trigger at_once into streamt7 as select _wstart, count(*) from ts1 interval(10s) ; +sql create stream streams7 trigger at_once IGNORE EXPIRED 0 into streamt7 as select _wstart, count(*) from ts1 interval(10s) ; sql insert into ts1 values(1648791211000,1,2,3); sql_error insert into ts1 values(-1648791211000,1,2,3); @@ -887,4 +887,20 @@ if $rows != 1 then goto loop19 endi +print select * from information_schema.ins_stream_tasks; +sql select * from information_schema.ins_stream_tasks; + +if $rows == 0 then + print =====rows=$rows + return -1 +endi + +print select * from information_schema.ins_streams; +sql select * from information_schema.ins_streams; + +if $rows == 0 then + print =====rows=$rows + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/deleteInterval.sim b/tests/script/tsim/stream/deleteInterval.sim index 7532b2d5de..9540e448d4 100644 --- a/tests/script/tsim/stream/deleteInterval.sim +++ b/tests/script/tsim/stream/deleteInterval.sim @@ -16,7 +16,7 @@ sql drop database if exists test; sql create database test vgroups 1; sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 interval(10s); +sql create stream streams0 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 interval(10s); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sleep 200 @@ -193,7 +193,7 @@ sql use test2; sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); @@ -419,7 +419,7 @@ sql use test3; sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams3 trigger at_once into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s); +sql create stream streams3 trigger at_once IGNORE EXPIRED 0 into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); diff --git a/tests/script/tsim/stream/deleteSession.sim b/tests/script/tsim/stream/deleteSession.sim index c3c64a5977..3645939178 100644 --- a/tests/script/tsim/stream/deleteSession.sim +++ b/tests/script/tsim/stream/deleteSession.sim @@ -16,7 +16,7 @@ sql drop database if exists test; sql create database test vgroups 1; sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 session(ts, 5s); +sql create stream streams0 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 session(ts, 5s); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sleep 200 @@ -191,7 +191,7 @@ sql use test2; sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); @@ -422,7 +422,7 @@ sql use test3; sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams3 trigger at_once into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s); +sql create stream streams3 trigger at_once IGNORE EXPIRED 0 into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s); sql insert into t1 values(1648791210000,1,1,1,NULL); sql insert into t1 values(1648791210001,2,2,2,NULL); @@ -532,8 +532,8 @@ sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -print create stream streams4 trigger at_once into streamt4 as select _wstart, count(*) c1 from st partition by tbname session(ts, 2s); -sql create stream streams4 trigger at_once into streamt4 as select _wstart, count(*) c1 from st partition by tbname session(ts, 2s); +print create stream streams4 trigger at_once IGNORE EXPIRED 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname session(ts, 2s); +sql create stream streams4 trigger at_once IGNORE EXPIRED 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname session(ts, 2s); sql insert into t1 values(1648791210000,1,2,3); sql insert into t1 values(1648791220000,2,2,3); diff --git a/tests/script/tsim/stream/deleteState.sim b/tests/script/tsim/stream/deleteState.sim index 45d9bc3e39..dd74b73dce 100644 --- a/tests/script/tsim/stream/deleteState.sim +++ b/tests/script/tsim/stream/deleteState.sim @@ -16,7 +16,7 @@ sql drop database if exists test; sql create database test vgroups 1; sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 state_window(a); +sql create stream streams0 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 state_window(a); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sleep 200 @@ -197,8 +197,8 @@ sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -print create stream streams4 trigger at_once into streamt4 as select _wstart, count(*) c1 from st partition by tbname state_window(c); -sql create stream streams4 trigger at_once into streamt4 as select _wstart, count(*) c1 from st partition by tbname state_window(c); +print create stream streams4 trigger at_once IGNORE EXPIRED 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname state_window(c); +sql create stream streams4 trigger at_once IGNORE EXPIRED 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname state_window(c); sql insert into t1 values(1648791210000,1,2,1); sql insert into t1 values(1648791220000,2,2,2); diff --git a/tests/script/tsim/stream/distributeIntervalRetrive0.sim b/tests/script/tsim/stream/distributeIntervalRetrive0.sim index ae2f9afdb5..529a2a1b30 100644 --- a/tests/script/tsim/stream/distributeIntervalRetrive0.sim +++ b/tests/script/tsim/stream/distributeIntervalRetrive0.sim @@ -44,7 +44,7 @@ sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); -sql create stream stream_t1 trigger at_once into streamtST1 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); +sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 into streamtST1 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); sleep 1000 @@ -243,7 +243,7 @@ sql use test1; sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams1 trigger at_once into streamt1 as select _wstart as c0, count(*) c1, count(a) c2 from st interval(10s) ; +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart as c0, count(*) c1, count(a) c2 from st interval(10s) ; sql insert into t1 values(1648791211000,1,2,3); diff --git a/tests/script/tsim/stream/distributeSession0.sim b/tests/script/tsim/stream/distributeSession0.sim index d752f5c29c..25ac479a38 100644 --- a/tests/script/tsim/stream/distributeSession0.sim +++ b/tests/script/tsim/stream/distributeSession0.sim @@ -39,7 +39,7 @@ sql use test; sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); -sql create stream stream_t1 trigger at_once into streamtST as select _wstart, count(*) c1, sum(a) c2 , max(b) c3 from st session(ts, 10s) ; +sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 into streamtST as select _wstart, count(*) c1, sum(a) c2 , max(b) c3 from st session(ts, 10s) ; sleep 1000 diff --git a/tests/script/tsim/stream/fillHistoryBasic1.sim b/tests/script/tsim/stream/fillHistoryBasic1.sim index 772a09c017..e7a8da90e2 100644 --- a/tests/script/tsim/stream/fillHistoryBasic1.sim +++ b/tests/script/tsim/stream/fillHistoryBasic1.sim @@ -17,7 +17,7 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream stream1 trigger at_once fill_history 1 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); +sql create stream stream1 trigger at_once fill_history 1 IGNORE EXPIRED 0 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791223001,2,2,3,1.1); @@ -479,7 +479,7 @@ sql insert into t1 values(1648791233002,3,2,3,2.1); sql insert into t1 values(1648791243003,4,2,3,3.1); sql insert into t1 values(1648791213004,4,2,3,4.1); -sql create stream stream2 trigger at_once fill_history 1 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); +sql create stream stream2 trigger at_once fill_history 1 IGNORE EXPIRED 0 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); sleep 5000 sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; diff --git a/tests/script/tsim/stream/fillHistoryBasic2.sim b/tests/script/tsim/stream/fillHistoryBasic2.sim index 3af198259d..2f6c3ea92d 100644 --- a/tests/script/tsim/stream/fillHistoryBasic2.sim +++ b/tests/script/tsim/stream/fillHistoryBasic2.sim @@ -79,7 +79,7 @@ sql insert into ts3 values(1648791243005,4,42,3,3.1) (1648791243003,4,2,33,3.1) sql insert into ts4 values(1648791243005,4,42,3,3.1) (1648791243003,4,2,33,3.1) (1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) (1648791233004,13,12,13,2.1) ; sql insert into ts3 values(1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) (1648791223001,22,22,83,1.1) ; -sql create stream stream_t1 trigger at_once fill_history 1 watermark 1d into streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); +sql create stream stream_t1 trigger at_once fill_history 1 watermark 1d IGNORE EXPIRED 0 into streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); sleep 1000 @@ -211,7 +211,7 @@ sql insert into ts1 values(1648791222001,2,2,3); sql insert into ts2 values(1648791211000,1,2,3); sql insert into ts2 values(1648791222001,2,2,3); -sql create stream stream_t2 trigger at_once fill_history 1 watermark 20s into streamtST1 as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ; +sql create stream stream_t2 trigger at_once fill_history 1 watermark 20s IGNORE EXPIRED 0 into streamtST1 as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ; $loop_count = 0 loop2: @@ -241,7 +241,7 @@ sql use test3; sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); -sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from st interval(10s) ; +sql create stream stream_t3 trigger at_once IGNORE EXPIRED 0 into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from st interval(10s) ; sql insert into ts1 values(1648791211000,1,2,3); sleep 50 diff --git a/tests/script/tsim/stream/fillHistoryBasic3.sim b/tests/script/tsim/stream/fillHistoryBasic3.sim index db8d6bc2d0..44d7ee9d9e 100644 --- a/tests/script/tsim/stream/fillHistoryBasic3.sim +++ b/tests/script/tsim/stream/fillHistoryBasic3.sim @@ -17,7 +17,7 @@ sql create table t2 using st tags(2,2,2); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); -sql create stream streams2 trigger at_once fill_history 1 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); +sql create stream streams2 trigger at_once fill_history 1 IGNORE EXPIRED 0 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); sleep 3000 diff --git a/tests/script/tsim/stream/fillIntervalDelete0.sim b/tests/script/tsim/stream/fillIntervalDelete0.sim index 1c0647d57b..41b018a862 100644 --- a/tests/script/tsim/stream/fillIntervalDelete0.sim +++ b/tests/script/tsim/stream/fillIntervalDelete0.sim @@ -16,11 +16,11 @@ sql drop database if exists test1; sql create database test1 vgroups 1; sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); -sql create stream streams1 trigger at_once into streamt1 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(NULL); -sql create stream streams2 trigger at_once into streamt2 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value,100,200,300); -sql create stream streams3 trigger at_once into streamt3 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); -sql create stream streams4 trigger at_once into streamt4 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); -sql create stream streams5 trigger at_once into streamt5 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(NULL); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into streamt2 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value,100,200,300); +sql create stream streams3 trigger at_once IGNORE EXPIRED 0 into streamt3 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); +sql create stream streams4 trigger at_once IGNORE EXPIRED 0 into streamt4 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); +sql create stream streams5 trigger at_once IGNORE EXPIRED 0 into streamt5 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); sql insert into t1 values(1648791213000,1,1,1,1.0,'aaa'); sleep 200 @@ -256,11 +256,11 @@ sql use test6; sql create stable st(ts timestamp, a int, b int , c int, d double, s varchar(20)) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(1,1,1); -sql create stream streams6 trigger at_once into streamt6 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(NULL); -sql create stream streams7 trigger at_once into streamt7 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value,100,200,300); -sql create stream streams8 trigger at_once into streamt8 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); -sql create stream streams9 trigger at_once into streamt9 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); -sql create stream streams10 trigger at_once into streamt10 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); +sql create stream streams6 trigger at_once IGNORE EXPIRED 0 into streamt6 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(NULL); +sql create stream streams7 trigger at_once IGNORE EXPIRED 0 into streamt7 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value,100,200,300); +sql create stream streams8 trigger at_once IGNORE EXPIRED 0 into streamt8 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); +sql create stream streams9 trigger at_once IGNORE EXPIRED 0 into streamt9 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); +sql create stream streams10 trigger at_once IGNORE EXPIRED 0 into streamt10 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); sql insert into t1 values(1648791210000,1,1,1,1.0,'aaa'); sql insert into t1 values(1648791217000,1,1,1,1.0,'aaa'); diff --git a/tests/script/tsim/stream/fillIntervalDelete1.sim b/tests/script/tsim/stream/fillIntervalDelete1.sim index 0206b88fdc..108f5f862d 100644 --- a/tests/script/tsim/stream/fillIntervalDelete1.sim +++ b/tests/script/tsim/stream/fillIntervalDelete1.sim @@ -18,11 +18,11 @@ sql drop database if exists test1; sql create database test1 vgroups 1; sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); -sql create stream streams1 trigger at_once into streamt1 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(NULL); -sql create stream streams2 trigger at_once into streamt2 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value,100,200,300); -sql create stream streams3 trigger at_once into streamt3 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); -sql create stream streams4 trigger at_once into streamt4 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); -sql create stream streams5 trigger at_once into streamt5 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(NULL); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into streamt2 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value,100,200,300); +sql create stream streams3 trigger at_once IGNORE EXPIRED 0 into streamt3 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); +sql create stream streams4 trigger at_once IGNORE EXPIRED 0 into streamt4 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); +sql create stream streams5 trigger at_once IGNORE EXPIRED 0 into streamt5 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); sql insert into t1 values(1648791210000,0,0,0,0.0,'aaa'); sql insert into t1 values(1648791213000,1,1,1,1.0,'bbb'); @@ -221,11 +221,11 @@ sql use test6; sql create stable st(ts timestamp, a int, b int , c int, d double, s varchar(20)) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(1,1,1); -sql create stream streams6 trigger at_once into streamt6 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(NULL); -sql create stream streams7 trigger at_once into streamt7 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value,100,200,300); -sql create stream streams8 trigger at_once into streamt8 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); -sql create stream streams9 trigger at_once into streamt9 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); -sql create stream streams10 trigger at_once into streamt10 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); +sql create stream streams6 trigger at_once IGNORE EXPIRED 0 into streamt6 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(NULL); +sql create stream streams7 trigger at_once IGNORE EXPIRED 0 into streamt7 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value,100,200,300); +sql create stream streams8 trigger at_once IGNORE EXPIRED 0 into streamt8 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); +sql create stream streams9 trigger at_once IGNORE EXPIRED 0 into streamt9 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); +sql create stream streams10 trigger at_once IGNORE EXPIRED 0 into streamt10 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); sql insert into t1 values(1648791210000,1,1,1,1.0,'aaa'); sql insert into t1 values(1648791215000,6,8,8,8.0,'bbb'); @@ -353,11 +353,11 @@ sql drop database if exists test7; sql create database test7 vgroups 1; sql use test7; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); -sql create stream streams11 trigger at_once into streamt11 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(NULL); -sql create stream streams12 trigger at_once into streamt12 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(value,100.0,200); -sql create stream streams13 trigger at_once into streamt13 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(next); -sql create stream streams14 trigger at_once into streamt14 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(prev); -sql create stream streams15 trigger at_once into streamt15 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(linear); +sql create stream streams11 trigger at_once IGNORE EXPIRED 0 into streamt11 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(NULL); +sql create stream streams12 trigger at_once IGNORE EXPIRED 0 into streamt12 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(value,100.0,200); +sql create stream streams13 trigger at_once IGNORE EXPIRED 0 into streamt13 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(next); +sql create stream streams14 trigger at_once IGNORE EXPIRED 0 into streamt14 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(prev); +sql create stream streams15 trigger at_once IGNORE EXPIRED 0 into streamt15 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(linear); sql insert into t1 values(1648791210000,1,1,1,1.0,'aaa'); diff --git a/tests/script/tsim/stream/fillIntervalLinear.sim b/tests/script/tsim/stream/fillIntervalLinear.sim index b9f301d81b..3fa369d8d5 100644 --- a/tests/script/tsim/stream/fillIntervalLinear.sim +++ b/tests/script/tsim/stream/fillIntervalLinear.sim @@ -16,7 +16,7 @@ sql drop database if exists test1; sql create database test1 vgroups 1; sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); -sql create stream streams1 trigger at_once into streamt1 as select _wstart as ts, max(a)+sum(c), avg(b), first(s), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart as ts, max(a)+sum(c), avg(b), first(s), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); sql insert into t1 values(1648791213000,4,4,4,4.0,'aaa') (1648791216000,5,5,5,5.0,'bbb'); sql insert into t1 values(1648791210000,1,1,1,1.0,'ccc') (1648791219000,2,2,2,2.0,'ddd') (1648791222000,3,3,3,3.0,'eee'); @@ -205,7 +205,7 @@ sql drop database if exists test2; sql create database test2 vgroups 1; sql use test2; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); -sql create stream streams2 trigger at_once into streamt2 as select _wstart as ts, max(a)+sum(c), avg(b), first(s), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into streamt2 as select _wstart as ts, max(a)+sum(c), avg(b), first(s), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); sql insert into t1 values(1648791210000,1,1,1,1.0,'ccc') (1648791219000,2,2,2,2.0,'ddd') (1648791222000,3,3,3,3.0,'eee'); sql insert into t1 values(1648791213000,4,4,4,4.0,'aaa') (1648791216000,5,5,5,5.0,'bbb'); @@ -393,7 +393,7 @@ sql drop database if exists test3; sql create database test3 vgroups 1; sql use test3; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); -sql create stream streams3 trigger at_once into streamt3 as select _wstart as ts, max(a), b+c, s, b+1, 1 from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(linear); +sql create stream streams3 trigger at_once IGNORE EXPIRED 0 into streamt3 as select _wstart as ts, max(a), b+c, s, b+1, 1 from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(linear); sql insert into t1 values(1648791215000,1,1,1,1.0,'aaa'); sql insert into t1 values(1648791217000,2,2,2,2.0,'bbb'); sql insert into t1 values(1648791211000,3,3,3,3.0,'ccc'); diff --git a/tests/script/tsim/stream/fillIntervalPartitionBy.sim b/tests/script/tsim/stream/fillIntervalPartitionBy.sim index 168452fdc8..6a11b9952c 100644 --- a/tests/script/tsim/stream/fillIntervalPartitionBy.sim +++ b/tests/script/tsim/stream/fillIntervalPartitionBy.sim @@ -18,11 +18,11 @@ sql use test1; sql create stable st(ts timestamp, a int, b int , c int, d double, s varchar(20)) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams1 trigger at_once into streamt1 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(NULL); -sql create stream streams2 trigger at_once into streamt2 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(value,100,200,300); -sql create stream streams3 trigger at_once into streamt3 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(next); -sql create stream streams4 trigger at_once into streamt4 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(prev); -sql create stream streams5 trigger at_once into streamt5 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(linear); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(NULL); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into streamt2 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(value,100,200,300); +sql create stream streams3 trigger at_once IGNORE EXPIRED 0 into streamt3 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(next); +sql create stream streams4 trigger at_once IGNORE EXPIRED 0 into streamt4 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(prev); +sql create stream streams5 trigger at_once IGNORE EXPIRED 0 into streamt5 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(linear); sql insert into t1 values(1648791210000,0,0,0,0.0,'aaa'); sql insert into t1 values(1648791213000,1,1,1,1.0,'bbb'); diff --git a/tests/script/tsim/stream/fillIntervalPrevNext.sim b/tests/script/tsim/stream/fillIntervalPrevNext.sim index 99f7dcb5cb..ec963e1d4a 100644 --- a/tests/script/tsim/stream/fillIntervalPrevNext.sim +++ b/tests/script/tsim/stream/fillIntervalPrevNext.sim @@ -15,8 +15,8 @@ sql drop database if exists test1; sql create database test1 vgroups 1; sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); -sql create stream streams1 trigger at_once into streamt1 as select _wstart as ts, count(*) c1, max(b)+sum(a) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); -sql create stream streams2 trigger at_once into streamt2 as select _wstart as ts, count(*) c1, max(a)+min(c), avg(b) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart as ts, count(*) c1, max(b)+sum(a) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into streamt2 as select _wstart as ts, count(*) c1, max(a)+min(c), avg(b) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); sql insert into t1 values(1648791213000,4,4,4,4.0,'aaa') (1648791215000,5,5,5,5.0,'aaa'); sql insert into t1 values(1648791211000,1,1,1,1.0,'aaa') (1648791217000,2,2,2,2.0,'aaa') (1648791220000,3,3,3,3.0,'aaa'); @@ -263,8 +263,8 @@ sql drop database if exists test5; sql create database test5 vgroups 1; sql use test5; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); -sql create stream streams5 trigger at_once into streamt5 as select _wstart as ts, count(*) c1, max(b)+sum(a) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); -sql create stream streams6 trigger at_once into streamt6 as select _wstart as ts, count(*) c1, max(a)+min(c), avg(b) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); +sql create stream streams5 trigger at_once IGNORE EXPIRED 0 into streamt5 as select _wstart as ts, count(*) c1, max(b)+sum(a) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); +sql create stream streams6 trigger at_once IGNORE EXPIRED 0 into streamt6 as select _wstart as ts, count(*) c1, max(a)+min(c), avg(b) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); sql insert into t1 values(1648791211000,1,1,1,1.0,'aaa') (1648791217000,2,2,2,2.0,'aaa') (1648791220000,3,3,3,3.0,'aaa'); sql insert into t1 values(1648791213000,4,4,4,4.0,'aaa') (1648791215000,5,5,5,5.0,'aaa'); diff --git a/tests/script/tsim/stream/fillIntervalPrevNext1.sim b/tests/script/tsim/stream/fillIntervalPrevNext1.sim index 8058065bcf..40ef895c5a 100644 --- a/tests/script/tsim/stream/fillIntervalPrevNext1.sim +++ b/tests/script/tsim/stream/fillIntervalPrevNext1.sim @@ -16,8 +16,8 @@ sql drop database if exists test7; sql create database test7 vgroups 1; sql use test7; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); -sql create stream streams7 trigger at_once into streamt7 as select _wstart as ts, max(a), b+c, s from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(prev); -sql create stream streams8 trigger at_once into streamt8 as select _wstart as ts, max(a), 1, b+1 from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(next); +sql create stream streams7 trigger at_once IGNORE EXPIRED 0 into streamt7 as select _wstart as ts, max(a), b+c, s from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(prev); +sql create stream streams8 trigger at_once IGNORE EXPIRED 0 into streamt8 as select _wstart as ts, max(a), 1, b+1 from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(next); sql insert into t1 values(1648791215000,1,1,1,1.0,'aaa'); sql insert into t1 values(1648791217000,2,2,2,2.0,'bbb'); sql insert into t1 values(1648791211000,3,3,3,3.0,'ccc'); diff --git a/tests/script/tsim/stream/fillIntervalRange.sim b/tests/script/tsim/stream/fillIntervalRange.sim index a0905141f2..0e0dfb46d8 100644 --- a/tests/script/tsim/stream/fillIntervalRange.sim +++ b/tests/script/tsim/stream/fillIntervalRange.sim @@ -13,7 +13,7 @@ sql create database test vgroups 1; sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20));; -sql create stream streams1 trigger at_once into streamt as select _wstart ts, count(*) c1 from t1 interval(1s) fill(NULL); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart ts, count(*) c1 from t1 interval(1s) fill(NULL); sql insert into t1 values(1648791211000,1,2,3,1.0,'aaa'); sleep 100 sql insert into t1 values(1648795308000,1,2,3,1.0,'aaa'); @@ -126,10 +126,10 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); print create stream streams1 trigger at_once into streamt as select _wstart ts, max(a) c1 from t1 interval(1s) fill(linear); -sql create stream streams1 trigger at_once into streamt as select _wstart ts, max(a) c1 from t1 interval(1s) fill(linear); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart ts, max(a) c1 from t1 interval(1s) fill(linear); print create stream streams2 trigger at_once into streamt2 as select _wstart ts, max(a) c1 from t1 interval(1s) fill(prev); -sql create stream streams2 trigger at_once into streamt2 as select _wstart ts, max(a) c1 from t1 interval(1s) fill(prev); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into streamt2 as select _wstart ts, max(a) c1 from t1 interval(1s) fill(prev); sql insert into t1 values(1648791211000,1,2,3,1.0,'aaa'); sleep 100 diff --git a/tests/script/tsim/stream/fillIntervalValue.sim b/tests/script/tsim/stream/fillIntervalValue.sim index 2cd419397f..b447e9a559 100644 --- a/tests/script/tsim/stream/fillIntervalValue.sim +++ b/tests/script/tsim/stream/fillIntervalValue.sim @@ -13,8 +13,8 @@ sql create database test vgroups 1; sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20));; -sql create stream streams1 trigger at_once into streamt as select _wstart ts, count(*) c1 from t1 where ts > 1648791210000 and ts < 1648791413000 interval(10s) fill(value, 100); -sql create stream streams1a trigger at_once into streamta as select _wstart ts, count(*) c1 from t1 where ts > 1648791210000 and ts < 1648791413000 interval(10s) fill(value_f, 100); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart ts, count(*) c1 from t1 where ts > 1648791210000 and ts < 1648791413000 interval(10s) fill(value, 100); +sql create stream streams1a trigger at_once IGNORE EXPIRED 0 into streamta as select _wstart ts, count(*) c1 from t1 where ts > 1648791210000 and ts < 1648791413000 interval(10s) fill(value_f, 100); sql insert into t1 values(1648791213000,1,2,3,1.0,'aaa'); sleep 100 sql insert into t1 values(1648791233000,1,2,3,1.0,'aaa'); @@ -146,7 +146,7 @@ sql drop database if exists test2; sql create database test2 vgroups 1; sql use test2; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); -sql create stream streams2 trigger at_once into streamt2 as select _wstart as ts, count(*) c1, max(b)+sum(a) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value, 100,200); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into streamt2 as select _wstart as ts, count(*) c1, max(b)+sum(a) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value, 100,200); sql insert into t1 values(1648791211000,1,1,1,1.0,'aaa') (1648791217000,2,2,2,2.0,'aaa') (1648791220000,3,3,3,3.0,'aaa'); sql insert into t1 values(1648791213000,4,4,4,4.0,'aaa') (1648791215000,5,5,5,5.0,'aaa'); @@ -280,7 +280,7 @@ sql drop database if exists test3; sql create database test3 vgroups 1; sql use test3; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); -sql create stream streams3 trigger at_once into streamt3 as select _wstart as ts, max(b), a+b, c from t1 where ts >= 1648791200000 and ts < 1648791261000 interval(10s) sliding(3s) fill(value, 100,200,300); +sql create stream streams3 trigger at_once IGNORE EXPIRED 0 into streamt3 as select _wstart as ts, max(b), a+b, c from t1 where ts >= 1648791200000 and ts < 1648791261000 interval(10s) sliding(3s) fill(value, 100,200,300); sql insert into t1 values(1648791220000,1,1,1,1.0,'aaa'); sleep 100 @@ -471,8 +471,8 @@ sql create stable st(ts timestamp,a int,b int,c int, d double, s varchar(20) ) t sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams4 trigger at_once into streamt4 as select _wstart ts, count(*) c1, concat(tbname, 'aaa') as pname, timezone() from st where ts > 1648791000000 and ts < 1648793000000 partition by tbname interval(10s) fill(NULL); -sql create stream streams4a trigger at_once into streamt4a as select _wstart ts, count(*) c1, concat(tbname, 'aaa') as pname, timezone() from st where ts > 1648791000000 and ts < 1648793000000 partition by tbname interval(10s) fill(NULL_F); +sql create stream streams4 trigger at_once IGNORE EXPIRED 0 into streamt4 as select _wstart ts, count(*) c1, concat(tbname, 'aaa') as pname, timezone() from st where ts > 1648791000000 and ts < 1648793000000 partition by tbname interval(10s) fill(NULL); +sql create stream streams4a trigger at_once IGNORE EXPIRED 0 into streamt4a as select _wstart ts, count(*) c1, concat(tbname, 'aaa') as pname, timezone() from st where ts > 1648791000000 and ts < 1648793000000 partition by tbname interval(10s) fill(NULL_F); sql insert into t1 values(1648791213000,1,2,3,1.0,'aaa'); sql insert into t1 values(1648791233000,1,2,3,1.0,'aaa'); sql insert into t1 values(1648791273000,1,2,3,1.0,'aaa'); diff --git a/tests/script/tsim/stream/ignoreCheckUpdate.sim b/tests/script/tsim/stream/ignoreCheckUpdate.sim index 7f99c534c8..2cd0117feb 100644 --- a/tests/script/tsim/stream/ignoreCheckUpdate.sim +++ b/tests/script/tsim/stream/ignoreCheckUpdate.sim @@ -12,9 +12,9 @@ sql create database test vgroups 1; sql use test; sql create table t1(ts timestamp, a int, b int , c int); -print create stream streams0 trigger at_once ignore update 1 into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 interval(10s); +print create stream streams0 trigger at_once IGNORE EXPIRED 0 ignore update 1 into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 interval(10s); -sql create stream streams0 trigger at_once ignore update 1 into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 interval(10s); +sql create stream streams0 trigger at_once IGNORE EXPIRED 0 ignore update 1 into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 interval(10s); sql insert into t1 values(1648791213000,1,1,1); sql insert into t1 values(1648791213000,2,2,2); diff --git a/tests/script/tsim/stream/partitionby.sim b/tests/script/tsim/stream/partitionby.sim index bc2c07b951..e63459e97d 100644 --- a/tests/script/tsim/stream/partitionby.sim +++ b/tests/script/tsim/stream/partitionby.sim @@ -12,7 +12,7 @@ sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); -sql create stream stream_t1 trigger at_once into test0.streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st partition by ta,tb,tc interval(10s); +sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 into test0.streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st partition by ta,tb,tc interval(10s); sql insert into ts1 values(1648791213001,1,12,3,1.0); sql insert into ts2 values(1648791213001,1,12,3,1.0); @@ -67,7 +67,7 @@ sql create table ts1 using st tags(1,2,3); sql create table ts2 using st tags(1,3,4); sql create table ts3 using st tags(1,4,5); -sql create stream streams1 trigger at_once into streamt as select _wstart, count(*) c1, count(a) c2 from st partition by ta,tb,tc interval(10s); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart, count(*) c1, count(a) c2 from st partition by ta,tb,tc interval(10s); sql insert into ts1 values(1648791211000,1,2,3); @@ -98,7 +98,7 @@ sql create stable st(ts timestamp,a int,b int,c int,id int) tags(ta int,tb int,t sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); -sql create stream stream_t2 trigger at_once watermark 20s into streamtST as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by ta interval(10s) ; +sql create stream stream_t2 trigger at_once watermark 20s IGNORE EXPIRED 0 into streamtST as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by ta interval(10s) ; sql insert into ts1 values(1648791211000,1,2,3,1); sql insert into ts1 values(1648791222001,2,2,3,2); sql insert into ts2 values(1648791211000,1,2,3,3); diff --git a/tests/script/tsim/stream/partitionby1.sim b/tests/script/tsim/stream/partitionby1.sim index b29666cad7..c8bb25e0dd 100644 --- a/tests/script/tsim/stream/partitionby1.sim +++ b/tests/script/tsim/stream/partitionby1.sim @@ -11,7 +11,7 @@ sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); -sql create stream stream_t1 trigger at_once into streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st partition by tbname interval(10s); +sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 into streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st partition by tbname interval(10s); sql insert into ts1 values(1648791213001,1,12,3,1.0); sql insert into ts2 values(1648791213001,1,12,3,1.0); @@ -43,7 +43,7 @@ sql create table ts1 using st tags(1,2,3); sql create table ts2 using st tags(1,3,4); sql create table ts3 using st tags(1,4,5); -sql create stream streams1 trigger at_once into streamt as select _wstart, count(*) c1, count(a) c2 from st partition by tbname interval(10s); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart, count(*) c1, count(a) c2 from st partition by tbname interval(10s); sql insert into ts1 values(1648791211000,1,2,3); @@ -74,7 +74,7 @@ sql create stable st(ts timestamp,a int,b int,c int,id int) tags(ta int,tb int,t sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); -sql create stream stream_t2 trigger at_once into streamtST as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by tbname interval(10s) ; +sql create stream stream_t2 trigger at_once IGNORE EXPIRED 0 into streamtST as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by tbname interval(10s) ; sql insert into ts1 values(1648791211000,1,2,3,1); sql insert into ts1 values(1648791222001,2,2,3,2); sql insert into ts2 values(1648791211000,1,2,3,3); diff --git a/tests/script/tsim/stream/partitionbyColumnInterval.sim b/tests/script/tsim/stream/partitionbyColumnInterval.sim index 2e57e8d699..94053990e4 100644 --- a/tests/script/tsim/stream/partitionbyColumnInterval.sim +++ b/tests/script/tsim/stream/partitionbyColumnInterval.sim @@ -16,7 +16,7 @@ sql drop database if exists test; sql create database test vgroups 1; sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a interval(10s); +sql create stream streams0 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a interval(10s); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); @@ -197,7 +197,7 @@ sql drop database if exists test1; sql create database test1 vgroups 1; sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream streams1 trigger at_once into streamt1 as select _wstart c1, count(*) c2, max(c) c3, _group_key(a+b) c4 from t1 partition by a+b interval(10s); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart c1, count(*) c2, max(c) c3, _group_key(a+b) c4 from t1 partition by a+b interval(10s); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); @@ -284,7 +284,7 @@ sql use test2; sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); @@ -481,7 +481,7 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create table t3 using st tags(2,2,2); sql create table t4 using st tags(2,2,2); -sql create stream streams4 trigger at_once into test.streamt4 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); +sql create stream streams4 trigger at_once IGNORE EXPIRED 0 into test.streamt4 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); sql insert into t1 values(1648791213000,2,2,3,1.0); sql insert into t2 values(1648791213000,2,2,3,1.0); @@ -571,7 +571,7 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create table t3 using st tags(2,2,2); sql create table t4 using st tags(2,2,2); -sql create stream streams5 trigger at_once into test.streamt5 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); +sql create stream streams5 trigger at_once IGNORE EXPIRED 0 into test.streamt5 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t2 values(1648791213000,2,2,3,1.0); diff --git a/tests/script/tsim/stream/partitionbyColumnSession.sim b/tests/script/tsim/stream/partitionbyColumnSession.sim index 1742d52cf0..bb3f6015c7 100644 --- a/tests/script/tsim/stream/partitionbyColumnSession.sim +++ b/tests/script/tsim/stream/partitionbyColumnSession.sim @@ -16,7 +16,7 @@ sql drop database if exists test; sql create database test vgroups 1; sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a session(ts, 5s); +sql create stream streams0 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a session(ts, 5s); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); @@ -196,7 +196,7 @@ sql drop database if exists test1; sql create database test1 vgroups 1; sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream streams1 trigger at_once into streamt1 as select _wstart c1, count(*) c2, max(c) c3, _group_key(a+b) c4 from t1 partition by a+b session(ts, 5s); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart c1, count(*) c2, max(c) c3, _group_key(a+b) c4 from t1 partition by a+b session(ts, 5s); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); @@ -282,7 +282,7 @@ sql use test2; sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a session(ts, 5s); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a session(ts, 5s); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); @@ -478,7 +478,7 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create table t3 using st tags(2,2,2); sql create table t4 using st tags(2,2,2); -sql create stream streams4 trigger at_once into test.streamt4 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a session(ts, 5s); +sql create stream streams4 trigger at_once IGNORE EXPIRED 0 into test.streamt4 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a session(ts, 5s); sql insert into t1 values(1648791213000,2,2,3,1.0); sql insert into t2 values(1648791213000,2,2,3,1.0); diff --git a/tests/script/tsim/stream/partitionbyColumnState.sim b/tests/script/tsim/stream/partitionbyColumnState.sim index 75d01b17ec..62262a490c 100644 --- a/tests/script/tsim/stream/partitionbyColumnState.sim +++ b/tests/script/tsim/stream/partitionbyColumnState.sim @@ -11,7 +11,7 @@ sql drop database if exists test; sql create database test vgroups 1; sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a state_window(b); +sql create stream streams0 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a state_window(b); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); @@ -191,7 +191,7 @@ sql drop database if exists test1; sql create database test1 vgroups 1; sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d int); -sql create stream streams1 trigger at_once into streamt1 as select _wstart c1, count(*) c2, max(d) c3, _group_key(a+b) c4 from t1 partition by a+b state_window(c); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart c1, count(*) c2, max(d) c3, _group_key(a+b) c4 from t1 partition by a+b state_window(c); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); diff --git a/tests/script/tsim/stream/schedSnode.sim b/tests/script/tsim/stream/schedSnode.sim index 2caecf50a2..6a4d6f79bb 100644 --- a/tests/script/tsim/stream/schedSnode.sim +++ b/tests/script/tsim/stream/schedSnode.sim @@ -20,7 +20,7 @@ sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); -sql create stream stream_t1 trigger at_once into target.streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); +sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 into target.streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); sleep 1000 diff --git a/tests/script/tsim/stream/session0.sim b/tests/script/tsim/stream/session0.sim index 5e95428e0a..622c5f7c6d 100644 --- a/tests/script/tsim/stream/session0.sim +++ b/tests/script/tsim/stream/session0.sim @@ -17,7 +17,7 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double,id int); -sql create stream streams1 trigger at_once into streamt as select _wstart, count(*) c1, sum(a), max(a), min(d), stddev(a), last(a), first(d), max(id) s from t1 session(ts,10s); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart, count(*) c1, sum(a), max(a), min(d), stddev(a), last(a), first(d), max(id) s from t1 session(ts,10s); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL,1); sql insert into t1 values(1648791223001,10,2,3,1.1,2); sql insert into t1 values(1648791233002,3,2,3,2.1,3); @@ -179,7 +179,7 @@ endi sql create database test2 vgroups 1; sql use test2; sql create table t2(ts timestamp, a int, b int , c int, d double, id int); -sql create stream streams2 trigger at_once watermark 1d into streamt2 as select _wstart,apercentile(a,30) c1, apercentile(a,70), apercentile(a,20,"t-digest") c2, apercentile(a,60,"t-digest") c3, max(id) c4 from t2 session(ts,10s); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 watermark 1d into streamt2 as select _wstart,apercentile(a,30) c1, apercentile(a,70), apercentile(a,20,"t-digest") c2, apercentile(a,60,"t-digest") c3, max(id) c4 from t2 session(ts,10s); sql insert into t2 values(1648791213001,1,1,3,1.0,1); sql insert into t2 values(1648791213002,2,2,6,3.4,2); sql insert into t2 values(1648791213003,4,9,3,4.8,3); @@ -229,13 +229,13 @@ endi sql create database test3 vgroups 1; sql use test3; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream streams3 trigger at_once watermark 1d into streamt3 as select _wstart, min(b), a,c from t1 session(ts,10s); -sql create stream streams4 trigger at_once watermark 1d into streamt4 as select _wstart, max(b), a,c from t1 session(ts,10s); -# sql create stream streams5 trigger at_once watermark 1d into streamt5 as select _wstart, top(b,3), a,c from t1 session(ts,10s); -# sql create stream streams6 trigger at_once watermark 1d into streamt6 as select _wstart, bottom(b,3), a,c from t1 session(ts,10s); -# sql create stream streams7 trigger at_once watermark 1d into streamt7 as select _wstart, spread(a), elapsed(ts), hyperloglog(a) from t1 session(ts,10s); -sql create stream streams7 trigger at_once watermark 1d into streamt7 as select _wstart, spread(a), hyperloglog(a) from t1 session(ts,10s); -# sql create stream streams8 trigger at_once watermark 1d into streamt8 as select _wstart, histogram(a,"user_input", "[1,3,5,7]", 1), histogram(a,"user_input", "[1,3,5,7]", 0) from t1 session(ts,10s); +sql create stream streams3 trigger at_once watermark 1d IGNORE EXPIRED 0 into streamt3 as select _wstart, min(b), a,c from t1 session(ts,10s); +sql create stream streams4 trigger at_once watermark 1d IGNORE EXPIRED 0 into streamt4 as select _wstart, max(b), a,c from t1 session(ts,10s); +# sql create stream streams5 trigger at_once watermark 1d IGNORE EXPIRED 0 into streamt5 as select _wstart, top(b,3), a,c from t1 session(ts,10s); +# sql create stream streams6 trigger at_once watermark 1d IGNORE EXPIRED 0 into streamt6 as select _wstart, bottom(b,3), a,c from t1 session(ts,10s); +# sql create stream streams7 trigger at_once watermark 1d IGNORE EXPIRED 0 into streamt7 as select _wstart, spread(a), elapsed(ts), hyperloglog(a) from t1 session(ts,10s); +sql create stream streams7 trigger at_once watermark 1d IGNORE EXPIRED 0 into streamt7 as select _wstart, spread(a), hyperloglog(a) from t1 session(ts,10s); +# sql create stream streams8 trigger at_once watermark 1d IGNORE EXPIRED 0 into streamt8 as select _wstart, histogram(a,"user_input", "[1,3,5,7]", 1), histogram(a,"user_input", "[1,3,5,7]", 0) from t1 session(ts,10s); sql insert into t1 values(1648791213001,1,1,1,1.0); sql insert into t1 values(1648791213002,2,3,2,3.4); sql insert into t1 values(1648791213003,4,9,3,4.8); diff --git a/tests/script/tsim/stream/session1.sim b/tests/script/tsim/stream/session1.sim index f535fd619f..3ad7c6f04e 100644 --- a/tests/script/tsim/stream/session1.sim +++ b/tests/script/tsim/stream/session1.sim @@ -17,7 +17,7 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double,id int); -sql create stream streams2 trigger at_once into streamt as select _wstart, count(*) c1, sum(a), min(b), max(id) s from t1 session(ts,10s); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart, count(*) c1, sum(a), min(b), max(id) s from t1 session(ts,10s); sql insert into t1 values(1648791210000,1,1,1,1.1,1); sql insert into t1 values(1648791220000,2,2,2,2.1,2); sql insert into t1 values(1648791230000,3,3,3,3.1,3); @@ -200,7 +200,7 @@ endi sql create database test1 vgroups 1; sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream streams3 trigger at_once into streamt3 as select _wstart, count(*) c1 from t1 where a > 5 session(ts, 5s); +sql create stream streams3 trigger at_once IGNORE EXPIRED 0 into streamt3 as select _wstart, count(*) c1 from t1 where a > 5 session(ts, 5s); sql insert into t1 values(1648791213000,1,2,3,1.0); $loop_count = 0 diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index 8287274cd2..e12e3c6686 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -17,10 +17,10 @@ sql use test; sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams1 trigger at_once into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s) sliding (5s); -sql create stream streams2 trigger at_once watermark 1d into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s) sliding (5s); -sql create stream stream_t1 trigger at_once into streamtST as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s) sliding (5s); -sql create stream stream_t2 trigger at_once watermark 1d into streamtST2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s) sliding (5s); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s) sliding (5s); +sql create stream streams2 trigger at_once watermark 1d IGNORE EXPIRED 0 into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s) sliding (5s); +sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 into streamtST as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s) sliding (5s); +sql create stream stream_t2 trigger at_once watermark 1d IGNORE EXPIRED 0 into streamtST2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s) sliding (5s); sql insert into t1 values(1648791210000,1,2,3,1.0); sql insert into t1 values(1648791216000,2,2,3,1.1); @@ -309,8 +309,8 @@ sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams11 trigger at_once into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s); -sql create stream streams12 trigger at_once into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); +sql create stream streams11 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s); +sql create stream streams12 trigger at_once IGNORE EXPIRED 0 into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791223001,2,2,3,1.1); @@ -442,9 +442,9 @@ sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams21 trigger at_once into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s); -sql create stream streams22 trigger at_once into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); -sql create stream streams23 trigger at_once into streamt3 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(20s) sliding(10s); +sql create stream streams21 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s); +sql create stream streams22 trigger at_once IGNORE EXPIRED 0 into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); +sql create stream streams23 trigger at_once IGNORE EXPIRED 0 into streamt3 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(20s) sliding(10s); sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791223001,2,2,2,1.1); @@ -683,7 +683,7 @@ sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams4 trigger at_once into streamt4 as select _wstart as ts, count(*),min(a) c1 from st interval(10s) sliding(5s); +sql create stream streams4 trigger at_once IGNORE EXPIRED 0 into streamt4 as select _wstart as ts, count(*),min(a) c1 from st interval(10s) sliding(5s); sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791243000,2,1,1,1.0); diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim index 07abdc0040..d009f742b7 100644 --- a/tests/script/tsim/stream/state0.sim +++ b/tests/script/tsim/stream/state0.sim @@ -17,9 +17,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double, id int); -print create stream streams1 trigger at_once into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); +print create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); -sql create stream streams1 trigger at_once into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); sql insert into t1 values(1648791213000,1,2,3,1.0,1); sql insert into t1 values(1648791213000,1,2,3,1.0,2); @@ -453,9 +453,9 @@ sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double, id int); -print create stream streams2 trigger at_once into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); +print create stream streams2 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); -sql create stream streams2 trigger at_once into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); sql insert into t1 values(1648791212000,2,2,3,1.0,1); sql insert into t1 values(1648791213000,1,2,3,1.0,1); @@ -501,9 +501,9 @@ sql use test3; sql create table t1(ts timestamp, a int, b int , c int, d double, id int); -print create stream streams3 trigger at_once into streamt3 as select _wstart, count(*) c1, sum(b) c3 from t1 state_window(a); +print create stream streams3 trigger at_once IGNORE EXPIRED 0 into streamt3 as select _wstart, count(*) c1, sum(b) c3 from t1 state_window(a); -sql create stream streams3 trigger at_once into streamt3 as select _wstart, count(*) c1, sum(b) c3 from t1 state_window(a); +sql create stream streams3 trigger at_once IGNORE EXPIRED 0 into streamt3 as select _wstart, count(*) c1, sum(b) c3 from t1 state_window(a); sql insert into t1 values(1648791212000,1,2,3,1.0,1); sql insert into t1 values(1648791213000,2,2,3,1.0,1); sql insert into t1 values(1648791214000,3,2,4,1.0,2); @@ -553,9 +553,9 @@ sql create table st (ts timestamp, c1 tinyint, c2 smallint) tags (t1 tinyint) ; sql create table t1 using st tags (-81) ; sql create table t2 using st tags (-81) ; -print create stream if not exists streams4 trigger window_close into streamt4 as select _wstart AS startts, min(c1),count(c1) from t1 state_window(c1); +print create stream if not exists streams4 trigger window_close IGNORE EXPIRED 0 into streamt4 as select _wstart AS startts, min(c1),count(c1) from t1 state_window(c1); -sql create stream if not exists streams4 trigger window_close into streamt4 as select _wstart AS startts, min(c1),count(c1) from t1 state_window(c1); +sql create stream if not exists streams4 trigger window_close IGNORE EXPIRED 0 into streamt4 as select _wstart AS startts, min(c1),count(c1) from t1 state_window(c1); sql insert into t1 (ts, c1) values (1668073288209, 11); sql insert into t1 (ts, c1) values (1668073288210, 11); @@ -742,9 +742,9 @@ sql create table tb (ts timestamp, a int); sql insert into tb values (now + 1m , 1 ); sql create table b (c timestamp, d int, e int , f int, g double); -print create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a); +print create stream streams0 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a); -sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a); +sql create stream streams0 trigger at_once IGNORE EXPIRED 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a); sql insert into b values(1648791213000,NULL,NULL,NULL,NULL); sql select * from streamt order by c1, c2, c3; diff --git a/tests/script/tsim/stream/triggerInterval0.sim b/tests/script/tsim/stream/triggerInterval0.sim index b522dcf035..1c62f689ac 100644 --- a/tests/script/tsim/stream/triggerInterval0.sim +++ b/tests/script/tsim/stream/triggerInterval0.sim @@ -15,7 +15,7 @@ print $data00 $data01 $data02 sql use test sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream streams1 trigger window_close into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); +sql create stream streams1 trigger window_close IGNORE EXPIRED 0 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); sql insert into t1 values(1648791213001,1,2,3,1.0); sleep 300 diff --git a/tests/script/tsim/stream/triggerSession0.sim b/tests/script/tsim/stream/triggerSession0.sim index 4c664cf7c7..81a016be2b 100644 --- a/tests/script/tsim/stream/triggerSession0.sim +++ b/tests/script/tsim/stream/triggerSession0.sim @@ -15,7 +15,7 @@ print $data00 $data01 $data02 sql use test; sql create table t2(ts timestamp, a int, b int , c int, d double); -sql create stream streams2 trigger window_close into streamt2 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t2 session(ts, 10s); +sql create stream streams2 trigger window_close IGNORE EXPIRED 0 into streamt2 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t2 session(ts, 10s); sql insert into t2 values(1648791213000,1,2,3,1.0); sql insert into t2 values(1648791222999,1,2,3,1.0); diff --git a/tests/script/tsim/stream/udTableAndTag0.sim b/tests/script/tsim/stream/udTableAndTag0.sim index 8bf34dc54c..e3ab344bbe 100644 --- a/tests/script/tsim/stream/udTableAndTag0.sim +++ b/tests/script/tsim/stream/udTableAndTag0.sim @@ -20,8 +20,8 @@ sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -#sql_error create stream streams1 trigger at_once into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); -sql create stream streams1 trigger at_once into result.streamt SUBTABLE(concat("aaa-", tbname)) as select _wstart, count(*) c1 from st partition by tbname interval(10s); +#sql_error create stream streams1 trigger at_once IGNORE EXPIRED 0 into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into result.streamt SUBTABLE(concat("aaa-", tbname)) as select _wstart, count(*) c1 from st partition by tbname interval(10s); sql insert into t1 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,1,2,3); @@ -88,7 +88,7 @@ sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams2 trigger at_once into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as cc interval(10s); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as cc interval(10s); sql insert into t1 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,1,2,3); @@ -173,7 +173,7 @@ sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams3 trigger at_once into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", tbname)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as dd, tbname interval(10s); +sql create stream streams3 trigger at_once IGNORE EXPIRED 0 into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", tbname)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as dd, tbname interval(10s); sql insert into t1 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,1,2,3); @@ -285,7 +285,7 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create table t3 using st tags(3,3,3); -sql create stream streams4 trigger at_once into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", tbname)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as dd, tbname interval(10s); +sql create stream streams4 trigger at_once IGNORE EXPIRED 0 into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", tbname)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as dd, tbname interval(10s); sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); @@ -404,7 +404,7 @@ sql create table t1 using st tags("1",1,1); sql create table t2 using st tags("2",2,2); sql create table t3 using st tags("3",3,3); -sql create stream streams6 trigger at_once into result6.streamt6 TAGS(dd int) as select _wstart, count(*) c1 from st partition by concat(ta, "0") as dd, tbname interval(10s); +sql create stream streams6 trigger at_once IGNORE EXPIRED 0 into result6.streamt6 TAGS(dd int) as select _wstart, count(*) c1 from st partition by concat(ta, "0") as dd, tbname interval(10s); sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); diff --git a/tests/script/tsim/stream/udTableAndTag1.sim b/tests/script/tsim/stream/udTableAndTag1.sim index 4229de2cf0..fbedaa6e4e 100644 --- a/tests/script/tsim/stream/udTableAndTag1.sim +++ b/tests/script/tsim/stream/udTableAndTag1.sim @@ -20,8 +20,8 @@ sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -#sql_error create stream streams1 trigger at_once into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); -sql create stream streams1 trigger at_once into result.streamt SUBTABLE( concat("aaa-", cast(a as varchar(10) ) ) ) as select _wstart, count(*) c1 from st partition by a interval(10s); +#sql_error create stream streams1 trigger at_once IGNORE EXPIRED 0 into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into result.streamt SUBTABLE( concat("aaa-", cast(a as varchar(10) ) ) ) as select _wstart, count(*) c1 from st partition by a interval(10s); print ===== insert into 1 sql insert into t1 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,2,2,3); @@ -87,7 +87,7 @@ sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams2 trigger at_once into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("col-", cast(a as varchar(10) ) ) as cc interval(10s); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("col-", cast(a as varchar(10) ) ) as cc interval(10s); print ===== insert into 2 sql insert into t1 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,2,2,3); @@ -171,7 +171,7 @@ sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams3 trigger at_once into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", cast(a as varchar(10) ) ) ) as select _wstart, count(*) c1 from st partition by concat("col-", cast(a as varchar(10) ) ) as dd, a interval(10s); +sql create stream streams3 trigger at_once IGNORE EXPIRED 0 into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", cast(a as varchar(10) ) ) ) as select _wstart, count(*) c1 from st partition by concat("col-", cast(a as varchar(10) ) ) as dd, a interval(10s); print ===== insert into 3 sql insert into t1 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,2,2,3); @@ -283,7 +283,7 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create table t3 using st tags(3,3,3); -sql create stream streams4 trigger at_once into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", dd)) as select _wstart, count(*) c1 from st partition by concat("t", cast(a as varchar(10) ) ) as dd interval(10s); +sql create stream streams4 trigger at_once IGNORE EXPIRED 0 into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", dd)) as select _wstart, count(*) c1 from st partition by concat("t", cast(a as varchar(10) ) ) as dd interval(10s); sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); diff --git a/tests/script/tsim/stream/udTableAndTag2.sim b/tests/script/tsim/stream/udTableAndTag2.sim index bacc301ad0..c0e72712df 100644 --- a/tests/script/tsim/stream/udTableAndTag2.sim +++ b/tests/script/tsim/stream/udTableAndTag2.sim @@ -20,7 +20,7 @@ sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams1 trigger at_once into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); print ===== insert into 1 sql insert into t1 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,2,2,3); @@ -93,7 +93,7 @@ sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams2 trigger at_once into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st interval(10s); print ===== insert into 2 sql insert into t1 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,2,2,3); @@ -195,7 +195,7 @@ sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams3 trigger at_once into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", "1") ) as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams3 trigger at_once IGNORE EXPIRED 0 into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", "1") ) as select _wstart, count(*) c1 from st interval(10s); print ===== insert into 3 sql insert into t1 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,2,2,3); @@ -305,7 +305,7 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create table t3 using st tags(3,3,3); -sql create stream streams4 trigger at_once into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", "1")) as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams4 trigger at_once IGNORE EXPIRED 0 into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", "1")) as select _wstart, count(*) c1 from st interval(10s); sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); @@ -375,9 +375,9 @@ sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams51 trigger at_once into result5.streamt51 SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); -sql create stream streams52 trigger at_once into result5.streamt52 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st interval(10s); -sql create stream streams53 trigger at_once into result5.streamt53 TAGS(dd varchar(100)) SUBTABLE(concat("aaa-", "1") ) as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams51 trigger at_once IGNORE EXPIRED 0 into result5.streamt51 SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams52 trigger at_once IGNORE EXPIRED 0 into result5.streamt52 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams53 trigger at_once IGNORE EXPIRED 0 into result5.streamt53 TAGS(dd varchar(100)) SUBTABLE(concat("aaa-", "1") ) as select _wstart, count(*) c1 from st interval(10s); sql insert into t1 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,2,2,3); diff --git a/tests/system-test/1-insert/database_pre_suf.py b/tests/system-test/1-insert/database_pre_suf.py index 488dfebff5..a6ff95ab3f 100755 --- a/tests/system-test/1-insert/database_pre_suf.py +++ b/tests/system-test/1-insert/database_pre_suf.py @@ -108,7 +108,7 @@ class TDTestCase: # create stream - tdSql.execute('''create stream current_stream into stream_max_stable_1 as select _wstart as startts, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s);''') + tdSql.execute('''create stream current_stream trigger at_once IGNORE EXPIRED 0 into stream_max_stable_1 as select _wstart as startts, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s);''') # insert data for i in range(num_random*n): diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py index c15a9bbc35..a8bfea2741 100644 --- a/tests/system-test/1-insert/drop.py +++ b/tests/system-test/1-insert/drop.py @@ -138,14 +138,14 @@ class TDTestCase: stream_name = tdCom.getLongName(5,"letters") tdSql.execute(f'create table {stbname} (ts timestamp,c0 int) tags(t0 int)') tdSql.execute(f'create table tb using {stbname} tags(1)') - tdSql.execute(f'create stream {stream_name} into stb as select * from {self.dbname}.{stbname} partition by tbname') + tdSql.execute(f'create stream {stream_name} trigger at_once ignore expired 0 into stb as select * from {self.dbname}.{stbname} partition by tbname') tdSql.query(f'select * from information_schema.ins_streams where stream_name = "{stream_name}"') print(tdSql.queryResult) - tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} into stb as select * from {self.dbname}.{stbname} partition by tbname') + tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} trigger at_once ignore expired 0 into stb as select * from {self.dbname}.{stbname} partition by tbname') tdSql.execute(f'drop stream {stream_name}') - tdSql.execute(f'create stream {stream_name} into stb1 as select * from tb') + tdSql.execute(f'create stream {stream_name} trigger at_once ignore expired 0 into stb1 as select * from tb') tdSql.query(f'select * from information_schema.ins_streams where stream_name = "{stream_name}"') - tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} into stb1 as select * from tb') + tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} trigger at_once ignore expired 0 into stb1 as select * from tb') tdSql.execute(f'drop database {self.dbname}') def run(self): self.drop_ntb_check() diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py index 66c2fdd14f..04c69ad618 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py @@ -143,7 +143,8 @@ class TDTestCase: stableName= '%s_%d'%(paraDict['stbName'],i) newTdSql=tdCom.newTdSql() threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) - threads.append(threading.Thread(target=self.reCreateUser,args=(newTdSql,i,"user","passwd"))) + createTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=self.reCreateUser,args=(createTdSql,i,"user","passwd"))) for tr in threads: tr.start() diff --git a/tests/system-test/7-tmq/tmqDelete-1ctb.py b/tests/system-test/7-tmq/tmqDelete-1ctb.py index 4b45b1a834..b09efdd1e6 100644 --- a/tests/system-test/7-tmq/tmqDelete-1ctb.py +++ b/tests/system-test/7-tmq/tmqDelete-1ctb.py @@ -80,16 +80,16 @@ class TDTestCase: tdLog.debug("del data ............ [OK]") return - def threadFunctionForDeletaData(self, **paraDict): + def threadFunctionForDeletaData(self, paraDict): # create new connector for new tdSql instance in my thread newTdSql = tdCom.newTdSql() self.delData(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["startTs"],paraDict["endTs"],paraDict["ctbStartIdx"]) return - def asyncDeleteData(self, paraDict): - pThread = threading.Thread(target=self.threadFunctionForDeletaData, kwargs=paraDict) - pThread.start() - return pThread + # def asyncDeleteData(self, paraDict): + # pThread = threading.Thread(target=self.threadFunctionForDeletaData, kwargs=paraDict) + # pThread.start() + # return pThread def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") @@ -109,7 +109,7 @@ class TDTestCase: 'batchNum': 3000, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'endTs': 0, - 'pollDelay': 5, + 'pollDelay': 15, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -194,7 +194,7 @@ class TDTestCase: 'rowsPerTbl': 10000, 'batchNum': 5000, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 5, + 'pollDelay': 15, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -296,7 +296,7 @@ class TDTestCase: 'rowsPerTbl': 10000, 'batchNum': 5000, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 5, + 'pollDelay': 15, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -340,7 +340,8 @@ class TDTestCase: # del some data rowsOfDelete = int(self.rowsPerTbl / 4 ) paraDict["endTs"] = paraDict["startTs"] + rowsOfDelete - 1 - pDeleteThread = self.asyncDeleteData(paraDict) + # pDeleteThread = self.asyncDeleteData(paraDict) + self.threadFunctionForDeletaData(paraDict) tdLog.info("start consume processor") tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index a8bc93dd0c..a8986351b7 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -86,8 +86,8 @@ SWords shellCommands[] = { " " " ;", 0, 0, NULL}, - {"create dnode ", 0, 0, NULL}, - {"create index ", 0, 0, NULL}, + {"create dnode ", 0, 0, NULL}, + {"create index on ()", 0, 0, NULL}, {"create mnode on dnode ;", 0, 0, NULL}, {"create qnode on dnode ;", 0, 0, NULL}, {"create stream into as select", 0, 0, NULL}, // 26 append sub sql diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 3d458d90c1..1f25eae366 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -409,7 +409,7 @@ int buildStable(TAOS* pConn, TAOS_RES* pRes) { taos_free_result(pRes); pRes = taos_query(pConn, - "create stream meters_summary_s into meters_summary as select _wstart, max(current) as current, " + "create stream meters_summary_s trigger at_once IGNORE EXPIRED 0 into meters_summary as select _wstart, max(current) as current, " "groupid, location from meters partition by groupid, location interval(10m)"); if (taos_errno(pRes) != 0) { printf("failed to create super table meters_summary, reason:%s\n", taos_errstr(pRes));