From 274d310f333bfefd1cfdd38ec381504b99a46a11 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 19 Sep 2022 17:36:51 +0800 Subject: [PATCH 01/71] feat: retention should not block data commit --- source/dnode/vnode/CMakeLists.txt | 2 +- source/dnode/vnode/src/inc/vnodeInt.h | 5 + source/dnode/vnode/src/tsdb/tsdbFS.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRetention.c | 24 +++- source/dnode/vnode/src/tsdb/tsdbRetention2.c | 140 +++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeOpen.c | 12 ++ source/dnode/vnode/src/vnd/vnodeSvr.c | 83 ++++++++++- 7 files changed, 262 insertions(+), 6 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbRetention2.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 7a99d26683..092c861766 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -49,7 +49,7 @@ target_sources( "src/tsdb/tsdbUtil.c" "src/tsdb/tsdbSnapshot.c" "src/tsdb/tsdbCacheRead.c" - "src/tsdb/tsdbRetention.c" + "src/tsdb/tsdbRetention2.c" "src/tsdb/tsdbDiskData.c" "src/tsdb/tsdbCompress.c" "src/tsdb/tsdbCompact.c" diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0e85e7bfb6..7a779454c6 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -58,6 +58,7 @@ typedef struct STQ STQ; typedef struct SVState SVState; typedef struct SVBufPool SVBufPool; typedef struct SQWorker SQHandle; +typedef struct STrimDbHandle STrimDbHandle; typedef struct STsdbKeepCfg STsdbKeepCfg; typedef struct SMetaSnapReader SMetaSnapReader; typedef struct SMetaSnapWriter SMetaSnapWriter; @@ -301,6 +302,9 @@ struct STsdbKeepCfg { int32_t keep1; int32_t keep2; }; +struct STrimDbHandle { + volatile int8_t state; // 0 not in trim, 1 in trim +}; struct SVnode { char* path; @@ -325,6 +329,7 @@ struct SVnode { bool restored; tsem_t syncSem; SQHandle* pQuery; + STrimDbHandle trimDbH; }; #define TD_VID(PVNODE) ((PVNODE)->config.vgId) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 10926ae6ad..bd3ebfdcd4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -1045,7 +1045,7 @@ _err: return code; } -int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) { +int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) { int32_t code = 0; int32_t nRef; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 2c68c57176..02d0d12ea2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -16,10 +16,28 @@ #include "tsdb.h" static bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { + if (taosArrayGetSize(pTsdb->fs.aDFileSet) == 0) { + return false; + } + + SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, 0); + if (tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now) < 0) { + return true; + } + + if (tfsGetLevel(pTsdb->pVnode->pTfs) < 2) { + return false; + } + + STsdbKeepCfg *keepCfg = &pTsdb->keepCfg; + if (keepCfg->keep0 == keepCfg->keep1 && keepCfg->keep1 == keepCfg->keep2) { + return false; + } + for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - SDiskID did; + pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); + int32_t expLevel = tsdbFidLevel(pSet->fid, keepCfg, now); + SDiskID did; if (expLevel == pSet->diskId.level) continue; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention2.c b/source/dnode/vnode/src/tsdb/tsdbRetention2.c new file mode 100644 index 0000000000..4bdae09d88 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbRetention2.c @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +static bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { + if (taosArrayGetSize(pTsdb->fs.aDFileSet) == 0) { + return false; + } + + SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, 0); + if (tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now) < 0) { + return true; + } + + if (tfsGetLevel(pTsdb->pVnode->pTfs) < 2) { + return false; + } + + STsdbKeepCfg *keepCfg = &pTsdb->keepCfg; + if (keepCfg->keep0 == keepCfg->keep1 && keepCfg->keep1 == keepCfg->keep2) { + return false; + } + + for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { + pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); + int32_t expLevel = tsdbFidLevel(pSet->fid, keepCfg, now); + SDiskID did; + + if (expLevel == pSet->diskId.level) continue; + + if (expLevel < 0) { + return true; + } else { + if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { + return false; + } + + if (did.level == pSet->diskId.level) continue; + + return true; + } + } + + return false; +} + +/** + * @brief Data migration between multi-tier storage, including remove expired data. + * 1) firstly, remove expired DFileSet; + * 2) partition the tsdbFS by the expLevel and fileSize(e.g. 500G, configurable), and migrate DFileSet groups between multi-tier storage; + * 3) update the tsdbFS and CURRENT in the same transaction; + * 4) finish + * @param pTsdb + * @param now + * @return int32_t + */ +int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { + int32_t code = 0; + + if (!tsdbShouldDoRetention(pTsdb, now)) { + return code; + } + + // do retention + STsdbFS fs; + + code = tsdbFSCopy(pTsdb, &fs); + if (code) goto _err; + + for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); iSet++) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); + int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); + SDiskID did; + + if (expLevel < 0) { + taosMemoryFree(pSet->pHeadF); + taosMemoryFree(pSet->pDataF); + taosMemoryFree(pSet->aSttF[0]); + taosMemoryFree(pSet->pSmaF); + taosArrayRemove(fs.aDFileSet, iSet); + iSet--; + } else { + if (expLevel == 0) continue; + if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { + code = terrno; + goto _exit; + } + + if (did.level == pSet->diskId.level) continue; + + // copy file to new disk (todo) + SDFileSet fSet = *pSet; + fSet.diskId = did; + + code = tsdbDFileSetCopy(pTsdb, pSet, &fSet); + if (code) goto _err; + + code = tsdbFSUpsertFSet(&fs, &fSet); + if (code) goto _err; + } + } + + // do change fs + code = tsdbFSCommit1(pTsdb, &fs); + if (code) goto _err; + + taosThreadRwlockWrlock(&pTsdb->rwLock); + + code = tsdbFSCommit2(pTsdb, &fs); + if (code) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _err; + } + + taosThreadRwlockUnlock(&pTsdb->rwLock); + + tsdbFSDestroy(&fs); + +_exit: + return code; + +_err: + tsdbError("vgId:%d, tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + ASSERT(0); + // tsdbFSRollback(pTsdb->pFS); + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 4ccfea4051..481c8b66be 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -186,6 +186,17 @@ void vnodePreClose(SVnode *pVnode) { } } +static void vnodeTrimDbClose(SVnode *pVnode) { + int32_t nLoops = 0; + while (atomic_load_8(&pVnode->trimDbH.state) != 0) { + if (++nLoops > 1000) { + vTrace("vgId:%d, wait for trimDb task to finish", TD_VID(pVnode)); + sched_yield(); + nLoops = 0; + } + } +} + void vnodeClose(SVnode *pVnode) { if (pVnode) { vnodeCommit(pVnode); @@ -197,6 +208,7 @@ void vnodeClose(SVnode *pVnode) { smaClose(pVnode->pSma); metaClose(pVnode->pMeta); vnodeCloseBufPool(pVnode); + vnodeTrimDbClose(pVnode); // destroy handle tsem_destroy(&(pVnode->canCommit)); tsem_destroy(&pVnode->syncSem); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 6e9eba306a..728b5b4cda 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -378,6 +378,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { pMetaRsp->precision = pVnode->config.tsdbCfg.precision; } +#if 0 static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; SVTrimDbReq trimReq = {0}; @@ -400,6 +401,85 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, _exit: return code; } +#endif + +typedef struct { + SVnode *pVnode; + SVTrimDbReq trimReq; +} SVndTrimDbReq; + +void *vnodeProcessTrimReqFunc(void *param) { + int32_t code = 0; + int8_t oldVal = 0; + SVndTrimDbReq *pReq = (SVndTrimDbReq *)param; + SVnode *pVnode = pReq->pVnode; + + setThreadName("vnode-trim"); + + // process + code = tsdbDoRetention(pVnode->pTsdb, pReq->trimReq.timestamp); + if (code) goto _exit; + + code = smaDoRetention(pVnode->pSma, pReq->trimReq.timestamp); + if (code) goto _exit; + +_exit: + vInfo("vgId:%d, trim vnode thread finished, time:%d", TD_VID(pVnode), pReq->trimReq.timestamp); + oldVal = atomic_val_compare_exchange_8(&pVnode->trimDbH.state, 1, 0); + ASSERT(oldVal == 1); + taosMemoryFree(pReq); + return NULL; +} + +static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + int32_t code = 0; + SVndTrimDbReq *pVndTrimReq = taosMemoryMalloc(sizeof(SVndTrimDbReq)); + STrimDbHandle *pHandle = &pVnode->trimDbH; + + if (!pVndTrimReq) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + pVndTrimReq->pVnode = pVnode; + + if (tDeserializeSVTrimDbReq(pReq, len, &pVndTrimReq->trimReq) != 0) { + taosMemoryFree(pVndTrimReq); + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (atomic_val_compare_exchange_8(&pHandle->state, 0, 1) != 0) { + vInfo("vgId:%d, trim vnode request will not be processed since duplicated req, time:%d", TD_VID(pVnode), + pVndTrimReq->trimReq.timestamp); + taosMemoryFree(pVndTrimReq); + goto _exit; + } + + vInfo("vgId:%d, trim vnode request will be processed, time:%d", TD_VID(pVnode), pVndTrimReq->trimReq.timestamp); + + TdThreadAttr thAttr = {0}; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_DETACHED); + + TdThread tid; + if (taosThreadCreate(&tid, &thAttr, vnodeProcessTrimReqFunc, (void *)pVndTrimReq) != 0) { + code = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(pVndTrimReq); + taosThreadAttrDestroy(&thAttr); + int8_t oldVal = atomic_val_compare_exchange_8(&pHandle->state, 1, 0); + ASSERT(oldVal == 1); + vError("vgId:%d, failed to create pthread for trim vnode since %s", TD_VID(pVnode), tstrerror(code)); + goto _exit; + } + vDebug("vgId:%d, success to create pthread for trim vnode", TD_VID(pVnode)); + + taosThreadAttrDestroy(&thAttr); + +_exit: + terrno = code; + return code; +} static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); @@ -921,7 +1001,8 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq } if (taosArrayGetSize(newTbUids) > 0) { - vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), (int32_t)taosArrayGetSize(newTbUids)); + vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), + (int32_t)taosArrayGetSize(newTbUids)); } tqUpdateTbUidList(pVnode->pTq, newTbUids, true); From b8ea469afff1ec6303a3684dc8812cb6d4486c04 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 23 Sep 2022 16:42:36 +0800 Subject: [PATCH 02/71] enh(tsdb): data migrate should not block data r/w and commit --- include/common/tmsg.h | 3 +- source/common/src/tmsg.c | 6 +- source/dnode/mnode/impl/src/mndDb.c | 4 +- source/dnode/vnode/src/inc/tsdb.h | 12 + source/dnode/vnode/src/inc/vnodeInt.h | 10 +- source/dnode/vnode/src/sma/smaRollup.c | 5 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 40 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 443 ++++++++++++++++++- source/dnode/vnode/src/tsdb/tsdbOpen.c | 3 + source/dnode/vnode/src/tsdb/tsdbRetention2.c | 309 ++++++++++--- source/dnode/vnode/src/vnd/vnodeSvr.c | 17 +- 11 files changed, 769 insertions(+), 83 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ac6ce82ffb..a05cd907b0 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -873,7 +873,8 @@ int32_t tSerializeSTrimDbReq(void* buf, int32_t bufLen, STrimDbReq* pReq); int32_t tDeserializeSTrimDbReq(void* buf, int32_t bufLen, STrimDbReq* pReq); typedef struct { - int32_t timestamp; + int64_t timestamp; // unit: millisecond + int32_t maxSpeed; // 0 no limit, unit: bit/s } SVTrimDbReq; int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 0990815024..8c4c552db2 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2706,7 +2706,8 @@ int32_t tSerializeSVTrimDbReq(void *buf, int32_t bufLen, SVTrimDbReq *pReq) { tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI32(&encoder, pReq->timestamp) < 0) return -1; + if (tEncodeI64(&encoder, pReq->timestamp) < 0) return -1; + if (tEncodeI32(&encoder, pReq->maxSpeed) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2719,7 +2720,8 @@ int32_t tDeserializeSVTrimDbReq(void *buf, int32_t bufLen, SVTrimDbReq *pReq) { tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->timestamp) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->timestamp) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->maxSpeed) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 39ec3484db..ccc4c8293a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1389,7 +1389,7 @@ static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; - SVTrimDbReq trimReq = {.timestamp = taosGetTimestampSec()}; + SVTrimDbReq trimReq = {.timestamp = taosGetTimestampMs(), .maxSpeed = 1048576 << 5}; // TODO: use specified maxSpeed int32_t reqLen = tSerializeSVTrimDbReq(NULL, 0, &trimReq); int32_t contLen = reqLen + sizeof(SMsgHead); @@ -1413,7 +1413,7 @@ static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) { if (code != 0) { mError("vgId:%d, failed to send vnode-trim request to vnode since 0x%x", pVgroup->vgId, code); } else { - mDebug("vgId:%d, send vnode-trim request to vnode, time:%d", pVgroup->vgId, trimReq.timestamp); + mDebug("vgId:%d, send vnode-trim request to vnode, time:%" PRIi64, pVgroup->vgId, trimReq.timestamp); } sdbRelease(pSdb, pVgroup); } diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 817089f237..66df364b4b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -58,6 +58,7 @@ typedef struct SDelFWriter SDelFWriter; typedef struct SDelFReader SDelFReader; typedef struct SRowIter SRowIter; typedef struct STsdbFS STsdbFS; +typedef struct STsdbTrimHdl STsdbTrimHdl; typedef struct SRowMerger SRowMerger; typedef struct STsdbReadSnap STsdbReadSnap; typedef struct SBlockInfo SBlockInfo; @@ -243,6 +244,7 @@ int32_t tsdbFSClose(STsdb *pTsdb); int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS); void tsdbFSDestroy(STsdbFS *pFS); int32_t tDFileSetCmprFn(const void *p1, const void *p2); +int32_t tsdbFSUpdDel(STsdb *pTsdb, STsdbFS *pFS, STsdbFS *pFSNew, int32_t maxFid); int32_t tsdbFSCommit1(STsdb *pTsdb, STsdbFS *pFS); int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFS); int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS); @@ -318,6 +320,12 @@ struct STsdbFS { SArray *aDFileSet; // SArray }; +struct STsdbTrimHdl { + volatile int8_t state; // 0 idle 1 in use + volatile int32_t maxRetentFid; + volatile int32_t minCommitFid; +}; + struct STsdb { char *path; SVnode *pVnode; @@ -326,6 +334,7 @@ struct STsdb { SMemTable *mem; SMemTable *imem; STsdbFS fs; + STsdbTrimHdl trimHdl; SLRUCache *lruCache; TdThreadMutex lruMutex; }; @@ -559,6 +568,9 @@ struct SDFileSet { SSttFile *aSttF[TSDB_MAX_STT_TRIGGER]; }; +#define SET_DFSET_EXPIRED(d) ((d)->diskId.id = -1) +#define IS_DFSET_EXPIRED(d) ((d)->diskId.id == -1) + struct SRowIter { TSDBROW *pRow; STSchema *pTSchema; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 54dcb66c21..d9acb6da21 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -58,7 +58,7 @@ typedef struct STQ STQ; typedef struct SVState SVState; typedef struct SVBufPool SVBufPool; typedef struct SQWorker SQHandle; -typedef struct STrimDbHandle STrimDbHandle; +typedef struct SVTrimDbHdl SVTrimDbHdl; typedef struct STsdbKeepCfg STsdbKeepCfg; typedef struct SMetaSnapReader SMetaSnapReader; typedef struct SMetaSnapWriter SMetaSnapWriter; @@ -145,7 +145,7 @@ int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepC int tsdbClose(STsdb** pTsdb); int32_t tsdbBegin(STsdb* pTsdb); int32_t tsdbCommit(STsdb* pTsdb); -int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now); +int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now, int32_t maxSpeed); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, @@ -200,7 +200,7 @@ int32_t smaSyncPostCommit(SSma* pSma); int32_t smaAsyncPreCommit(SSma* pSma); int32_t smaAsyncCommit(SSma* pSma); int32_t smaAsyncPostCommit(SSma* pSma); -int32_t smaDoRetention(SSma* pSma, int64_t now); +int32_t smaDoRetention(SSma* pSma, int64_t now, int32_t maxSpeed); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); @@ -302,7 +302,7 @@ struct STsdbKeepCfg { int32_t keep1; int32_t keep2; }; -struct STrimDbHandle { +struct SVTrimDbHdl { volatile int8_t state; // 0 not in trim, 1 in trim }; @@ -329,7 +329,7 @@ struct SVnode { bool restored; tsem_t syncSem; SQHandle* pQuery; - STrimDbHandle trimDbH; + SVTrimDbHdl trimDbH; }; #define TD_VID(PVNODE) ((PVNODE)->config.vgId) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index c6ec31cc21..d6c3e774ce 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -661,9 +661,10 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { * * @param pSma * @param now + * @param maxSpeed * @return int32_t */ -int32_t smaDoRetention(SSma *pSma, int64_t now) { +int32_t smaDoRetention(SSma *pSma, int64_t now, int32_t maxSpeed) { int32_t code = TSDB_CODE_SUCCESS; if (!VND_IS_RSMA(pSma->pVnode)) { return code; @@ -671,7 +672,7 @@ int32_t smaDoRetention(SSma *pSma, int64_t now) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { - code = tsdbDoRetention(pSma->pRSmaTsdb[i], now); + code = tsdbDoRetention(pSma->pRSmaTsdb[i], now, maxSpeed); if (code) goto _end; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index a619b9f2e4..0a2d37bdbc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -756,6 +756,30 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } + + if (pTsdb->imem->nRow > 0) { + int32_t minCommitFid = tsdbKeyFid(pTsdb->imem->minKey, pCommitter->minutes, pCommitter->precision); + int32_t nLoops = 0; + + _wait_retention_end: + while (atomic_load_32(&pTsdb->trimHdl.maxRetentFid) >= minCommitFid) { + if (++nLoops > 1000) { + nLoops = 0; + sched_yield(); + } + } + if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { + if (atomic_load_32(&pTsdb->trimHdl.maxRetentFid) >= minCommitFid) { + atomic_store_8(&pTsdb->trimHdl.state, 0); + goto _wait_retention_end; + } + atomic_store_32(&pTsdb->trimHdl.minCommitFid, minCommitFid); + atomic_store_8(&pTsdb->trimHdl.state, 0); + } else { + goto _wait_retention_end; + } + } + code = tsdbFSCopy(pTsdb, &pCommitter->fs); if (code) goto _err; @@ -962,14 +986,23 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { int32_t code = 0; STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; + bool inTrim = atomic_load_8(&pTsdb->pVnode->trimDbH.state); ASSERT(eno == 0); + if(inTrim) taosThreadRwlockWrlock(&pTsdb->rwLock); + + int64_t startTime = taosGetTimestampMs(); code = tsdbFSCommit1(pTsdb, &pCommitter->fs); - if (code) goto _err; + if (code) { + if(inTrim) taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _err; + } + int64_t endTime = taosGetTimestampMs(); + tsdbInfo("vgId:%d, tsdb end commit - commit1 fsSize:%d cost: %" PRIi64 " ms", TD_VID(pTsdb->pVnode), (int32_t)taosArrayGetSize(pCommitter->fs.aDFileSet), endTime - startTime); // lock - taosThreadRwlockWrlock(&pTsdb->rwLock); + if(!inTrim) taosThreadRwlockWrlock(&pTsdb->rwLock); // commit or rollback code = tsdbFSCommit2(pTsdb, &pCommitter->fs); @@ -977,6 +1010,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { taosThreadRwlockUnlock(&pTsdb->rwLock); goto _err; } + tsdbInfo("vgId:%d, tsdb end commit - commit2 cost: %" PRIi64 " ms", TD_VID(pTsdb->pVnode), taosGetTimestampMs() - endTime); pTsdb->imem = NULL; @@ -986,6 +1020,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { tsdbUnrefMemTable(pMemTable); tsdbFSDestroy(&pCommitter->fs); taosArrayDestroy(pCommitter->aTbDataP); + atomic_store_32(&pTsdb->trimHdl.minCommitFid, INT32_MAX); // if (pCommitter->toMerge) { // code = tsdbMerge(pTsdb); @@ -996,6 +1031,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { return code; _err: + atomic_store_32(&pTsdb->trimHdl.minCommitFid, INT32_MAX); tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index bd3ebfdcd4..45d28f4135 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -711,7 +711,7 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { _exit: return code; } - +#if 0 int32_t tsdbFSCommit1(STsdb *pTsdb, STsdbFS *pFSNew) { int32_t code = 0; char tfname[TSDB_FILENAME_LEN]; @@ -1045,6 +1045,447 @@ _err: return code; } +#endif + +#if 1 + +/** + * @brief Update or delete DFileSet in pFS according to DFileSet (fid <= maxFid) in pFSNew. + * + * @param pTsdb + * @param pFS + * @param pFSNew + * @param maxFid + * @return int32_t + */ +int32_t tsdbFSUpdDel(STsdb *pTsdb, STsdbFS *pFS, STsdbFS *pFSNew, int32_t maxFid) { + int32_t code = 0; + int32_t nRef = 0; + char fname[TSDB_FILENAME_LEN]; + + int32_t iOld = 0; + int32_t iNew = 0; + while (true) { + int32_t nOld = taosArrayGetSize(pFS->aDFileSet); + int32_t nNew = taosArrayGetSize(pFSNew->aDFileSet); + SDFileSet fSet; + int8_t sameDisk; + + if (iOld >= nOld && iNew >= nNew) break; + + SDFileSet *pSetOld = (iOld < nOld) ? taosArrayGet(pFS->aDFileSet, iOld) : NULL; + SDFileSet *pSetNew = (iNew < nNew) ? taosArrayGet(pFSNew->aDFileSet, iNew) : NULL; + + if (pSetNew && (pSetNew->fid > maxFid)) break; + + if (pSetOld && pSetNew) { + if (pSetOld->fid == pSetNew->fid) { + if (IS_DFSET_EXPIRED(pSetNew)) goto _remove_old; + goto _merge_migrate; + } else if (pSetOld->fid < pSetNew->fid) { + ++iOld; + } else { + ++iNew; + } + continue; + } else { + break; + } + + _merge_migrate: + sameDisk = ((pSetOld->diskId.level == pSetNew->diskId.level) && (pSetOld->diskId.id == pSetNew->diskId.id)); + + ASSERT(pSetOld->pHeadF->commitID == pSetNew->pHeadF->commitID); + ASSERT(pSetOld->pHeadF->size == pSetNew->pHeadF->size); + ASSERT(pSetOld->pHeadF->offset == pSetNew->pHeadF->offset); + ASSERT(!sameDisk); + + // head + *pSetOld->pHeadF = *pSetNew->pHeadF; + pSetOld->pHeadF->nRef = 1; + + // data + ASSERT(pSetOld->pDataF->size == pSetNew->pDataF->size); + *pSetOld->pDataF = *pSetNew->pDataF; + pSetOld->pDataF->nRef = 1; + + // sma + ASSERT(pSetOld->pSmaF->size == pSetNew->pSmaF->size); + *pSetOld->pSmaF = *pSetNew->pSmaF; + pSetOld->pSmaF->nRef = 1; + + // stt + ASSERT(pSetOld->nSttF == pSetNew->nSttF); + for (int32_t iStt = 0; iStt < pSetOld->nSttF; ++iStt) { + ASSERT(pSetOld->aSttF[iStt]->size == pSetNew->aSttF[iStt]->size); + ASSERT(pSetOld->aSttF[iStt]->offset == pSetNew->aSttF[iStt]->offset); + + *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; + pSetOld->aSttF[iStt]->nRef = 1; + } + + // set diskId + pSetOld->diskId = pSetNew->diskId; + + iOld++; + iNew++; + continue; + + _remove_old: + taosMemoryFree(pSetOld->pHeadF); + taosMemoryFree(pSetOld->pDataF); + for (int32_t iStt = 0; iStt < pSetOld->nSttF; ++iStt) { + taosMemoryFree(pSetOld->aSttF[iStt]); + } + taosMemoryFree(pSetOld->pSmaF); + taosArrayRemove(pFS->aDFileSet, iOld); + iNew++; + continue; + } + + return code; + +_err: + tsdbError("vgId:%d, tsdb fs upd/del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + + +int32_t tsdbFSCommit1(STsdb *pTsdb, STsdbFS *pFSNew) { + int32_t code = 0; + char tfname[TSDB_FILENAME_LEN]; + char fname[TSDB_FILENAME_LEN]; + + snprintf(tfname, TSDB_FILENAME_LEN - 1, "%s%s%s%sCURRENT.t", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), TD_DIRSEP, + pTsdb->path, TD_DIRSEP); + snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sCURRENT", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), TD_DIRSEP, + pTsdb->path, TD_DIRSEP); + + // gnrt CURRENT.t + code = tsdbGnrtCurrent(pTsdb, pFSNew, tfname); + if (code) goto _err; + + // rename + code = taosRenameFile(tfname, fname); + if (code) { + code = TAOS_SYSTEM_ERROR(code); + goto _err; + } + + return code; + +_err: + tsdbError("vgId:%d, tsdb fs commit phase 1 failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { + int32_t code = 0; + int32_t nRef; + char fname[TSDB_FILENAME_LEN]; + + // del + if (pFSNew->pDelFile) { + SDelFile *pDelFile = pTsdb->fs.pDelFile; + + if (pDelFile == NULL || (pDelFile->commitID != pFSNew->pDelFile->commitID)) { + pTsdb->fs.pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile)); + if (pTsdb->fs.pDelFile == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + *pTsdb->fs.pDelFile = *pFSNew->pDelFile; + pTsdb->fs.pDelFile->nRef = 1; + + if (pDelFile) { + nRef = atomic_sub_fetch_32(&pDelFile->nRef, 1); + if (nRef == 0) { + tsdbDelFileName(pTsdb, pDelFile, fname); + taosRemoveFile(fname); + taosMemoryFree(pDelFile); + } + } + } + } else { + ASSERT(pTsdb->fs.pDelFile == NULL); + } + + // data + int32_t iOld = 0; + int32_t iNew = 0; + while (true) { + int32_t nOld = taosArrayGetSize(pTsdb->fs.aDFileSet); + int32_t nNew = taosArrayGetSize(pFSNew->aDFileSet); + SDFileSet fSet; + int8_t sameDisk; + + if (iOld >= nOld && iNew >= nNew) break; + + SDFileSet *pSetOld = (iOld < nOld) ? taosArrayGet(pTsdb->fs.aDFileSet, iOld) : NULL; + SDFileSet *pSetNew = (iNew < nNew) ? taosArrayGet(pFSNew->aDFileSet, iNew) : NULL; + + if (pSetOld && pSetNew) { + if (pSetOld->fid == pSetNew->fid) { + goto _merge_old_and_new; + } else if (pSetOld->fid < pSetNew->fid) { + goto _remove_old; + } else { + goto _add_new; + } + } else if (pSetOld) { + goto _remove_old; + } else { + goto _add_new; + } + + _merge_old_and_new: + sameDisk = ((pSetOld->diskId.level == pSetNew->diskId.level) && (pSetOld->diskId.id == pSetNew->diskId.id)); + + // head + fSet.pHeadF = pSetOld->pHeadF; + if ((!sameDisk) || (pSetOld->pHeadF->commitID != pSetNew->pHeadF->commitID)) { + pSetOld->pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); + if (pSetOld->pHeadF == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->pHeadF = *pSetNew->pHeadF; + pSetOld->pHeadF->nRef = 1; + + nRef = atomic_sub_fetch_32(&fSet.pHeadF->nRef, 1); + if (nRef == 0) { + tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pHeadF, fname); + taosRemoveFile(fname); + taosMemoryFree(fSet.pHeadF); + } + } else { + ASSERT(fSet.pHeadF->size == pSetNew->pHeadF->size); + ASSERT(fSet.pHeadF->offset == pSetNew->pHeadF->offset); + } + + // data + fSet.pDataF = pSetOld->pDataF; + if ((!sameDisk) || (pSetOld->pDataF->commitID != pSetNew->pDataF->commitID)) { + pSetOld->pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile)); + if (pSetOld->pDataF == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->pDataF = *pSetNew->pDataF; + pSetOld->pDataF->nRef = 1; + + nRef = atomic_sub_fetch_32(&fSet.pDataF->nRef, 1); + if (nRef == 0) { + tsdbDataFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pDataF, fname); + taosRemoveFile(fname); + taosMemoryFree(fSet.pDataF); + } + } else { + ASSERT(pSetOld->pDataF->size <= pSetNew->pDataF->size); + pSetOld->pDataF->size = pSetNew->pDataF->size; + } + + // sma + fSet.pSmaF = pSetOld->pSmaF; + if ((!sameDisk) || (pSetOld->pSmaF->commitID != pSetNew->pSmaF->commitID)) { + pSetOld->pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); + if (pSetOld->pSmaF == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->pSmaF = *pSetNew->pSmaF; + pSetOld->pSmaF->nRef = 1; + + nRef = atomic_sub_fetch_32(&fSet.pSmaF->nRef, 1); + if (nRef == 0) { + tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pSmaF, fname); + taosRemoveFile(fname); + taosMemoryFree(fSet.pSmaF); + } + } else { + ASSERT(pSetOld->pSmaF->size <= pSetNew->pSmaF->size); + pSetOld->pSmaF->size = pSetNew->pSmaF->size; + } + + // stt + if (sameDisk) { + if (pSetNew->nSttF > pSetOld->nSttF) { + ASSERT(pSetNew->nSttF = pSetOld->nSttF + 1); + pSetOld->aSttF[pSetOld->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[pSetOld->nSttF] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aSttF[pSetOld->nSttF] = *pSetNew->aSttF[pSetOld->nSttF]; + pSetOld->aSttF[pSetOld->nSttF]->nRef = 1; + pSetOld->nSttF++; + } else if (pSetNew->nSttF < pSetOld->nSttF) { + ASSERT(pSetNew->nSttF == 1); + for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + SSttFile *pSttFile = pSetOld->aSttF[iStt]; + nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); + if (nRef == 0) { + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); + taosRemoveFile(fname); + taosMemoryFree(pSttFile); + } + pSetOld->aSttF[iStt] = NULL; + } + + pSetOld->nSttF = 1; + pSetOld->aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aSttF[0] = *pSetNew->aSttF[0]; + pSetOld->aSttF[0]->nRef = 1; + } else { + for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + if (pSetOld->aSttF[iStt]->commitID != pSetNew->aSttF[iStt]->commitID) { + SSttFile *pSttFile = pSetOld->aSttF[iStt]; + nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); + if (nRef == 0) { + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); + taosRemoveFile(fname); + taosMemoryFree(pSttFile); + } + + pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[iStt] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; + pSetOld->aSttF[iStt]->nRef = 1; + } else { + ASSERT(pSetOld->aSttF[iStt]->size == pSetNew->aSttF[iStt]->size); + ASSERT(pSetOld->aSttF[iStt]->offset == pSetNew->aSttF[iStt]->offset); + } + } + } + } else { + ASSERT(pSetOld->nSttF == pSetNew->nSttF); + for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + SSttFile *pSttFile = pSetOld->aSttF[iStt]; + nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); + if (nRef == 0) { + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); + taosRemoveFile(fname); + taosMemoryFree(pSttFile); + } + + pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[iStt] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; + pSetOld->aSttF[iStt]->nRef = 1; + } + } + + if (!sameDisk) { + pSetOld->diskId = pSetNew->diskId; + } + + iOld++; + iNew++; + continue; + + _remove_old: + nRef = atomic_sub_fetch_32(&pSetOld->pHeadF->nRef, 1); + if (nRef == 0) { + tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pHeadF, fname); + taosRemoveFile(fname); + taosMemoryFree(pSetOld->pHeadF); + } + + nRef = atomic_sub_fetch_32(&pSetOld->pDataF->nRef, 1); + if (nRef == 0) { + tsdbDataFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pDataF, fname); + taosRemoveFile(fname); + taosMemoryFree(pSetOld->pDataF); + } + + nRef = atomic_sub_fetch_32(&pSetOld->pSmaF->nRef, 1); + if (nRef == 0) { + tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pSmaF, fname); + taosRemoveFile(fname); + taosMemoryFree(pSetOld->pSmaF); + } + + for (int8_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + nRef = atomic_sub_fetch_32(&pSetOld->aSttF[iStt]->nRef, 1); + if (nRef == 0) { + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aSttF[iStt], fname); + taosRemoveFile(fname); + taosMemoryFree(pSetOld->aSttF[iStt]); + } + } + + taosArrayRemove(pTsdb->fs.aDFileSet, iOld); + continue; + + _add_new: + fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nSttF = 1}; + + // head + fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); + if (fSet.pHeadF == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *fSet.pHeadF = *pSetNew->pHeadF; + fSet.pHeadF->nRef = 1; + + // data + fSet.pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile)); + if (fSet.pDataF == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *fSet.pDataF = *pSetNew->pDataF; + fSet.pDataF->nRef = 1; + + // sma + fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); + if (fSet.pSmaF == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *fSet.pSmaF = *pSetNew->pSmaF; + fSet.pSmaF->nRef = 1; + + // stt + ASSERT(pSetNew->nSttF == 1); + fSet.aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (fSet.aSttF[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *fSet.aSttF[0] = *pSetNew->aSttF[0]; + fSet.aSttF[0]->nRef = 1; + + if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + iOld++; + iNew++; + continue; + } + + return code; + +_err: + tsdbError("vgId:%d, tsdb fs commit phase 2 failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + +#endif + int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) { int32_t code = 0; int32_t nRef; diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index ec760e3c57..71d745972e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -71,6 +71,9 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee goto _err; } + pTsdb->trimHdl.maxRetentFid = INT32_MIN; + pTsdb->trimHdl.minCommitFid = INT32_MAX; + tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d", TD_VID(pVnode), pTsdb->path, pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2); diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention2.c b/source/dnode/vnode/src/tsdb/tsdbRetention2.c index 4bdae09d88..61ed29368a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention2.c @@ -15,16 +15,18 @@ #include "tsdb.h" -static bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { - if (taosArrayGetSize(pTsdb->fs.aDFileSet) == 0) { - return false; - } +enum { RETENTION_NO = 0, RETENTION_EXPIRED = 1, RETENTION_MIGRATE = 2 }; - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, 0); - if (tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now) < 0) { - return true; - } +#define MIGRATE_MIN_FSIZE (1048576 << 9) // 512 MB +#define MIGRATE_MAX_SPEED (1048576 << 5) // 32 MB +#define MIGRATE_MIN_COST (5) // second +static bool tsdbShouldDoMigrate(STsdb *pTsdb); +static int32_t tsdbShouldDoRetention(STsdb *pTsdb, int64_t now); +static int32_t tsdbProcessExpire(STsdb *pTsdb, int64_t now, int32_t retention); +static int32_t tsdbProcessMigrate(STsdb *pTsdb, int64_t now, int32_t maxSpeed, int32_t retention); + +static bool tsdbShouldDoMigrate(STsdb *pTsdb) { if (tfsGetLevel(pTsdb->pVnode->pTfs) < 2) { return false; } @@ -33,67 +35,203 @@ static bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { if (keepCfg->keep0 == keepCfg->keep1 && keepCfg->keep1 == keepCfg->keep2) { return false; } + return true; +} - for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { +static int32_t tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { + int32_t retention = RETENTION_NO; + if (taosArrayGetSize(pTsdb->fs.aDFileSet) == 0) { + return retention; + } + + SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, 0); + if (tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now) < 0) { + retention |= RETENTION_EXPIRED; + } + + if (!tsdbShouldDoMigrate(pTsdb)) { + return retention; + } + + for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); ++iSet) { pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, keepCfg, now); - SDiskID did; + int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); if (expLevel == pSet->diskId.level) continue; - if (expLevel < 0) { - return true; - } else { - if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { - return false; - } - - if (did.level == pSet->diskId.level) continue; - - return true; + if (expLevel > 0) { + retention |= RETENTION_MIGRATE; + break; } } - return false; + return retention; } -/** - * @brief Data migration between multi-tier storage, including remove expired data. - * 1) firstly, remove expired DFileSet; - * 2) partition the tsdbFS by the expLevel and fileSize(e.g. 500G, configurable), and migrate DFileSet groups between multi-tier storage; - * 3) update the tsdbFS and CURRENT in the same transaction; - * 4) finish - * @param pTsdb - * @param now - * @return int32_t - */ -int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { +static int32_t tsdbProcessExpire(STsdb *pTsdb, int64_t now, int32_t retention) { int32_t code = 0; + int32_t nLoops = 0; + int32_t maxFid = INT32_MIN; + STsdbFS fs = {0}; + STsdbFS fsLatest = {0}; - if (!tsdbShouldDoRetention(pTsdb, now)) { - return code; + if (!(retention & RETENTION_EXPIRED)) { + goto _exit; } - // do retention - STsdbFS fs; - code = tsdbFSCopy(pTsdb, &fs); - if (code) goto _err; + if (code) goto _exit; - for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); iSet++) { + int32_t fsSize = taosArrayGetSize(fs.aDFileSet); + for (int32_t iSet = 0; iSet < fsSize; iSet++) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); SDiskID did; if (expLevel < 0) { - taosMemoryFree(pSet->pHeadF); - taosMemoryFree(pSet->pDataF); - taosMemoryFree(pSet->aSttF[0]); - taosMemoryFree(pSet->pSmaF); - taosArrayRemove(fs.aDFileSet, iSet); - iSet--; + SET_DFSET_EXPIRED(pSet); + if (pSet->fid > maxFid) maxFid = pSet->fid; } else { - if (expLevel == 0) continue; + break; + } + } + + if (maxFid == INT32_MIN) goto _exit; + +_wait_commit_end: + while (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) { + if (++nLoops > 1000) { + nLoops = 0; + sched_yield(); + printf("%s:%d sche_yield() minCommitFid:%d maxFid:%d\n", __func__, __LINE__, pTsdb->trimHdl.minCommitFid, maxFid); + } + } + if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { + if (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) { + atomic_store_8(&pTsdb->trimHdl.state, 0); + goto _wait_commit_end; + } + atomic_store_32(&pTsdb->trimHdl.maxRetentFid, maxFid); + atomic_store_8(&pTsdb->trimHdl.state, 0); + } else { + goto _wait_commit_end; + } + +_merge_fs: + taosThreadRwlockWrlock(&pTsdb->rwLock); + if ((code = tsdbFSCopy(pTsdb, &fsLatest))) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _exit; + } + // 1) merge tsdbFSNew and pTsdb->fs + if ((code = tsdbFSUpdDel(pTsdb, &fsLatest, &fs, maxFid))) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _exit; + } + // 2) save CURRENT + if ((code = tsdbFSCommit1(pTsdb, &fsLatest))) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _exit; + } + // 3) apply the tsdbFS to pTsdb->fs + if ((code = tsdbFSCommit2(pTsdb, &fsLatest))) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _exit; + } + taosThreadRwlockUnlock(&pTsdb->rwLock); + +_exit: + tsdbFSDestroy(&fs); + tsdbFSDestroy(&fsLatest); + if (code != 0) { + tsdbError("vgId:%d, tsdb do retention(expire) failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + ASSERT(0); + } + return code; +} + +/** + * @brief + * + * @param pTsdb + * @param now + * @param retention + * @return int32_t + */ +static int32_t tsdbProcessMigrate(STsdb *pTsdb, int64_t now, int32_t maxSpeed, int32_t retention) { + int32_t code = 0; + int32_t nLoops = 0; + int32_t maxFid = INT32_MIN; + int64_t fSize = 0; + STsdbFS fs = {0}; + STsdbFS fsLatest = {0}; + + if (!(retention & RETENTION_MIGRATE)) { + goto _exit; + } + +_migrate_loop: + // reset + maxFid = INT32_MIN; + fSize = 0; + + code = tsdbFSCopy(pTsdb, &fs); + if (code) goto _exit; + + int32_t fsSize = taosArrayGetSize(fs.aDFileSet); + for (int32_t iSet = 0; iSet < fsSize; ++iSet) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); + int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); + SDiskID did; + + if (expLevel > 0) { + ASSERT(pSet->fid > maxFid); + maxFid = pSet->fid; + fSize += (pSet->pDataF->size + pSet->pHeadF->size + pSet->pSmaF->size); + if (fSize / MIGRATE_MAX_SPEED > MIGRATE_MIN_COST) { + break; + } + for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) { + fSize += pSet->aSttF[iStt]->size; + } + if (fSize / MIGRATE_MAX_SPEED > MIGRATE_MIN_COST) { + break; + } + } + } + + if (maxFid == INT32_MIN) goto _exit; + +_wait_commit_end: + while (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) { + if (++nLoops > 1000) { + nLoops = 0; + sched_yield(); + printf("%s:%d sche_yield()\n", __func__, __LINE__); + } + } + if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { + if (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) { + atomic_store_8(&pTsdb->trimHdl.state, 0); + goto _wait_commit_end; + } + atomic_store_32(&pTsdb->trimHdl.maxRetentFid, maxFid); + atomic_store_8(&pTsdb->trimHdl.state, 0); + } else { + goto _wait_commit_end; + } + + // migrate + for (int32_t iSet = 0; iSet < fsSize; ++iSet) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); + int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); + SDiskID did; + + if (pSet->fid > maxFid) break; + + if (expLevel < 0) { + SET_DFSET_EXPIRED(pSet); + } else if (expLevel > 0) { if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { code = terrno; goto _exit; @@ -101,40 +239,87 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { if (did.level == pSet->diskId.level) continue; - // copy file to new disk (todo) + // copy file to new disk SDFileSet fSet = *pSet; fSet.diskId = did; code = tsdbDFileSetCopy(pTsdb, pSet, &fSet); - if (code) goto _err; + if (code) goto _exit; code = tsdbFSUpsertFSet(&fs, &fSet); - if (code) goto _err; + if (code) goto _exit; } } - // do change fs - code = tsdbFSCommit1(pTsdb, &fs); - if (code) goto _err; - +_merge_fs: taosThreadRwlockWrlock(&pTsdb->rwLock); - code = tsdbFSCommit2(pTsdb, &fs); - if (code) { + if ((code = tsdbFSCopy(pTsdb, &fsLatest))) { taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; + goto _exit; + } + // 1) merge tsdbFSNew and pTsdb->fs + if ((code = tsdbFSUpdDel(pTsdb, &fsLatest, &fs, maxFid))) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _exit; + } + // 2) save CURRENT + if ((code = tsdbFSCommit1(pTsdb, &fsLatest))) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _exit; + } + // 3) apply the tsdbFS to pTsdb->fs + if ((code = tsdbFSCommit2(pTsdb, &fsLatest))) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _exit; } - taosThreadRwlockUnlock(&pTsdb->rwLock); tsdbFSDestroy(&fs); _exit: + if (code != 0) { + tsdbError("vgId:%d, tsdb do retention(migrate) failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + ASSERT(0); + } return code; +} -_err: - tsdbError("vgId:%d, tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - ASSERT(0); - // tsdbFSRollback(pTsdb->pFS); +/** + * @brief Data migration between multi-tier storage, including remove expired data. + * 1) firstly, remove expired DFileSet; + * 2) partition the tsdbFS by the expLevel and the estimated cost(e.g. 5s) to copy, and migrate + * DFileSet groups between multi-tier storage; + * 3) update the tsdbFS and CURRENT in the same transaction; + * 4) finish the migration + * @param pTsdb + * @param now + * @param maxSpeed + * @return int32_t + */ +int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now, int32_t maxSpeed) { + int32_t code = 0; + int32_t retention = RETENTION_NO; + + retention = tsdbShouldDoRetention(pTsdb, now); + if (retention == RETENTION_NO) { + goto _exit; + } + + // step 1: process expire + code = tsdbProcessExpire(pTsdb, now, retention); + if (code < 0) goto _exit; + + // step 2: process multi-tier migration + code = tsdbProcessMigrate(pTsdb, now, maxSpeed, retention); + if (code < 0) goto _exit; + +_exit: + pTsdb->trimHdl.maxRetentFid = INT32_MIN; + if (code != 0) { + tsdbError("vgId:%d, tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + ASSERT(0); + // tsdbFSRollback(pTsdb->pFS); + } return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 728b5b4cda..0263740e4a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -417,24 +417,29 @@ void *vnodeProcessTrimReqFunc(void *param) { setThreadName("vnode-trim"); // process - code = tsdbDoRetention(pVnode->pTsdb, pReq->trimReq.timestamp); + code = tsdbDoRetention(pVnode->pTsdb, pReq->trimReq.timestamp, pReq->trimReq.maxSpeed); if (code) goto _exit; - code = smaDoRetention(pVnode->pSma, pReq->trimReq.timestamp); + code = smaDoRetention(pVnode->pSma, pReq->trimReq.timestamp, pReq->trimReq.maxSpeed); if (code) goto _exit; - _exit: - vInfo("vgId:%d, trim vnode thread finished, time:%d", TD_VID(pVnode), pReq->trimReq.timestamp); oldVal = atomic_val_compare_exchange_8(&pVnode->trimDbH.state, 1, 0); ASSERT(oldVal == 1); taosMemoryFree(pReq); + if (code) { + vError("vgId:%d, trim vnode thread failed since %s, time:%" PRIi64, TD_VID(pVnode), tstrerror(code), + pReq->trimReq.timestamp); + } else { + vInfo("vgId:%d, trim vnode thread finished, time:%" PRIi64, TD_VID(pVnode), pReq->trimReq.timestamp); + } + return NULL; } static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; SVndTrimDbReq *pVndTrimReq = taosMemoryMalloc(sizeof(SVndTrimDbReq)); - STrimDbHandle *pHandle = &pVnode->trimDbH; + SVTrimDbHdl *pHandle = &pVnode->trimDbH; if (!pVndTrimReq) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -450,7 +455,7 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, } if (atomic_val_compare_exchange_8(&pHandle->state, 0, 1) != 0) { - vInfo("vgId:%d, trim vnode request will not be processed since duplicated req, time:%d", TD_VID(pVnode), + vInfo("vgId:%d, trim vnode request will not be processed since duplicated req, time:%" PRIi64, TD_VID(pVnode), pVndTrimReq->trimReq.timestamp); taosMemoryFree(pVndTrimReq); goto _exit; From 797b9376e7b32a52605488a18d942f3f1dcebe17 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 23 Sep 2022 17:27:48 +0800 Subject: [PATCH 03/71] chore: code optimization for fs destroy --- source/dnode/vnode/src/inc/tsdb.h | 3 ++- source/dnode/vnode/src/tsdb/tsdbFS.c | 4 ++-- source/dnode/vnode/src/tsdb/tsdbRetention2.c | 6 ++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 66df364b4b..262bd083ef 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -321,7 +321,8 @@ struct STsdbFS { }; struct STsdbTrimHdl { - volatile int8_t state; // 0 idle 1 in use + volatile int8_t state; // 0 idle 1 in use + volatile int8_t limitSpeed; // 0 no limit, 1 with limit volatile int32_t maxRetentFid; volatile int32_t minCommitFid; }; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 45d28f4135..8fdbd68af3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -250,7 +250,7 @@ _err: void tsdbFSDestroy(STsdbFS *pFS) { if (pFS->pDelFile) { - taosMemoryFree(pFS->pDelFile); + taosMemoryFreeClear(pFS->pDelFile); } for (int32_t iSet = 0; iSet < taosArrayGetSize(pFS->aDFileSet); iSet++) { @@ -263,7 +263,7 @@ void tsdbFSDestroy(STsdbFS *pFS) { } } - taosArrayDestroy(pFS->aDFileSet); + pFS->aDFileSet = taosArrayDestroy(pFS->aDFileSet); } static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention2.c b/source/dnode/vnode/src/tsdb/tsdbRetention2.c index 61ed29368a..a5a16a9602 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention2.c @@ -174,6 +174,8 @@ _migrate_loop: // reset maxFid = INT32_MIN; fSize = 0; + tsdbFSDestroy(&fs); + tsdbFSDestroy(&fsLatest); code = tsdbFSCopy(pTsdb, &fs); if (code) goto _exit; @@ -275,9 +277,9 @@ _merge_fs: } taosThreadRwlockUnlock(&pTsdb->rwLock); - tsdbFSDestroy(&fs); - _exit: + tsdbFSDestroy(&fs); + tsdbFSDestroy(&fsLatest); if (code != 0) { tsdbError("vgId:%d, tsdb do retention(migrate) failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); ASSERT(0); From ed2b3678a66cda734c7634e1f62c5c6f671adaa5 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 23 Sep 2022 18:57:13 +0800 Subject: [PATCH 04/71] feat(tsdb): migrate support speed limit --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 + .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 52 +++++++++++++++++-- source/dnode/vnode/src/tsdb/tsdbRetention2.c | 2 +- 4 files changed, 51 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 262bd083ef..6611baa2f4 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -265,7 +265,7 @@ int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast); -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); +int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, int32_t maxSpeed); // SDataFReader int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 0a2d37bdbc..bc9c37fc76 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -763,6 +763,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { _wait_retention_end: while (atomic_load_32(&pTsdb->trimHdl.maxRetentFid) >= minCommitFid) { + atomic_val_compare_exchange_8(&pTsdb->trimHdl.limitSpeed, 1, 0); if (++nLoops > 1000) { nLoops = 0; sched_yield(); @@ -778,6 +779,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { } else { goto _wait_retention_end; } + atomic_store_8(&pTsdb->trimHdl.limitSpeed, 1); } code = tsdbFSCopy(pTsdb, &pCommitter->fs); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 5fe0b408b1..615b383e65 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -607,7 +607,43 @@ _err: return code; } -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { +/** + * @brief send file with limited speed(rough control) + * + * @param pFileOut + * @param pFileIn + * @param size + * @param speed 0 no limit, unit: B/s + * @return int64_t + */ +static int64_t tsdbFSendFile(TdFilePtr pOutFD, TdFilePtr pInFD, int64_t size, int32_t speed) { + if (speed <= 0) { + return taosFSendFile(pOutFD, pInFD, 0, size); + } + + int64_t offset = 0; + int64_t nBytes = 0; + int64_t startMs = 0; + int64_t endMs = 0; + int64_t cost = 0; + while ((offset + speed) < size) { + startMs = taosGetTimestampMs(); + nBytes += taosFSendFile(pOutFD, pInFD, &offset, speed); + cost = taosGetTimestampMs() - startMs; + + if (cost < 0) { + taosMsleep(1000); + } else if (cost < 1000) { + taosMsleep(1000 - cost); + } + } + if (offset < size) { + nBytes += taosFSendFile(pOutFD, pInFD, &offset, size - offset); + } + return nBytes; +} + +int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, int32_t maxSpeed) { int32_t code = 0; int64_t n; int64_t size; @@ -616,6 +652,12 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { int32_t szPage = pTsdb->pVnode->config.szPage; char fNameFrom[TSDB_FILENAME_LEN]; char fNameTo[TSDB_FILENAME_LEN]; + int32_t speed = 0; + + if (atomic_load_8(&pTsdb->trimHdl.limitSpeed)) { + ASSERT(maxSpeed > 0); + speed = maxSpeed; + } // head tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); @@ -630,7 +672,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage)); + n = tsdbFSendFile(pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage), speed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -651,7 +693,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - n = taosFSendFile(pOutFD, PInFD, 0, LOGIC_TO_FILE_OFFSET(pSetFrom->pDataF->size, szPage)); + n = tsdbFSendFile(pOutFD, PInFD, LOGIC_TO_FILE_OFFSET(pSetFrom->pDataF->size, szPage), speed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -672,7 +714,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage)); + n = tsdbFSendFile(pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage), speed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -694,7 +736,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage)); + n = tsdbFSendFile(pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage), speed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention2.c b/source/dnode/vnode/src/tsdb/tsdbRetention2.c index a5a16a9602..34b8e0018f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention2.c @@ -245,7 +245,7 @@ _wait_commit_end: SDFileSet fSet = *pSet; fSet.diskId = did; - code = tsdbDFileSetCopy(pTsdb, pSet, &fSet); + code = tsdbDFileSetCopy(pTsdb, pSet, &fSet, maxSpeed); if (code) goto _exit; code = tsdbFSUpsertFSet(&fs, &fSet); From 132425b1947e00dc14d64bb4bb9c290e337af57d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 23 Sep 2022 19:00:38 +0800 Subject: [PATCH 05/71] chore: remove obsolete codes --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 615b383e65..bde628662d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -624,7 +624,6 @@ static int64_t tsdbFSendFile(TdFilePtr pOutFD, TdFilePtr pInFD, int64_t size, in int64_t offset = 0; int64_t nBytes = 0; int64_t startMs = 0; - int64_t endMs = 0; int64_t cost = 0; while ((offset + speed) < size) { startMs = taosGetTimestampMs(); From b101e425c34403ee242aa39de1a19bc93c3cc258 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 25 Sep 2022 22:30:48 +0800 Subject: [PATCH 06/71] chore: more logic for migrate with speed limited --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index bde628662d..8a4618992b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -622,13 +622,17 @@ static int64_t tsdbFSendFile(TdFilePtr pOutFD, TdFilePtr pInFD, int64_t size, in } int64_t offset = 0; + int64_t tBytes = 0; int64_t nBytes = 0; int64_t startMs = 0; int64_t cost = 0; while ((offset + speed) < size) { startMs = taosGetTimestampMs(); - nBytes += taosFSendFile(pOutFD, pInFD, &offset, speed); + if ((nBytes = taosFSendFile(pOutFD, pInFD, &offset, speed)) < 0) { + return nBytes; + } cost = taosGetTimestampMs() - startMs; + tBytes += nBytes; if (cost < 0) { taosMsleep(1000); @@ -637,9 +641,12 @@ static int64_t tsdbFSendFile(TdFilePtr pOutFD, TdFilePtr pInFD, int64_t size, in } } if (offset < size) { - nBytes += taosFSendFile(pOutFD, pInFD, &offset, size - offset); + if ((nBytes = taosFSendFile(pOutFD, pInFD, &offset, size - offset)) < 0) { + return nBytes; + } + tBytes += nBytes; } - return nBytes; + return tBytes; } int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, int32_t maxSpeed) { From d4fb75df8bb60f5a5f853e645b238dbbb6a04332 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 26 Sep 2022 14:02:57 +0800 Subject: [PATCH 07/71] chore: merge 3.0 and migrate fix for multi-tier storage --- Jenkinsfile | 2 +- Jenkinsfile2 | 4 +- cmake/cmake.options | 34 +- docs/en/05-get-started/01-docker.md | 2 +- docs/en/05-get-started/03-package.md | 2 +- .../14-reference/02-rest-api/02-rest-api.mdx | 4 +- docs/zh/08-connector/02-rest-api.mdx | 4 +- include/common/tdataformat.h | 15 +- include/common/tmsg.h | 10 + include/libs/executor/executor.h | 11 +- include/libs/nodes/plannodes.h | 1 + include/libs/qcom/query.h | 15 +- include/libs/qworker/qworker.h | 26 +- include/libs/scheduler/scheduler.h | 1 + include/util/tdef.h | 1 + include/util/tutil.h | 1 + packaging/docker/dockerManifest.sh | 89 +- packaging/docker/dockerbuild.sh | 22 - packaging/release.bat | 3 - source/client/src/clientEnv.c | 1 + source/client/src/clientHb.c | 28 +- source/client/src/clientImpl.c | 19 +- source/client/src/clientMain.c | 1 + source/common/src/tdataformat.c | 40 +- source/common/src/tmsg.c | 17 +- source/common/src/trow.c | 6 +- source/common/test/dataformatTest.cpp | 12 +- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 2 +- source/dnode/mnode/impl/src/mndAcct.c | 4 +- source/dnode/mnode/impl/src/mndBnode.c | 8 +- source/dnode/mnode/impl/src/mndCluster.c | 4 +- source/dnode/mnode/impl/src/mndDb.c | 26 +- source/dnode/mnode/impl/src/mndDnode.c | 8 +- source/dnode/mnode/impl/src/mndFunc.c | 12 +- source/dnode/mnode/impl/src/mndMain.c | 26 +- source/dnode/mnode/impl/src/mndQnode.c | 8 +- source/dnode/mnode/impl/src/mndQuery.c | 2 +- source/dnode/mnode/impl/src/mndSma.c | 16 +- source/dnode/mnode/impl/src/mndSnode.c | 8 +- source/dnode/mnode/impl/src/mndStb.c | 66 +- source/dnode/mnode/impl/src/mndStream.c | 20 +- source/dnode/mnode/impl/src/mndSubscribe.c | 4 +- source/dnode/mnode/impl/src/mndSync.c | 14 +- source/dnode/mnode/impl/src/mndTopic.c | 12 +- source/dnode/mnode/impl/src/mndTrans.c | 8 +- source/dnode/mnode/impl/src/mndUser.c | 16 +- source/dnode/mnode/impl/src/mndVgroup.c | 32 +- source/dnode/mnode/sdb/src/sdb.c | 14 +- source/dnode/mnode/sdb/src/sdbFile.c | 40 +- source/dnode/mnode/sdb/src/sdbHash.c | 2 +- source/dnode/mnode/sdb/src/sdbRaw.c | 4 +- source/dnode/mnode/sdb/src/sdbRow.c | 4 +- source/dnode/qnode/src/qnode.c | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 16 +- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 13 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 3 + source/dnode/vnode/src/tsdb/tsdbRead.c | 119 ++- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 8 +- source/dnode/vnode/src/tsdb/tsdbRetention2.c | 4 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 24 +- source/dnode/vnode/src/vnd/vnodeQuery.c | 2 +- source/libs/command/inc/commandInt.h | 2 + source/libs/command/src/explain.c | 61 +- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executor.c | 6 +- source/libs/executor/src/executorimpl.c | 82 +- source/libs/executor/src/timewindowoperator.c | 12 +- source/libs/function/src/builtinsimpl.c | 4 +- source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/parser/src/parInsertData.c | 2 +- source/libs/qworker/inc/qwInt.h | 20 +- source/libs/qworker/inc/qwMsg.h | 2 +- source/libs/qworker/src/qwMsg.c | 4 +- source/libs/qworker/src/qworker.c | 212 +++-- source/libs/qworker/test/qworkerTests.cpp | 16 +- source/libs/scheduler/CMakeLists.txt | 2 +- source/libs/scheduler/inc/schInt.h | 12 +- source/libs/scheduler/src/schJob.c | 1 + source/libs/scheduler/src/schRemote.c | 120 +-- source/libs/scheduler/src/schTask.c | 180 +++- source/libs/scheduler/src/scheduler.c | 4 + source/libs/sync/CMakeLists.txt | 4 +- source/libs/transport/src/thttp.c | 39 +- tests/system-test/99-TDcase/TD-19201.py | 797 ++++++++++++++++++ tests/system-test/fulltest.sh | 93 ++ 86 files changed, 1937 insertions(+), 664 deletions(-) create mode 100644 tests/system-test/99-TDcase/TD-19201.py diff --git a/Jenkinsfile b/Jenkinsfile index 4b84e1f88e..0842795f0a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -79,7 +79,7 @@ def pre_test(){ rm -rf debug mkdir debug cd debug - cmake .. > /dev/null + cmake .. -DBUILD_TEST=true > /dev/null make -j4> /dev/null ''' diff --git a/Jenkinsfile2 b/Jenkinsfile2 index b95b3ff86b..ac152e2aa4 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -173,7 +173,7 @@ def pre_test_build_mac() { ''' sh ''' cd ${WK}/debug - cmake .. + cmake .. -DBUILD_TEST=true make -j8 ''' sh ''' @@ -302,7 +302,7 @@ def pre_test_build_win() { set CL=/MP8 echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake" time /t - cmake .. -G "NMake Makefiles JOM" || exit 7 + cmake .. -G "NMake Makefiles JOM" -DBUILD_TEST=true || exit 7 echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6" time /t jom -j 6 || exit 8 diff --git a/cmake/cmake.options b/cmake/cmake.options index 3baccde4d7..60ff00affc 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -2,6 +2,12 @@ # Deps options # ========================================================= +option( + BUILD_TEST + "If build unit tests using googletest" + OFF +) + IF(${TD_WINDOWS}) MESSAGE("build pthread Win32") @@ -45,12 +51,6 @@ IF(${TD_WINDOWS}) "If build wingetopt on Windows" ON ) - - option( - BUILD_TEST - "If build unit tests using googletest" - ON - ) option( TDENGINE_3 @@ -65,28 +65,8 @@ IF(${TD_WINDOWS}) ) ELSEIF (TD_DARWIN_64) - add_definitions(-DCOMPILER_SUPPORTS_CXX13) - option( - BUILD_TEST - "If build unit tests using googletest" - ON - ) -ELSE () - include(CheckCXXCompilerFlag) - CHECK_CXX_COMPILER_FLAG("-std=c++13" COMPILER_SUPPORTS_CXX13) - IF(${COMPILER_SUPPORTS_CXX13}) + IF(${BUILD_TEST}) add_definitions(-DCOMPILER_SUPPORTS_CXX13) - option( - BUILD_TEST - "If build unit tests using googletest" - ON - ) - ELSE () - option( - BUILD_TEST - "If build unit tests using googletest" - OFF - ) ENDIF () ENDIF () diff --git a/docs/en/05-get-started/01-docker.md b/docs/en/05-get-started/01-docker.md index be61dfeb8e..d0874c331e 100644 --- a/docs/en/05-get-started/01-docker.md +++ b/docs/en/05-get-started/01-docker.md @@ -5,8 +5,8 @@ title: Quick Install on Docker This document describes how to install TDengine in a Docker container and perform queries and inserts. +- The easiest way to explore TDengine is through [TDengine Cloud](http://cloud.tdengine.com). - To get started with TDengine in a non-containerized environment, see [Quick Install from Package](../../get-started/package). -- For a fully managed solution, see the [TDengine Cloud documentation](/cloud/). - If you want to view the source code, build TDengine yourself, or contribute to the project, see the [TDengine GitHub repository](https://github.com/taosdata/TDengine). ## Run TDengine diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index da06aca148..25a92573fa 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -9,8 +9,8 @@ import PkgListV3 from "/components/PkgListV3"; This document describes how to install TDengine on Linux and Windows and perform queries and inserts. +- The easiest way to explore TDengine is through [TDengine Cloud](http://cloud.tdengine.com). - To get started with TDengine on Docker, see [Quick Install on Docker](../../get-started/docker). -- For a fully managed solution, see the [TDengine Cloud documentation](/cloud/). - If you want to view the source code, build TDengine yourself, or contribute to the project, see the [TDengine GitHub repository](https://github.com/taosdata/TDengine). The full package of TDengine includes the TDengine Server (`taosd`), TDengine Client (`taosc`), taosAdapter for connecting with third-party systems and providing a RESTful interface, a command-line interface (CLI, taos), and some tools. Note that taosAdapter supports Linux only. In addition to connectors for multiple languages, TDengine also provides a [REST API](../../reference/rest-api) through [taosAdapter](../../reference/taosadapter). diff --git a/docs/en/14-reference/02-rest-api/02-rest-api.mdx b/docs/en/14-reference/02-rest-api/02-rest-api.mdx index ce28ee87d9..da26b34c6f 100644 --- a/docs/en/14-reference/02-rest-api/02-rest-api.mdx +++ b/docs/en/14-reference/02-rest-api/02-rest-api.mdx @@ -123,7 +123,7 @@ where `TOKEN` is the string after Base64 encoding of `{username}:{password}`, e. ### HTTP body structure -#### Successful Operation +#### Successful Insert Operation Example: @@ -143,7 +143,7 @@ Description: - rows: (`int`) Only returns `1`. - data: (`[][]any`) Returns the number of rows affected. -#### Successful Query +#### Successful Query Operation Example: diff --git a/docs/zh/08-connector/02-rest-api.mdx b/docs/zh/08-connector/02-rest-api.mdx index 12a9cf5347..a8e1682301 100644 --- a/docs/zh/08-connector/02-rest-api.mdx +++ b/docs/zh/08-connector/02-rest-api.mdx @@ -125,7 +125,7 @@ curl -L -u username:password -d "" :/rest/sql/[db_name] ### HTTP body 结构 -#### 正确执行 +#### 正确执行插入 样例： @@ -145,7 +145,7 @@ curl -L -u username:password -d "" :/rest/sql/[db_name] - rows：（`int`）只返回 `1`。 - data：（`[][]any`）返回受影响行数。 -#### 正确查询 +#### 正确执行查询 样例： diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 5b49560175..ac94b0af7d 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -66,10 +66,18 @@ int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type); int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type); // SColVal ================================ -#define COL_VAL_NONE(CID, TYPE) ((SColVal){.cid = (CID), .type = (TYPE), .isNone = 1}) -#define COL_VAL_NULL(CID, TYPE) ((SColVal){.cid = (CID), .type = (TYPE), .isNull = 1}) +#define CV_FLAG_VALUE ((int8_t)0x0) +#define CV_FLAG_NONE ((int8_t)0x1) +#define CV_FLAG_NULL ((int8_t)0x2) + +#define COL_VAL_NONE(CID, TYPE) ((SColVal){.cid = (CID), .type = (TYPE), .flag = CV_FLAG_NONE}) +#define COL_VAL_NULL(CID, TYPE) ((SColVal){.cid = (CID), .type = (TYPE), .flag = CV_FLAG_NULL}) #define COL_VAL_VALUE(CID, TYPE, V) ((SColVal){.cid = (CID), .type = (TYPE), .value = (V)}) +#define COL_VAL_IS_NONE(CV) ((CV)->flag == CV_FLAG_NONE) +#define COL_VAL_IS_NULL(CV) ((CV)->flag == CV_FLAG_NULL) +#define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE) + // STSRow2 ================================ #define TSROW_LEN(PROW, V) tGetI32v((uint8_t *)(PROW)->data, (V) ? &(V) : NULL) #define TSROW_SVER(PROW, V) tGetI32v((PROW)->data + TSROW_LEN(PROW, NULL), (V) ? &(V) : NULL) @@ -175,8 +183,7 @@ struct SValue { struct SColVal { int16_t cid; int8_t type; - int8_t isNone; - int8_t isNull; + int8_t flag; SValue value; }; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a05cd907b0..56bca4cc50 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1425,6 +1425,14 @@ typedef struct { SExplainExecInfo* subplanInfo; } SExplainRsp; +typedef struct { + SExplainRsp rsp; + uint64_t qId; + uint64_t tId; + int64_t rId; + int32_t eId; +} SExplainLocalRsp; + typedef struct STableScanAnalyzeInfo { uint64_t totalRows; uint64_t totalCheckedRows; @@ -1439,6 +1447,7 @@ typedef struct STableScanAnalyzeInfo { int32_t tSerializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp); int32_t tDeserializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp); +void tFreeSExplainRsp(SExplainRsp *pRsp); typedef struct { char fqdn[TSDB_FQDN_LEN]; // end point, hostname:port @@ -2338,6 +2347,7 @@ int32_t tSerializeSClientHbBatchReq(void* buf, int32_t bufLen, const SClientHbBa int32_t tDeserializeSClientHbBatchReq(void* buf, int32_t bufLen, SClientHbBatchReq* pReq); static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq) { + if (pReq == NULL) return; SClientHbBatchReq* req = (SClientHbBatchReq*)pReq; taosArrayDestroyEx(req->reqs, tFreeClientHbReq); taosMemoryFree(pReq); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 25a6221fcb..8c1d957381 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -29,6 +29,15 @@ typedef void* DataSinkHandle; struct SRpcMsg; struct SSubplan; +typedef int32_t (*localFetchFp)(void *, uint64_t, uint64_t, uint64_t, int64_t, int32_t, void**, SArray*); + +typedef struct { + void *handle; + bool localExec; + localFetchFp fp; + SArray *explainRes; +} SLocalFetch; + typedef struct { void* tqReader; void* meta; @@ -127,7 +136,7 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table * @param handle * @return */ -int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds); +int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, SLocalFetch *pLocal); int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds); /** diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 31d67a85a7..a531eeffde 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -396,6 +396,7 @@ typedef struct SDownstreamSourceNode { uint64_t schedId; int32_t execId; int32_t fetchMsgType; + bool localExec; } SDownstreamSourceNode; typedef struct SExchangePhysiNode { diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 44a9e10679..e9f3864f67 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -52,6 +52,7 @@ typedef enum { #define QUERY_POLICY_VNODE 1 #define QUERY_POLICY_HYBRID 2 #define QUERY_POLICY_QNODE 3 +#define QUERY_POLICY_CLIENT 4 typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema @@ -269,43 +270,43 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define qFatal(...) \ do { \ if (qDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("QRY FATAL ", DEBUG_FATAL, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); \ } \ } while (0) #define qError(...) \ do { \ if (qDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("QRY ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); \ } \ } while (0) #define qWarn(...) \ do { \ if (qDebugFlag & DEBUG_WARN) { \ - taosPrintLog("QRY WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); \ } \ } while (0) #define qInfo(...) \ do { \ if (qDebugFlag & DEBUG_INFO) { \ - taosPrintLog("QRY ", DEBUG_INFO, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); \ } \ } while (0) #define qDebug(...) \ do { \ if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ } \ } while (0) #define qTrace(...) \ do { \ if (qDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("QRY ", DEBUG_TRACE, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \ } \ } while (0) #define qDebugL(...) \ do { \ if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLongString("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ } \ } while (0) diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 87aefe5187..99f5189228 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -29,6 +29,7 @@ enum { NODE_TYPE_QNODE, NODE_TYPE_SNODE, NODE_TYPE_MNODE, + NODE_TYPE_CLIENT, }; typedef struct SQWorkerCfg { @@ -55,7 +56,24 @@ typedef struct { uint64_t numOfErrors; } SQWorkerStat; -int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, const SMsgCb *pMsgCb); +typedef struct SQWMsgInfo { + int8_t taskType; + int8_t explain; + int8_t needFetch; +} SQWMsgInfo; + +typedef struct SQWMsg { + void *node; + int32_t code; + int32_t msgType; + void *msg; + int32_t msgLen; + SQWMsgInfo msgInfo; + SRpcHandleInfo connInfo; +} SQWMsg; + + +int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const SMsgCb *pMsgCb); int32_t qWorkerAbortPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg); @@ -77,10 +95,14 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_ int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SDeleteRes *pRes); -void qWorkerDestroy(void **qWorkerMgmt); +void qWorkerDestroy(void **qWorkerMgmt); int32_t qWorkerGetStat(SReadHandle *handle, void *qWorkerMgmt, SQWorkerStat *pStat); +int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId, SQWMsg *qwMsg, SArray *explainRes); + +int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId, void** pRsp, SArray* explainRes); + #ifdef __cplusplus } #endif diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 738d057e6a..077c23c1b5 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -64,6 +64,7 @@ typedef bool (*schedulerChkKillFp)(void* param); typedef struct SSchedulerReq { bool syncReq; + bool localReq; SRequestConnInfo *pConn; SArray *pNodeList; SQueryPlan *pDag; diff --git a/include/util/tdef.h b/include/util/tdef.h index 0c7c2ea3a3..43fd31afa7 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -483,6 +483,7 @@ enum { #define SNODE_HANDLE -2 #define VNODE_HANDLE -3 #define BNODE_HANDLE -4 +#define CLIENT_HANDLE -5 #define TSDB_CONFIG_OPTION_LEN 32 #define TSDB_CONFIG_VALUE_LEN 64 diff --git a/include/util/tutil.h b/include/util/tutil.h index 32a88b37ec..c22495b75f 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -62,6 +62,7 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar tMD5Final(&context); char buf[TSDB_PASSWORD_LEN + 1]; + buf[TSDB_PASSWORD_LEN] = 0; sprintf(buf, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], diff --git a/packaging/docker/dockerManifest.sh b/packaging/docker/dockerManifest.sh index 71788423f6..8f71e30fbd 100755 --- a/packaging/docker/dockerManifest.sh +++ b/packaging/docker/dockerManifest.sh @@ -1,6 +1,7 @@ #!/bin/bash set -e #set -x +set -v # dockerbuild.sh # -n [version number] @@ -11,8 +12,9 @@ set -e version="" passWord="" verType="" +dockerLatest="n" -while getopts "hn:p:V:" arg +while getopts "hn:p:V:a:" arg do case $arg in n) @@ -29,9 +31,15 @@ do ;; h) echo "Usage: `basename $0` -n [version number] " - echo " -p [password for docker hub] " + echo " -p [password for docker hub] " + echo " -V [stable |beta] " + echo " -a [y | n ] " exit 0 ;; + a) + #echo "dockerLatest=$OPTARG" + dockerLatest=$(echo $OPTARG) + ;; ?) #unknow option echo "unkonw argument" exit 1 @@ -41,42 +49,55 @@ done echo "version=${version}" -#docker manifest rm tdengine/tdengine -#docker manifest rm tdengine/tdengine:${version} -if [ "$verType" == "beta" ]; then - docker manifest create -a tdengine/tdengine-beta:${version} tdengine/tdengine-amd64-beta:${version} tdengine/tdengine-aarch64-beta:${version} tdengine/tdengine-aarch32-beta:${version} - docker manifest create -a tdengine/tdengine-beta:latest tdengine/tdengine-amd64-beta:latest tdengine/tdengine-aarch64-beta:latest tdengine/tdengine-aarch32-beta:latest - docker manifest rm tdengine/tdengine-beta:${version} - docker manifest rm tdengine/tdengine-beta:latest - docker manifest create -a tdengine/tdengine-beta:${version} tdengine/tdengine-amd64-beta:${version} tdengine/tdengine-aarch64-beta:${version} tdengine/tdengine-aarch32-beta:${version} - docker manifest create -a tdengine/tdengine-beta:latest tdengine/tdengine-amd64-beta:latest tdengine/tdengine-aarch64-beta:latest tdengine/tdengine-aarch32-beta:latest - docker manifest inspect tdengine/tdengine:latest - docker manifest inspect tdengine/tdengine:${version} - docker login -u tdengine -p ${passWord} #replace the docker registry username and password - docker manifest push tdengine/tdengine-beta:${version} - docker manifest push tdengine/tdengine-beta:latest -elif [ "$verType" == "stable" ]; then - docker manifest create -a tdengine/tdengine:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} - docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest - docker manifest rm tdengine/tdengine:latest - docker manifest rm tdengine/tdengine:${version} - docker manifest create -a tdengine/tdengine:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} - docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest - docker manifest inspect tdengine/tdengine:latest - docker manifest inspect tdengine/tdengine:${version} - docker login -u tdengine -p ${passWord} #replace the docker registry username and password - docker manifest push tdengine/tdengine:${version} - docker manifest push tdengine/tdengine:latest -else +if [ "$verType" == "stable" ]; then + verType=stable + dockerinput=TDengine-server-${version}-Linux-$cpuType.tar.gz + dockerinput_x64=TDengine-server-${version}-Linux-amd64.tar.gz + dockerim=tdengine/tdengine + dockeramd64=tdengine/tdengine-amd64 + dockeraarch64=tdengine/tdengine-aarch64 + dockeraarch32=tdengine/tdengine-aarch32 +elif [ "$verType" == "beta" ];then + verType=beta + tagVal=ver-${version}-beta + dockerinput=TDengine-server-${version}-${verType}-Linux-$cpuType.tar.gz + dockerinput_x64=TDengine-server-${version}-${verType}-Linux-amd64.tar.gz + dockerim=tdengine/tdengine-beta + dockeramd64=tdengine/tdengine-amd64-beta + dockeraarch64=tdengine/tdengine-aarch64-beta + dockeraarch32=tdengine/tdengine-aarch32-beta + else echo "unknow verType, nor stabel or beta" exit 1 fi -# docker manifest create -a tdengine/${dockername}:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version} -# docker manifest create -a tdengine/${dockername}:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest +username="tdengine" -# docker login -u tdengine -p ${passWord} #replace the docker registry username and password +# generate docker verison +echo "generate ${dockerim}:${version}" +docker manifest create -a ${dockerim}:${version} ${dockeramd64}:${version} ${dockeraarch64}:${version} +docker manifest inspect ${dockerim}:${version} +docker manifest rm ${dockerim}:${version} +docker manifest create -a ${dockerim}:${version} ${dockeramd64}:${version} ${dockeraarch64}:${version} +docker manifest inspect ${dockerim}:${version} +docker login -u ${username} -p ${passWord} +docker manifest push ${dockerim}:${version} + + +# generate docker latest +echo "generate ${dockerim}:latest " + +if [ ${dockerLatest} == 'y' ] ;then + echo "docker manifest create -a ${dockerim}:latest ${dockeramd64}:latest ${dockeraarch64}:latest" + docker manifest create -a ${dockerim}:latest ${dockeramd64}:latest ${dockeraarch64}:latest + docker manifest inspect ${dockerim}:latest + docker manifest rm ${dockerim}:latest + docker manifest create -a ${dockerim}:latest ${dockeramd64}:latest ${dockeraarch64}:latest + docker manifest inspect ${dockerim}:latest + docker login -u tdengine -p ${passWord} #replace the docker registry username and password + docker manifest push ${dockerim}:latest + docker pull tdengine/tdengine:latest + +fi -# docker manifest push tdengine/tdengine:latest -# # how set latest version ??? diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh index 541ae6ec13..8b0b0c190c 100755 --- a/packaging/docker/dockerbuild.sh +++ b/packaging/docker/dockerbuild.sh @@ -149,26 +149,4 @@ rm -rf temp1.data if [ ${dockerLatest} == 'y' ] ;then docker tag tdengine/tdengine-${dockername}:${version} tdengine/tdengine-${dockername}:latest docker push tdengine/tdengine-${dockername}:latest - echo ">>>>>>>>>>>>> check whether tdengine/tdengine-${dockername}:latest has been published correctly" - docker run -d --name doctestla -p 7030-7049:6030-6049 -p 7030-7049:6030-6049/udp tdengine/tdengine-${dockername}:latest - sleep 2 - curl -u root:taosdata -d 'show variables;' 127.0.0.1:7041/rest/sql > temp2.data - version_latest=` cat temp2.data |jq .data| jq '.[]' |grep "version" -A 2 -B 1 | jq ".[1]" ` - echo "${version_latest}" - if [ "${version_latest}" == "\"${version}\"" ] ; then - echo "docker version is right " - else - echo "docker version is wrong " - exit 1 - fi fi -rm -rf temp2.data - -if [ -n "$(docker ps -aq)" ] ;then - echo "delte docker process" - docker stop $(docker ps -aq) - docker rm $(docker ps -aq) -fi - -cd ${scriptDir} -rm -f ${pkgFile} diff --git a/packaging/release.bat b/packaging/release.bat index 591227382f..b87ae68e2b 100644 --- a/packaging/release.bat +++ b/packaging/release.bat @@ -44,8 +44,6 @@ cmake ../../ -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_TOOLS=tru cmake --build . rd /s /Q C:\TDengine cmake --install . -for /r c:\TDengine %%i in (*.dll) do signtool sign /f D:\\123.pfx /p taosdata %%i -for /r c:\TDengine %%i in (*.exe) do signtool sign /f D:\\123.pfx /p taosdata %%i if not %errorlevel% == 0 ( call :RUNFAILED build x64 failed & exit /b 1) cd %package_dir% iscc /DMyAppInstallName="%packagServerName_x64%" /DMyAppVersion="%2" /DMyAppExcludeSource="" tools\tdengine.iss /O..\release @@ -53,7 +51,6 @@ if not %errorlevel% == 0 ( call :RUNFAILED package %packagServerName_x64% faile iscc /DMyAppInstallName="%packagClientName_x64%" /DMyAppVersion="%2" /DMyAppExcludeSource="taosd.exe" tools\tdengine.iss /O..\release if not %errorlevel% == 0 ( call :RUNFAILED package %packagClientName_x64% failed & exit /b 1) -for /r ..\release %%i in (*.exe) do signtool sign /f d:\\123.pfx /p taosdata %%i goto EXIT0 :USAGE diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 624abe6da5..6a4ef3d821 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -27,6 +27,7 @@ #include "trpc.h" #include "tsched.h" #include "ttime.h" +#include "qworker.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index acf87f2aca..096e2cdac8 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -606,28 +606,34 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { int32_t connKeyCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); pBatchReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq)); + int64_t rid = -1; int32_t code = 0; - void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL); + + void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL); + + SClientHbReq *pOneReq = pIter; + SClientHbKey *connKey = pOneReq ? &pOneReq->connKey : NULL; + if (connKey != NULL) rid = connKey->tscRid; + + STscObj *pTscObj = (STscObj *)acquireTscObj(rid); + if (pTscObj == NULL) { + tFreeClientHbBatchReq(pBatchReq); + return NULL; + } + while (pIter != NULL) { - SClientHbReq *pOneReq = pIter; - pOneReq = taosArrayPush(pBatchReq->reqs, pOneReq); - code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, &pOneReq->clusterId, pOneReq); if (code) { pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); + pOneReq = pIter; continue; } - // hbClearClientHbReq(pOneReq); - pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); + pOneReq = pIter; } - - // if (code) { - // taosArrayDestroyEx(pBatchReq->reqs, hbFreeReq); - // taosMemoryFreeClear(pBatchReq); - // } + releaseTscObj(rid); return pBatchReq; } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 034f78e5f6..be907ea1e2 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -379,7 +379,7 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) { } bool qnodeRequired(SRequestObj* pRequest) { - if (QUERY_POLICY_VNODE == tsQueryPolicy) { + if (QUERY_POLICY_VNODE == tsQueryPolicy || QUERY_POLICY_CLIENT == tsQueryPolicy) { return false; } @@ -483,7 +483,8 @@ void setResPrecision(SReqResultInfo* pResInfo, int32_t precision) { int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SArray* pDbVgList) { SArray* nodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); - + char *policy = (tsQueryPolicy == QUERY_POLICY_VNODE) ? "vnode" : "client"; + int32_t dbNum = taosArrayGetSize(pDbVgList); for (int32_t i = 0; i < dbNum; ++i) { SArray* pVg = taosArrayGetP(pDbVgList, i); @@ -504,20 +505,20 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr int32_t vnodeNum = taosArrayGetSize(nodeList); if (vnodeNum > 0) { - tscDebug("0x%" PRIx64 " vnode policy, use vnode list, num:%d", pRequest->requestId, vnodeNum); + tscDebug("0x%" PRIx64 " %s policy, use vnode list, num:%d", pRequest->requestId, policy, vnodeNum); goto _return; } int32_t mnodeNum = taosArrayGetSize(pMnodeList); if (mnodeNum <= 0) { - tscDebug("0x%" PRIx64 " vnode policy, empty node list", pRequest->requestId); + tscDebug("0x%" PRIx64 " %s policy, empty node list", pRequest->requestId, policy); goto _return; } void* pData = taosArrayGet(pMnodeList, 0); taosArrayAddBatch(nodeList, pData, mnodeNum); - tscDebug("0x%" PRIx64 " vnode policy, use mnode list, num:%d", pRequest->requestId, mnodeNum); + tscDebug("0x%" PRIx64 " %s policy, use mnode list, num:%d", pRequest->requestId, policy, mnodeNum); _return: @@ -561,7 +562,8 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray int32_t code = 0; switch (tsQueryPolicy) { - case QUERY_POLICY_VNODE: { + case QUERY_POLICY_VNODE: + case QUERY_POLICY_CLIENT: { if (pResultMeta) { pDbVgList = taosArrayInit(4, POINTER_BYTES); @@ -622,7 +624,8 @@ int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* int32_t code = 0; switch (tsQueryPolicy) { - case QUERY_POLICY_VNODE: { + case QUERY_POLICY_VNODE: + case QUERY_POLICY_CLIENT: { int32_t dbNum = taosArrayGetSize(pRequest->dbList); if (dbNum > 0) { SCatalog* pCtg = NULL; @@ -682,6 +685,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList .requestObjRefId = pRequest->self}; SSchedulerReq req = { .syncReq = true, + .localReq = (tsQueryPolicy == QUERY_POLICY_CLIENT), .pConn = &conn, .pNodeList = pNodeList, .pDag = pDag, @@ -1061,6 +1065,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM .pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; SSchedulerReq req = { .syncReq = false, + .localReq = (tsQueryPolicy == QUERY_POLICY_CLIENT), .pConn = &conn, .pNodeList = pNodeList, .pDag = pDag, diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 831212bdee..650b16f855 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -26,6 +26,7 @@ #include "tref.h" #include "trpc.h" #include "version.h" +#include "qworker.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 15a369fe40..c0ae99806e 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -212,9 +212,9 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S if (pColVal->cid == pTColumn->colId) { iColVal++; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { flags |= TSROW_HAS_NONE; - } else if (pColVal->isNull) { + } else if (COL_VAL_IS_NULL(pColVal)) { flags |= TSROW_HAS_NULL; maxIdx = nkv; nTag++; @@ -398,9 +398,9 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S if (pColVal->cid == pTColumn->colId) { iColVal++; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { goto _set_none; - } else if (pColVal->isNull) { + } else if (COL_VAL_IS_NULL(pColVal)) { goto _set_null; } else { goto _set_value; @@ -1222,9 +1222,9 @@ _exit: static FORCE_INLINE int32_t tColDataAppendValue0(SColData *pColData, SColVal *pColVal) { // 0 int32_t code = 0; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { pColData->flag = HAS_NONE; - } else if (pColVal->isNull) { + } else if (COL_VAL_IS_NULL(pColVal)) { pColData->flag = HAS_NULL; } else { pColData->flag = HAS_VALUE; @@ -1239,7 +1239,7 @@ _exit: static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pColVal) { // HAS_NONE int32_t code = 0; - if (!pColVal->isNone) { + if (!COL_VAL_IS_NONE(pColVal)) { int32_t nBit = BIT1_SIZE(pColData->nVal + 1); code = tRealloc(&pColData->pBitMap, nBit); @@ -1248,7 +1248,7 @@ static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pC memset(pColData->pBitMap, 0, nBit); SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - if (pColVal->isNull) { + if (COL_VAL_IS_NULL(pColVal)) { pColData->flag |= HAS_NULL; } else { pColData->flag |= HAS_VALUE; @@ -1279,12 +1279,12 @@ _exit: static FORCE_INLINE int32_t tColDataAppendValue2(SColData *pColData, SColVal *pColVal) { // HAS_NULL int32_t code = 0; - if (!pColVal->isNull) { + if (!COL_VAL_IS_NULL(pColVal)) { int32_t nBit = BIT1_SIZE(pColData->nVal + 1); code = tRealloc(&pColData->pBitMap, nBit); if (code) goto _exit; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { pColData->flag |= HAS_NONE; memset(pColData->pBitMap, 255, nBit); @@ -1321,12 +1321,12 @@ _exit: static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pColVal) { // HAS_NULL|HAS_NONE int32_t code = 0; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) goto _exit; SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - } else if (pColVal->isNull) { + } else if (COL_VAL_IS_NULL(pColVal)) { code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) goto _exit; @@ -1371,8 +1371,8 @@ _exit: static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pColVal) { // HAS_VALUE int32_t code = 0; - if (pColVal->isNone || pColVal->isNull) { - if (pColVal->isNone) { + if (!COL_VAL_IS_VALUE(pColVal)) { + if (COL_VAL_IS_NONE(pColVal)) { pColData->flag |= HAS_NONE; } else { pColData->flag |= HAS_NULL; @@ -1399,7 +1399,7 @@ _exit: static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NONE int32_t code = 0; - if (pColVal->isNull) { + if (COL_VAL_IS_NULL(pColVal)) { pColData->flag |= HAS_NULL; uint8_t *pBitMap = NULL; @@ -1417,7 +1417,7 @@ static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pC code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) goto _exit; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { SET_BIT1(pColData->pBitMap, pColData->nVal, 0); } else { SET_BIT1(pColData->pBitMap, pColData->nVal, 1); @@ -1434,7 +1434,7 @@ _exit: static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NULL int32_t code = 0; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { pColData->flag |= HAS_NONE; uint8_t *pBitMap = NULL; @@ -1452,7 +1452,7 @@ static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pC code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) goto _exit; - if (pColVal->isNull) { + if (COL_VAL_IS_NULL(pColVal)) { SET_BIT1(pColData->pBitMap, pColData->nVal, 0); } else { SET_BIT1(pColData->pBitMap, pColData->nVal, 1); @@ -1473,9 +1473,9 @@ static FORCE_INLINE int32_t tColDataAppendValue7(SColData *pColData, code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) goto _exit; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { SET_BIT2(pColData->pBitMap, pColData->nVal, 0); - } else if (pColVal->isNull) { + } else if (COL_VAL_IS_NULL(pColVal)) { SET_BIT2(pColData->pBitMap, pColData->nVal, 1); } else { SET_BIT2(pColData->pBitMap, pColData->nVal, 2); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8c4c552db2..05ef3606a2 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4367,7 +4367,7 @@ int32_t tDeserializeSExplainRsp(void *buf, int32_t bufLen, SExplainRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->numOfPlans) < 0) return -1; if (pRsp->numOfPlans > 0) { - pRsp->subplanInfo = taosMemoryMalloc(pRsp->numOfPlans * sizeof(SExplainExecInfo)); + pRsp->subplanInfo = taosMemoryCalloc(pRsp->numOfPlans, sizeof(SExplainExecInfo)); if (pRsp->subplanInfo == NULL) return -1; } for (int32_t i = 0; i < pRsp->numOfPlans; ++i) { @@ -4375,7 +4375,7 @@ int32_t tDeserializeSExplainRsp(void *buf, int32_t bufLen, SExplainRsp *pRsp) { if (tDecodeDouble(&decoder, &pRsp->subplanInfo[i].totalCost) < 0) return -1; if (tDecodeU64(&decoder, &pRsp->subplanInfo[i].numOfRows) < 0) return -1; if (tDecodeU32(&decoder, &pRsp->subplanInfo[i].verboseLen) < 0) return -1; - if (tDecodeBinary(&decoder, (uint8_t **)&pRsp->subplanInfo[i].verboseInfo, &pRsp->subplanInfo[i].verboseLen) < 0) + if (tDecodeBinaryAlloc(&decoder, &pRsp->subplanInfo[i].verboseInfo, NULL) < 0) return -1; } @@ -4385,6 +4385,19 @@ int32_t tDeserializeSExplainRsp(void *buf, int32_t bufLen, SExplainRsp *pRsp) { return 0; } +void tFreeSExplainRsp(SExplainRsp *pRsp) { + if (NULL == pRsp) { + return; + } + + for (int32_t i = 0; i < pRsp->numOfPlans; ++i) { + SExplainExecInfo *pExec = pRsp->subplanInfo + i; + taosMemoryFree(pExec->verboseInfo); + } + + taosMemoryFreeClear(pRsp->subplanInfo); +} + int32_t tSerializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 9880fe362e..f23a2b386e 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -620,7 +620,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { ASSERT(pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID); } else { if (IS_VAR_DATA_TYPE(pTColumn->type)) { - if (pColVal && !pColVal->isNone && !pColVal->isNull) { + if (pColVal && COL_VAL_IS_VALUE(pColVal)) { varDataLen += (pColVal->value.nData + sizeof(VarDataLenT)); if (maxVarDataLen < (pColVal->value.nData + sizeof(VarDataLenT))) { maxVarDataLen = pColVal->value.nData + sizeof(VarDataLenT); @@ -680,9 +680,9 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { const void *val = NULL; if (iColVal < nColVal) { pColVal = (SColVal *)taosArrayGet(pArray, iColVal); - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { valType = TD_VTYPE_NONE; - } else if (pColVal->isNull) { + } else if (COL_VAL_IS_NULL(pColVal)) { valType = TD_VTYPE_NULL; } else if (IS_VAR_DATA_TYPE(pTColumn->type)) { varDataSetLen(varBuf, pColVal->value.nData); diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index 65f21bee40..6d77180610 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -133,11 +133,11 @@ static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) { SColVal colVal = {0}; colVal.cid = PRIMARYKEY_TIMESTAMP_COL_ID + i; if (strncasecmp(data[i], NONE_CSTR, NONE_LEN) == 0) { - colVal.isNone = 1; + colVal.flag = CV_FLAG_NONE; taosArrayPush(*pArray, &colVal); continue; } else if (strncasecmp(data[i], NULL_CSTR, NULL_LEN) == 0) { - colVal.isNull = 1; + colVal.flag = CV_FLAG_NULL; taosArrayPush(*pArray, &colVal); continue; } @@ -204,11 +204,11 @@ static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) { } int32_t debugPrintSColVal(SColVal *cv, int8_t type) { - if (cv->isNone) { + if (COL_VAL_IS_NONE(cv)) { printf("None "); return 0; } - if (cv->isNull) { + if (COL_VAL_IS_NULL(cv)) { printf("Null "); return 0; } @@ -298,11 +298,11 @@ void debugPrintTSRow(STSRow2 *row, STSchema *pTSchema, const char *tags, int32_t static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) { ASSERT(rawVal); - if (cv->isNone) { + if (COL_VAL_IS_NONE(cv)) { EXPECT_STRCASEEQ(rawVal, NONE_CSTR); return 0; } - if (cv->isNull) { + if (COL_VAL_IS_NULL(cv)) { EXPECT_STRCASEEQ(rawVal, NULL_CSTR); return 0; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 0a42f06081..d7df30bc75 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -244,7 +244,7 @@ int32_t vmPutMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); if (pMsg == NULL) { - rpcFreeCont(pMsg->pCont); + rpcFreeCont(pRpc->pCont); pRpc->pCont = NULL; return -1; } diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index e0713e1570..5279a86d83 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -79,14 +79,14 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) { if (pRaw == NULL) return -1; sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mDebug("acct:%s, will be created when deploying, raw:%p", acctObj.acct, pRaw); + mInfo("acct:%s, will be created when deploying, raw:%p", acctObj.acct, pRaw); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "create-acct"); if (pTrans == NULL) { mError("acct:%s, failed to create since %s", acctObj.acct, terrstr()); return -1; } - mDebug("trans:%d, used to create acct:%s", pTrans->id, acctObj.acct); + mInfo("trans:%d, used to create acct:%s", pTrans->id, acctObj.acct); if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { mError("trans:%d, failed to commit redo log since %s", pTrans->id, terrstr()); diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 6cabf6d57a..9cae83152a 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -249,7 +249,7 @@ static int32_t mndCreateBnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-bnode"); if (pTrans == NULL) goto _OVER; - mDebug("trans:%d, used to create bnode:%d", pTrans->id, pCreate->dnodeId); + mInfo("trans:%d, used to create bnode:%d", pTrans->id, pCreate->dnodeId); if (mndSetCreateBnodeRedoLogs(pTrans, &bnodeObj) != 0) goto _OVER; if (mndSetCreateBnodeUndoLogs(pTrans, &bnodeObj) != 0) goto _OVER; if (mndSetCreateBnodeCommitLogs(pTrans, &bnodeObj) != 0) goto _OVER; @@ -276,7 +276,7 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("bnode:%d, start to create", createReq.dnodeId); + mInfo("bnode:%d, start to create", createReq.dnodeId); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { goto _OVER; } @@ -357,7 +357,7 @@ static int32_t mndDropBnode(SMnode *pMnode, SRpcMsg *pReq, SBnodeObj *pObj) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-bnode"); if (pTrans == NULL) goto _OVER; - mDebug("trans:%d, used to drop bnode:%d", pTrans->id, pObj->id); + mInfo("trans:%d, used to drop bnode:%d", pTrans->id, pObj->id); if (mndSetDropBnodeRedoLogs(pTrans, pObj) != 0) goto _OVER; if (mndSetDropBnodeCommitLogs(pTrans, pObj) != 0) goto _OVER; if (mndSetDropBnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) goto _OVER; @@ -381,7 +381,7 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("bnode:%d, start to drop", dropReq.dnodeId); + mInfo("bnode:%d, start to drop", dropReq.dnodeId); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 5ba39b4f1a..7a3dde3cf3 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -233,14 +233,14 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { if (pRaw == NULL) return -1; sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mDebug("cluster:%" PRId64 ", will be created when deploying, raw:%p", clusterObj.id, pRaw); + mInfo("cluster:%" PRId64 ", will be created when deploying, raw:%p", clusterObj.id, pRaw); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "create-cluster"); if (pTrans == NULL) { mError("cluster:%" PRId64 ", failed to create since %s", clusterObj.id, terrstr()); return -1; } - mDebug("trans:%d, used to create cluster:%" PRId64, pTrans->id, clusterObj.id); + mInfo("trans:%d, used to create cluster:%" PRId64, pTrans->id, clusterObj.id); if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { mError("trans:%d, failed to commit redo log since %s", pTrans->id, terrstr()); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index ccc4c8293a..c5f6d179d8 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -544,7 +544,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "create-db"); if (pTrans == NULL) goto _OVER; // mndTransSetSerial(pTrans); - mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); + mInfo("trans:%d, used to create db:%s", pTrans->id, pCreate->db); mndTransSetDbName(pTrans, dbObj.name, NULL); mndTransSetOper(pTrans, MND_OPER_CREATE_DB); @@ -580,7 +580,7 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups); + mInfo("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups); if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) { goto _OVER; } @@ -588,7 +588,7 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { pDb = mndAcquireDb(pMnode, createReq.db); if (pDb != NULL) { if (createReq.ignoreExist) { - mDebug("db:%s, already exist, ignore exist is set", createReq.db); + mInfo("db:%s, already exist, ignore exist is set", createReq.db); code = 0; goto _OVER; } else { @@ -597,7 +597,7 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { } } else if (terrno == TSDB_CODE_SDB_OBJ_CREATING) { if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) { - mDebug("db:%s, is creating and response after trans finished", createReq.db); + mInfo("db:%s, is creating and response after trans finished", createReq.db); code = TSDB_CODE_ACTION_IN_PROGRESS; goto _OVER; } else { @@ -775,7 +775,7 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *pNew) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "alter-db"); if (pTrans == NULL) return -1; - mDebug("trans:%d, used to alter db:%s", pTrans->id, pOld->name); + mInfo("trans:%d, used to alter db:%s", pTrans->id, pOld->name); int32_t code = -1; mndTransSetDbName(pTrans, pOld->name, NULL); @@ -802,7 +802,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("db:%s, start to alter", alterReq.db); + mInfo("db:%s, start to alter", alterReq.db); pDb = mndAcquireDb(pMnode, alterReq.db); if (pDb == NULL) { @@ -1030,7 +1030,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-db"); if (pTrans == NULL) goto _OVER; - mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name); + mInfo("trans:%d, used to drop db:%s", pTrans->id, pDb->name); mndTransSetDbName(pTrans, pDb->name, NULL); if (mndCheckTopicExist(pMnode, pDb) < 0) goto _OVER; @@ -1079,7 +1079,7 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("db:%s, start to drop", dropReq.db); + mInfo("db:%s, start to drop", dropReq.db); pDb = mndAcquireDb(pMnode, dropReq.db); if (pDb == NULL) { @@ -1306,11 +1306,11 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable) { - mDebug("db:%s, version and numOfTable not changed", pDbVgVersion->dbFName); + mInfo("db:%s, version and numOfTable not changed", pDbVgVersion->dbFName); mndReleaseDb(pMnode, pDb); continue; } else { - mDebug("db:%s, vgroup version changed from %d to %d", pDbVgVersion->dbFName, pDbVgVersion->vgVersion, + mInfo("db:%s, vgroup version changed from %d to %d", pDbVgVersion->dbFName, pDbVgVersion->vgVersion, pDb->vgVersion); } @@ -1363,7 +1363,7 @@ static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("db:%s, start to compact", compactReq.db); + mInfo("db:%s, start to compact", compactReq.db); pDb = mndAcquireDb(pMnode, compactReq.db); if (pDb == NULL) { @@ -1413,7 +1413,7 @@ static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) { if (code != 0) { mError("vgId:%d, failed to send vnode-trim request to vnode since 0x%x", pVgroup->vgId, code); } else { - mDebug("vgId:%d, send vnode-trim request to vnode, time:%" PRIi64, pVgroup->vgId, trimReq.timestamp); + mInfo("vgId:%d, send vnode-trim request to vnode, time:%d", pVgroup->vgId, trimReq.timestamp); } sdbRelease(pSdb, pVgroup); } @@ -1432,7 +1432,7 @@ static int32_t mndProcessTrimDbReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("db:%s, start to trim", trimReq.db); + mInfo("db:%s, start to trim", trimReq.db); pDb = mndAcquireDb(pMnode, trimReq.db); if (pDb == NULL) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index c6fdf23023..4a8436513f 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -106,7 +106,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-dnode"); if (pTrans == NULL) goto _OVER; - mDebug("trans:%d, used to create dnode:%s on first deploy", pTrans->id, dnodeObj.ep); + mInfo("trans:%d, used to create dnode:%s on first deploy", pTrans->id, dnodeObj.ep); pRaw = mndDnodeActionEncode(&dnodeObj); if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; @@ -326,7 +326,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { if (statusReq.dnodeId == 0) { pDnode = mndAcquireDnodeByEp(pMnode, statusReq.dnodeEp); if (pDnode == NULL) { - mDebug("dnode:%s, not created yet", statusReq.dnodeEp); + mInfo("dnode:%s, not created yet", statusReq.dnodeEp); goto _OVER; } } else { @@ -435,7 +435,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { mInfo("dnode:%d, from offline to online, memory avail:%" PRId64 " total:%" PRId64 " cores:%.2f", pDnode->id, statusReq.memAvail, statusReq.memTotal, statusReq.numOfCores); } else { - mDebug("dnode:%d, send dnode epset, online:%d dnodeVer:%" PRId64 ":%" PRId64 " reboot:%d", pDnode->id, online, + mInfo("dnode:%d, send dnode epset, online:%d dnodeVer:%" PRId64 ":%" PRId64 " reboot:%d", pDnode->id, online, statusReq.dnodeVer, dnodeVer, reboot); } @@ -490,7 +490,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq, "create-dnode"); if (pTrans == NULL) goto _OVER; - mDebug("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep); + mInfo("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep); pRaw = mndDnodeActionEncode(&dnodeObj); if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 429d840202..48ab485a2c 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -222,7 +222,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-func"); if (pTrans == NULL) goto _OVER; - mDebug("trans:%d, used to create func:%s", pTrans->id, pCreate->name); + mInfo("trans:%d, used to create func:%s", pTrans->id, pCreate->name); SSdbRaw *pRedoRaw = mndFuncActionEncode(&func); if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; @@ -252,7 +252,7 @@ static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "drop-func"); if (pTrans == NULL) goto _OVER; - mDebug("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); + mInfo("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; @@ -286,7 +286,7 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("func:%s, start to create", createReq.name); + mInfo("func:%s, start to create", createReq.name); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { goto _OVER; } @@ -294,7 +294,7 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { pFunc = mndAcquireFunc(pMnode, createReq.name); if (pFunc != NULL) { if (createReq.igExists) { - mDebug("func:%s, already exist, ignore exist is set", createReq.name); + mInfo("func:%s, already exist, ignore exist is set", createReq.name); code = 0; goto _OVER; } else { @@ -349,7 +349,7 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("func:%s, start to drop", dropReq.name); + mInfo("func:%s, start to drop", dropReq.name); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { goto _OVER; } @@ -362,7 +362,7 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { pFunc = mndAcquireFunc(pMnode, dropReq.name); if (pFunc == NULL) { if (dropReq.igNotExists) { - mDebug("func:%s, not exist, ignore not exist is set", dropReq.name); + mInfo("func:%s, not exist, ignore not exist is set", dropReq.name); code = 0; goto _OVER; } else { diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 2221718023..5aad4af9ac 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -65,7 +65,7 @@ static void mndPullupTrans(SMnode *pMnode) { } } -static void mndTtlTimer(SMnode *pMnode) { +static void mndPullupTtl(SMnode *pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TTL_TIMER, .pCont = pReq, .contLen = contLen}; @@ -90,7 +90,7 @@ static void mndPullupTelem(SMnode *pMnode) { } } -static void mndGrantHeartBeat(SMnode *pMnode) { +static void mndPullupGrant(SMnode *pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { @@ -121,7 +121,7 @@ static void *mndThreadFp(void *param) { if (mndGetStop(pMnode)) break; if (lastTime % (tsTtlPushInterval * 10) == 1) { - mndTtlTimer(pMnode); + mndPullupTtl(pMnode); } if (lastTime % (tsTransPullupInterval * 10) == 0) { @@ -137,7 +137,7 @@ static void *mndThreadFp(void *param) { } if (lastTime % (tsGrantHBInterval * 10) == 0) { - mndGrantHeartBeat(pMnode); + mndPullupGrant(pMnode); } if ((lastTime % (tsUptimeInterval * 10)) == ((tsUptimeInterval - 1) * 10)) { @@ -300,7 +300,7 @@ static void mndCleanupSteps(SMnode *pMnode, int32_t pos) { for (int32_t s = pos; s >= 0; s--) { SMnodeStep *pStep = taosArrayGet(pMnode->pSteps, s); - mDebug("%s will cleanup", pStep->name); + mInfo("%s will cleanup", pStep->name); if (pStep->cleanupFp != NULL) { (*pStep->cleanupFp)(pMnode); } @@ -324,7 +324,7 @@ static int32_t mndExecSteps(SMnode *pMnode) { terrno = code; return -1; } else { - mDebug("%s is initialized", pStep->name); + mInfo("%s is initialized", pStep->name); tmsgReportStartup(pStep->name, "initialized"); } } @@ -341,7 +341,7 @@ static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { } SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { - mDebug("start to open mnode in %s", path); + mInfo("start to open mnode in %s", path); SMnode *pMnode = taosMemoryCalloc(1, sizeof(SMnode)); if (pMnode == NULL) { @@ -390,7 +390,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { return NULL; } - mDebug("mnode open successfully "); + mInfo("mnode open successfully "); return pMnode; } @@ -400,24 +400,24 @@ void mndPreClose(SMnode *pMnode) { syncLeaderTransfer(pMnode->syncMgmt.sync); /* - mDebug("vgId:1, mnode start leader transfer"); + mInfo("vgId:1, mnode start leader transfer"); // wait for leader transfer finish while (!atomic_load_8(&(pMnode->syncMgmt.leaderTransferFinish))) { taosMsleep(10); - mDebug("vgId:1, mnode waiting for leader transfer"); + mInfo("vgId:1, mnode waiting for leader transfer"); } - mDebug("vgId:1, mnode finish leader transfer"); + mInfo("vgId:1, mnode finish leader transfer"); */ } } void mndClose(SMnode *pMnode) { if (pMnode != NULL) { - mDebug("start to close mnode"); + mInfo("start to close mnode"); mndCleanupSteps(pMnode, -1); taosMemoryFreeClear(pMnode->path); taosMemoryFreeClear(pMnode); - mDebug("mnode is closed"); + mInfo("mnode is closed"); } } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index cd6e91b154..cbd398e361 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -251,7 +251,7 @@ static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-qnode"); if (pTrans == NULL) goto _OVER; - mDebug("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId); + mInfo("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId); if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) goto _OVER; if (mndSetCreateQnodeUndoLogs(pTrans, &qnodeObj) != 0) goto _OVER; if (mndSetCreateQnodeCommitLogs(pTrans, &qnodeObj) != 0) goto _OVER; @@ -278,7 +278,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("qnode:%d, start to create", createReq.dnodeId); + mInfo("qnode:%d, start to create", createReq.dnodeId); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { goto _OVER; } @@ -367,7 +367,7 @@ static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-qnode"); if (pTrans == NULL) goto _OVER; - mDebug("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id); + mInfo("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id); if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -389,7 +389,7 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("qnode:%d, start to drop", dropReq.dnodeId); + mInfo("qnode:%d, start to drop", dropReq.dnodeId); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 654f46ec85..0e897de4e7 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -170,7 +170,7 @@ _exit: } int32_t mndInitQuery(SMnode *pMnode) { - if (qWorkerInit(NODE_TYPE_MNODE, MNODE_HANDLE, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) { + if (qWorkerInit(NODE_TYPE_MNODE, MNODE_HANDLE, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) { mError("failed to init qworker in mnode since %s", terrstr()); return -1; } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 3fdd3995f9..90d05e8f21 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -591,7 +591,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea if (pTrans == NULL) goto _OVER; mndTransSetDbName(pTrans, pDb->name, NULL); mndTransSetSerial(pTrans); - mDebug("trans:%d, used to create sma:%s stream:%s", pTrans->id, pCreate->name, streamObj.name); + mInfo("trans:%d, used to create sma:%s stream:%s", pTrans->id, pCreate->name, streamObj.name); if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupRedoLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; @@ -603,8 +603,8 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea if (mndPersistStream(pMnode, pTrans, &streamObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - mDebug("mndSma: create sma index %s %" PRIi64 " on stb:%" PRIi64 ", dstSuid:%" PRIi64 " dstTb:%s dstVg:%d", - pCreate->name, smaObj.uid, smaObj.stbUid, smaObj.dstTbUid, smaObj.dstTbName, smaObj.dstVgId); + mInfo("sma:%s, uid:%" PRIi64 " create on stb:%" PRIi64 ", dstSuid:%" PRIi64 " dstTb:%s dstVg:%d", pCreate->name, + smaObj.uid, smaObj.stbUid, smaObj.dstTbUid, smaObj.dstTbName, smaObj.dstVgId); code = 0; @@ -665,7 +665,7 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("sma:%s, start to create", createReq.name); + mInfo("sma:%s, start to create", createReq.name); if (mndCheckCreateSmaReq(&createReq) != 0) { goto _OVER; } @@ -689,7 +689,7 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { pSma = mndAcquireSma(pMnode, createReq.name); if (pSma != NULL) { if (createReq.igExists) { - mDebug("sma:%s, already exist in sma:%s, ignore exist is set", createReq.name, pSma->name); + mInfo("sma:%s, already exist in sma:%s, ignore exist is set", createReq.name, pSma->name); code = 0; goto _OVER; } else { @@ -802,7 +802,7 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-sma"); if (pTrans == NULL) goto _OVER; - mDebug("trans:%d, used to drop sma:%s", pTrans->id, pSma->name); + mInfo("trans:%d, used to drop sma:%s", pTrans->id, pSma->name); mndTransSetDbName(pTrans, pDb->name, NULL); mndTransSetSerial(pTrans); @@ -929,12 +929,12 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("sma:%s, start to drop", dropReq.name); + mInfo("sma:%s, start to drop", dropReq.name); pSma = mndAcquireSma(pMnode, dropReq.name); if (pSma == NULL) { if (dropReq.igNotExists) { - mDebug("sma:%s, not exist, ignore not exist is set", dropReq.name); + mInfo("sma:%s, not exist, ignore not exist is set", dropReq.name); code = 0; goto _OVER; } else { diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 5f4231aa6a..6912709970 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -256,7 +256,7 @@ static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode"); if (pTrans == NULL) goto _OVER; - mDebug("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); + mInfo("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); if (mndSetCreateSnodeRedoLogs(pTrans, &snodeObj) != 0) goto _OVER; if (mndSetCreateSnodeUndoLogs(pTrans, &snodeObj) != 0) goto _OVER; @@ -287,7 +287,7 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("snode:%d, start to create", createReq.dnodeId); + mInfo("snode:%d, start to create", createReq.dnodeId); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { goto _OVER; } @@ -378,7 +378,7 @@ static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode"); if (pTrans == NULL) goto _OVER; - mDebug("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); + mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -403,7 +403,7 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("snode:%d, start to drop", dropReq.dnodeId); + mInfo("snode:%d, start to drop", dropReq.dnodeId); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index ecb24becba..e0b5bb1abf 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -803,7 +803,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "create-stb"); if (pTrans == NULL) goto _OVER; - mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); + mInfo("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) goto _OVER; if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -854,7 +854,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) { if (code != 0) { mError("vgId:%d, failed to send drop ttl table request to vnode since 0x%x", pVgroup->vgId, code); } else { - mDebug("vgId:%d, send drop ttl table request to vnode, time:%d", pVgroup->vgId, ttlReq.timestamp); + mInfo("vgId:%d, send drop ttl table request to vnode, time:%d", pVgroup->vgId, ttlReq.timestamp); } sdbRelease(pSdb, pVgroup); } @@ -943,7 +943,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("stb:%s, start to create", createReq.name); + mInfo("stb:%s, start to create", createReq.name); if (mndCheckCreateStbReq(&createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; @@ -953,7 +953,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { if (pStb != NULL) { if (createReq.igExists) { if (createReq.source == TD_REQ_FROM_APP) { - mDebug("stb:%s, already exist, ignore exist is set", createReq.name); + mInfo("stb:%s, already exist, ignore exist is set", createReq.name); code = 0; goto _OVER; } else if (pStb->uid != createReq.suid) { @@ -1144,7 +1144,7 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p pSchema->colId = pNew->nextColId; pNew->nextColId++; - mDebug("stb:%s, start to add tag %s", pNew->name, pSchema->name); + mInfo("stb:%s, start to add tag %s", pNew->name, pSchema->name); } pNew->tagVer++; @@ -1159,7 +1159,7 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); if (pIter == NULL) break; - mDebug("topic:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d, subType:%d sql:%s", + mInfo("topic:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d, subType:%d sql:%s", pTopic->name, stbFullName, suid, colId, pTopic->subType, pTopic->sql); if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { sdbRelease(pSdb, pTopic); @@ -1177,11 +1177,11 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, SNode *pNode = NULL; FOREACH(pNode, pNodeList) { SColumnNode *pCol = (SColumnNode *)pNode; - mDebug("topic:%s, check colId:%d tableId:%" PRId64 " ctbStbUid:%" PRId64, pTopic->name, pCol->colId, + mInfo("topic:%s, check colId:%d tableId:%" PRId64 " ctbStbUid:%" PRId64, pTopic->name, pCol->colId, pCol->tableId, pTopic->ctbStbUid); if (pCol->tableId != suid && pTopic->ctbStbUid != suid) { - mDebug("topic:%s, check colId:%d passed", pTopic->name, pCol->colId); + mInfo("topic:%s, check colId:%d passed", pTopic->name, pCol->colId); goto NEXT; } if (pCol->colId > 0 && pCol->colId == colId) { @@ -1192,7 +1192,7 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, mError("topic:%s, check colId:%d conflicted", pTopic->name, pCol->colId); return -1; } - mDebug("topic:%s, check colId:%d passed", pTopic->name, pCol->colId); + mInfo("topic:%s, check colId:%d passed", pTopic->name, pCol->colId); } NEXT: @@ -1224,7 +1224,7 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName SColumnNode *pCol = (SColumnNode *)pNode; if (pCol->tableId != suid) { - mDebug("stream:%s, check colId:%d passed", pStream->name, pCol->colId); + mInfo("stream:%s, check colId:%d passed", pStream->name, pCol->colId); goto NEXT; } if (pCol->colId > 0 && pCol->colId == colId) { @@ -1235,7 +1235,7 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName mError("stream:%s, check colId:%d conflicted", pStream->name, pCol->colId); return -1; } - mDebug("stream:%s, check colId:%d passed", pStream->name, pCol->colId); + mInfo("stream:%s, check colId:%d passed", pStream->name, pCol->colId); } NEXT: @@ -1254,7 +1254,7 @@ static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName, pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); if (pIter == NULL) break; - mDebug("tsma:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d, sql:%s", pSma->name, + mInfo("tsma:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d, sql:%s", pSma->name, stbFullName, suid, colId, pSma->sql); SNode *pAst = NULL; @@ -1270,10 +1270,10 @@ static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName, SNode *pNode = NULL; FOREACH(pNode, pNodeList) { SColumnNode *pCol = (SColumnNode *)pNode; - mDebug("tsma:%s, check colId:%d tableId:%" PRId64, pSma->name, pCol->colId, pCol->tableId); + mInfo("tsma:%s, check colId:%d tableId:%" PRId64, pSma->name, pCol->colId, pCol->tableId); if ((pCol->tableId != suid) && (pSma->stbUid != suid)) { - mDebug("tsma:%s, check colId:%d passed", pSma->name, pCol->colId); + mInfo("tsma:%s, check colId:%d passed", pSma->name, pCol->colId); goto NEXT; } if ((pCol->colId) > 0 && (pCol->colId == colId)) { @@ -1284,7 +1284,7 @@ static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName, mError("tsma:%s, check colId:%d conflicted", pSma->name, pCol->colId); return -1; } - mDebug("tsma:%s, check colId:%d passed", pSma->name, pCol->colId); + mInfo("tsma:%s, check colId:%d passed", pSma->name, pCol->colId); } NEXT: @@ -1329,7 +1329,7 @@ static int32_t mndDropSuperTableTag(SMnode *pMnode, const SStbObj *pOld, SStbObj pNew->numOfTags--; pNew->tagVer++; - mDebug("stb:%s, start to drop tag %s", pNew->name, tagName); + mInfo("stb:%s, start to drop tag %s", pNew->name, tagName); return 0; } @@ -1374,7 +1374,7 @@ static int32_t mndAlterStbTagName(SMnode *pMnode, const SStbObj *pOld, SStbObj * memcpy(pSchema->name, newTagName, TSDB_COL_NAME_LEN); pNew->tagVer++; - mDebug("stb:%s, start to modify tag %s to %s", pNew->name, oldTagName, newTagName); + mInfo("stb:%s, start to modify tag %s to %s", pNew->name, oldTagName, newTagName); return 0; } @@ -1409,7 +1409,7 @@ static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj pTag->bytes = pField->bytes; pNew->tagVer++; - mDebug("stb:%s, start to modify tag len %s to %d", pNew->name, pField->name, pField->bytes); + mInfo("stb:%s, start to modify tag len %s to %d", pNew->name, pField->name, pField->bytes); return 0; } @@ -1443,7 +1443,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray pSchema->colId = pNew->nextColId; pNew->nextColId++; - mDebug("stb:%s, start to add column %s", pNew->name, pSchema->name); + mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name); } pNew->colVer++; @@ -1480,7 +1480,7 @@ static int32_t mndDropSuperTableColumn(SMnode *pMnode, const SStbObj *pOld, SStb pNew->numOfColumns--; pNew->colVer++; - mDebug("stb:%s, start to drop col %s", pNew->name, colName); + mInfo("stb:%s, start to drop col %s", pNew->name, colName); return 0; } @@ -1524,7 +1524,7 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO pCol->bytes = pField->bytes; pNew->colVer++; - mDebug("stb:%s, start to modify col len %s to %d", pNew->name, pField->name, pField->bytes); + mInfo("stb:%s, start to modify col len %s to %d", pNew->name, pField->name, pField->bytes); return 0; } @@ -1846,7 +1846,7 @@ static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbOb STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "alter-stb"); if (pTrans == NULL) goto _OVER; - mDebug("trans:%d, used to alter stb:%s", pTrans->id, pStb->name); + mInfo("trans:%d, used to alter stb:%s", pTrans->id, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name); if (needRsp) { @@ -1942,7 +1942,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("stb:%s, start to alter", alterReq.name); + mInfo("stb:%s, start to alter", alterReq.name); if (mndCheckAlterStbReq(&alterReq) != 0) goto _OVER; pDb = mndAcquireDbByStb(pMnode, alterReq.name); @@ -2045,7 +2045,7 @@ static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *p STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "drop-stb"); if (pTrans == NULL) goto _OVER; - mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); + mInfo("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name); if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER; @@ -2169,12 +2169,12 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("stb:%s, start to drop", dropReq.name); + mInfo("stb:%s, start to drop", dropReq.name); pStb = mndAcquireStb(pMnode, dropReq.name); if (pStb == NULL) { if (dropReq.igNotExists) { - mDebug("stb:%s, not exist, ignore not exist is set", dropReq.name); + mInfo("stb:%s, not exist, ignore not exist is set", dropReq.name); code = 0; goto _OVER; } else { @@ -2237,17 +2237,17 @@ static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { } if (0 == strcmp(infoReq.dbFName, TSDB_INFORMATION_SCHEMA_DB)) { - mDebug("information_schema table:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); + mInfo("information_schema table:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); if (mndBuildInsTableSchema(pMnode, infoReq.dbFName, infoReq.tbName, sysinfo, &metaRsp) != 0) { goto _OVER; } } else if (0 == strcmp(infoReq.dbFName, TSDB_PERFORMANCE_SCHEMA_DB)) { - mDebug("performance_schema table:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); + mInfo("performance_schema table:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); if (mndBuildPerfsTableSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { goto _OVER; } } else { - mDebug("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); + mInfo("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp, NULL) != 0) { goto _OVER; } @@ -2294,17 +2294,17 @@ static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) { } if (0 == strcmp(cfgReq.dbFName, TSDB_INFORMATION_SCHEMA_DB)) { - mDebug("information_schema table:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); + mInfo("information_schema table:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); if (mndBuildInsTableCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp) != 0) { goto _OVER; } } else if (0 == strcmp(cfgReq.dbFName, TSDB_PERFORMANCE_SCHEMA_DB)) { - mDebug("performance_schema table:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); + mInfo("performance_schema table:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); if (mndBuildPerfsTableCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp) != 0) { goto _OVER; } } else { - mDebug("stb:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); + mInfo("stb:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); if (mndBuildStbCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp) != 0) { goto _OVER; } @@ -2363,7 +2363,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t STableMetaRsp metaRsp = {0}; int32_t smaVer = 0; - mDebug("stb:%s.%s, start to retrieve meta", pStbVersion->dbFName, pStbVersion->stbName); + mInfo("stb:%s.%s, start to retrieve meta", pStbVersion->dbFName, pStbVersion->stbName); if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp, &smaVer) != 0) { metaRsp.numOfColumns = -1; metaRsp.suid = pStbVersion->suid; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index a039b54b80..c6f46b28fc 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -266,7 +266,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, SNode *pAst = NULL; SQueryPlan *pPlan = NULL; - mDebug("stream:%s to create", pCreate->name); + mInfo("stream:%s to create", pCreate->name); memcpy(pObj->name, pCreate->name, TSDB_STREAM_FNAME_LEN); pObj->createTime = taosGetTimestampMs(); pObj->updateTime = pObj->createTime; @@ -285,7 +285,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, SDbObj *pSourceDb = mndAcquireDb(pMnode, pCreate->sourceDB); if (pSourceDb == NULL) { /*ASSERT(0);*/ - mDebug("stream:%s failed to create, source db %s not exist", pCreate->name, pObj->sourceDb); + mInfo("stream:%s failed to create, source db %s not exist", pCreate->name, pObj->sourceDb); terrno = TSDB_CODE_MND_DB_NOT_EXIST; return -1; } @@ -295,7 +295,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, SDbObj *pTargetDb = mndAcquireDbByStb(pMnode, pObj->targetSTbName); if (pTargetDb == NULL) { - mDebug("stream:%s failed to create, target db %s not exist", pCreate->name, pObj->targetDb); + mInfo("stream:%s failed to create, target db %s not exist", pCreate->name, pObj->targetDb); terrno = TSDB_CODE_MND_DB_NOT_EXIST; return -1; } @@ -638,7 +638,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("stream:%s, start to create, sql:%s", createStreamReq.name, createStreamReq.sql); + mInfo("stream:%s, start to create, sql:%s", createStreamReq.name, createStreamReq.sql); if (mndCheckCreateStreamReq(&createStreamReq) != 0) { mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); @@ -648,7 +648,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { pStream = mndAcquireStream(pMnode, createStreamReq.name); if (pStream != NULL) { if (createStreamReq.igExists) { - mDebug("stream:%s, already exist, ignore exist is set", createStreamReq.name); + mInfo("stream:%s, already exist, ignore exist is set", createStreamReq.name); code = 0; goto _OVER; } else { @@ -672,7 +672,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } mndTransSetDbName(pTrans, createStreamReq.sourceDB, streamObj.targetDb); // hack way - mDebug("trans:%d, used to create stream:%s", pTrans->id, createStreamReq.name); + mInfo("trans:%d, used to create stream:%s", pTrans->id, createStreamReq.name); // create stb for stream if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) { @@ -746,7 +746,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { if (pStream == NULL) { if (dropReq.igNotExists) { - mDebug("stream:%s, not exist, ignore not exist is set", dropReq.name); + mInfo("stream:%s, not exist, ignore not exist is set", dropReq.name); sdbRelease(pMnode->pSdb, pStream); return 0; } else { @@ -765,7 +765,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { sdbRelease(pMnode->pSdb, pStream); return -1; } - mDebug("trans:%d, used to drop stream:%s", pTrans->id, dropReq.name); + mInfo("trans:%d, used to drop stream:%s", pTrans->id, dropReq.name); // drop all tasks if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { @@ -810,7 +810,7 @@ static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq) { if (pStream == NULL) { if (recoverReq.igNotExists) { - mDebug("stream:%s, not exist, ignore not exist is set", recoverReq.name); + mInfo("stream:%s, not exist, ignore not exist is set", recoverReq.name); sdbRelease(pMnode->pSdb, pStream); return 0; } else { @@ -829,7 +829,7 @@ static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq) { sdbRelease(pMnode->pSdb, pStream); return -1; } - mDebug("trans:%d, used to drop stream:%s", pTrans->id, recoverReq.name); + mInfo("trans:%d, used to drop stream:%s", pTrans->id, recoverReq.name); // broadcast to recover all tasks if (mndRecoverStreamTasks(pMnode, pTrans, pStream) < 0) { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 43eae3f60c..21e45407fd 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -658,7 +658,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pReq) { SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, dropReq.cgroup, dropReq.topic); if (pSub == NULL) { if (dropReq.igNotExists) { - mDebug("cgroup:%s on topic:%s, not exist, ignore not exist is set", dropReq.cgroup, dropReq.topic); + mInfo("cgroup:%s on topic:%s, not exist, ignore not exist is set", dropReq.cgroup, dropReq.topic); return 0; } else { terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; @@ -681,7 +681,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pReq) { return -1; } - mDebug("trans:%d, used to drop cgroup:%s on topic %s", pTrans->id, dropReq.cgroup, dropReq.topic); + mInfo("trans:%d, used to drop cgroup:%s on topic %s", pTrans->id, dropReq.cgroup, dropReq.topic); if (mndDropOffsetBySubKey(pMnode, pTrans, pSub->key) < 0) { ASSERT(0); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index e8b75e6a94..3655289f99 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -105,11 +105,11 @@ void mndRestoreFinish(struct SSyncFSM *pFsm) { SMnode *pMnode = pFsm->data; if (!pMnode->deploy) { - mInfo("mnode sync restore finished, and will handle outstanding transactions"); + mInfo("vgId:1, sync restore finished, and will handle outstanding transactions"); mndTransPullup(pMnode); mndSetRestore(pMnode, true); } else { - mInfo("mnode sync restore finished"); + mInfo("vgId:1, sync restore finished"); } } @@ -228,7 +228,7 @@ int32_t mndInitSync(SMnode *pMnode) { syncInfo.isStandBy = pMgmt->standby; syncInfo.snapshotStrategy = SYNC_STRATEGY_STANDARD_SNAPSHOT; - mInfo("start to open mnode sync, standby:%d", pMgmt->standby); + mInfo("vgId:1, start to open sync, standby:%d", pMgmt->standby); if (pMgmt->standby || pMgmt->replica.id > 0) { SSyncCfg *pCfg = &syncInfo.syncCfg; pCfg->replicaNum = 1; @@ -236,7 +236,7 @@ int32_t mndInitSync(SMnode *pMnode) { SNodeInfo *pNode = &pCfg->nodeInfo[0]; tstrncpy(pNode->nodeFqdn, pMgmt->replica.fqdn, sizeof(pNode->nodeFqdn)); pNode->nodePort = pMgmt->replica.port; - mInfo("mnode ep:%s:%u", pNode->nodeFqdn, pNode->nodePort); + mInfo("vgId:1, ep:%s:%u", pNode->nodeFqdn, pNode->nodePort); } tsem_init(&pMgmt->syncSem, 0, 0); @@ -255,14 +255,14 @@ int32_t mndInitSync(SMnode *pMnode) { setHeartbeatTimerMS(pMgmt->sync, 300); */ - mDebug("mnode-sync is opened, id:%" PRId64, pMgmt->sync); + mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync); return 0; } void mndCleanupSync(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; syncStop(pMgmt->sync); - mDebug("mnode-sync is stopped, id:%" PRId64, pMgmt->sync); + mInfo("mnode-sync is stopped, id:%" PRId64, pMgmt->sync); tsem_destroy(&pMgmt->syncSem); memset(pMgmt, 0, sizeof(SSyncMgmt)); @@ -314,7 +314,7 @@ void mndSyncStart(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; syncSetMsgCb(pMgmt->sync, &pMnode->msgCb); syncStart(pMgmt->sync); - mInfo("mnode sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby); + mInfo("vgId:1, sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby); } void mndSyncStop(SMnode *pMnode) { diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 8fe2d1af15..b9647a28fb 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -358,7 +358,7 @@ static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) { } static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) { - mDebug("topic:%s to create", pCreate->name); + mInfo("topic:%s to create", pCreate->name); SMqTopicObj topicObj = {0}; tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN); tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN); @@ -448,7 +448,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * taosMemoryFreeClear(topicObj.physicalPlan); return -1; } - mDebug("trans:%d, used to create topic:%s", pTrans->id, pCreate->name); + mInfo("trans:%d, used to create topic:%s", pTrans->id, pCreate->name); SSdbRaw *pCommitRaw = mndTopicActionEncode(&topicObj); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { @@ -541,7 +541,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("topic:%s, start to create, sql:%s", createTopicReq.name, createTopicReq.sql); + mInfo("topic:%s, start to create, sql:%s", createTopicReq.name, createTopicReq.sql); if (mndCheckCreateTopicReq(&createTopicReq) != 0) { mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr()); @@ -551,7 +551,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { pTopic = mndAcquireTopic(pMnode, createTopicReq.name); if (pTopic != NULL) { if (createTopicReq.igExists) { - mDebug("topic:%s, already exist, ignore exist is set", createTopicReq.name); + mInfo("topic:%s, already exist, ignore exist is set", createTopicReq.name); code = 0; goto _OVER; } else { @@ -619,7 +619,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { SMqTopicObj *pTopic = mndAcquireTopic(pMnode, dropReq.name); if (pTopic == NULL) { if (dropReq.igNotExists) { - mDebug("topic:%s, not exist, ignore not exist is set", dropReq.name); + mInfo("topic:%s, not exist, ignore not exist is set", dropReq.name); return 0; } else { terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; @@ -671,7 +671,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { return -1; } - mDebug("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); + mInfo("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); #if 0 if (mndDropOffsetByTopic(pMnode, pTrans, dropReq.name) < 0) { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 9d918e2abd..b26fb16043 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -321,7 +321,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) action.pRaw = taosMemoryMalloc(dataLen); if (action.pRaw == NULL) goto _OVER; - // mTrace("raw:%p, is created", pData); + mTrace("raw:%p, is created", pData); SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; action.pRaw = NULL; @@ -357,7 +357,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) action.pRaw = taosMemoryMalloc(dataLen); if (action.pRaw == NULL) goto _OVER; - // mTrace("raw:%p, is created", action.pRaw); + mTrace("raw:%p, is created", action.pRaw); SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; action.pRaw = NULL; @@ -393,7 +393,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) action.pRaw = taosMemoryMalloc(dataLen); if (action.pRaw == NULL) goto _OVER; - // mTrace("raw:%p, is created", action.pRaw); + mTrace("raw:%p, is created", action.pRaw); SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); if (taosArrayPush(pTrans->commitActions, &action) == NULL) goto _OVER; action.pRaw = NULL; @@ -824,7 +824,7 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { pNew->id, pNew->dbname, pNew->stbname, pNew->conflict, pTrans->id, pTrans->dbname, pTrans->stbname, pTrans->conflict); } else { - mDebug("trans:%d, db:%s stb:%s type:%d, not conflict with trans:%d db:%s stb:%s type:%d", pNew->id, + mInfo("trans:%d, db:%s stb:%s type:%d, not conflict with trans:%d db:%s stb:%s type:%d", pNew->id, pNew->dbname, pNew->stbname, pNew->conflict, pTrans->id, pTrans->dbname, pTrans->stbname, pTrans->conflict); } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index c4adb03e21..5512fa4107 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -79,14 +79,14 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char if (pRaw == NULL) return -1; sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mDebug("user:%s, will be created when deploying, raw:%p", userObj.user, pRaw); + mInfo("user:%s, will be created when deploying, raw:%p", userObj.user, pRaw); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "create-user"); if (pTrans == NULL) { mError("user:%s, failed to create since %s", userObj.user, terrstr()); return -1; } - mDebug("trans:%d, used to create user:%s", pTrans->id, userObj.user); + mInfo("trans:%d, used to create user:%s", pTrans->id, userObj.user); if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { mError("trans:%d, failed to commit redo log since %s", pTrans->id, terrstr()); @@ -304,7 +304,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate mError("user:%s, failed to create since %s", pCreate->user, terrstr()); return -1; } - mDebug("trans:%d, used to create user:%s", pTrans->id, pCreate->user); + mInfo("trans:%d, used to create user:%s", pTrans->id, pCreate->user); SSdbRaw *pCommitRaw = mndUserActionEncode(&userObj); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { @@ -336,7 +336,7 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("user:%s, start to create", createReq.user); + mInfo("user:%s, start to create", createReq.user); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) { goto _OVER; } @@ -388,7 +388,7 @@ static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SRpc mError("user:%s, failed to alter since %s", pOld->user, terrstr()); return -1; } - mDebug("trans:%d, used to alter user:%s", pTrans->id, pOld->user); + mInfo("trans:%d, used to alter user:%s", pTrans->id, pOld->user); SSdbRaw *pCommitRaw = mndUserActionEncode(pNew); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { @@ -446,7 +446,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("user:%s, start to alter", alterReq.user); + mInfo("user:%s, start to alter", alterReq.user); if (alterReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; @@ -603,7 +603,7 @@ static int32_t mndDropUser(SMnode *pMnode, SRpcMsg *pReq, SUserObj *pUser) { mError("user:%s, failed to drop since %s", pUser->user, terrstr()); return -1; } - mDebug("trans:%d, used to drop user:%s", pTrans->id, pUser->user); + mInfo("trans:%d, used to drop user:%s", pTrans->id, pUser->user); SSdbRaw *pCommitRaw = mndUserActionEncode(pUser); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { @@ -634,7 +634,7 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("user:%s, start to drop", dropReq.user); + mInfo("user:%s, start to drop", dropReq.user); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index cc9ed5f9ba..c1bc340819 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -404,7 +404,7 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2 pDnode->numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); pDnode->memUsed = mndGetVnodesMemory(pMnode, pDnode->id); - mDebug("dnode:%d, vnodes:%d supportVnodes:%d isMnode:%d online:%d memory avail:%" PRId64 " used:%" PRId64, pDnode->id, + mInfo("dnode:%d, vnodes:%d supportVnodes:%d isMnode:%d online:%d memory avail:%" PRId64 " used:%" PRId64, pDnode->id, pDnode->numOfVnodes, pDnode->numOfSupportVnodes, isMnode, online, pDnode->memAvail, pDnode->memUsed); if (isMnode) { @@ -855,7 +855,7 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); - mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + mInfo("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); } SVnodeGid *pVgid = &pVgroup->vnodeGid[pVgroup->replica]; @@ -905,7 +905,7 @@ int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); - mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + mInfo("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); } int32_t code = -1; @@ -1174,7 +1174,7 @@ int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t del static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t newDnodeId) { - mDebug("vgId:%d, will add 1 vnode, replica:%d dnode:%d", pVgroup->vgId, pVgroup->replica, newDnodeId); + mInfo("vgId:%d, will add 1 vnode, replica:%d dnode:%d", pVgroup->vgId, pVgroup->replica, newDnodeId); SVnodeGid *pGid = &pVgroup->vnodeGid[pVgroup->replica]; pVgroup->replica++; @@ -1190,7 +1190,7 @@ static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDb static int32_t mndAddDecVgroupReplicaFromTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t delDnodeId) { - mDebug("vgId:%d, will remove 1 vnode, replica:%d dnode:%d", pVgroup->vgId, pVgroup->replica, delDnodeId); + mInfo("vgId:%d, will remove 1 vnode, replica:%d dnode:%d", pVgroup->vgId, pVgroup->replica, delDnodeId); SVnodeGid *pGid = NULL; SVnodeGid delGid = {0}; @@ -1226,7 +1226,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "red-vgroup"); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); - mDebug("trans:%d, used to redistribute vgroup, vgId:%d", pTrans->id, pVgroup->vgId); + mInfo("trans:%d, used to redistribute vgroup, vgId:%d", pTrans->id, pVgroup->vgId); SVgObj newVg = {0}; memcpy(&newVg, pVgroup, sizeof(SVgObj)); @@ -1609,7 +1609,7 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "split-vgroup"); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); - mDebug("trans:%d, used to split vgroup, vgId:%d", pTrans->id, pVgroup->vgId); + mInfo("trans:%d, used to split vgroup, vgId:%d", pTrans->id, pVgroup->vgId); SVgObj newVg1 = {0}; memcpy(&newVg1, pVgroup, sizeof(SVgObj)); @@ -1666,7 +1666,7 @@ static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { SVgObj *pVgroup = NULL; SDbObj *pDb = NULL; - mDebug("vgId:%d, start to split", vgId); + mInfo("vgId:%d, start to split", vgId); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) { goto _OVER; } @@ -1777,13 +1777,13 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "balance-vgroup"); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); - mDebug("trans:%d, used to balance vgroup", pTrans->id); + mInfo("trans:%d, used to balance vgroup", pTrans->id); while (1) { taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); - mDebug("dnode:%d, equivalent vnodes:%d support:%d, score:%f", pDnode->id, pDnode->numOfVnodes, + mInfo("dnode:%d, equivalent vnodes:%d support:%d, score:%f", pDnode->id, pDnode->numOfVnodes, pDnode->numOfSupportVnodes, (float)pDnode->numOfVnodes / pDnode->numOfSupportVnodes); } @@ -1792,7 +1792,7 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { float srcScore = (float)(pSrc->numOfVnodes - 1) / pSrc->numOfSupportVnodes; float dstScore = (float)(pDst->numOfVnodes + 1) / pDst->numOfSupportVnodes; - mDebug("trans:%d, after balance, src dnode:%d score:%f, dst dnode:%d score:%f", pTrans->id, pSrc->id, srcScore, + mInfo("trans:%d, after balance, src dnode:%d score:%f, dst dnode:%d score:%f", pTrans->id, pSrc->id, srcScore, pDst->id, dstScore); if (srcScore > dstScore - 0.000001) { @@ -1803,20 +1803,20 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { numOfVgroups++; continue; } else { - mDebug("trans:%d, no vgroup need to balance from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id); + mInfo("trans:%d, no vgroup need to balance from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id); break; } } else { - mDebug("trans:%d, no vgroup need to balance any more", pTrans->id); + mInfo("trans:%d, no vgroup need to balance any more", pTrans->id); break; } } if (numOfVgroups <= 0) { - mDebug("no need to balance vgroup"); + mInfo("no need to balance vgroup"); code = 0; } else { - mDebug("start to balance vgroup, numOfVgroups:%d", numOfVgroups); + mInfo("start to balance vgroup, numOfVgroups:%d", numOfVgroups); if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = TSDB_CODE_ACTION_IN_PROGRESS; } @@ -1865,7 +1865,7 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { if (pArray == NULL) goto _OVER; if (taosArrayGetSize(pArray) < 2) { - mDebug("no need to balance vgroup since dnode num less than 2"); + mInfo("no need to balance vgroup since dnode num less than 2"); code = 0; } else { code = mndBalanceVgroup(pMnode, pReq, pArray); diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 3db0087334..5393c42da3 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -19,7 +19,7 @@ static int32_t sdbCreateDir(SSdb *pSdb); SSdb *sdbInit(SSdbOpt *pOption) { - mDebug("start to init sdb in %s", pOption->path); + mInfo("start to init sdb in %s", pOption->path); SSdb *pSdb = taosMemoryCalloc(1, sizeof(SSdb)); if (pSdb == NULL) { @@ -61,12 +61,12 @@ SSdb *sdbInit(SSdbOpt *pOption) { pSdb->commitConfig = -1; pSdb->pMnode = pOption->pMnode; taosThreadMutexInit(&pSdb->filelock, NULL); - mDebug("sdb init successfully"); + mInfo("sdb init success"); return pSdb; } void sdbCleanup(SSdb *pSdb) { - mDebug("start to cleanup sdb"); + mInfo("start to cleanup sdb"); sdbWriteFile(pSdb, 0); @@ -103,12 +103,12 @@ void sdbCleanup(SSdb *pSdb) { pSdb->hashObjs[i] = NULL; memset(&pSdb->locks[i], 0, sizeof(pSdb->locks[i])); - mDebug("sdb table:%s is cleaned up", sdbTableName(i)); + mInfo("sdb table:%s is cleaned up", sdbTableName(i)); } taosThreadMutexDestroy(&pSdb->filelock); taosMemoryFree(pSdb); - mDebug("sdb is cleaned up"); + mInfo("sdb is cleaned up"); } int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) { @@ -139,7 +139,7 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) { pSdb->maxId[sdbType] = 0; pSdb->hashObjs[sdbType] = hash; - mDebug("sdb table:%s is initialized", sdbTableName(sdbType)); + mInfo("sdb table:%s is initialized", sdbTableName(sdbType)); return 0; } @@ -175,7 +175,7 @@ void sdbGetCommitInfo(SSdb *pSdb, int64_t *index, int64_t *term, int64_t *config *index = pSdb->commitIndex; *term = pSdb->commitTerm; *config = pSdb->commitConfig; -#if 0 +#if 1 mTrace("mnode current info, apply index:%" PRId64 " term:%" PRId64 " config:%" PRId64 ", commit index:%" PRId64 " term:%" PRId64 " config:%" PRId64, pSdb->applyIndex, pSdb->applyTerm, pSdb->applyConfig, *index, *term, *config); diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 00659939e9..ff0c4b26ef 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -23,25 +23,25 @@ #define SDB_FILE_VER 1 static int32_t sdbDeployData(SSdb *pSdb) { - mDebug("start to deploy sdb"); + mInfo("start to deploy sdb"); for (int32_t i = SDB_MAX - 1; i >= 0; --i) { SdbDeployFp fp = pSdb->deployFps[i]; if (fp == NULL) continue; - mDebug("start to deploy sdb:%s", sdbTableName(i)); + mInfo("start to deploy sdb:%s", sdbTableName(i)); if ((*fp)(pSdb->pMnode) != 0) { mError("failed to deploy sdb:%s since %s", sdbTableName(i), terrstr()); return -1; } } - mDebug("sdb deploy successfully"); + mInfo("sdb deploy success"); return 0; } static void sdbResetData(SSdb *pSdb) { - mDebug("start to reset sdb"); + mInfo("start to reset sdb"); for (ESdbType i = 0; i < SDB_MAX; ++i) { SHashObj *hash = pSdb->hashObjs[i]; @@ -64,7 +64,7 @@ static void sdbResetData(SSdb *pSdb) { taosHashClear(pSdb->hashObjs[i]); pSdb->tableVer[i] = 0; pSdb->maxId[i] = 0; - mDebug("sdb:%s is reset", sdbTableName(i)); + mInfo("sdb:%s is reset", sdbTableName(i)); } pSdb->applyIndex = -1; @@ -73,7 +73,7 @@ static void sdbResetData(SSdb *pSdb) { pSdb->commitIndex = -1; pSdb->commitTerm = -1; pSdb->commitConfig = -1; - mDebug("sdb reset successfully"); + mInfo("sdb reset success"); } static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { @@ -229,7 +229,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { char file[PATH_MAX] = {0}; snprintf(file, sizeof(file), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); - mDebug("start to read sdb file:%s", file); + mInfo("start to read sdb file:%s", file); SSdbRaw *pRaw = taosMemoryMalloc(TSDB_MAX_MSG_SIZE + 100); if (pRaw == NULL) { @@ -306,7 +306,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { pSdb->commitTerm = pSdb->applyTerm; pSdb->commitConfig = pSdb->applyConfig; memcpy(pSdb->tableVer, tableVer, sizeof(tableVer)); - mDebug("read sdb file:%s successfully, commit index:%" PRId64 " term:%" PRId64 " config:%" PRId64, file, + mInfo("read sdb file:%s success, commit index:%" PRId64 " term:%" PRId64 " config:%" PRId64, file, pSdb->commitIndex, pSdb->commitTerm, pSdb->commitConfig); _OVER: @@ -339,7 +339,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { char curfile[PATH_MAX] = {0}; snprintf(curfile, sizeof(curfile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); - mDebug("start to write sdb file, apply index:%" PRId64 " term:%" PRId64 " config:%" PRId64 ", commit index:%" PRId64 + mInfo("start to write sdb file, apply index:%" PRId64 " term:%" PRId64 " config:%" PRId64 ", commit index:%" PRId64 " term:%" PRId64 " config:%" PRId64 ", file:%s", pSdb->applyIndex, pSdb->applyTerm, pSdb->applyConfig, pSdb->commitIndex, pSdb->commitTerm, pSdb->commitConfig, curfile); @@ -361,7 +361,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { SdbEncodeFp encodeFp = pSdb->encodeFps[i]; if (encodeFp == NULL) continue; - mDebug("write %s to sdb file, total %d rows", sdbTableName(i), sdbGetSize(pSdb, i)); + mInfo("write %s to sdb file, total %d rows", sdbTableName(i), sdbGetSize(pSdb, i)); SHashObj *hash = pSdb->hashObjs[i]; TdThreadRwlock *pLock = &pSdb->locks[i]; @@ -437,7 +437,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { pSdb->commitIndex = pSdb->applyIndex; pSdb->commitTerm = pSdb->applyTerm; pSdb->commitConfig = pSdb->applyConfig; - mDebug("write sdb file successfully, commit index:%" PRId64 " term:%" PRId64 " config:%" PRId64 " file:%s", + mInfo("write sdb file success, commit index:%" PRId64 " term:%" PRId64 " config:%" PRId64 " file:%s", pSdb->commitIndex, pSdb->commitTerm, pSdb->commitConfig, curfile); } @@ -519,7 +519,7 @@ static void sdbCloseIter(SSdbIter *pIter) { pIter->name = NULL; } - mDebug("sdbiter:%p, is closed, total:%" PRId64, pIter, pIter->total); + mInfo("sdbiter:%p, is closed, total:%" PRId64, pIter, pIter->total); taosMemoryFree(pIter); } @@ -556,7 +556,7 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *ter if (term != NULL) *term = commitTerm; if (config != NULL) *config = commitConfig; - mDebug("sdbiter:%p, is created to read snapshot, commit index:%" PRId64 " term:%" PRId64 " config:%" PRId64 + mInfo("sdbiter:%p, is created to read snapshot, commit index:%" PRId64 " term:%" PRId64 " config:%" PRId64 " file:%s", pIter, commitIndex, commitTerm, commitConfig, pIter->name); return 0; @@ -568,7 +568,7 @@ int32_t sdbStopRead(SSdb *pSdb, SSdbIter *pIter) { } int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len) { - int32_t maxlen = 100; + int32_t maxlen = 4096; void *pBuf = taosMemoryCalloc(1, maxlen); if (pBuf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -584,14 +584,14 @@ int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len) { taosMemoryFree(pBuf); return -1; } else if (readlen == 0) { - mDebug("sdbiter:%p, read snapshot to the end, total:%" PRId64, pIter, pIter->total); + mInfo("sdbiter:%p, read snapshot to the end, total:%" PRId64, pIter, pIter->total); *ppBuf = NULL; *len = 0; taosMemoryFree(pBuf); return 0; } else { // (readlen <= maxlen) pIter->total += readlen; - mDebug("sdbiter:%p, read:%d bytes from snapshot, total:%" PRId64, pIter, readlen, pIter->total); + mInfo("sdbiter:%p, read:%d bytes from snapshot, total:%" PRId64, pIter, readlen, pIter->total); *ppBuf = pBuf; *len = readlen; return 0; @@ -610,7 +610,7 @@ int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter) { } *ppIter = pIter; - mDebug("sdbiter:%p, is created to write snapshot, file:%s", pIter, pIter->name); + mInfo("sdbiter:%p, is created to write snapshot, file:%s", pIter, pIter->name); return 0; } @@ -619,7 +619,7 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i if (!isApply) { sdbCloseIter(pIter); - mDebug("sdbiter:%p, not apply to sdb", pIter); + mInfo("sdbiter:%p, not apply to sdb", pIter); return 0; } @@ -655,7 +655,7 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i pSdb->commitIndex = index; } - mDebug("sdbiter:%p, successfully applyed to sdb", pIter); + mInfo("sdbiter:%p, success applyed to sdb", pIter); return 0; } @@ -668,6 +668,6 @@ int32_t sdbDoWrite(SSdb *pSdb, SSdbIter *pIter, void *pBuf, int32_t len) { } pIter->total += writelen; - mDebug("sdbiter:%p, write:%d bytes to snapshot, total:%" PRId64, pIter, writelen, pIter->total); + mInfo("sdbiter:%p, write:%d bytes to snapshot, total:%" PRId64, pIter, writelen, pIter->total); return 0; } diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index c579f82a9d..a102541aca 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -83,7 +83,7 @@ const char *sdbStatusName(ESdbStatus status) { } void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper) { -#if 0 +#if 1 EKeyType keyType = pSdb->keyTypes[pRow->type]; if (keyType == SDB_KEY_BINARY) { diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index 95985cd3d9..3a16ee3f13 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -37,7 +37,7 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { pRaw->sver = sver; pRaw->dataLen = dataLen; -#if 0 +#if 1 mTrace("raw:%p, is created, len:%d table:%s", pRaw, dataLen, sdbTableName(type)); #endif return pRaw; @@ -45,7 +45,7 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { void sdbFreeRaw(SSdbRaw *pRaw) { if (pRaw != NULL) { -#if 0 +#if 1 mTrace("raw:%p, is freed", pRaw); #endif taosMemoryFree(pRaw); diff --git a/source/dnode/mnode/sdb/src/sdbRow.c b/source/dnode/mnode/sdb/src/sdbRow.c index b362ee3a45..c078e7eb21 100644 --- a/source/dnode/mnode/sdb/src/sdbRow.c +++ b/source/dnode/mnode/sdb/src/sdbRow.c @@ -23,7 +23,7 @@ SSdbRow *sdbAllocRow(int32_t objSize) { return NULL; } -#if 0 +#if 1 mTrace("row:%p, is created, len:%d", pRow->pObj, objSize); #endif return pRow; @@ -47,7 +47,7 @@ void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc) { sdbPrintOper(pSdb, pRow, "free"); -#if 0 +#if 1 mTrace("row:%p, is freed", pRow->pObj); #endif taosMemoryFreeClear(pRow); diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index b65189153e..efdc8b4693 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -26,7 +26,7 @@ SQnode *qndOpen(const SQnodeOpt *pOption) { return NULL; } - if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) { + if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, (void **)&pQnode->pQuery, &pOption->msgCb)) { taosMemoryFreeClear(pQnode); return NULL; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index d6c3e774ce..4f5f741b9e 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -691,7 +691,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma while (1) { uint64_t ts; - int32_t code = qExecTaskOpt(taskInfo, pResList, &ts); + int32_t code = qExecTaskOpt(taskInfo, pResList, &ts, NULL); if (code < 0) { if (code == TSDB_CODE_QRY_IN_EXEC) { break; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 8452d14c86..32272b541c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -272,8 +272,8 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb SColVal colVal = {0}; tTSRowGetVal(row, pTSchema, iCol, &colVal); - if (colVal.isNone || colVal.isNull) { - if (keyTs == tTsVal1->ts && !tColVal->isNone && !tColVal->isNull) { + if (!COL_VAL_IS_VALUE(&colVal)) { + if (keyTs == tTsVal1->ts && COL_VAL_IS_VALUE(tColVal)) { invalidate = true; break; @@ -1062,7 +1062,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo goto _err; } - if (pColVal->isNone && !setNoneCol) { + if (COL_VAL_IS_NONE(pColVal) && !setNoneCol) { noneCol = iCol; setNoneCol = true; } @@ -1087,9 +1087,9 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo SColVal *tColVal = (SColVal *)taosArrayGet(pColArray, iCol); tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); - if (tColVal->isNone && !pColVal->isNone) { + if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { taosArraySet(pColArray, iCol, pColVal); - } else if (tColVal->isNone && pColVal->isNone && !setNoneCol) { + } else if (COL_VAL_IS_NONE(tColVal) && COL_VAL_IS_NONE(pColVal) && !setNoneCol) { noneCol = iCol; setNoneCol = true; } @@ -1161,7 +1161,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { goto _err; } - if ((pColVal->isNone || pColVal->isNull) && !setNoneCol) { + if (!COL_VAL_IS_VALUE(pColVal) && !setNoneCol) { noneCol = iCol; setNoneCol = true; } @@ -1181,9 +1181,9 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { SColVal *tColVal = (SColVal *)taosArrayGet(pColArray, iCol); tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); - if ((tColVal->isNone || tColVal->isNull) && (!pColVal->isNone && !pColVal->isNull)) { + if (!COL_VAL_IS_VALUE(tColVal) && COL_VAL_IS_VALUE(pColVal)) { taosArraySet(pColArray, iCol, &(SLastCol){.ts = rowTs, .colVal = *pColVal}); - } else if ((tColVal->isNone || tColVal->isNull) && (pColVal->isNone || pColVal->isNull) && !setNoneCol) { + } else if (!COL_VAL_IS_VALUE(tColVal) && !COL_VAL_IS_VALUE(pColVal) && !setNoneCol) { noneCol = iCol; setNoneCol = true; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index ea9a7ec7d9..5f981649f3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -45,7 +45,7 @@ static void saveOneRow(STSRow* pRow, SSDataBlock* pBlock, SCacheRowsReader* pRea tTSRowGetVal(pRow, pReader->pSchema, slotId, &colVal); if (IS_VAR_DATA_TYPE(colVal.type)) { - if (colVal.isNull || colVal.isNone) { + if (!COL_VAL_IS_VALUE(&colVal)) { colDataAppendNULL(pColInfoData, numOfRows); } else { varDataSetLen(pReader->transferBuf[slotId], colVal.value.nData); @@ -53,7 +53,7 @@ static void saveOneRow(STSRow* pRow, SSDataBlock* pBlock, SCacheRowsReader* pRea colDataAppend(pColInfoData, numOfRows, pReader->transferBuf[slotId], false); } } else { - colDataAppend(pColInfoData, numOfRows, (const char*)&colVal.value, colVal.isNull || colVal.isNone); + colDataAppend(pColInfoData, numOfRows, (const char*)&colVal.value, !COL_VAL_IS_VALUE(&colVal)); } } } @@ -84,7 +84,7 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, SArray* pTableIdList p->transferBuf = taosMemoryCalloc(p->pSchema->numOfCols, POINTER_BYTES); if (p->transferBuf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t i = 0; i < p->pSchema->numOfCols; ++i) { @@ -117,7 +117,8 @@ int32_t tsdbCacherowsReaderClose(void* pReader) { return TSDB_CODE_SUCCESS; } -static int32_t doExtractCacheRow(SCacheRowsReader* pr, SLRUCache* lruCache, uint64_t uid, STSRow** pRow, LRUHandle** h) { +static int32_t doExtractCacheRow(SCacheRowsReader* pr, SLRUCache* lruCache, uint64_t uid, STSRow** pRow, + LRUHandle** h) { int32_t code = TSDB_CODE_SUCCESS; if ((pr->type & CACHESCAN_RETRIEVE_LAST_ROW) == CACHESCAN_RETRIEVE_LAST_ROW) { code = tsdbCacheGetLastrowH(lruCache, uid, pr->pVnode->pTsdb, h); @@ -166,7 +167,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 STableKeyInfo* pKeyInfo = taosArrayGet(pr->pTableList, i); code = doExtractCacheRow(pr, lruCache, pKeyInfo->uid, &pRow, &h); - if (code != TSDB_CODE_SUCCESS) { + if (code != TSDB_CODE_SUCCESS) { return code; } @@ -194,7 +195,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 for (int32_t i = pr->tableIndex; i < numOfTables; ++i) { STableKeyInfo* pKeyInfo = taosArrayGet(pr->pTableList, i); code = doExtractCacheRow(pr, lruCache, pKeyInfo->uid, &pRow, &h); - if (code != TSDB_CODE_SUCCESS) { + if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 8fdbd68af3..53d04cc9c1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -664,6 +664,9 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { } } + // update the diskId + pDFileSet->diskId = pSet->diskId; + goto _exit; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 681b25f676..a2f21d4f6e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -662,7 +662,7 @@ static void setBlockAllDumped(SFileBlockDumpInfo* pDumpInfo, int64_t maxKey, int static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal, SBlockLoadSuppInfo* pSup) { if (IS_VAR_DATA_TYPE(pColVal->type)) { - if (pColVal->isNull || pColVal->isNone) { + if (!COL_VAL_IS_VALUE(pColVal)) { colDataAppendNULL(pColInfoData, rowIndex); } else { varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData); @@ -671,7 +671,7 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_ colDataAppend(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); } } else { - colDataAppend(pColInfoData, rowIndex, (const char*)&pColVal->value, pColVal->isNull || pColVal->isNone); + colDataAppend(pColInfoData, rowIndex, (const char*)&pColVal->value, !COL_VAL_IS_VALUE(pColVal)); } } @@ -760,51 +760,43 @@ static int doBinarySearchKey(TSKEY* keyList, int num, int pos, TSKEY key, int or s = pos; // check - assert(pos >=0 && pos < num); + assert(pos >= 0 && pos < num); assert(num > 0); if (order == TSDB_ORDER_ASC) { // find the first position which is smaller than the key - e = num - 1; - if (key < keyList[pos]) - return -1; + e = num - 1; + if (key < keyList[pos]) return -1; while (1) { // check can return - if (key >= keyList[e]) - return e; - if (key <= keyList[s]) - return s; - if (e - s <= 1) - return s; + if (key >= keyList[e]) return e; + if (key <= keyList[s]) return s; + if (e - s <= 1) return s; // change start or end position - int mid = s + (e - s + 1)/2; + int mid = s + (e - s + 1) / 2; if (keyList[mid] > key) e = mid; - else if(keyList[mid] < key) + else if (keyList[mid] < key) s = mid; else return mid; } - } else { // DESC + } else { // DESC // find the first position which is bigger than the key - e = 0; - if (key > keyList[pos]) - return -1; + e = 0; + if (key > keyList[pos]) return -1; while (1) { // check can return - if (key <= keyList[e]) - return e; - if (key >= keyList[s]) - return s; - if (s - e <= 1) - return s; + if (key <= keyList[e]) return e; + if (key >= keyList[s]) return s; + if (s - e <= 1) return s; // change start or end position - int mid = s - (s - e + 1)/2; + int mid = s - (s - e + 1) / 2; if (keyList[mid] < key) e = mid; - else if(keyList[mid] > key) + else if (keyList[mid] > key) s = mid; else return mid; @@ -815,7 +807,7 @@ static int doBinarySearchKey(TSKEY* keyList, int num, int pos, TSKEY key, int or int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SDataBlk* pBlock, int32_t pos) { // NOTE: reverse the order to find the end position in data block int32_t endPos = -1; - bool asc = ASCENDING_TRAVERSE(pReader->order); + bool asc = ASCENDING_TRAVERSE(pReader->order); if (asc && pReader->window.ekey >= pBlock->maxKey.ts) { endPos = pBlock->nRow - 1; @@ -851,8 +843,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn } else if (!asc && pReader->window.ekey >= pBlock->maxKey.ts) { pDumpInfo->rowIndex = pBlock->nRow - 1; } else { - int32_t pos = asc? pBlock->nRow-1:0; - int32_t order = (pReader->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; + int32_t pos = asc ? pBlock->nRow - 1 : 0; + int32_t order = (pReader->order == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC; pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, pReader->window.skey, order); } @@ -865,13 +857,13 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn endIndex += step; int32_t remain = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex); - if (remain > pReader->capacity) { // output buffer check + if (remain > pReader->capacity) { // output buffer check remain = pReader->capacity; } int32_t rowIndex = 0; - int32_t i = 0; + int32_t i = 0; SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i); if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { if (asc) { @@ -895,7 +887,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn if (pData->cid < pColData->info.colId) { colIndex += 1; } else if (pData->cid == pColData->info.colId) { - if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL|HAS_NONE)) { + if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) { colDataAppendNNULL(pColData, 0, remain); } else { if (IS_NUMERIC_TYPE(pColData->info.type) && asc) { @@ -940,7 +932,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; setBlockAllDumped(pDumpInfo, ts, pReader->order); } else { - int64_t k = asc? pBlock->maxKey.ts:pBlock->minKey.ts; + int64_t k = asc ? pBlock->maxKey.ts : pBlock->minKey.ts; setBlockAllDumped(pDumpInfo, k, pReader->order); } @@ -950,8 +942,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t unDumpedRows = asc ? pBlock->nRow - pDumpInfo->rowIndex : pDumpInfo->rowIndex + 1; tsdbDebug("%p copy file block to sdatablock, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", - pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, remain, unDumpedRows, - pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, remain, + unDumpedRows, pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); return TSDB_CODE_SUCCESS; } @@ -1287,14 +1279,14 @@ static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, cons if (p->version >= pBlock->minVer) { if (i < num - 1) { TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1); -// if (i + 1 == num - 1) { // pnext is the last point - if (pnext->ts >= pBlock->minKey.ts) { - return true; -// } -// } else { -// if (pnext->ts >= pBlock->minKey.ts) { -// return true; -// } + // if (i + 1 == num - 1) { // pnext is the last point + if (pnext->ts >= pBlock->minKey.ts) { + return true; + // } + // } else { + // if (pnext->ts >= pBlock->minKey.ts) { + // return true; + // } } } else { // it must be the last point ASSERT(p->version == 0); @@ -1755,7 +1747,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsLast = getCurrentKeyInLastBlock(pLastBlockReader); } - int64_t key = hasDataInFileBlock(pBlockData, pDumpInfo)? pBlockData->aTSKEY[pDumpInfo->rowIndex]:INT64_MIN; + int64_t key = hasDataInFileBlock(pBlockData, pDumpInfo) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN; TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY ik = TSDBROW_KEY(piRow); @@ -2023,12 +2015,12 @@ static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { } static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } -bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo) { - if (pBlockData->nRow > 0) { - ASSERT(pBlockData->nRow == pDumpInfo->totalRows); +bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo) { + if (pBlockData->nRow > 0) { + ASSERT(pBlockData->nRow == pDumpInfo->totalRows); } - return pBlockData->nRow > 0 && (!pDumpInfo->allDumped); + return pBlockData->nRow > 0 && (!pDumpInfo->allDumped); } int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, @@ -2101,11 +2093,12 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { if (pBlockInfo != NULL) { pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); - TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); + TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); // it is a clean block, load it directly if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader)) { - if (pReader->order == TSDB_ORDER_ASC || (pReader->order == TSDB_ORDER_DESC && (!hasDataInLastBlock(pLastBlockReader)))) { + if (pReader->order == TSDB_ORDER_ASC || + (pReader->order == TSDB_ORDER_DESC && (!hasDataInLastBlock(pLastBlockReader)))) { copyBlockDataToSDataBlock(pReader, pBlockScanInfo); goto _end; } @@ -2158,12 +2151,12 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } } - _end: +_end: pResBlock->info.uid = pBlockScanInfo->uid; blockDataUpdateTsWindow(pResBlock, 0); setComposedBlockFlag(pReader, true); - double el = (taosGetTimestampUs() - st)/1000.0; + double el = (taosGetTimestampUs() - st) / 1000.0; pReader->cost.composedBlocks += 1; pReader->cost.buildComposedBlockTime += el; @@ -3228,7 +3221,8 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S tColDataGetValue(pData, rowIndex, &cv); doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); j += 1; - } else if (pData->cid > pCol->info.colId) { // the specified column does not exist in file block, fill with null data + } else if (pData->cid > + pCol->info.colId) { // the specified column does not exist in file block, fill with null data colDataAppendNULL(pCol, outputRowIndex); } @@ -3302,7 +3296,6 @@ void* tsdbGetIvtIdx(SMeta* pMeta) { uint64_t getReaderMaxVersion(STsdbReader* pReader) { return pReader->verRange.maxVer; } - static int32_t doOpenReaderImpl(STsdbReader* pReader) { SDataBlockIter* pBlockIter = &pReader->status.blockIter; @@ -3375,19 +3368,20 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl // NOTE: the endVersion in pCond is the data version not schema version, so pCond->endVersion is not correct here. if (pCond->suid != 0) { - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, -1); + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, -1); if (pReader->pSchema == NULL) { - tsdbError("failed to get table schema, suid:%"PRIu64", ver:%"PRId64" , %s", pReader->suid, -1, pReader->idStr); + tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:%" PRId64 " , %s", pReader->suid, -1, + pReader->idStr); } } else if (taosArrayGetSize(pTableList) > 0) { STableKeyInfo* pKey = taosArrayGet(pTableList, 0); pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1); if (pReader->pSchema == NULL) { - tsdbError("failed to get table schema, uid:%"PRIu64", ver:%"PRId64" , %s", pKey->uid, -1, pReader->idStr); + tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:%" PRId64 " , %s", pKey->uid, -1, pReader->idStr); } } - STsdbReader* p = pReader->innerReader[0] != NULL? pReader->innerReader[0]:pReader; + STsdbReader* p = pReader->innerReader[0] != NULL ? pReader->innerReader[0] : pReader; int32_t numOfTables = taosArrayGetSize(pTableList); pReader->status.pTableMap = createDataBlockScanInfo(p, pTableList->pData, numOfTables); @@ -3507,7 +3501,8 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbDebug( "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 - " SMA-time:%.2f ms, fileBlocks:%" PRId64 ", fileBlocks-load-time:%.2f ms, " + " SMA-time:%.2f ms, fileBlocks:%" PRId64 + ", fileBlocks-load-time:%.2f ms, " "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64 ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb %s", pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks, @@ -3765,8 +3760,10 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { } } - tsdbDebug("%p reset reader, suid:%" PRIu64 ", numOfTables:%d, skey:%"PRId64", query range:%" PRId64 " - %" PRId64 " in query %s", - pReader, pReader->suid, numOfTables, pCond->twindows.skey, pReader->window.skey, pReader->window.ekey, pReader->idStr); + tsdbDebug("%p reset reader, suid:%" PRIu64 ", numOfTables:%d, skey:%" PRId64 ", query range:%" PRId64 " - %" PRId64 + " in query %s", + pReader, pReader->suid, numOfTables, pCond->twindows.skey, pReader->window.skey, pReader->window.ekey, + pReader->idStr); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 8a4618992b..db39d98b9e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -668,7 +668,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i // head tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo); - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -689,7 +689,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i // data tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom); tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo); - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -710,7 +710,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i // sma tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom); tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo); - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -732,7 +732,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i for (int8_t iStt = 0; iStt < pSetFrom->nSttF; iStt++) { tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[iStt], fNameFrom); tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[iStt], fNameTo); - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention2.c b/source/dnode/vnode/src/tsdb/tsdbRetention2.c index 34b8e0018f..47fe3c5384 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention2.c @@ -319,9 +319,11 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now, int32_t maxSpeed) { _exit: pTsdb->trimHdl.maxRetentFid = INT32_MIN; if (code != 0) { - tsdbError("vgId:%d, tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, tsdb do retention:%d failed since %s", TD_VID(pTsdb->pVnode), retention, tstrerror(code)); ASSERT(0); // tsdbFSRollback(pTsdb->pFS); + } else { + tsdbInfo("vgId:%d, tsdb do retention:%d succeed", TD_VID(pTsdb->pVnode), retention); } return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 64c150484b..6068093538 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -509,16 +509,24 @@ void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minK *maxKey = *minKey + minutes * tsTickPerMin[precision] - 1; } +/** + * @brief get fid level by keep and days. + * + * @param fid + * @param pKeepCfg + * @param now millisecond + * @return int32_t + */ int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now) { int32_t aFid[3]; TSKEY key; if (pKeepCfg->precision == TSDB_TIME_PRECISION_MILLI) { - now = now * 1000; + // now = now * 1000; } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_MICRO) { - now = now * 1000000l; + now = now * 1000l; } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) { - now = now * 1000000000l; + now = now * 1000000l; } else { ASSERT(0); } @@ -705,12 +713,12 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); if (key.version > pMerger->version) { - if (!pColVal->isNone) { + if (!COL_VAL_IS_NONE(pColVal)) { taosArraySet(pMerger->pArray, iCol, pColVal); } } else if (key.version < pMerger->version) { SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol); - if (tColVal->isNone && !pColVal->isNone) { + if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { taosArraySet(pMerger->pArray, iCol, pColVal); } } else { @@ -776,12 +784,12 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { tsdbRowGetColVal(pRow, pMerger->pTSchema, iCol, pColVal); if (key.version > pMerger->version) { - if (!pColVal->isNone) { + if (!COL_VAL_IS_NONE(pColVal)) { taosArraySet(pMerger->pArray, iCol, pColVal); } } else if (key.version < pMerger->version) { SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol); - if (tColVal->isNone && !pColVal->isNone) { + if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { taosArraySet(pMerger->pArray, iCol, pColVal); } } else { @@ -1505,7 +1513,7 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { tColDataGetValue(pColData, iVal, pColVal); - if (pColVal->isNone || pColVal->isNull) { + if (!COL_VAL_IS_VALUE(pColVal)) { pColAgg->numOfNull++; } else { switch (pColData->type) { diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 8cfe1d8adf..825bf8a0a7 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -16,7 +16,7 @@ #include "vnd.h" int vnodeQueryOpen(SVnode *pVnode) { - return qWorkerInit(NODE_TYPE_VNODE, TD_VID(pVnode), NULL, (void **)&pVnode->pQuery, &pVnode->msgCb); + return qWorkerInit(NODE_TYPE_VNODE, TD_VID(pVnode), (void **)&pVnode->pQuery, &pVnode->msgCb); } void vnodeQueryClose(SVnode *pVnode) { qWorkerDestroy((void **)&pVnode->pQuery); } diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index 706985f894..ad677a81ff 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -99,8 +99,10 @@ extern "C" { typedef struct SExplainGroup { int32_t nodeNum; + int32_t nodeIdx; int32_t physiPlanExecNum; int32_t physiPlanExecIdx; + bool singleChannel; SRWLatch lock; SSubplan *plan; SArray *nodeExecInfo; //Array diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index a1a8ed66e6..c5de44e1f6 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -21,14 +21,14 @@ #include "tdatablock.h" int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes); -int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level); +int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel); void qExplainFreeResNode(SExplainResNode *resNode) { if (NULL == resNode) { return; } - taosMemoryFreeClear(resNode->pExecInfo); + taosArrayDestroy(resNode->pExecInfo); SNode *node = NULL; FOREACH(node, resNode->pChildren) { qExplainFreeResNode((SExplainResNode *)node); } @@ -56,8 +56,9 @@ void qExplainFreeCtx(SExplainCtx *pCtx) { int32_t num = taosArrayGetSize(group->nodeExecInfo); for (int32_t i = 0; i < num; ++i) { SExplainRsp *rsp = taosArrayGet(group->nodeExecInfo, i); - taosMemoryFreeClear(rsp->subplanInfo); + tFreeSExplainRsp(rsp); } + taosArrayDestroy(group->nodeExecInfo); } pIter = taosHashIterate(pCtx->groupHash, pIter); @@ -66,6 +67,7 @@ void qExplainFreeCtx(SExplainCtx *pCtx) { taosHashCleanup(pCtx->groupHash); taosArrayDestroy(pCtx->rows); + taosMemoryFreeClear(pCtx->tbuf); taosMemoryFree(pCtx); } @@ -248,7 +250,7 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo return TSDB_CODE_SUCCESS; } -int32_t qExplainGenerateResNodeExecInfo(SArray **pExecInfo, SExplainGroup *group) { +int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, SExplainGroup *group) { *pExecInfo = taosArrayInit(group->nodeNum, sizeof(SExplainExecInfo)); if (NULL == (*pExecInfo)) { qError("taosArrayInit %d explainExecInfo failed", group->nodeNum); @@ -256,17 +258,28 @@ int32_t qExplainGenerateResNodeExecInfo(SArray **pExecInfo, SExplainGroup *group } SExplainRsp *rsp = NULL; - for (int32_t i = 0; i < group->nodeNum; ++i) { - rsp = taosArrayGet(group->nodeExecInfo, i); - /* - if (group->physiPlanExecIdx >= rsp->numOfPlans) { - qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); - return TSDB_CODE_QRY_APP_ERROR; - } + if (group->singleChannel) { + if (0 == group->physiPlanExecIdx) { + group->nodeIdx = 0; + } + + rsp = taosArrayGet(group->nodeExecInfo, group->nodeIdx++); + if (group->physiPlanExecIdx >= rsp->numOfPlans) { + qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); + return TSDB_CODE_QRY_APP_ERROR; + } + + taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); + } else { + for (int32_t i = 0; i < group->nodeNum; ++i) { + rsp = taosArrayGet(group->nodeExecInfo, i); + if (group->physiPlanExecIdx >= rsp->numOfPlans) { + qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); + return TSDB_CODE_QRY_APP_ERROR; + } - taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); - */ - taosArrayPush(*pExecInfo, rsp->subplanInfo); + taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); + } } ++group->physiPlanExecIdx; @@ -291,7 +304,7 @@ int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplai resNode->pNode = pNode; if (group->nodeExecInfo) { - QRY_ERR_JRET(qExplainGenerateResNodeExecInfo(&resNode->pExecInfo, group)); + QRY_ERR_JRET(qExplainGenerateResNodeExecInfo(pNode, &resNode->pExecInfo, group)); } QRY_ERR_JRET(qExplainGenerateResChildren(pNode, group, &resNode->pChildren)); @@ -801,7 +814,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } } - QRY_ERR_RET(qExplainAppendGroupResRows(ctx, pExchNode->srcStartGroupId, level + 1)); + QRY_ERR_RET(qExplainAppendGroupResRows(ctx, pExchNode->srcStartGroupId, level + 1, pExchNode->singleChannel)); break; } case QUERY_NODE_PHYSICAL_PLAN_SORT: { @@ -1533,7 +1546,7 @@ int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32 return TSDB_CODE_SUCCESS; } -int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level) { +int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel) { SExplainResNode *node = NULL; int32_t code = 0; SExplainCtx *ctx = (SExplainCtx *)pCtx; @@ -1544,6 +1557,9 @@ int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level) { QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } + group->singleChannel = singleChannel; + group->physiPlanExecIdx = 0; + QRY_ERR_RET(qExplainGenerateResNode(group->plan->pNode, group, &node)); QRY_ERR_JRET(qExplainResNodeToRows(node, ctx, level)); @@ -1707,7 +1723,7 @@ int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) { } int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { - QRY_ERR_RET(qExplainAppendGroupResRows(pCtx, pCtx->rootGroupId, 0)); + QRY_ERR_RET(qExplainAppendGroupResRows(pCtx, pCtx->rootGroupId, 0, false)); QRY_ERR_RET(qExplainAppendPlanRows(pCtx)); QRY_ERR_RET(qExplainGetRspFromCtx(pCtx, pRsp)); @@ -1723,7 +1739,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId)); if (NULL == group) { qError("group %d not in groupHash", groupId); - taosMemoryFreeClear(pRspMsg->subplanInfo); + tFreeSExplainRsp(pRspMsg); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } @@ -1732,7 +1748,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t group->nodeExecInfo = taosArrayInit(group->nodeNum, sizeof(SExplainRsp)); if (NULL == group->nodeExecInfo) { qError("taosArrayInit %d explainExecInfo failed", group->nodeNum); - taosMemoryFreeClear(pRspMsg->subplanInfo); + tFreeSExplainRsp(pRspMsg); taosWUnLockLatch(&group->lock); QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1742,7 +1758,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t } else if (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum) { qError("group execInfo already full, size:%d, nodeNum:%d", (int32_t)taosArrayGetSize(group->nodeExecInfo), group->nodeNum); - taosMemoryFreeClear(pRspMsg->subplanInfo); + tFreeSExplainRsp(pRspMsg); taosWUnLockLatch(&group->lock); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -1751,13 +1767,14 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t if (group->physiPlanExecNum != pRspMsg->numOfPlans) { qError("physiPlanExecNum %d mismatch with others %d in group %d", pRspMsg->numOfPlans, group->physiPlanExecNum, groupId); - taosMemoryFreeClear(pRspMsg->subplanInfo); + tFreeSExplainRsp(pRspMsg); taosWUnLockLatch(&group->lock); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } taosArrayPush(group->nodeExecInfo, pRspMsg); + groupDone = (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum); taosWUnLockLatch(&group->lock); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 6484b6cb31..dc98b05479 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -183,6 +183,7 @@ typedef struct SExecTaskInfo { EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] SSubplan* pSubplan; struct SOperatorInfo* pRoot; + SLocalFetch localFetch; } SExecTaskInfo; enum { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index be8bdf7308..373cb451f4 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -479,10 +479,14 @@ static void freeBlock(void* param) { blockDataDestroy(pBlock); } -int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { +int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, SLocalFetch* pLocal) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; int64_t threadId = taosGetSelfPthreadId(); + if (pLocal) { + memcpy(&pTaskInfo->localFetch, pLocal, sizeof(*pLocal)); + } + taosArrayClearEx(pResList, freeBlock); int64_t curOwner = 0; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 31a7b0facf..7fcd41dc3e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1776,49 +1776,57 @@ void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) { size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); - if (NULL == pMsg) { - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return pTaskInfo->code; - } - SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY); - qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId, - sourceIndex, totalSources); - - pMsg->header.vgId = htonl(pSource->addr.nodeId); - pMsg->sId = htobe64(pSource->schedId); - pMsg->taskId = htobe64(pSource->taskId); - pMsg->queryId = htobe64(pTaskInfo->id.queryId); - pMsg->execId = htonl(pSource->execId); - - // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - taosMemoryFreeClear(pMsg); - qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return pTaskInfo->code; - } - SFetchRspHandleWrapper* pWrapper = taosMemoryCalloc(1, sizeof(SFetchRspHandleWrapper)); pWrapper->exchangeId = pExchangeInfo->self; pWrapper->sourceIndex = sourceIndex; - pMsgSendInfo->param = pWrapper; - pMsgSendInfo->paramFreeFp = taosMemoryFree; - pMsgSendInfo->msgInfo.pData = pMsg; - pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); - pMsgSendInfo->msgType = pSource->fetchMsgType; - pMsgSendInfo->fp = loadRemoteDataCallback; + if (pSource->localExec) { + SDataBuf pBuf = {0}; + int32_t code = (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId, pSource->taskId, 0, pSource->execId, &pBuf.pData, pTaskInfo->localFetch.explainRes); + loadRemoteDataCallback(pWrapper, &pBuf, code); + taosMemoryFree(pWrapper); + } else { + SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); + if (NULL == pMsg) { + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return pTaskInfo->code; + } - int64_t transporterId = 0; - int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo); + qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId, + sourceIndex, totalSources); + + pMsg->header.vgId = htonl(pSource->addr.nodeId); + pMsg->sId = htobe64(pSource->schedId); + pMsg->taskId = htobe64(pSource->taskId); + pMsg->queryId = htobe64(pTaskInfo->id.queryId); + pMsg->execId = htonl(pSource->execId); + + // send the fetch remote task result reques + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + taosMemoryFreeClear(pMsg); + qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return pTaskInfo->code; + } + + pMsgSendInfo->param = pWrapper; + pMsgSendInfo->paramFreeFp = taosMemoryFree; + pMsgSendInfo->msgInfo.pData = pMsg; + pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); + pMsgSendInfo->msgType = pSource->fetchMsgType; + pMsgSendInfo->fp = loadRemoteDataCallback; + + int64_t transporterId = 0; + int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo); + } + return TSDB_CODE_SUCCESS; } @@ -3531,7 +3539,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { - pOperator = createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo); + pOperator = createExchangeOperatorInfo(pHandle ? pHandle->pMsgCb->clientRpc : NULL, (SExchangePhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; if (pHandle->vnode) { @@ -4047,8 +4055,10 @@ void doDestroyTask(SExecTaskInfo* pTaskInfo) { cleanupTableSchemaInfo(&pTaskInfo->schemaInfo); cleanupStreamInfo(&pTaskInfo->streamInfo); - nodesDestroyNode((SNode*)pTaskInfo->pSubplan); - + if (!pTaskInfo->localFetch.localExec) { + nodesDestroyNode((SNode*)pTaskInfo->pSubplan); + } + taosMemoryFreeClear(pTaskInfo->sql); taosMemoryFreeClear(pTaskInfo->id.str); taosMemoryFreeClear(pTaskInfo); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 9e51f3638e..4671570802 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2016,7 +2016,11 @@ static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock pkey->isNull = false; char* val = colDataGetData(pColInfoData, rowIndex); - memcpy(pkey->pData, val, pkey->bytes); + if (!IS_VAR_DATA_TYPE(pkey->type)) { + memcpy(pkey->pData, val, pkey->bytes); + } else { + memcpy(pkey->pData, val, varDataLen(val)); + } } } @@ -2034,7 +2038,11 @@ static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock pkey->isNull = false; char* val = colDataGetData(pColInfoData, rowIndex); - memcpy(pkey->pData, val, pkey->bytes); + if (!IS_VAR_DATA_TYPE(pkey->type)) { + memcpy(pkey->pData, val, pkey->bytes); + } else { + memcpy(pkey->pData, val, varDataLen(val)); + } } } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 9177d15fae..d092c259da 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1558,7 +1558,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { } } else if (type == TSDB_DATA_TYPE_FLOAT) { float* pData = (float*)pCol->pData; - double* val = (double*)&pBuf->v; + float* val = (float*)&pBuf->v; for (int32_t i = start; i < start + numOfRows; ++i) { if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) { @@ -1625,7 +1625,7 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pEntryInfo->isNullRes = (pEntryInfo->numOfRes == 0) ? 1 : 0; if (pCol->info.type == TSDB_DATA_TYPE_FLOAT) { - float v = *(double*)&pRes->v; + float v = *(float*)&pRes->v; colDataAppend(pCol, currentRow, (const char*)&v, pEntryInfo->isNullRes); } else { colDataAppend(pCol, currentRow, (const char*)&pRes->v, pEntryInfo->isNullRes); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index dbe08bb7b2..c145952778 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -627,6 +627,7 @@ static int32_t downstreamSourceCopy(const SDownstreamSourceNode* pSrc, SDownstre COPY_SCALAR_FIELD(schedId); COPY_SCALAR_FIELD(execId); COPY_SCALAR_FIELD(fetchMsgType); + COPY_SCALAR_FIELD(localExec); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 3ea5e81872..e8c877bed2 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -505,7 +505,7 @@ static int32_t tdBlockRowMerge(STableMeta* pTableMeta, SBlockKeyTuple* pEndKeyTp SColVal colVal = {0}; for (int32_t j = 0; j < nDupRows; ++j) { tTSRowGetVal((pEndKeyTp - j)->payloadAddr, pSchema, i, &colVal); - if (!colVal.isNone) { + if (!COL_VAL_IS_NONE(&colVal)) { break; } } diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 0cb1bbe6c3..3a32263599 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -29,7 +29,7 @@ extern "C" { #include "executor.h" #include "trpc.h" -#define QW_DEFAULT_SCHEDULER_NUMBER 10000 +#define QW_DEFAULT_SCHEDULER_NUMBER 100 #define QW_DEFAULT_TASK_NUMBER 10000 #define QW_DEFAULT_SCH_TASK_NUMBER 10000 #define QW_DEFAULT_SHORT_RUN_TIMES 2 @@ -83,22 +83,6 @@ typedef struct SQWDebug { extern SQWDebug gQWDebug; -typedef struct SQWMsgInfo { - int8_t taskType; - int8_t explain; - int8_t needFetch; -} SQWMsgInfo; - -typedef struct SQWMsg { - void *node; - int32_t code; - int32_t msgType; - char *msg; - int32_t msgLen; - SQWMsgInfo msgInfo; - SRpcHandleInfo connInfo; -} SQWMsg; - typedef struct SQWHbParam { bool inUse; int32_t qwrId; @@ -133,6 +117,7 @@ typedef struct SQWTaskCtx { int8_t taskType; int8_t explain; int8_t needFetch; + int8_t localExec; int32_t msgType; int32_t fetchType; int32_t execId; @@ -150,6 +135,7 @@ typedef struct SQWTaskCtx { int8_t events[QW_EVENT_MAX]; + SArray *explainRes; void *taskHandle; void *sinkHandle; STbVerInfo tbInfo; diff --git a/source/libs/qworker/inc/qwMsg.h b/source/libs/qworker/inc/qwMsg.h index 3ff5b5950f..b46c5d6baf 100644 --- a/source/libs/qworker/inc/qwMsg.h +++ b/source/libs/qworker/inc/qwMsg.h @@ -42,7 +42,7 @@ int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t c int32_t qwBuildAndSendExplainRsp(SRpcHandleInfo *pConn, SArray* pExecList); int32_t qwBuildAndSendErrorRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code); void qwFreeFetchRsp(void *msg); -int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp); +int32_t qwMallocFetchRsp(int8_t rpcMalloc, int32_t length, SRetrieveTableRsp **rsp); int32_t qwBuildAndSendHbRsp(SRpcHandleInfo *pConn, SSchedulerHbRsp *rsp, int32_t code); int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn); int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t sId, SRpcHandleInfo *pConn); diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index c38c9ac1cf..e4271dfcda 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -9,10 +9,10 @@ #include "tmsg.h" #include "tname.h" -int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp) { +int32_t qwMallocFetchRsp(int8_t rpcMalloc, int32_t length, SRetrieveTableRsp **rsp) { int32_t msgSize = sizeof(SRetrieveTableRsp) + length; - SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcReallocCont(*rsp, msgSize); + SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)(rpcMalloc ? rpcReallocCont(*rsp, msgSize) : taosMemoryRealloc(*rsp, msgSize)); if (NULL == pRsp) { qError("rpcMallocCont %d failed", msgSize); QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index af9399bbdc..3c102938d0 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -63,11 +63,25 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { SArray *execInfoList = taosArrayInit(4, sizeof(SExplainExecInfo)); QW_ERR_RET(qGetExplainExecInfo(taskHandle, execInfoList)); - SRpcHandleInfo connInfo = ctx->ctrlConnInfo; - connInfo.ahandle = NULL; - int32_t code = qwBuildAndSendExplainRsp(&connInfo, execInfoList); - taosArrayDestroyEx(execInfoList, freeItem); - QW_ERR_RET(code); + if (ctx->localExec) { + SExplainLocalRsp localRsp = {0}; + localRsp.rsp.numOfPlans = taosArrayGetSize(execInfoList); + SExplainExecInfo *pExec = taosMemoryCalloc(localRsp.rsp.numOfPlans, sizeof(SExplainExecInfo)); + memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo)); + localRsp.rsp.subplanInfo = pExec; + localRsp.qId = qId; + localRsp.tId = tId; + localRsp.rId = rId; + localRsp.eId = eId; + taosArrayPush(ctx->explainRes, &localRsp); + taosArrayDestroy(execInfoList); + } else { + SRpcHandleInfo connInfo = ctx->ctrlConnInfo; + connInfo.ahandle = NULL; + int32_t code = qwBuildAndSendExplainRsp(&connInfo, execInfoList); + taosArrayDestroyEx(execInfoList, freeItem); + QW_ERR_RET(code); + } } if (!ctx->needFetch) { @@ -86,6 +100,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { int32_t execNum = 0; qTaskInfo_t taskHandle = ctx->taskHandle; DataSinkHandle sinkHandle = ctx->sinkHandle; + SLocalFetch localFetch = {(void*)mgmt, ctx->localExec, qWorkerProcessLocalFetch, ctx->explainRes}; SArray *pResList = taosArrayInit(4, POINTER_BYTES); while (true) { @@ -94,8 +109,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { // if *taskHandle is NULL, it's killed right now if (taskHandle) { qwDbgSimulateSleep(); - - code = qExecTaskOpt(taskHandle, pResList, &useconds); + code = qExecTaskOpt(taskHandle, pResList, &useconds, &localFetch); if (code) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) { QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -235,7 +249,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); if (NULL == rsp) { - QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); + QW_ERR_RET(qwMallocFetchRsp(!ctx->localExec, len, &rsp)); *pOutput = output; } else { pOutput->queryEnd = output.queryEnd; @@ -256,7 +270,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, *dataLen += len; - QW_ERR_RET(qwMallocFetchRsp(*dataLen, &rsp)); + QW_ERR_RET(qwMallocFetchRsp(!ctx->localExec, *dataLen, &rsp)); output.pData = rsp->data + *dataLen - len; code = dsGetDataBlock(ctx->sinkHandle, &output); @@ -480,16 +494,18 @@ _return: } if (QW_PHASE_POST_QUERY == phase && ctx) { - ctx->queryRsped = true; - - bool rsped = false; - SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; - qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); - qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); - if (!rsped) { - qwBuildAndSendQueryRsp(input->msgType + 1, &ctx->ctrlConnInfo, code, ctx); - QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); + if (!ctx->localExec) { + bool rsped = false; + SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; + qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); + qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); + if (!rsped) { + qwBuildAndSendQueryRsp(input->msgType + 1, &ctx->ctrlConnInfo, code, ctx); + QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); + } } + + ctx->queryRsped = true; } if (ctx) { @@ -518,11 +534,6 @@ int32_t qwAbortPrerocessQuery(QW_FPARAMS_DEF) { int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; - bool queryRsped = false; - SSubplan *plan = NULL; - SQWPhaseInput input = {0}; - qTaskInfo_t pTaskInfo = NULL; - DataSinkHandle sinkHandle = NULL; SQWTaskCtx *ctx = NULL; QW_ERR_JRET(qwRegisterQueryBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo)); @@ -562,6 +573,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { ctx->explain = qwMsg->msgInfo.explain; ctx->needFetch = qwMsg->msgInfo.needFetch; ctx->msgType = qwMsg->msgType; + ctx->localExec = false; // QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); @@ -584,19 +596,12 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - // QW_ERR_JRET(qwBuildAndSendQueryRsp(&qwMsg->connInfo, code)); - // QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); - - // queryRsped = true; - ctx->level = plan->level; atomic_store_ptr(&ctx->taskHandle, pTaskInfo); atomic_store_ptr(&ctx->sinkHandle, sinkHandle); - if (pTaskInfo && sinkHandle) { - qwSaveTbVersionInfo(pTaskInfo, ctx); - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, NULL)); - } + qwSaveTbVersionInfo(pTaskInfo, ctx); + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, NULL)); _return: @@ -606,11 +611,6 @@ _return: input.msgType = qwMsg->msgType; code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); - // if (!queryRsped) { - // qwBuildAndSendQueryRsp(&qwMsg->connInfo, code); - // QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); - //} - QW_RET(TSDB_CODE_SUCCESS); } @@ -1006,8 +1006,8 @@ _return: QW_RET(TSDB_CODE_SUCCESS); } -int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, const SMsgCb *pMsgCb) { - if (NULL == qWorkerMgmt || pMsgCb->mgmt == NULL) { +int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const SMsgCb *pMsgCb) { + if (NULL == qWorkerMgmt || (pMsgCb && pMsgCb->mgmt == NULL)) { qError("invalid param to init qworker"); QW_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1030,22 +1030,9 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - if (cfg) { - mgmt->cfg = *cfg; - if (0 == mgmt->cfg.maxSchedulerNum) { - mgmt->cfg.maxSchedulerNum = QW_DEFAULT_SCHEDULER_NUMBER; - } - if (0 == mgmt->cfg.maxTaskNum) { - mgmt->cfg.maxTaskNum = QW_DEFAULT_TASK_NUMBER; - } - if (0 == mgmt->cfg.maxSchTaskNum) { - mgmt->cfg.maxSchTaskNum = QW_DEFAULT_SCH_TASK_NUMBER; - } - } else { - mgmt->cfg.maxSchedulerNum = QW_DEFAULT_SCHEDULER_NUMBER; - mgmt->cfg.maxTaskNum = QW_DEFAULT_TASK_NUMBER; - mgmt->cfg.maxSchTaskNum = QW_DEFAULT_SCH_TASK_NUMBER; - } + mgmt->cfg.maxSchedulerNum = QW_DEFAULT_SCHEDULER_NUMBER; + mgmt->cfg.maxTaskNum = QW_DEFAULT_TASK_NUMBER; + mgmt->cfg.maxSchTaskNum = QW_DEFAULT_SCH_TASK_NUMBER; mgmt->schHash = taosHashInit(mgmt->cfg.maxSchedulerNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); @@ -1070,7 +1057,11 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW mgmt->nodeType = nodeType; mgmt->nodeId = nodeId; - mgmt->msgCb = *pMsgCb; + if (pMsgCb) { + mgmt->msgCb = *pMsgCb; + } else { + memset(&mgmt->msgCb, 0, sizeof(mgmt->msgCb)); + } mgmt->refId = taosAddRef(gQwMgmt.qwRef, mgmt); if (mgmt->refId < 0) { @@ -1153,3 +1144,112 @@ int32_t qWorkerGetStat(SReadHandle *handle, void *qWorkerMgmt, SQWorkerStat *pSt return TSDB_CODE_SUCCESS; } + +int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId, SQWMsg *qwMsg, SArray *explainRes) { + SQWorker *mgmt = (SQWorker*)pMgmt; + int32_t code = 0; + SQWTaskCtx *ctx = NULL; + SSubplan *plan = (SSubplan *)qwMsg->msg; + SQWPhaseInput input = {0}; + qTaskInfo_t pTaskInfo = NULL; + DataSinkHandle sinkHandle = NULL; + SReadHandle rHandle = {0}; + + QW_ERR_JRET(qwAddTaskCtx(QW_FPARAMS())); + QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT)); + + QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, NULL)); + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); + + ctx->taskType = qwMsg->msgInfo.taskType; + ctx->explain = qwMsg->msgInfo.explain; + ctx->needFetch = qwMsg->msgInfo.needFetch; + ctx->msgType = qwMsg->msgType; + ctx->localExec = true; + ctx->explainRes = explainRes; + + rHandle.pMsgCb = taosMemoryCalloc(1, sizeof(SMsgCb)); + rHandle.pMsgCb->clientRpc = qwMsg->connInfo.handle; + + code = qCreateExecTask(&rHandle, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, NULL, OPTR_EXEC_MODEL_BATCH); + if (code) { + QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); + QW_ERR_JRET(code); + } + + if (NULL == sinkHandle || NULL == pTaskInfo) { + QW_TASK_ELOG("create task result error, taskHandle:%p, sinkHandle:%p", pTaskInfo, sinkHandle); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + ctx->level = plan->level; + atomic_store_ptr(&ctx->taskHandle, pTaskInfo); + atomic_store_ptr(&ctx->sinkHandle, sinkHandle); + + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, NULL)); + +_return: + + taosMemoryFree(rHandle.pMsgCb); + + input.code = code; + input.msgType = qwMsg->msgType; + code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); + + if (ctx) { + QW_UPDATE_RSP_CODE(ctx, code); + qwReleaseTaskCtx(mgmt, ctx); + } + + QW_RET(code); +} + +int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId, void** pRsp, SArray* explainRes) { + SQWorker *mgmt = (SQWorker*)pMgmt; + int32_t code = 0; + int32_t dataLen = 0; + SQWTaskCtx *ctx = NULL; + void *rsp = NULL; + bool queryStop = false; + + SQWPhaseInput input = {0}; + + QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, NULL)); + + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + + ctx->msgType = TDMT_SCH_MERGE_FETCH; + ctx->explainRes = explainRes; + + SOutputData sOutput = {0}; + + while (true) { + QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); + + if (NULL == rsp) { + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryStop)); + + continue; + } else { + bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); + + qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); + if (qComplete) { + atomic_store_8((int8_t *)&ctx->queryEnd, true); + } + + break; + } + } + +_return: + + *pRsp = rsp; + + input.code = code; + code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_FETCH, &input, NULL); + + QW_RET(code); +} + + diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 1f76ea1e7e..60d6594c1b 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -877,7 +877,7 @@ TEST(seqTest, normalCase) { SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); + code = qWorkerInit(NODE_TYPE_VNODE, 1, &mgmt, &msgCb); ASSERT_EQ(code, 0); code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc, 0); @@ -913,7 +913,7 @@ TEST(seqTest, cancelFirst) { SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); + code = qWorkerInit(NODE_TYPE_VNODE, 1, &mgmt, &msgCb); ASSERT_EQ(code, 0); code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc, 0); @@ -950,7 +950,7 @@ TEST(seqTest, randCase) { SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); + code = qWorkerInit(NODE_TYPE_VNODE, 1, &mgmt, &msgCb); ASSERT_EQ(code, 0); int32_t t = 0; @@ -1021,7 +1021,7 @@ TEST(seqTest, multithreadRand) { SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); + code = qWorkerInit(NODE_TYPE_VNODE, 1, &mgmt, &msgCb); ASSERT_EQ(code, 0); TdThreadAttr thattr; @@ -1084,7 +1084,7 @@ TEST(rcTest, shortExecshortDelay) { SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); + code = qWorkerInit(NODE_TYPE_VNODE, 1, &mgmt, &msgCb); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 0; @@ -1168,7 +1168,7 @@ TEST(rcTest, longExecshortDelay) { SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); + code = qWorkerInit(NODE_TYPE_VNODE, 1, &mgmt, &msgCb); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 1000000; @@ -1254,7 +1254,7 @@ TEST(rcTest, shortExeclongDelay) { SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); + code = qWorkerInit(NODE_TYPE_VNODE, 1, &mgmt, &msgCb); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 0; @@ -1338,7 +1338,7 @@ TEST(rcTest, dropTest) { SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); + code = qWorkerInit(NODE_TYPE_VNODE, 1, &mgmt, &msgCb); ASSERT_EQ(code, 0); tsem_init(&qwtTestQuerySem, 0, 0); diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index 88180391dd..3288120b67 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -9,7 +9,7 @@ target_include_directories( target_link_libraries( scheduler - PUBLIC os util nodes planner qcom common catalog transport command + PUBLIC os util nodes planner qcom common catalog transport command qworker executor ) if(${BUILD_TEST}) diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index a62531a875..7ced4f626c 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -151,6 +151,7 @@ typedef struct SSchedulerMgmt { SSchStat stat; SRWLatch hbLock; SHashObj *hbConnections; + void *queryMgmt; } SSchedulerMgmt; typedef struct SSchCallbackParamHeader { @@ -235,8 +236,10 @@ typedef struct SSchTask { typedef struct SSchJobAttr { EExplainMode explainMode; bool queryJob; + bool insertJob; bool needFetch; bool needFlowCtrl; + bool localExec; } SSchJobAttr; typedef struct { @@ -263,7 +266,7 @@ typedef struct SSchJob { SHashObj *taskList; SHashObj *execTasks; // executing and executed tasks, key:taskid, value:SQueryTask* SHashObj *flowCtrl; // key is ep, element is SSchFlowControl - + SExplainCtx *explainCtx; int8_t status; SQueryNodeAddr resNode; @@ -304,6 +307,8 @@ extern SSchedulerMgmt schMgmt; #define SCH_IS_DATA_BIND_QRY_TASK(task) ((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) #define SCH_IS_DATA_BIND_TASK(task) (((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) || ((task)->plan->subplanType == SUBPLAN_TYPE_MODIFY)) #define SCH_IS_LEAF_TASK(_job, _task) (((_task)->level->level + 1) == (_job)->levelNum) +#define SCH_IS_DATA_MERGE_TASK(task) (!SCH_IS_DATA_BIND_TASK(task)) +#define SCH_IS_LOCAL_EXEC_TASK(_job, _task) ((_job)->attr.localExec && SCH_IS_QUERY_JOB(_job) && (!SCH_IS_INSERT_JOB(_job)) && (!SCH_IS_DATA_BIND_QRY_TASK(_task))) #define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st) #define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status) @@ -326,8 +331,9 @@ extern SSchedulerMgmt schMgmt; #define SCH_FETCH_TYPE(_pSrcTask) (SCH_IS_DATA_BIND_QRY_TASK(_pSrcTask) ? TDMT_SCH_FETCH : TDMT_SCH_MERGE_FETCH) #define SCH_TASK_NEED_FETCH(_task) ((_task)->plan->subplanType != SUBPLAN_TYPE_MODIFY) -#define SCH_SET_JOB_TYPE(_job, type) do { if ((type) != SUBPLAN_TYPE_MODIFY) { (_job)->attr.queryJob = true; } } while (0) +#define SCH_SET_JOB_TYPE(_job, type) do { if ((type) != SUBPLAN_TYPE_MODIFY) { (_job)->attr.queryJob = true; } else { (_job)->attr.insertJob = true; } } while (0) #define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob) +#define SCH_IS_INSERT_JOB(_job) ((_job)->attr.insertJob) #define SCH_JOB_NEED_FETCH(_job) ((_job)->attr.needFetch) #define SCH_JOB_NEED_WAIT(_job) (!SCH_IS_QUERY_JOB(_job)) #define SCH_JOB_NEED_DROP(_job) (SCH_IS_QUERY_JOB(_job)) @@ -502,6 +508,8 @@ void schDirectPostJobRes(SSchedulerReq* pReq, int32_t errCode); int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode); int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode); bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync); +int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rspCode); +int32_t schProcessExplainRsp(SSchJob *pJob, SSchTask *pTask, SExplainRsp *rsp); extern SSchDebug gSCHDebug; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 9880490594..5e47c0a0ed 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -720,6 +720,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { } pJob->attr.explainMode = pReq->pDag->explainInfo.mode; + pJob->attr.localExec = pReq->localReq; pJob->conn = *pReq->pConn; if (pReq->sql) { pJob->sql = strdup(pReq->sql); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index c6dc57d752..1b59c06140 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -72,6 +72,71 @@ int32_t schValidateRspMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { return TSDB_CODE_SUCCESS; } +int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rspCode) { + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; + int32_t code = 0; + + SCH_ERR_JRET(rspCode); + + if (NULL == msg) { + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (SCH_IS_EXPLAIN_JOB(pJob)) { + if (rsp->completed) { + SRetrieveTableRsp *pRsp = NULL; + SCH_ERR_JRET(qExecExplainEnd(pJob->explainCtx, &pRsp)); + if (pRsp) { + SCH_ERR_JRET(schProcessOnExplainDone(pJob, pTask, pRsp)); + } + + taosMemoryFreeClear(msg); + + return TSDB_CODE_SUCCESS; + } + + SCH_ERR_JRET(schLaunchFetchTask(pJob)); + + taosMemoryFreeClear(msg); + + return TSDB_CODE_SUCCESS; + } + + if (pJob->fetchRes) { + SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->fetchRes); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); + } + + atomic_store_ptr(&pJob->fetchRes, rsp); + atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows)); + + if (rsp->completed) { + SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); + } + + SCH_TASK_DLOG("got fetch rsp, rows:%d, complete:%d", htonl(rsp->numOfRows), rsp->completed); + + msg = NULL; + schProcessOnDataFetched(pJob); + +_return: + + taosMemoryFreeClear(msg); + + SCH_RET(code); +} + +int32_t schProcessExplainRsp(SSchJob *pJob, SSchTask *pTask, SExplainRsp *rsp) { + SRetrieveTableRsp *pRsp = NULL; + SCH_ERR_RET(qExplainUpdateExecInfo(pJob->explainCtx, rsp, pTask->plan->id.groupId, &pRsp)); + + if (pRsp) { + SCH_ERR_RET(schProcessOnExplainDone(pJob, pTask, pRsp)); + } + + return TSDB_CODE_SUCCESS; +} + // Note: no more task error processing, handled in function internal int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; @@ -301,65 +366,20 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SExplainRsp rsp = {0}; if (tDeserializeSExplainRsp(msg, msgSize, &rsp)) { - taosMemoryFree(rsp.subplanInfo); + tFreeSExplainRsp(&rsp); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SRetrieveTableRsp *pRsp = NULL; - SCH_ERR_JRET(qExplainUpdateExecInfo(pJob->explainCtx, &rsp, pTask->plan->id.groupId, &pRsp)); + SCH_ERR_JRET(schProcessExplainRsp(pJob, pTask, &rsp)); - if (pRsp) { - SCH_ERR_JRET(schProcessOnExplainDone(pJob, pTask, pRsp)); - } + taosMemoryFreeClear(msg); break; } case TDMT_SCH_FETCH_RSP: case TDMT_SCH_MERGE_FETCH_RSP: { - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; - - SCH_ERR_JRET(rspCode); - if (NULL == msg) { - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - - if (SCH_IS_EXPLAIN_JOB(pJob)) { - if (rsp->completed) { - SRetrieveTableRsp *pRsp = NULL; - SCH_ERR_JRET(qExecExplainEnd(pJob->explainCtx, &pRsp)); - if (pRsp) { - SCH_ERR_JRET(schProcessOnExplainDone(pJob, pTask, pRsp)); - } - - taosMemoryFreeClear(msg); - - return TSDB_CODE_SUCCESS; - } - - SCH_ERR_JRET(schLaunchFetchTask(pJob)); - - taosMemoryFreeClear(msg); - - return TSDB_CODE_SUCCESS; - } - - if (pJob->fetchRes) { - SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->fetchRes); - taosMemoryFreeClear(rsp); - SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); - } - - atomic_store_ptr(&pJob->fetchRes, rsp); - atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows)); - - if (rsp->completed) { - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); - } - - SCH_TASK_DLOG("got fetch rsp, rows:%d, complete:%d", htonl(rsp->numOfRows), rsp->completed); - + code = schProcessFetchRsp(pJob, pTask, msg, rspCode); msg = NULL; - - schProcessOnDataFetched(pJob); + SCH_ERR_JRET(code); break; } case TDMT_SCH_DROP_TASK_RSP: { diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 2cb227d84b..b6f96a188e 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -20,6 +20,7 @@ #include "tmsg.h" #include "tref.h" #include "trpc.h" +#include "qworker.h" #include "tglobal.h" void schFreeTask(SSchJob *pJob, SSchTask *pTask) { @@ -90,6 +91,10 @@ _return: } int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { + if (SCH_IS_LOCAL_EXEC_TASK(pJob, pTask)) { + return TSDB_CODE_SUCCESS; + } + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); if (NULL == addr) { SCH_TASK_ELOG("taosArrayGet candidate addr failed, idx:%d, size:%d", pTask->candidateIdx, @@ -230,7 +235,6 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) SCH_RET(errCode); } -// Note: no more task error processing, handled in function internal int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { bool moved = false; int32_t code = 0; @@ -295,6 +299,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { .execId = pTask->execId, .addr = pTask->succeedAddr, .fetchMsgType = SCH_FETCH_TYPE(pTask), + .localExec = SCH_IS_LOCAL_EXEC_TASK(pJob, pTask), }; qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source); SCH_UNLOCK(SCH_WRITE, &parent->planLock); @@ -825,6 +830,120 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) { return TSDB_CODE_SUCCESS; } +int32_t schHandleExplainRes(SArray *pExplainRes) { + int32_t code = 0; + int32_t resNum = taosArrayGetSize(pExplainRes); + if (resNum <= 0) { + goto _return; + } + + SSchTask *pTask = NULL; + SSchJob *pJob = NULL; + + for (int32_t i = 0; i < resNum; ++i) { + SExplainLocalRsp* localRsp = taosArrayGet(pExplainRes, i); + + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId); + + pJob = schAcquireJob(localRsp->rId); + if (NULL == pJob) { + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId, localRsp->tId, localRsp->rId); + SCH_ERR_JRET(TSDB_CODE_QRY_JOB_NOT_EXIST); + } + + int8_t status = 0; + if (schJobNeedToStop(pJob, &status)) { + SCH_TASK_DLOG("will not do further processing cause of job status %s", jobTaskStatusStr(status)); + schReleaseJob(pJob->refId); + SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); + } + + code = schGetTaskInJob(pJob, localRsp->tId, &pTask); + + if (TSDB_CODE_SUCCESS == code) { + code = schProcessExplainRsp(pJob, pTask, &localRsp->rsp); + } + + schReleaseJob(pJob->refId); + + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId, localRsp->tId, code); + + SCH_ERR_JRET(code); + + localRsp->rsp.numOfPlans = 0; + localRsp->rsp.subplanInfo = NULL; + pTask = NULL; + pJob = NULL; + } + +_return: + + for (int32_t i = 0; i < resNum; ++i) { + SExplainLocalRsp* localRsp = taosArrayGet(pExplainRes, i); + tFreeSExplainRsp(&localRsp->rsp); + } + + taosArrayDestroy(pExplainRes); + + SCH_RET(code); +} + +int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { + SSubplan *plan = pTask->plan; + int32_t code = 0; + + if (NULL == pTask->msg) { // TODO add more detailed reason for failure + code = qSubPlanToMsg(plan, &pTask->msg, &pTask->msgLen); + if (TSDB_CODE_SUCCESS != code) { + SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg, + pTask->msgLen); + SCH_ERR_RET(code); + } else if (tsQueryPlannerTrace) { + char *msg = NULL; + int32_t msgLen = 0; + qSubPlanToString(plan, &msg, &msgLen); + SCH_TASK_DLOGL("physical plan len:%d, %s", msgLen, msg); + taosMemoryFree(msg); + } + } + + SCH_ERR_RET(schSetTaskCandidateAddrs(pJob, pTask)); + + if (SCH_IS_QUERY_JOB(pJob)) { + SCH_ERR_RET(schEnsureHbConnection(pJob, pTask)); + } + + SCH_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); +} + +int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { + //SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); + if (NULL == schMgmt.queryMgmt) { + SCH_ERR_RET(qWorkerInit(NODE_TYPE_CLIENT, CLIENT_HANDLE, (void **)&schMgmt.queryMgmt, NULL)); + } + + SArray *explainRes = NULL; + SQWMsg qwMsg = {0}; + qwMsg.msgInfo.taskType = TASK_TYPE_TEMP; + qwMsg.msgInfo.explain = SCH_IS_EXPLAIN_JOB(pJob); + qwMsg.msgInfo.needFetch = SCH_TASK_NEED_FETCH(pTask); + qwMsg.msg = pTask->plan; + qwMsg.msgType = pTask->plan->msgType; + qwMsg.connInfo.handle = pJob->conn.pTrans; + + if (SCH_IS_EXPLAIN_JOB(pJob)) { + explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); + } + + SCH_ERR_RET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, pTask->execId, &qwMsg, explainRes)); + + if (SCH_IS_EXPLAIN_JOB(pJob)) { + SCH_ERR_RET(schHandleExplainRes(explainRes)); + } + + SCH_RET(schProcessOnTaskSuccess(pJob, pTask)); +} + int32_t schLaunchTaskImpl(void *param) { SSchTaskCtx *pCtx = (SSchTaskCtx *)param; SSchJob *pJob = schAcquireJob(pCtx->jobRid); @@ -848,7 +967,8 @@ int32_t schLaunchTaskImpl(void *param) { pTask->retryTimes++; pTask->waitRetry = false; - SCH_TASK_DLOG("start to launch task, execId %d, retry %d", pTask->execId, pTask->retryTimes); + SCH_TASK_DLOG("start to launch %s task, execId %d, retry %d", SCH_IS_LOCAL_EXEC_TASK(pJob, pTask) ? "LOCAL" : "REMOTE", + pTask->execId, pTask->retryTimes); SCH_LOG_TASK_START_TS(pTask); @@ -863,31 +983,12 @@ int32_t schLaunchTaskImpl(void *param) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXEC); } - SSubplan *plan = pTask->plan; - - if (NULL == pTask->msg) { // TODO add more detailed reason for failure - code = qSubPlanToMsg(plan, &pTask->msg, &pTask->msgLen); - if (TSDB_CODE_SUCCESS != code) { - SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg, - pTask->msgLen); - SCH_ERR_JRET(code); - } else if (tsQueryPlannerTrace) { - char *msg = NULL; - int32_t msgLen = 0; - qSubPlanToString(plan, &msg, &msgLen); - SCH_TASK_DLOGL("physical plan len:%d, %s", msgLen, msg); - taosMemoryFree(msg); - } + if (SCH_IS_LOCAL_EXEC_TASK(pJob, pTask)) { + SCH_ERR_JRET(schLaunchLocalTask(pJob, pTask)); + } else { + SCH_ERR_JRET(schLaunchRemoteTask(pJob, pTask)); } - SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); - - if (SCH_IS_QUERY_JOB(pJob)) { - SCH_ERR_JRET(schEnsureHbConnection(pJob, pTask)); - } - - SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); - _return: if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) { @@ -980,6 +1081,29 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { } } +int32_t schExecRemoteFetch(SSchJob *pJob, SSchTask *pTask) { + SCH_RET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, SCH_FETCH_TYPE(pJob->fetchTask))); +} + +int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { + void *pRsp = NULL; + SArray *explainRes = NULL; + + if (SCH_IS_EXPLAIN_JOB(pJob)) { + explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); + } + + SCH_ERR_RET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, pTask->execId, &pRsp, explainRes)); + + if (SCH_IS_EXPLAIN_JOB(pJob)) { + SCH_ERR_RET(schHandleExplainRes(explainRes)); + } + + SCH_ERR_RET(schProcessFetchRsp(pJob, pTask, pRsp, TSDB_CODE_SUCCESS)); + + return TSDB_CODE_SUCCESS; +} + // Note: no more error processing, handled in function internal int32_t schLaunchFetchTask(SSchJob *pJob) { int32_t code = 0; @@ -990,7 +1114,11 @@ int32_t schLaunchFetchTask(SSchJob *pJob) { return TSDB_CODE_SUCCESS; } - SCH_ERR_JRET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, SCH_FETCH_TYPE(pJob->fetchTask))); + if (SCH_IS_LOCAL_EXEC_TASK(pJob, pJob->fetchTask)) { + SCH_ERR_JRET(schExecLocalFetch(pJob, pJob->fetchTask)); + } else { + SCH_ERR_JRET(schExecRemoteFetch(pJob, pJob->fetchTask)); + } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index f1ec7f5e04..0ccaef3857 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -17,6 +17,7 @@ #include "schInt.h" #include "tmsg.h" #include "tref.h" +#include "qworker.h" SSchedulerMgmt schMgmt = { .jobRef = -1, @@ -192,4 +193,7 @@ void schedulerDestroy(void) { schMgmt.hbConnections = NULL; } SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); + + qWorkerDestroy(&schMgmt.queryMgmt); + schMgmt.queryMgmt = NULL; } diff --git a/source/libs/sync/CMakeLists.txt b/source/libs/sync/CMakeLists.txt index 551849c6f2..6025070cb7 100644 --- a/source/libs/sync/CMakeLists.txt +++ b/source/libs/sync/CMakeLists.txt @@ -15,6 +15,6 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -if(${BUILD_TEST}) +if(BUILD_TEST AND BUILD_SYNC_TEST) add_subdirectory(test) -endif(${BUILD_TEST}) +endif() diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 1559c85e23..c1d8dbef62 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -146,7 +146,7 @@ static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const if (nread < 0) { uError("http-report recv error:%s", uv_err_name(nread)); } else { - uTrace("http-report succ to recv %d bytes, just ignore it", nread); + uTrace("http-report succ to recv %d bytes", nread); } uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); } @@ -160,7 +160,16 @@ static void clientSentCb(uv_write_t* req, int32_t status) { } else { uTrace("http-report succ to send data"); } - uv_read_start((uv_stream_t*)&cli->tcp, clientAllocBuffCb, clientRecvCb); + status = uv_read_start((uv_stream_t*)&cli->tcp, clientAllocBuffCb, clientRecvCb); + if (status != 0) { + terrno = TAOS_SYSTEM_ERROR(status); + uError("http-report failed to recv data,reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); + if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { + uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); + } else { + destroyHttpClient(cli); + } + } } static void clientConnCb(uv_connect_t* req, int32_t status) { SHttpClient* cli = req->data; @@ -170,7 +179,16 @@ static void clientConnCb(uv_connect_t* req, int32_t status) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); return; } - uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->wbuf, 2, clientSentCb); + status = uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->wbuf, 2, clientSentCb); + if (0 != status) { + terrno = TAOS_SYSTEM_ERROR(status); + uError("http-report failed to send data,reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); + if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { + uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); + } else { + destroyHttpClient(cli); + } + } } static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) { @@ -220,13 +238,22 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 uv_tcp_init(loop, &cli->tcp); // set up timeout to avoid stuck; int32_t fd = taosCreateSocketWithTimeout(5); - uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); - int32_t ret = uv_tcp_connect(&cli->conn, &cli->tcp, (const struct sockaddr*)&dest, clientConnCb); + int ret = uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); if (ret != 0) { - uError("http-report failed to connect to server, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, cli->port); + uError("http-report failed to open socket, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, cli->port); destroyHttpClient(cli); uv_stop(loop); + terrno = TAOS_SYSTEM_ERROR(ret); + } else { + ret = uv_tcp_connect(&cli->conn, &cli->tcp, (const struct sockaddr*)&dest, clientConnCb); + if (ret != 0) { + uError("http-report failed to connect to http-server, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, + cli->port); + destroyHttpClient(cli); + uv_stop(loop); + terrno = TAOS_SYSTEM_ERROR(ret); + } } uv_run(loop, UV_RUN_DEFAULT); diff --git a/tests/system-test/99-TDcase/TD-19201.py b/tests/system-test/99-TDcase/TD-19201.py new file mode 100644 index 0000000000..26caa27815 --- /dev/null +++ b/tests/system-test/99-TDcase/TD-19201.py @@ -0,0 +1,797 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def create_tables(self): + tdSql.execute(f"drop database if exists table_numeric_max_min") + tdSql.execute(f"create database if not exists table_numeric_max_min keep 36500 replica 1") + tdSql.execute(f"use table_numeric_max_min") + tdSql.execute(f"use table_numeric_max_min") + tdSql.execute(f"create stable stable_1 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp)") + tdSql.execute(f"create stable stable_2 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp)") + tdSql.execute(f"create stable stable_null_data (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp)") + tdSql.execute(f"create stable stable_null_childtable (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp)") + tdSql.execute(f"create table stable_1_1 using stable_1 tags('stable_1_1', '1875819221' , '590058475305213213', '27794' , '-9' , 0 , 'binary1.RwUtFTPELUljKAhTomOd' , 'nchar1.jUSliorTipXBPWUFvgmL' , '70702407145428.500000', '3942240905.627770' ,'0')") + tdSql.execute(f"create table stable_1_2 using stable_1 tags('stable_1_2', '-1129886283' , '8365334906094716813', '-6150' , '57' , 1 , 'binary1.vTmIEcSbytCJAERpWCGd' , 'nchar1.voiUIweoRUDKNWYNjrBV' , '5437.641507', '77141629449.138901' , '1999-09-09 09:09:09.090')") + tdSql.execute(f"create table stable_1_3 using stable_1 tags('stable_1_3', '-2147483647' , '-9223372036854775807' , '-32767' , '-127' , false , 'binary3' , 'nchar3nchar3' , '-3.3' , '-33.33' , '2099-09-09 09:09:09.090')") + tdSql.execute(f"create table stable_1_4 using stable_1 tags('stable_1_4', '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' ,'0')") + tdSql.execute(f"create table stable_1_5 using stable_1 tags('stable_1_5', '1771737352' , '8519944309455480374', '-8770' , '-7' , 1 , 'binary1.SJUKdVsgafZagrqnYdTh' , 'nchar1.ONFhJIwhhLWJUtLKvKmw' , '31182473.161482', '64.321472' ,'-339837172')") + tdSql.execute(f"create table stable_1_6 using stable_1 tags('stable_1_6', '987461012' , '-776424343185148156', '-6733' , '-115' , 1 , 'binary1.YQTZHyzKNWJIZrDeOKdP' , 'nchar1.BxnvJVUAFklEphKHWIqP' , '60.387150', '385878886707.695007' ,'-1313802358')") + tdSql.execute(f"create table stable_2_1 using stable_2 tags('stable_2_1', '1972317427' , '-3820929692060830425', '18206' , '89' , 1 , 'binary2.InbFxJVQLmIseMeYntEH' , 'nchar2.wkGXcuYevFTHgydogliI' , '656538.907187', '11.667827' ,'0')") + tdSql.execute(f"create table stable_2_2 using stable_2 tags('stable_2_2' , '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' ,'0')") + tdSql.execute(f"create table stable_2_3 using stable_2 tags('stable_2_3', '121025327' , '-5067951734787165505', '-4541' , '39' , 1 , 'binary2.PWqqHvgkGRmSULvNblZM' , 'nchar2.BsSqoSHfdDzTWBIDTxIW' , '722578830492.845947', '-62936658111.557999' ,'1719325653')") + tdSql.execute(f"create table stable_2_4 using stable_2 tags('stable_2_4', '-1703445602' , '952583276175911157', '-15847' , '-1' , 1 , 'binary2.ZIRXiGPcWsDDYadmVHbC' , 'nchar2.CkMRgnmReMXbRfKiaHos' , '-2.806483', '620882751.119021' ,'1462034599')") + tdSql.execute(f"create table stable_2_5 using stable_2 tags('stable_2_5', '2022084941' , '-172782721714864542', '-31211' , '-13' , 1 , 'binary2.RgbEIzefrTuDfyRolGgQ' , 'nchar2.ahMsbNznUpydshGChFWR' , '9230.698315', '11569418.757060' ,'1789286591')") + tdSql.execute(f"create table stable_2_6 using stable_2 tags('stable_2_6', '2139131012' , '-2151050146076640638', '11205' , '98' , 1 , 'binary2.XdcwVjbqUuKEFWdeAOGK' , 'nchar2.adnShgPhNhoFUgVrzrIF' , '4896671.448837', '-663422197.440925' ,'-591925015')") + tdSql.execute(f"create table stable_null_data_1 using stable_null_data tags('stable_null_data_1', '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' ,'0')") + tdSql.execute(f"create table regular_table_1 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp)") + tdSql.execute(f"create table regular_table_2 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp)") + tdSql.execute(f"create table regular_table_3 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp)") + tdSql.execute(f"create table regular_table_null (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp)") + + return + + def insert_data(self): + tdLog.debug("start to insert data ............") + + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630000000000, -143708439, -6740796029571244151, 15012, -31, -934058894431.915039, -3695933.334631, 0, 'binary.wEvcyPTYhxxKhVFrShTZ', 'nchar.河南省成都市高明邱街y座 294108', 1630000000000)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630000000000, -1643794439, -3164709410897301333, -7208, -11, -1306531857188.310059, 632395.211968, 0, 'binary.jrpjGVDqTHJXDbUIxEir', 'nchar.青海省贵阳县牧野邯郸路U座 687168', 1630000000000)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1629999999999, 1048090051, 4753139546176318956, 31490, 65, -8328268324.680830, 74037727.447481, 1, 'binary.QFXcjYxRIgvVcaabbAuo', 'nchar.湖南省畅市璧山石家庄路O座 936681', 1630000000000)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630000000000, 1054267597, 332752484638053804, 1388, 112, -125.582297, -76003.954672, 1, 'binary.YNzAxeYGxNpFArjidcOh', 'nchar.江西省雪市六枝特深圳街b座 108494', 1630000000000)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630000000001, -115380468, -2294264221485096401, -20382, -90, -3775423439181.850098, -95439449.928655, 1, 'binary.ilLTZcPTGRSwIjACLGlM', 'nchar.台湾省瑜市平山李街D座 331780', 1630000000001)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630000000001, -504963560, -8159067702893064842, -5491, -49, 841872.351000, -84748281935263.906250, 1, 'binary.DtKESUYTjuakZGPtBWsn', 'nchar.山东省西宁县房山北镇街k座 852004', 1630000000001)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630000000000, 1962198354, 5726511532140217455, 20962, -124, -35534.621562, 655.911379, 0, 'binary.kczEOqXlmYDzQxVrxiri', 'nchar.天津市秀英市锡山成都街c座 934752', 1630000000000)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630015000000, -2044434228, -1228575710134602773, 23424, 71, 6.886844, 71.519941, 0, 'binary.iVmwRqGqAbGImLpoUiKX', 'nchar.山东省兰州市山亭卜路d座 110251', 1630000000001)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630015000000, -1561126129, 4402494144547686327, 19018, -31, 857260.740185, -750.240044, 0, 'binary.qPQkCflOVhWCAnuCCHWY', 'nchar.广东省秀荣市山亭南京路C座 785792', 1630000000001)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630014999999, 1095266727, 2746241631637453300, 7253, 92, -237902303804.821014, 46908.997377, 1, 'binary.NMQGxjbGhMVVbcQEBaEK', 'nchar.台湾省东莞市高坪南京街u座 150902', 1630000000001)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630015000000, 2130829317, 2085871782840505044, 2484, 30, -3.105125, 29879090388.666199, 1, 'binary.yfnegmvETJTxPrIrvxxe', 'nchar.福建省兴安盟县西夏澳门路p座 460995', 1630000000001)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630015000001, -1664825220, -5631789961756669368, -23059, -76, 8441.841532, -6705.596239, 1, 'binary.LZlUlCryyKuhhwGGlSjc', 'nchar.四川省巢湖县长寿西宁街O座 989989', 1630000000002)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630015000001, -1067652799, -5826504339509635297, -390, -15, -66135776826527.101562, -8198121301.465600, 1, 'binary.bppXThGxTXGyhKdEpRgX', 'nchar.四川省桂芳县长寿杭州路M座 546474', 1630000000002)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630015000000, 1596913082, 1646557549281147083, -14492, -72, 33.692574, 72141847615.397003, 0, 'binary.XxVywbRiXEabPDUoMgxT', 'nchar.重庆市凤兰市城北呼和浩特街O座 959006', 1630000000001)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630030000000, 1109067209, 8318147433525574130, 291, 81, 308262.371110, 3017294.523108, 0, 'binary.mJPDrXToNeEldaahHrjD', 'nchar.新疆维吾尔自治区广州市清河沈阳街F座 287894', 1630000000002)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630030000000, -995915290, 3161170011949889743, -3415, -62, -5111655245631.230469, -7257779.688047, 0, 'binary.vzcGfGPzmKOqwWlstVmQ', 'nchar.河北省海燕县怀柔齐齐哈尔路O座 679791', 1630000000002)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630029999999, 1270783858, 3618027349082548235, 25887, 84, -659813588408.829956, -32621042966661.398438, 1, 'binary.GqXavHpdBwhbCDrQnpSF', 'nchar.湖北省永安市金平通辽路o座 143728', 1630000000002)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630030000000, 1739749630, 1610294704102234156, 18492, 117, -24323056593048.300781, 46.116231, 1, 'binary.hzRmZmuYAUsAdQrSEyFm', 'nchar.贵州省杰市南长王街Q座 377635', 1630000000002)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630030000001, -1718628594, -4057665590789803346, -27617, -32, -84372491943.149796, 5698.767553, 1, 'binary.wPOmeglMuNsatgqVOYvh', 'nchar.新疆维吾尔自治区深圳市长寿梁路l座 415101', 1630000000003)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630030000001, -97007549, -1557528468040059015, -6101, -18, -34048.334225, -1657.736116, 1, 'binary.eDYjVVXEPBpiilaVwOrx', 'nchar.贵州省成市六枝特傅街t座 803260', 1630000000003)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630030000000, -2121650398, -1192567467640899873, 27738, 2, -89612476.306992, -5984440.654033, 0, 'binary.BOOyBOTaGGGWppOLcCbQ', 'nchar.天津市峰县朝阳梁街X座 288727', 1630000000002)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630045000000, 1148326681, -6925816541216354608, -31227, 54, 222564.451289, 8431.504529, 0, 'binary.NHZMOcXRwhSPvmIIxdxp', 'nchar.台湾省台北市魏都路街Y座 785205', 1630000000003)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630045000000, 544052556, 66604009301449217, 21345, 113, 765168602.280518, -97989380.717398, 0, 'binary.CwzIKVOzfevwyQikTtty', 'nchar.江苏省郑州县新城骆街E座 897275', 1630000000003)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630044999999, 781814072, 820039380718257377, 16412, 8, 13534073596162.000000, 84710586503.486099, 1, 'binary.LBBuZbsGbCPwrtpgpJUU', 'nchar.西藏自治区西宁县牧野孙路U座 873211', 1630000000003)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630045000000, 2133537352, 8495070517087119217, 650, 4, -99578297.552409, 635328734398.352051, 1, 'binary.NfQiEIznVNaygWdjgrpf', 'nchar.山西省阜新县锡山台北路j座 854425', 1630000000003)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630045000001, -222510400, -1899555459266145413, -1874, -18, -116.302821, -2969179.976125, 1, 'binary.BgnCkJAICiNgxqiaowKU', 'nchar.上海市淑珍市城东上海路v座 534884', 1630000000004)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630045000001, -995678859, -7818673459856301613, -15799, -53, 4590.410976, -289305504318.122009, 1, 'binary.jzxySgQRnfthYtrsClss', 'nchar.香港特别行政区洁市东丽白路X座 942700', 1630000000004)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630045000000, 2039518409, -7951010656620124852, 26555, 97, 317885662.762503, -15671218405798.900391, 0, 'binary.PpJrRRqrlVkrvHDFhGfc', 'nchar.广东省长春县沈河银川街p座 226735', 1630000000003)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630060000000, 723117353, -1590889099228121365, -2895, -118, -9424.662668, 106.531721, 0, 'binary.TublFbdwZUkjDXBnzoCS', 'nchar.内蒙古自治区成都市淄川梁路O座 490394', 1630000000004)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630060000000, -1943729504, 4725330849524291079, 19350, 23, -72996737.774954, 99.647626, 0, 'binary.NaxwyUiYqMDJixXFkrwl', 'nchar.青海省佛山县浔阳辽阳路i座 673884', 1630000000004)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630059999999, 260526220, 1293392929073191769, 195, 84, -889913.874227, -86079294.573690, 1, 'binary.uSifMBsrTaQhLuYRzSNB', 'nchar.河北省台北市梁平石家庄路p座 882177', 1630000000004)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630060000000, 1100804375, 8343686791181684001, 11516, 82, -898520.749607, 8.704132, 1, 'binary.mlUobRUmHCNOOqCgXLAC', 'nchar.上海市潮州县魏都永安街n座 239177', 1630000000004)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630060000001, -1734497021, -6543273238001990224, -9471, -105, 5.215463, -8.406843, 1, 'binary.UnQSdMFNesxovxaQqywB', 'nchar.河北省兰英市高明西安街O座 898407', 1630000000005)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630060000001, -356952133, -5811125050930035045, -10106, -86, -59731.343942, -6.711605, 1, 'binary.BucKdpqIcWfBATWFIUwQ', 'nchar.台湾省梅市华龙葛路f座 841443', 1630000000005)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630060000000, -1448303077, 7581281332610752593, -28597, 65, 42652369.542399, 19027309795.674400, 0, 'binary.kNqlJbVupSKDoutNSkZj', 'nchar.内蒙古自治区梅市龙潭郑街P座 821111', 1630000000004)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630075000000, -1735114764, -910694996111564041, 17973, 73, 210.783307, 78353441914751.406250, 0, 'binary.TCDlFpIyMpXfcghfWlUe', 'nchar.湖南省惠州市浔阳章街y座 769412', 1630000000005)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630075000000, -1569047795, 6154014930112626762, -23107, 71, 7415.321612, -333.790673, 0, 'binary.cQDlPOzuViCfYAjUdgwt', 'nchar.吉林省婷婷市西夏西宁路m座 909021', 1630000000005)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630074999999, 856956546, 6275962241273936020, 9270, 21, -33.848277, 64751237042062.703125, 1, 'binary.ctAKBcRCfQQDgayszuAG', 'nchar.江苏省宁市清河李路g座 724944', 1630000000005)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630075000000, 1448326946, 7972199845302312059, 705, 105, -204.655811, 732152.198035, 1, 'binary.BcZdTPFgPbkxBUcFljtp', 'nchar.宁夏回族自治区峰县沈北新长春街L座 533383', 1630000000005)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630075000001, -2035071163, -7787122277706108419, -7254, -103, 5674.411412, 577.478622, 1, 'binary.evhulVVFzQzmcxSxrhKQ', 'nchar.内蒙古自治区亮市合川长沙路n座 765501', 1630000000006)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630075000001, -306773260, -7374114523451218038, -9388, -82, -62404928835266.203125, 42386299338.397598, 1, 'binary.esbdUcokubRSnOqkJkLB', 'nchar.山东省琴市上街马鞍山路C座 147584', 1630000000006)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630075000000, -400440831, 3437601667225177538, 20276, -39, -16726360314300.500000, -4598873.151255, 0, 'binary.OkLYkfrvFAWiFDpqiaxV', 'nchar.广东省佛山市璧山孟街L座 706570', 1630000000005)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630090000000, 1973623075, -488129617113651385, -13853, 30, 75272169.904987, 27857.549440, 0, 'binary.xkRVpVawmLMQlKCdvZtZ', 'nchar.江西省磊市怀柔太原路K座 851187', 1630000000006)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630090000000, -252137090, -5296831169315465834, -20550, -20, -3.941531, -75700534.472326, 0, 'binary.BpKasTBDYsehOyjPzszU', 'nchar.上海市红市西峰张路E座 415889', 1630000000006)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630089999999, 2024548336, 4139843660031507838, 19827, 13, -2167586.132392, -25417449654.380001, 1, 'binary.HAdhzWKQINYLaYkXNWMA', 'nchar.河南省汕尾市沙市张街w座 551925', 1630000000006)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630090000000, 2107913968, 1289804881586351882, 11892, 64, 5317489029302.500000, -87057162.779656, 1, 'binary.OgWLsotYlsjEVMRdLBuE', 'nchar.安徽省健县西峰太原路Q座 657330', 1630000000006)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630090000001, -1388002413, -8410442498260521376, -32608, -106, 253633520667.924011, 189217.183223, 1, 'binary.uYfWFNKpQafMupBPxsaQ', 'nchar.重庆市宜都县闵行简街i座 191919', 1630000000007)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630090000001, -1299395984, -5371389022375922971, -10699, -3, -77152176732086.500000, 3022372632154.790039, 1, 'binary.suubVqjYFZLbSTtupfJz', 'nchar.上海市桂珍市沈河重庆街m座 297301', 1630000000007)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630090000000, -2027322440, 2494209288710251536, 9722, 36, -5920.224246, -6082294200418.269531, 0, 'binary.AdrnDYFrIaLanobBNtfe', 'nchar.青海省邯郸市新城潮州街n座 431311', 1630000000006)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630105000000, 1693347900, 3148347996562430560, 7984, -125, 851.227689, 36582.261941, 0, 'binary.CRmkBadMfWkcImouLOYC', 'nchar.陕西省澳门市沙湾海口街B座 465135', 1630000000007)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630105000000, -993042639, -3563434487593064918, 25565, 92, 35189008.309030, 636.547745, 0, 'binary.OoULtmPoUYrnyIRmOomq', 'nchar.内蒙古自治区雷县涪城辽阳路M座 278853', 1630000000007)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630104999999, 896614799, 2243073176348662224, 28908, 61, -27224979.660738, -774.713640, 1, 'binary.eLMnpehchKlvWYooNwzT', 'nchar.山西省六盘水县兴山周路E座 606601', 1630000000007)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630105000000, 814428057, 1481917516738425350, 15453, 4, 48966471575006.898438, 963385.769898, 1, 'binary.ihMiqNMlQJJZzmmqQCxd', 'nchar.安徽省南昌县翔安济南路m座 375964', 1630000000007)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630105000001, -449300050, -9120117477577024283, -32402, -33, 10292695658.142700, -34437.176958, 1, 'binary.qKdRIooLiEPovATVrIjv', 'nchar.贵州省淑华县兴山西宁路l座 608705', 1630000000008)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630105000001, -432206150, -2588236186691485163, -23087, -15, 400818.639211, -4.716588, 1, 'binary.eZOuaAtkYBpffobznIIg', 'nchar.湖北省阜新县高港西安街U座 699140', 1630000000008)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630105000000, -1368554069, -7541187960916300774, 14988, -91, 1942225715.555290, -70334.201426, 0, 'binary.HvoKMzXkvOzwerEjbwEu', 'nchar.甘肃省佛山县丰都阜新路h座 248791', 1630000000007)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630120000000, 1041114692, -4272739428405588701, -2465, 88, 21784769476439.500000, -68577664683.573502, 0, 'binary.FplIhrrnZDEqcPpMXcCz', 'nchar.广东省亮市清河嘉禾路Q座 453757', 1630000000008)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630120000000, 176715584, -8079534335031834008, -22423, -105, 2684870082.726260, -8.267891, 0, 'binary.xAOmXPYnrbGdHxHqywad', 'nchar.甘肃省荆门市清浦马鞍山路r座 410741', 1630000000008)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630119999999, 640308875, 3240059255473088756, 31836, 80, -7728815076.892700, 750986226173.490967, 1, 'binary.OzGnekdKxbZxbKgGFJlg', 'nchar.澳门特别行政区台北市白云淮安路B座 375090', 1630000000008)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630120000000, 1092638007, 8935528528767608896, 4337, 23, -372778.775792, 93085.883825, 1, 'binary.vZySFqduHRgMRgGDCHMO', 'nchar.青海省旭县山亭陈路r座 820766', 1630000000008)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630120000001, -1961834598, -4976614618620345763, -15019, -7, -181775367037.890015, -489008.481333, 1, 'binary.FbQxRmecVULCPbyUFBtd', 'nchar.河北省志强县浔阳福州街G座 429013', 1630000000009)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630120000001, -1722034194, -2355585911534022326, -4297, -50, -55.449413, -615331713611.660034, 1, 'binary.JELgidqLYfIKlQGqJxFQ', 'nchar.河南省柳州县花溪田路q座 284391', 1630000000009)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630120000000, -1171799899, 6798705239630104147, 14631, 13, -690512100.797553, 4942216164.922990, 0, 'binary.JZyfPEyhblizmAILYppl', 'nchar.广东省玉英县孝南沈阳街Y座 637681', 1630000000008)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630135000000, -1384036824, 218630290551636514, 30524, -57, -887.357597, 6.720203, 0, 'binary.NpfAGysaedqwHzyjOEhb', 'nchar.澳门特别行政区六盘水市和平长沙街x座 821514', 1630000000009)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630135000000, -1305588265, -8021530421135445279, -32118, -94, -3231868745378.140137, -3384463304.343490, 0, 'binary.yAnQLRdPTIoYFmazSvtN', 'nchar.贵州省宜都县海港童路E座 199160', 1630000000009)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630134999999, 152017229, 5533762214494486905, 29777, 107, -597881444.549760, 60.902207, 1, 'binary.CoERNXcHahySItezBpen', 'nchar.云南省北京县海陵张路A座 568099', 1630000000009)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630135000000, 1236190014, 4653596223295984266, 14748, 21, -5951588996860.620117, 2234250837.617470, 1, 'binary.WwUGpNKIpNfbqTYAZTDT', 'nchar.广东省济南市长寿林街l座 798089', 1630000000009)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630135000001, -873242705, -7271514029553280325, -7516, -61, 66128079.834318, 77571831.128020, 1, 'binary.LLhLPAXTmmXelsILHdQZ', 'nchar.西藏自治区瑜县长寿天津路t座 566335', 1630000000010)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630135000001, -2107043391, -6872223014761334604, -253, -30, -2052472.350812, -816315756.554040, 1, 'binary.wvSnelJAPwZethZNEZqM', 'nchar.上海市兴安盟县徐汇陈街L座 910420', 1630000000010)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630135000000, -283811418, -890001232755261528, -15727, -29, 632302961.122940, 3372672571.351960, 0, 'binary.DNpxbprlJeuvywljvLWV', 'nchar.黑龙江省畅县大东佛山街o座 623470', 1630000000009)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630150000000, 1841425030, 5840114857227443220, 15419, 20, -176685681169.731995, -20366.126795, 0, 'binary.wpRXidgNPOERhEaZlJAM', 'nchar.青海省帅市平山梧州路s座 315192', 1630000000010)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630150000000, 1868258378, -829467537608086421, -15673, 43, -682501443.992492, 564.308967, 0, 'binary.PThYCRdfuhudrwOYOFgy', 'nchar.重庆市欣县丰都唐路B座 999593', 1630000000010)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630149999999, 7371013, 8606554902691526904, 11354, 74, 26482.297818, -6880189394.673160, 1, 'binary.OtbSjvYZOPWbIQOdIrKG', 'nchar.福建省波县璧山嘉禾路X座 190183', 1630000000010)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630150000000, 1574294578, 6360685624760963951, 11117, 71, -272.941943, 29.887578, 1, 'binary.djhaZVhHOWsrnxbiHVTk', 'nchar.西藏自治区南昌县清河拉萨街w座 620924', 1630000000010)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630150000001, -259884599, -3561655319278167435, -1554, -47, -0.233381, 45265486282274.000000, 1, 'binary.ILwvbvFGWLKDelTSCcDm', 'nchar.河北省哈尔滨市海港梧州路m座 889188', 1630000000011)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630150000001, -1515650476, -7802732131723416636, -12941, -64, -809.874644, 6032.826263, 1, 'binary.tAYvdysPqcwaHdGoSNgs', 'nchar.上海市玉兰市高明宁德街v座 296568', 1630000000011)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630150000000, 1110286890, -8373744956439073351, 15041, -53, 4069186274.937290, 2.194559, 0, 'binary.auztDxkjzltBpZNEImEj', 'nchar.贵州省平市璧山乐路a座 747181', 1630000000010)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630165000000, -270068972, 7039763496469502050, -17618, 23, -975020808834.901001, 20433408805144.398438, 0, 'binary.FNcyuyrPVVVfxbzDuGlf', 'nchar.宁夏回族自治区平市白云沈阳街Q座 933408', 1630000000011)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630165000000, 1508766963, -2166535776743943172, -8241, 42, 2.785578, 579582.453552, 0, 'binary.ManjhxWWPYfMDOZgGgDH', 'nchar.宁夏回族自治区石家庄市永川徐路s座 143325', 1630000000011)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630164999999, 1060176497, 5654208159595485509, 24751, 76, -33188910.345760, 3221.783209, 1, 'binary.cZSBdkTCfHzKdaUbRObM', 'nchar.河南省磊市友好方路y座 897409', 1630000000011)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630165000000, 497425977, 354065118101195712, 31551, 8, -615489296.966200, -71357728420.250000, 1, 'binary.yxlWGAHHdFrrqtbkxmms', 'nchar.西藏自治区惠州县安次汕尾街f座 271896', 1630000000011)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630165000001, -1895192712, -4432843295407575817, -20962, -98, -233660222.154913, 56771044.886008, 1, 'binary.CYsQdbzmYnNmadYZeaGH', 'nchar.重庆市阜新县永川惠州街l座 567817', 1630000000012)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630165000001, -716978435, -3103914421806026937, -21749, -50, 700892864.685490, 48303403313.167900, 1, 'binary.BWgPkqjghxVKeRoXrTZu', 'nchar.广西壮族自治区拉萨县萧山海门路k座 599659', 1630000000012)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630165000000, -1947418103, -2239330846377726020, -32039, -98, 56444198375439.101562, 1600824326231.459961, 0, 'binary.MDSyTDtKwgaMZDQPRBnU', 'nchar.宁夏回族自治区广州市蓟州马鞍山街C座 728186', 1630000000011)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630180000000, 418330248, -5409102699471857283, -15785, -3, 6935.516881, 35832047684.500603, 0, 'binary.KevLpYjpMOfugEsBCBlm', 'nchar.广东省宜都市蓟州周街Y座 950736', 1630000000012)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630180000000, 1176493205, 2459630938091775425, -16447, -108, -2248206343057.399902, -6078285.169543, 0, 'binary.hMRUCjOJuPOsbiZnpUAp', 'nchar.台湾省北镇市海港西安街Y座 510871', 1630000000012)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630179999999, 778869650, 849789444462698421, 10597, 31, 7103017.633025, 176806.190116, 1, 'binary.HYiRCyeEqGzFGwoHupCH', 'nchar.上海市杰县安次成都街f座 504387', 1630000000012)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630180000000, 976603072, 3464432716810727627, 31947, 3, -25689733.219437, 8778.999062, 1, 'binary.oIPQmjoWcexSWGXgBAoz', 'nchar.江苏省丽华县闵行广州路s座 381177', 1630000000012)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630180000001, -1070149378, -5440223162019446936, -3467, -31, 39057698787674.796875, -90.367916, 1, 'binary.VZKQmmtwMfYTGzlJOyzv', 'nchar.新疆维吾尔自治区丽丽县高坪乌鲁木齐路P座 898975', 1630000000013)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630180000001, -34074056, -4597641513205145794, -21694, -46, -1647875.388086, 944448733008.215942, 1, 'binary.tJCYGQwLiehXnYdGKEAX', 'nchar.广西壮族自治区巢湖县翔安周街s座 313835', 1630000000013)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630180000000, 1604229550, 1136238301593252013, -13346, 79, 44.613233, -20420040255472.898438, 0, 'binary.dTQiFvIdRuhkakGujxGq', 'nchar.广西壮族自治区冬梅市秀英张路h座 755432', 1630000000012)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630195000000, -1261626278, 8850592003715310781, -3464, -28, -7417877514167.179688, -337072829176.359009, 0, 'binary.DmaGJbQkLmJQaURuaowv', 'nchar.天津市梧州市友好韩路r座 509518', 1630000000013)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630195000000, 779648104, 7349237640085656137, 6427, -18, -8212845456.626070, -382.979362, 0, 'binary.CTvjwpLAPIpvFzGoTwwo', 'nchar.陕西省雷县房山济南街v座 822721', 1630000000013)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630194999999, 2139522991, 329464564820839576, 24696, 37, 29057737.124244, 207289.798705, 1, 'binary.WgIELLWDqNqWvdsblKCB', 'nchar.新疆维吾尔自治区马鞍山县孝南天津路c座 644425', 1630000000013)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630195000000, 1838525411, 7428851921488225564, 29432, 50, -682610786103.390015, -10041.651604, 1, 'binary.KxfGzIGCDurNlCmGKYcc', 'nchar.江苏省建军县淄川唐路A座 669136', 1630000000013)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630195000001, -1450415363, -2829926279480742311, -27438, -88, -9583575812.348600, 273024623.630804, 1, 'binary.THXDnYCLdvuTXRSUsMnH', 'nchar.河北省丽娟县吉区陈街o座 998303', 1630000000014)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630195000001, -886751600, -2534482963926108103, -10707, -39, -617571391893.729980, -67.220665, 1, 'binary.hTLReGUfQIJXEaSINBnV', 'nchar.河南省海口县涪城古路p座 921643', 1630000000014)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630195000000, 1570287998, -344238082251883349, -7319, 81, 90.287960, 88600160.145757, 0, 'binary.dtVTWjArbQtxafOTUJTM', 'nchar.云南省北镇县丰都张路W座 898660', 1630000000013)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630210000000, 665794589, 6004756980835485088, 2092, -123, -627.945609, -7528668680465.099609, 0, 'binary.IKfpbOnsEUqZvDUDXwxE', 'nchar.云南省深圳县高明黄街I座 754805', 1630000000014)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630210000000, 1588394658, -8959712112812621693, 2206, -36, -441691927006.486023, 646726.359195, 0, 'binary.aBvZFciAllhXZWLUFnpr', 'nchar.甘肃省建国县房山李路y座 514187', 1630000000014)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630209999999, 1202083307, 5412703541265854339, 12679, 43, -192710.317268, -669700577.626661, 1, 'binary.jzdrBvoIOBydYtlFDBfZ', 'nchar.吉林省淮安市南湖西安街g座 120691', 1630000000014)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630210000000, 593595969, 819751030048983732, 20912, 107, 9850514448.774269, -5078164092007.000000, 1, 'binary.bJypQCiflteoujptLzMI', 'nchar.湖北省广州县大东华路Q座 390816', 1630000000014)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630210000001, -808169724, -297187932768772011, -24985, -112, -407819568.753060, -79910028504558.000000, 1, 'binary.sUZnozuiMlgMQjgaKLWT', 'nchar.辽宁省太原市南湖刘街o座 982705', 1630000000015)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630210000001, -671630434, -2446424003708709538, -6161, -89, -894616811978.269043, -9662906448.356800, 1, 'binary.xabYsCWCjPMPlKTSVUhg', 'nchar.河北省婷市长寿成都街b座 907850', 1630000000015)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630210000000, 382558478, 5220322206204946702, -11796, -81, 300102736.783901, -1.980706, 0, 'binary.armYvyPdgPaDoNbFvsmU', 'nchar.海南省丹丹市永川戴街g座 757366', 1630000000014)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630225000000, 1556546304, 2632455039606144331, 12588, -78, -9911986.409861, 229879157290.842987, 0, 'binary.GKOAUmTHRsUYNGpJuJXS', 'nchar.西藏自治区浩市龙潭东莞街H座 896444', 1630000000015)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630225000000, -412103084, 7651422716161914550, -8997, -43, -7588824690.400900, 32094429957.228199, 0, 'binary.vHMehvIzcpBBVNWMluzW', 'nchar.安徽省重庆县朝阳兴城街o座 531109', 1630000000015)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630224999999, 202190259, 6158110401345061186, 25612, 56, 99429428549280.703125, 4845580.749217, 1, 'binary.QFvJnTjavOlzsXAEZVwH', 'nchar.重庆市彬县沈河通辽街r座 998791', 1630000000015)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630225000000, 1577801807, 7099278354328143832, 21437, 127, 13218371734602.199219, 161023981083.549988, 1, 'binary.wcmkWTSCZNJlHhnahEQx', 'nchar.广东省彬县永川顾路M座 592892', 1630000000015)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630225000001, -409805763, -6445166879074268180, -22964, -55, -228981.319085, 7949825.592390, 1, 'binary.sbSiqKmjPKvnXbSeKwXG', 'nchar.湖南省沈阳市安次李街u座 919293', 1630000000016)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630225000001, -1355452771, -6578771374830832013, -11963, -31, 7.979511, -2.355683, 1, 'binary.xkUAxXHQaMRjzWXGozUi', 'nchar.黑龙江省莹县龙潭刘街C座 459661', 1630000000016)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630225000000, 651625961, 351888809599313432, 19819, 10, 7236673390283.780273, -9467952650.400270, 0, 'binary.JHSbZaIRcqbSjVqzeLeL', 'nchar.河北省合肥市清河曹街a座 227339', 1630000000015)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630240000000, -635394652, -8268124534001451281, -4907, 93, -64765076507.232002, 8584607.867581, 0, 'binary.UPKoftKIqXYBGkgijulk', 'nchar.天津市永安县怀柔合山街K座 967698', 1630000000016)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630240000000, -213770053, 5635964652235949546, 25274, -124, -5.607006, 535.380902, 0, 'binary.wDdznQCwUKDSARatEpet', 'nchar.四川省北京市翔安佛山街I座 729861', 1630000000016)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630239999999, 238657263, 8869424583833435507, 27884, 75, 7565809000.422980, -416084252676.710022, 1, 'binary.OgrrbsocdxzSAsMKmGFW', 'nchar.辽宁省凤英市魏都李路J座 525191', 1630000000016)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630240000000, 1210124155, 7042734097723621808, 10874, 73, 86.768567, 6463014968.922540, 1, 'binary.oVXdkvYFywXquaOlhtXC', 'nchar.山西省兰州县花溪齐齐哈尔街s座 800225', 1630000000016)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630240000001, -1447960279, -635780358358789864, -5330, -51, -91432.263672, 2275481108331.560059, 1, 'binary.gMIOOqmPvHzbiBiPEVsv', 'nchar.香港特别行政区波市崇文惠州街C座 328563', 1630000000017)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630240000001, -1695106558, -3628563110892589597, -22424, -5, 5.150266, -77282937.851303, 1, 'binary.xJkaFVdbXowwlHYQZaUe', 'nchar.安徽省杭州市西峰邯郸路L座 436694', 1630000000017)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630240000000, 413277553, -2335706071365382890, 32108, -32, 4843.206092, 46356442353609.296875, 0, 'binary.CoOYGsXiwrFxtoGItDBS', 'nchar.广东省太原市崇文石家庄街q座 321264', 1630000000016)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630255000000, 1654794184, -8374474014569215734, -31917, -15, -938770678.951200, -6731352698.717200, 0, 'binary.QHlEvczXvDhrjeXKechh', 'nchar.广西壮族自治区杭州市城北合肥街Q座 560934', 1630000000017)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630255000000, 1566108371, -3808480661071967050, -30863, 42, -89576902.121811, 78224.319157, 0, 'binary.coyrtmcCFEMDoUzvhCay', 'nchar.江西省芳市涪城银川街b座 311496', 1630000000017)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630254999999, 954795210, 5603814446969530884, 969, 18, -5490774.644756, 178.908991, 1, 'binary.VOdZRucmuTuXzTGKuWfA', 'nchar.四川省宁县城东李路i座 842257', 1630000000017)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630255000000, 1152370442, 5925336630497472314, 12541, 125, -719570559.985753, -8.257668, 1, 'binary.huKmkwvJTPCAFDoXOBlW', 'nchar.安徽省柳州县安次昆明路i座 788628', 1630000000017)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630255000001, -185956520, -4103946855252239353, -25507, -108, -7.426104, 4211603.554600, 1, 'binary.LsRcBtiQqicggCLXZoWe', 'nchar.福建省玉华县滨城郭路B座 383017', 1630000000018)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630255000001, -1921474054, -6233300582184415910, -708, -43, -311807806.567927, -658203860771.300049, 1, 'binary.fZeaeMgVywkMCcqxATSP', 'nchar.陕西省杨市沙市吕路y座 145823', 1630000000018)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630255000000, 1570551972, 3518470870097214499, 31701, 67, -937555.256387, 928225.166342, 0, 'binary.JHXTHKFgRbIHElDZDhBC', 'nchar.云南省娜县合川钟街I座 842962', 1630000000017)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630270000000, 41007230, -53608277352743061, 27336, 106, -147300.927027, -84382353.649139, 0, 'binary.wuOctyBeTsCZynkMWYzu', 'nchar.辽宁省惠州市牧野刘路u座 359412', 1630000000018)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630270000000, -781808333, 579199728109057365, -6090, 83, 77577783.578131, -3329626275.966800, 0, 'binary.jVblcXbShCimKsOQPpIm', 'nchar.江西省太原市兴山辛集街G座 996182', 1630000000018)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630269999999, 1365142146, 4845996538283657204, 32039, 114, 75890.665555, 5702245.657774, 1, 'binary.EnWxuBUlakKcHrXrPBOj', 'nchar.重庆市荣县锡山蒋街H座 306410', 1630000000018)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630270000000, 333444667, 7150058468897111437, 14463, 56, 8105784073132.780273, 2424186854.253130, 1, 'binary.hWWfvRLXRhrTBXsFKBhN', 'nchar.海南省广州县长寿贵阳路i座 445153', 1630000000018)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630270000001, -210822579, -5173305434078560312, -32628, -58, -605574731.487583, 8.466106, 1, 'binary.ZKvMAWaDuBEInoXJgQSL', 'nchar.湖北省玲县清城杨街y座 553175', 1630000000019)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630270000001, -99120821, -6443231186326613201, -4035, -59, 967868262277.944946, -3544.231286, 1, 'binary.QBaMOSBKnDsAwBeYphtn', 'nchar.澳门特别行政区春梅县黄浦西安路O座 267015', 1630000000019)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630270000000, 1289868664, -3904737247840537443, -9207, 24, 87052030.454020, -5467838.797992, 0, 'binary.QtwNbVNKonvfueApUYkW', 'nchar.海南省强市花溪太原路L座 680108', 1630000000018)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630285000000, 1387304128, 4228361435380026760, -1238, -38, 819.343099, 3269274838.864710, 0, 'binary.QwtMztxuEtuQdccEPSgc', 'nchar.河北省沈阳县沈北新巢湖街F座 514745', 1630000000019)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630285000000, 1400828058, 7402159199902680827, 4622, -81, 83124160209331.093750, -597482.723308, 0, 'binary.GkCjoKlnQLMdOZQfOaRe', 'nchar.黑龙江省海口市高港北镇路g座 490967', 1630000000019)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630284999999, 952122349, 4226769910009513793, 1618, 46, -82338299659233.500000, -49022614631460.601562, 1, 'binary.lcXghoPWtfUTfSAAJDPS', 'nchar.湖北省巢湖市兴山关街E座 864057', 1630000000019)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630285000000, 749718159, 3908265828578345378, 4939, 2, -3187143413.393930, -63.396907, 1, 'binary.JgknllAiyOFQYSPvLBrT', 'nchar.广西壮族自治区齐齐哈尔市西峰冉街r座 460995', 1630000000019)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630285000001, -1152099753, -8078485240322452190, -23406, -63, 7434279511617.269531, -297650111.818850, 1, 'binary.vtvGwZIEcxVUBWDFVniL', 'nchar.香港特别行政区雷县六枝特卫街O座 501711', 1630000000020)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630285000001, -2025743219, -8360647266536686674, -31408, -127, 3198209717985.200195, -5167244033576.759766, 1, 'binary.EeDihkVVSciCssqEbEgY', 'nchar.湖北省波市高港王街x座 618754', 1630000000020)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630285000000, -651143475, 6089391424113410155, -28724, 72, -6172591503482.719727, 874675419692.713013, 0, 'binary.wAmJChouNetSkLgButdA', 'nchar.内蒙古自治区淑珍县南溪昆明街X座 457011', 1630000000019)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630300000000, -1036308321, 8505935902679868907, -12126, -56, 21041094684126.500000, -31779.452036, 0, 'binary.CHRMHcuiDKIrbquzMWnc', 'nchar.香港特别行政区桂英市徐汇马路y座 894271', 1630000000020)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630300000000, -1691555741, -2294462358761347751, -1794, -1, -4454607780157.769531, -197045759.620307, 0, 'binary.dciaYPKuPwWmZtbTNqAa', 'nchar.宁夏回族自治区沈阳市普陀李街p座 330494', 1630000000020)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630299999999, 1198995002, 3637649461872145723, 16760, 95, -2.986117, 21825715.342398, 1, 'binary.CobYIOXzviHBQmtndNfY', 'nchar.湖南省合肥市高坪天津街w座 641163', 1630000000020)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630300000000, 1641081252, 3497467759234706417, 16909, 61, -156530.187860, -60111517262.459099, 1, 'binary.WegyfVyvBuwzQpvVtxrJ', 'nchar.天津市天津县高明汕尾路L座 748705', 1630000000020)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630300000001, -1498342366, -7248641586651479050, -14472, -49, -8.737031, -916375489.622606, 1, 'binary.qfSLXCSfQexiQHdraTYY', 'nchar.海南省凤兰市普陀王街a座 920512', 1630000000021)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630300000001, -1529997535, -3950829483875051901, -14962, -125, 2566536996.957260, 46289841809.267998, 1, 'binary.xcQvVjJSZRlsYkqJLaqC', 'nchar.黑龙江省济南市永川苏街u座 855053', 1630000000021)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630300000000, 66936834, 6231741803696417438, -26509, 116, -36723525869968.101562, -140577503982.942993, 0, 'binary.JuGZrZpqrsboVygUyAtd', 'nchar.台湾省汕尾县萧山巢湖路n座 479255', 1630000000020)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630315000000, -1768408453, 3912858851624378906, -8457, 115, 63.229176, -5678913267191.299805, 0, 'binary.OaHLnnqkcUEKxOOHCYCJ', 'nchar.甘肃省济南县高明李街w座 328341', 1630000000021)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630315000000, -1676018488, 5929581737688896192, 587, 50, 8974753.896018, 9867720208144.640625, 0, 'binary.kPHIzCFGzzXfjYJPRBPj', 'nchar.广西壮族自治区齐齐哈尔县滨城杜路G座 832544', 1630000000021)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630314999999, 388708472, 3484566086576549522, 24020, 101, -3436412657.609220, 1.526276, 1, 'binary.VWdcyMxbxjRZCcDesjWH', 'nchar.重庆市娜县清浦周路Q座 318835', 1630000000021)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630315000000, 1742645526, 7447737287016433535, 11601, 102, 205650774.133360, -718740206925.855957, 1, 'binary.pLVCvWZQFPGWrURwfLcX', 'nchar.湖北省辽阳县静安李路s座 669158', 1630000000021)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630315000001, -1688761021, -3633609441651034972, -32192, -45, -6589.308664, -96579.525261, 1, 'binary.fsNWygGviERgbHDCTNoU', 'nchar.湖南省合肥市清河马鞍山路g座 155575', 1630000000022)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630315000001, -1972942235, -7595572329950088744, -21058, -64, -652542.999844, -6645809.678817, 1, 'binary.HeHuMgTlvUMAUlpdnygc', 'nchar.西藏自治区郑州县白云东莞路J座 860659', 1630000000022)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630315000000, 34583826, -5677401721211288846, 5759, 4, -922.875171, 41207973.369290, 0, 'binary.FgvACcdWyWjXDQzboRVN', 'nchar.山西省台北市南湖北镇路X座 676913', 1630000000021)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630330000000, 125729034, 5772587033883405659, 31451, 115, 1579351734.492360, 83475.538080, 0, 'binary.YAKBKrKRqlgNweLacnpB', 'nchar.青海省秀华市合川刘街r座 962617', 1630000000022)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630330000000, 1327440015, 5359031926120752338, 13595, -7, -143.262728, -8.120978, 0, 'binary.VJYIDxwcizIsqjyjvTpO', 'nchar.北京市柳县清河淮安路q座 744491', 1630000000022)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630329999999, 1046617386, 1012222098325008150, 22053, 93, 45723435049.348999, -4738289642.121180, 1, 'binary.OmMMfoOWmDphnVXsqVny', 'nchar.澳门特别行政区建华县永川郭路W座 929334', 1630000000022)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630330000000, 1996072991, 2291923905975087108, 25734, 49, -988.744553, 891307.838533, 1, 'binary.RIMwMykwFAdjljWnLgTu', 'nchar.天津市重庆县沙市西宁路T座 441908', 1630000000022)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630330000001, -509016842, -8755338571881630470, -16764, -101, 54206022913.869003, -38597163884.896202, 1, 'binary.miOrdOLIjNjowuqZLMJY', 'nchar.河北省福州县牧野西安街K座 544169', 1630000000023)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630330000001, -1043050284, -4065479627331509328, -30284, -44, -7442000934306.809570, 8404611937.272200, 1, 'binary.FYEOsVKepLPcPqRAMtQf', 'nchar.江苏省凤英市门头沟莫路D座 821625', 1630000000023)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630330000000, 1336266375, -3748199673349739489, -14794, -38, -98329032120.339996, 4354904615.450910, 0, 'binary.byOStgFhklLTIvsltIZw', 'nchar.甘肃省丽娟市安次哈尔滨路m座 552966', 1630000000022)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630345000000, -125236222, -7443235125783230646, -4372, -51, -2592298452.864900, -1806314197.669880, 0, 'binary.UJkpaKsbERScbXdQuhvo', 'nchar.宁夏回族自治区柳州县锡山赵街f座 139786', 1630000000023)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630345000000, -219965012, -4716987119042513689, 24416, -100, -3747315428739.759766, 0.494564, 0, 'binary.NOTNVTroRkOsLgVdkLoJ', 'nchar.重庆市娟县江北哈尔滨街w座 338249', 1630000000023)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630344999999, 1470657859, 2320764016731890530, 27688, 116, 12160748.177215, 83545.714436, 1, 'binary.QLjYhzfGDKcPmHFQjBub', 'nchar.安徽省婷市白云惠州街i座 922795', 1630000000023)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630345000000, 1908975565, 3788319142030902566, 28513, 73, -810951288254.207031, 704115304293.949951, 1, 'binary.WHaxQQMLidfdINYOwQiY', 'nchar.安徽省淑兰县萧山重庆街W座 385558', 1630000000023)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630345000001, -720454315, -3301539443338809381, -23269, -40, -354.491111, 9.482702, 1, 'binary.GaEJJOwaHtRTijYNSKrt', 'nchar.香港特别行政区玲县龙潭南宁街R座 455543', 1630000000024)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630345000001, -1952988609, -8224270172957627373, -12805, -23, -39.405009, 9889.998342, 1, 'binary.nnoIletebLWbHwpqrpjQ', 'nchar.陕西省深圳市长寿兴城路L座 522359', 1630000000024)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630345000000, -523958452, 2997994281167364719, 22138, 13, 8.626707, -2801928136.443440, 0, 'binary.yqSdjVqJlmWCSBRuPeAS', 'nchar.甘肃省惠州县和平田路N座 869008', 1630000000023)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630360000000, -677274672, -7849607379954530156, -24205, 84, -971756295956.161011, -13922706678085.400391, 0, 'binary.jwdoZKFVRIbdNbxvzglZ', 'nchar.北京市帅县金平沈路L座 955700', 1630000000024)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630360000000, 1246868406, 8440235059780981684, 17929, -72, -179883228781.260010, 74775.726422, 0, 'binary.AdgnDiCLdBHvjVsUEUlA', 'nchar.福建省张家港市怀柔唐街k座 206637', 1630000000024)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630359999999, 1874112564, 2471561821001658734, 21353, 86, 60186670.967332, 977025.978653, 1, 'binary.tNjvCgdWlmcOHVbLJRMN', 'nchar.福建省太原市闵行郑路j座 374763', 1630000000024)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630360000000, 1286025365, 9197590639676784477, 7479, 117, -78306895.453520, 16133.105382, 1, 'binary.oyzmHpPVRNdmGjSWoOye', 'nchar.安徽省佳县清河高路t座 131773', 1630000000024)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630360000001, -805620343, -9199526400762315204, -32703, -90, 255348664.922541, 1.265149, 1, 'binary.rMWNLTrygVpKqbvfAPEQ', 'nchar.西藏自治区大冶县沈河马路x座 692129', 1630000000025)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630360000001, -1016979209, -233992014932124431, -3372, -83, 1155.736906, 74.581949, 1, 'binary.wfBaavnTeXgzocUDpJkd', 'nchar.青海省兰英县南溪梁街d座 749974', 1630000000025)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630360000000, 1965870107, -2918841171553952995, -22593, -23, 8305333945954.679688, -3905.740237, 0, 'binary.WijQcImFTfQfgOAyFVuR', 'nchar.贵州省兰英市沙湾广州街R座 380656', 1630000000024)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630375000000, 1859005003, 1104214274917305717, 10798, 94, 850723.403288, -2.625752, 0, 'binary.uDiGDbJBCeWVuQCYGwQU', 'nchar.江西省海燕县西夏天津街h座 925745', 1630000000025)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630375000000, 249965459, 4153812250107690762, 8690, -85, -2.375649, -7556996798575.759766, 0, 'binary.OsRPmbVqrwwmstYOvsFp', 'nchar.宁夏回族自治区丹丹市淄川梧州路M座 486374', 1630000000025)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630374999999, 1624260248, 7015806862250068758, 4350, 97, 5194046419.133550, -8059822602.437020, 1, 'binary.OMehRclQHcJBTynDoScY', 'nchar.江苏省兴安盟县长寿拉萨街L座 799456', 1630000000025)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630375000000, 396319016, 1703630598675774845, 15206, 66, 82212219.419588, -694808505327.339966, 1, 'binary.ThuTfsNgNgkwqdluGnzy', 'nchar.新疆维吾尔自治区重庆市东丽天津街U座 711659', 1630000000025)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630375000001, -321755377, -7381963169677430909, -17059, -104, -50669392403.276001, 24555.641531, 1, 'binary.pYwaFBbMgRmreHVLLqHM', 'nchar.吉林省关岭县西峰王街A座 128393', 1630000000026)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630375000001, -3112247, -2326747004845963522, -32753, -15, -6.479060, -4.885868, 1, 'binary.nyUKFiEfATIOqiQYTGYd', 'nchar.北京市深圳市海陵邓街w座 923149', 1630000000026)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630375000000, 1848873864, -1425149577411932526, -15441, -97, 53983878.838902, 7.604429, 0, 'binary.AebpWwYtnpvpbQojcptM', 'nchar.福建省勇县大东董街D座 925934', 1630000000025)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630390000000, 620402706, -2092872879669844684, 22721, -99, -3.578269, 9.904233, 0, 'binary.kcxyJbYKMUthZCemDZwN', 'nchar.天津市淑英县淄川陶路n座 983382', 1630000000026)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630390000000, 1424324822, -5344921050768394808, -9933, -104, -579352179419.427002, -8535712823374.900391, 0, 'binary.iQVkXSwNBowzrsOuetLF', 'nchar.辽宁省兰州市蓟州长春路b座 767058', 1630000000026)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630389999999, 1338486910, 5553939443389037108, 7465, 21, 57919539684.215698, -1133897.445424, 1, 'binary.PewhoryOLSZsZOExXrsO', 'nchar.云南省东莞县秀英六盘水街f座 870761', 1630000000026)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630390000000, 1377263295, 2287159705001305018, 23692, 108, 839604115054.706055, 4528078.990663, 1, 'binary.NQMLbBaxRGpfdrcfKtci', 'nchar.广东省龙市魏都呼和浩特街X座 339810', 1630000000026)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630390000001, -1972396465, -2625500637464979513, -26565, -22, -441.336819, 4228263266694.529785, 1, 'binary.HCciTUVnuKaUQHOTLmoy', 'nchar.福建省潜江县长寿张街i座 756207', 1630000000027)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630390000001, -1919101583, -1025584328134825800, -12869, -73, 82107948133.142197, 8170338838.340360, 1, 'binary.SdhXWDcuORDtQtHNqinV', 'nchar.山东省淑珍县龙潭王街u座 917673', 1630000000027)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630390000000, -342515504, -850995699758580491, -27592, -104, 9983310041114.599609, -3552213.522875, 0, 'binary.mrZLEcTnUkwltOKwWmMG', 'nchar.陕西省六安县东城汪路a座 741921', 1630000000026)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630405000000, -697189031, -5174916001528455052, -18768, 31, 889278993.213705, -4519776483.921300, 0, 'binary.mjvvuLgBZRyYZvSnmrpW', 'nchar.广东省北京县淄川哈尔滨街k座 473589', 1630000000027)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630405000000, 1399897758, -5070718027080576434, -28441, 81, -6464798967823.299805, 64.959990, 0, 'binary.auzkgdvXEbwhZrzOnCgY', 'nchar.四川省岩市西峰兴安盟街L座 261109', 1630000000027)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630404999999, 586219277, 5261527665322286803, 15053, 17, -7473982.479707, 53758410580.825996, 1, 'binary.ejuJVeKrOYTBIOGtnvwN', 'nchar.黑龙江省英县璧山袁街c座 834323', 1630000000027)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630405000000, 2069647643, 8085374297895048889, 25867, 38, -9114097802.719160, -511.311795, 1, 'binary.hyODFAlyNDYzGBrgfzQx', 'nchar.河北省荆门市高明蒋路p座 794155', 1630000000027)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630405000001, -23773554, -991944997406075040, -17827, -70, -817975598.936861, -59647871.118851, 1, 'binary.HMzLCwXDnCBrSjyvRiij', 'nchar.四川省梅县牧野乌鲁木齐街u座 768713', 1630000000028)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630405000001, -777805682, -6317678604532276560, -15689, -38, -900453816.555716, 384.378368, 1, 'binary.QygspByIGreKmSuckTRN', 'nchar.新疆维吾尔自治区莉县永川巢湖街H座 128070', 1630000000028)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630405000000, 254749120, 1622387876144097468, 8950, 5, -81.624948, -24.366419, 0, 'binary.prUIqqWJFRMgzMaUTmax', 'nchar.新疆维吾尔自治区惠州县长寿南京路y座 661755', 1630000000027)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630420000000, -1223538957, 2286130958857879860, 13823, 69, 40014883538.375099, -65188739203798.500000, 0, 'binary.YhGtRmitXcCooBPlNxlh', 'nchar.北京市西安县兴山罗街c座 992434', 1630000000028)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630420000000, 1037952556, -1750883376714996603, 24124, 25, -182.347721, 547.995492, 0, 'binary.JYQaxIMJeszunPLADRsN', 'nchar.青海省玉珍县大东黄街N座 831217', 1630000000028)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630419999999, 403592186, 7560013542016462366, 168, 55, 9517875.498748, 352.106624, 1, 'binary.KbgXOhPmkagxFsFbQakf', 'nchar.澳门特别行政区郑州县沙市长沙路s座 614203', 1630000000028)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630420000000, 1368705327, 8252423483843671206, 16131, 92, -5515498174334.660156, -3.888975, 1, 'binary.MIPrismzSjiOFMNueQYp', 'nchar.福建省昆明县高港巢湖街t座 950444', 1630000000028)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630420000001, -1017841441, -1591486103588407573, -17251, -3, -2188221503680.879883, -926521.314582, 1, 'binary.BONmREpjVojUyNnkKVDr', 'nchar.辽宁省北京市浔阳潮州路g座 945777', 1630000000029)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630420000001, -1833376251, -9184585089936473286, -3917, -99, -92896209472.994095, -1.820004, 1, 'binary.PVpVjMOTKQwOyrnumWdy', 'nchar.北京市淮安市孝南潘街L座 593813', 1630000000029)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630420000000, 1948244273, -196734016981898350, -12592, 74, 572284.494489, 2864954.350333, 0, 'binary.hfQUVCcsvQaXPEtLGHpD', 'nchar.甘肃省想市翔安通辽街N座 611877', 1630000000028)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630435000000, -838875167, -1361899531005372465, 29323, 99, -1007837437.160790, 3992.134409, 0, 'binary.AXEKjEPNWZzTiqwFpkrg', 'nchar.宁夏回族自治区郑州市徐汇沈阳街X座 928833', 1630000000029)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630435000000, 1388029674, -546879967630233611, 2763, -36, 672919.301731, -18885480.555128, 0, 'binary.wxQQJhYFoSSMpDGXpiTJ', 'nchar.河北省波县沙市喻路d座 714578', 1630000000029)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630434999999, 2123582595, 7410514965760337418, 2706, 2, 62014125.916508, 2008325132.990800, 1, 'binary.JqrmmSVJyyJSphzDfdfF', 'nchar.浙江省浩市怀柔郑路c座 136184', 1630000000029)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630435000000, 1201222250, 7502546531668208416, 9700, 30, 71831491844532.593750, -589.390815, 1, 'binary.QjDXDUKqRtDbRoKaLzPh', 'nchar.重庆市玉梅市高坪大冶街Y座 589155', 1630000000029)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630435000001, -541998051, -1627876155987640389, -18145, -31, -566248120966.409058, 4.742559, 1, 'binary.DAHIBWzujRgGmXHITFdP', 'nchar.宁夏回族自治区博市高坪嘉禾街w座 519572', 1630000000030)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630435000001, -546978867, -2173360923517013860, -19473, -44, 59.410448, 5922.830565, 1, 'binary.RSrVhjmSLyoKdKZvXTIz', 'nchar.福建省海口县金平东莞街o座 455086', 1630000000030)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630435000000, -439057091, 8924713488149522354, 7802, 47, 48376534665906.500000, 45445.194835, 0, 'binary.ZDMaZnUxfgdroXGRliPY', 'nchar.吉林省建市沈河范路n座 318872', 1630000000029)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630450000000, -765134201, 347278210407306547, -999, 101, -9542079.208161, 81311055.831151, 0, 'binary.uytkzHZmorAgiDiISVoG', 'nchar.台湾省潜江县沈河罗街d座 335953', 1630000000030)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630450000000, -883583883, 8388741643434251423, -15135, -94, -68313.913298, -35.800171, 0, 'binary.tzspsEtclAyEBtTUGUVb', 'nchar.安徽省辛集县花溪周路x座 919405', 1630000000030)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630449999999, 863783720, 5196197078377636765, 15104, 13, 51450224872520.898438, 725329.992278, 1, 'binary.iZrgQLIwNNvEhiErAwkt', 'nchar.海南省宇县静安朱街Q座 212815', 1630000000030)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630450000000, 1599585284, 3659927052268251378, 27540, 113, 426035.852478, -9384.374028, 1, 'binary.yjEiWqUVlFokNWyaFBDd', 'nchar.贵州省海门县上街周路L座 539708', 1630000000030)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630450000001, -1507574370, -2553743849779578949, -26885, -73, -54390935620.480400, -8539853135856.200195, 1, 'binary.nVGhmvtMlkPpeaBpAFTM', 'nchar.宁夏回族自治区马鞍山市萧山潘路A座 970545', 1630000000031)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630450000001, -961124591, -9073578603354545587, -23272, -29, 425584796.348519, 190947665.184546, 1, 'binary.kXxbQgCutjoylguhQAdl', 'nchar.天津市旭县房山屈路c座 773300', 1630000000031)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630450000000, -1705974843, -2143380587844176814, -21730, 59, 5199526189.259490, 7705938.503009, 0, 'binary.bxSiEDFvjRcoPXYDTYCh', 'nchar.福建省淮安市和平赵街l座 610790', 1630000000030)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630465000000, -314548498, -5935072584738796094, -27392, -1, -274339.402115, 3511.621351, 0, 'binary.zPUYZbaXGMPJOywgECDZ', 'nchar.福建省兴安盟市上街昆明路j座 373799', 1630000000031)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630465000000, 1966145764, 6102444836691824974, -10115, -19, -5651247.555394, -57863153091.628502, 0, 'binary.ZrFDxoPexxwNZrlMQtvg', 'nchar.西藏自治区凯市长寿邯郸街i座 464310', 1630000000031)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630464999999, 1652311395, 2469620212179292220, 8776, 21, -3689.666358, 7851383246.205440, 1, 'binary.GFGXaNXjaQWBuiuIuflv', 'nchar.台湾省荆门市淄川永安街A座 461064', 1630000000031)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630465000000, 1692550978, 7365080219522096539, 27515, 70, -99952861.539751, 339330736.780068, 1, 'binary.bJznaiQdbEjfsGYBWcoX', 'nchar.新疆维吾尔自治区红梅市上街合肥街V座 387140', 1630000000031)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630465000001, -881848122, -3511404188725458925, -17230, -50, 6717458.476512, -951.668912, 1, 'binary.WKsWUZoBjTlKVGgAJxPx', 'nchar.青海省云县涪城永安街T座 147122', 1630000000032)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630465000001, -1473523142, -4395305224463915000, -3855, -100, 72183.581727, 7932375.897433, 1, 'binary.qNxynPRwuwQrqgyuePBu', 'nchar.安徽省沈阳市双滦张街V座 833089', 1630000000032)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630465000000, -158748589, 6425132610953044127, 18824, 57, -78085392.809785, -521675448.839484, 0, 'binary.TkNybjkEeqjVoGZKWfiU', 'nchar.河南省林县安次陈路l座 523653', 1630000000031)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630480000000, 780205345, -3394022500515803762, -8656, 59, 4353205.758237, 6894995444848.599609, 0, 'binary.dyNGyigdTXCBLJSGKHrJ', 'nchar.天津市秀梅县城北夏街Q座 373284', 1630000000032)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630480000000, 1285010274, 6775476870679767416, -30268, -78, 2376719.820365, -102060.306247, 0, 'binary.gEkadxughuSyhxVlowaj', 'nchar.甘肃省广州县淄川张路u座 694715', 1630000000032)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630479999999, 133942525, 4687491610368814212, 2473, 123, 74.707449, -37825106590154.500000, 1, 'binary.jFkkgXUPSCrZnMUmiDca', 'nchar.江西省海燕市怀柔辽阳街T座 593401', 1630000000032)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630480000000, 1198452303, 2827497746106779583, 15755, 3, -96312992172.283997, -7583.330552, 1, 'binary.OuAJuifTLwDmabcKuPpO', 'nchar.陕西省欢县永川李路U座 124763', 1630000000032)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630480000001, -1161482672, -642941094008198244, -13538, -101, -4415177912288.160156, 57577.407295, 1, 'binary.TMzTATElhmlhyqFvwFKJ', 'nchar.江西省红霞市城东车路h座 867833', 1630000000033)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630480000001, -2145746929, -2828855748044877841, -24180, -68, 3002911.812991, -5.963978, 1, 'binary.IOKXhDrLSveyxAAZcVVV', 'nchar.山东省桂芝市东城长春街f座 739636', 1630000000033)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630480000000, -2128865070, -3361908133638881363, -7698, -124, -2810320306.295400, 42424.469553, 0, 'binary.pRTQbYdbdqTUGzTBJjwW', 'nchar.四川省平县和平刘街G座 530429', 1630000000032)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630495000000, -179608812, -2830103688811366966, 17973, -6, -98267364621.484100, -34541396619.394699, 0, 'binary.VMzHtbnrmjokPDZCMXsk', 'nchar.广西壮族自治区永安市魏都海口街Y座 538635', 1630000000033)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630495000000, 17975727, -9147014879010075409, -29070, 11, 6629237.824116, -644456283208.812988, 0, 'binary.qxIcDfHdizkemidhxBOX', 'nchar.四川省金凤县魏都朱路Y座 167401', 1630000000033)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630494999999, 1748308690, 5607449654092965535, 32130, 79, 662142.922850, -4208712290.375350, 1, 'binary.bsqrWOhUgZliFzqtMCyZ', 'nchar.北京市兴安盟县门头沟夏街W座 853138', 1630000000033)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630495000000, 292621133, 8029177555041414497, 32719, 52, 9.459390, -85.260795, 1, 'binary.SptfDQZmkxZwzQDPUgly', 'nchar.山西省玉市白云合山街d座 653433', 1630000000033)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630495000001, -45395742, -3417033390164130042, -31766, -93, -22336078027069.101562, 68135086530908.703125, 1, 'binary.SEnorgonMIgtUKkNTOxP', 'nchar.澳门特别行政区永安县合川吴路W座 144142', 1630000000034)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630495000001, -1446286464, -2834239883196880371, -9824, -122, 515950931167.439026, 3301610613616.100098, 1, 'binary.pwZptdrFvhLwFfJFLqQh', 'nchar.浙江省桂荣县龙潭徐路e座 191807', 1630000000034)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630495000000, 147020729, 3276148986164408557, 21995, -8, -669813.120570, -66.191763, 0, 'binary.CDlhPrXllKDbKnpPbQCR', 'nchar.江西省桂香市大兴太原路v座 117465', 1630000000033)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630510000000, -138813503, 6315166587841671464, 1355, 40, 9602.262694, 861862121.594895, 0, 'binary.cvjJVfNMFXXyaKDjkvRu', 'nchar.山西省淮安县静安刘路E座 917860', 1630000000034)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630510000000, -1499926587, -5494769986822553194, -7513, -59, 20.566688, 164110090.400241, 0, 'binary.ncYYeVQupIbKXmfuwZoJ', 'nchar.浙江省昆明市滨城深圳街C座 408704', 1630000000034)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630509999999, 59717021, 5309467270318718403, 20281, 80, -690547726621.959961, -279234186984.130005, 1, 'binary.XBJIPnZODnweOOXmzRkO', 'nchar.重庆市南京市滨城罗街R座 920733', 1630000000034)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630510000000, 965932797, 2618901242071795537, 19035, 60, 88667158.776933, 57.714218, 1, 'binary.vPKtTfrDyvkwPapGKBHC', 'nchar.重庆市阳县高坪张路e座 430297', 1630000000034)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630510000001, -369410792, -8341233851372213560, -21268, -53, -68353.245312, 680.211243, 1, 'binary.HBmqhBQjhNYwAaLJWbpE', 'nchar.新疆维吾尔自治区坤市蓟州曾路Y座 389605', 1630000000035)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630510000001, -290124344, -5005783105138836089, -31038, -99, -4125833987.987010, -61865.560889, 1, 'binary.msJuffjbXNohLzDghECt', 'nchar.吉林省辽阳县淄川廖路y座 592416', 1630000000035)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630510000000, 1554805513, 5103394770308878784, -20138, -118, -158087627.334212, -54553.318555, 0, 'binary.QAiiKHDGWEvrjERhDojf', 'nchar.江苏省柳州县江北潜江街R座 415694', 1630000000034)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630525000000, -319429462, 2783408640725418994, -12995, 109, 8059.171767, 34270137051.186001, 0, 'binary.INFGvqMFdDTmdeFePZnQ', 'nchar.青海省南宁市璧山欧街K座 723228', 1630000000035)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630525000000, -2053704222, -387584738711581594, -28738, 100, 321.108570, 174292.288484, 0, 'binary.JEULkwBGXQMTtcrRCcru', 'nchar.重庆市南昌市东丽大冶路r座 919184', 1630000000035)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630524999999, 1042899672, 525664086008023453, 18210, 36, -992.375585, 67404918237736.601562, 1, 'binary.SyRZdWnAthyOUqpOtOos', 'nchar.内蒙古自治区辛集县华龙武汉街U座 797248', 1630000000035)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630525000000, 210832777, 8293861208931311807, 3500, 77, -4.291887, 8353937865.544660, 1, 'binary.JNasTUjHeTYxMpjwHQwp', 'nchar.黑龙江省济南市孝南广州路u座 818270', 1630000000035)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630525000001, -1784712756, -1900282826185833888, -12635, -10, -147742335517.860992, 1892142041.165020, 1, 'binary.FsBnuFMAwIdTjEjYkjBR', 'nchar.天津市丹丹市双滦马鞍山路L座 954700', 1630000000036)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630525000001, -1963762926, -3093406333417399437, -28946, -93, -361883.201813, -7970832707874.200195, 1, 'binary.AzpaKlaPZcHBJgfiMusA', 'nchar.台湾省桂香市兴山罗街I座 411116', 1630000000036)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630525000000, -1594406401, 2911814076154398648, 1593, -40, 94252.505169, -57.127701, 0, 'binary.UieVjNVQaDaURDdvbVxx', 'nchar.河南省建华市锡山马鞍山街d座 922365', 1630000000035)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630540000000, -298407008, -890073803487483026, 19138, 22, -13371145012.763000, 787260475.918045, 0, 'binary.ligoRoVSAHzFhUvIDwlE', 'nchar.上海市淑兰市锡山周街z座 377582', 1630000000036)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630540000000, 1742761629, -3538927150920177969, -26178, -17, -60236333852265.898438, -8776.586953, 0, 'binary.RxvCthLQrpgveOwWJJfo', 'nchar.辽宁省长沙市淄川石路J座 923025', 1630000000036)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630539999999, 656087591, 7671013800748633383, 21136, 114, -9294.519104, 4829386792163.620117, 1, 'binary.vsIjUPiZNVxiFWHzGqkg', 'nchar.湖南省石家庄县城东杨街b座 396607', 1630000000036)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630540000000, 1908331659, 2529406409884626842, 19478, 70, 33286729894380.898438, -3.973702, 1, 'binary.owpOIRLxxXjudcaMtZsk', 'nchar.重庆市彬县金平牛路B座 110982', 1630000000036)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630540000001, -70790307, -4087110708810857496, -6014, -118, -605686122.372439, 26511865.401597, 1, 'binary.GLAJYWukLZueieYRsidQ', 'nchar.湖北省峰县牧野辽阳路N座 528796', 1630000000037)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630540000001, -375748080, -5402179642725069125, -17546, -56, 333.228050, -8211802092271.910156, 1, 'binary.yhkrusrZjfABOqNjyutJ', 'nchar.河南省济南县长寿刘路V座 435013', 1630000000037)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630540000000, 1483404105, -1062308720694573985, 21132, 127, -3034420855288.169922, -4783009296713.500000, 0, 'binary.pkmpVTKNUbqmEYaWIxNI', 'nchar.吉林省石家庄市六枝特张路z座 262916', 1630000000036)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630555000000, 632563376, -2204241930370454289, 20442, -66, 68778435235.149796, 41338715505.798897, 0, 'binary.RsQnCVXQtzoeXuamUiBl', 'nchar.吉林省洁县永川香港街V座 762000', 1630000000037)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630555000000, -472397946, 1038938837241211981, -29204, 47, 3047173131.130830, 9172947943.508301, 0, 'binary.gSxMxJFnpWRALREpkZUQ', 'nchar.云南省辽阳市六枝特汕尾路m座 376809', 1630000000037)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630554999999, 396390446, 1620566595591926682, 11031, 85, -9305855762.999241, 1578928.160116, 1, 'binary.pgBGZOKYSliCxHFmGsyp', 'nchar.浙江省军市清浦许路C座 950185', 1630000000037)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630555000000, 1237088867, 9126129724607826846, 23721, 97, 147.865610, 55260948679.278999, 1, 'binary.EygLnzSPKfjPvserwjwg', 'nchar.青海省济南县普陀太原路P座 240312', 1630000000037)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630555000001, -1965530504, -871830532885341515, -23233, -76, -3.827964, 9.798035, 1, 'binary.ECLVeNvkmqcTHEoSSxYT', 'nchar.西藏自治区北京市长寿昆明路X座 708472', 1630000000038)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630555000001, -1484513338, -792998805013840059, -31538, -71, -8141323055.196050, -7510534665057.400391, 1, 'binary.jmfqESFcasFwykqiXlqj', 'nchar.贵州省重庆市吉区张家港路m座 856450', 1630000000038)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630555000000, 1069828446, 6908596510608104575, 28783, 110, -9162083944.840309, -102.142752, 0, 'binary.gkjpRGJEiQVghWoJbwzo', 'nchar.安徽省潜江县魏都李街r座 766818', 1630000000037)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630570000000, 446510409, -3656476974551396976, 13256, -104, 8171493376646.099609, -530097963872.411987, 0, 'binary.FEewTpfZgngFmPSTSvxJ', 'nchar.内蒙古自治区建华市锡山哈尔滨路U座 278315', 1630000000038)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630570000000, 1541307361, 3859916508801896099, 21134, 123, -1454095130.880070, -793.944907, 0, 'binary.vUtaMtwXoWgIARQjnHHj', 'nchar.广东省秀华县东丽天津街f座 273761', 1630000000038)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630569999999, 554982384, 4634928837125592948, 23064, 110, -16387837.580834, 67731033.476346, 1, 'binary.zpGkiHSJABZuoZlpNXMp', 'nchar.四川省晶县龙潭太原街d座 373693', 1630000000038)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630570000000, 1650730478, 2509551581387664974, 17823, 0, -810284073.820230, 9519108910338.419922, 1, 'binary.MykLktVEEpwPHBiBMoqk', 'nchar.山西省峰市门头沟香港路j座 412357', 1630000000038)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630570000001, -2032946157, -7551742451999725958, -11549, -110, 548121030.233320, 8537.353780, 1, 'binary.lyLVozekCioDlqntmATw', 'nchar.青海省太原市江北宁德街J座 782858', 1630000000039)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630570000001, -563992071, -3064141540636515026, -15744, -120, -430754463567.528015, -3116194745462.259766, 1, 'binary.sHiKXDptOBQSALxyVOWj', 'nchar.湖南省玉兰市龙潭大冶街I座 537334', 1630000000039)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630570000000, -2129764760, -4193554531789104653, -23767, 118, 93.790612, 786694048079.288940, 0, 'binary.WQewsnemfqUAFHeLJnXF', 'nchar.新疆维吾尔自治区东市滨城王街s座 713240', 1630000000038)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630585000000, 640199308, -5655660832137064380, -11147, 77, 16876033.858953, -7485.569430, 0, 'binary.bRHphFiZkeWtyDjXqNaO', 'nchar.天津市杨县高坪梁街B座 525434', 1630000000039)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630585000000, -1686982965, -8299508680322272166, 3619, -126, -64849.198682, -9337.273192, 0, 'binary.IGNxucSArjxWEALkrQRY', 'nchar.安徽省淑兰县城北齐齐哈尔街P座 697435', 1630000000039)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630584999999, 677947760, 1314864805338112674, 15237, 86, -929824456656.410034, -85.764381, 1, 'binary.vfXYvrHXFnDSAHwCndcp', 'nchar.青海省建华县花溪柳州街T座 100213', 1630000000039)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630585000000, 307125580, 5094829418785488158, 19844, 2, -54041115.787217, 6.233947, 1, 'binary.nAMUcXLvZLCQRHFwNvkT', 'nchar.广东省淑华市上街天津路S座 813901', 1630000000039)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630585000001, -494862894, -4253955294640876951, -19423, -20, -37835740177817.796875, 51554911990.855003, 1, 'binary.zffLjgnXcDlwdKAxADMr', 'nchar.河北省玉珍县西峰刁路C座 452461', 1630000000040)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630585000001, -437854754, -8916960191048231333, -6475, -87, 422259.957474, 81115866.296574, 1, 'binary.NlfhDIBFmzxTiVSybgon', 'nchar.新疆维吾尔自治区石家庄县高港大冶街f座 940073', 1630000000040)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630585000000, 895715893, 6676915082558096536, -2843, 80, 3019829720.850890, 45890083973973.703125, 0, 'binary.YtrhJOiBzFYUzUxxGHcB', 'nchar.河北省桂英县海陵重庆路X座 577152', 1630000000039)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630600000000, 790711145, -5535878799111175644, 24130, 71, -864521126.754360, -5257536.250149, 0, 'binary.uCOCqMGewUkNeUhAbNNf', 'nchar.上海市澳门县大兴海门路k座 319118', 1630000000040)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630600000000, -1515729891, 6489509016449838006, -19281, 68, -75090633218986.296875, 43026646570.672997, 0, 'binary.zhgeUDYkrfqFOzepcyTJ', 'nchar.北京市北京市南长拉萨街H座 770300', 1630000000040)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630599999999, 1558178384, 7355756171678259138, 18244, 101, -2195442510.130770, -623135577.309077, 1, 'binary.bKLXWDPaBjWRXgOeesiD', 'nchar.江苏省红市长寿贵阳路T座 962883', 1630000000040)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630600000000, 1292528970, 2464957048738107533, 21085, 23, -97350629.653519, 97632.613057, 1, 'binary.QhAZodfGPuZBebCyHSme', 'nchar.澳门特别行政区建平县山亭邯郸街x座 185831', 1630000000040)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630600000001, -475557451, -6826045973586477294, -7961, -56, 1705823848847.199951, -268959384930.907990, 1, 'binary.wnZulRregqgognSRRYWI', 'nchar.陕西省桂兰市孝南太原路b座 604090', 1630000000041)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630600000001, -1662397849, -6337230624292122601, -10537, -31, -7225617.287194, 9.181853, 1, 'binary.dIBenysHkntfJuvBkgWK', 'nchar.安徽省海口县闵行六安街S座 630253', 1630000000041)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630600000000, 485553847, 7808570548913367950, -25532, 111, 1267936395.925200, 965.329639, 0, 'binary.uUkJRhVNdIFbCCKdSxNu', 'nchar.云南省六盘水市沈河黄街r座 670992', 1630000000040)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630615000000, 1664157316, -3375551224102004419, 200, 88, -745584775385.793945, -699.239448, 0, 'binary.AyZCLhYiMxZgfYXlVciw', 'nchar.吉林省六盘水市清浦贺路O座 851729', 1630000000041)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630615000000, -1198323418, 1449001209366933334, -29356, 113, 970.596178, -8258648600.433700, 0, 'binary.wnEzpmjBgvHDFbCHrxCx', 'nchar.辽宁省淮安市沈北新拉萨街Q座 858646', 1630000000041)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630614999999, 632223220, 7440877353040054077, 26302, 94, -9662701.768525, -9843725.667981, 1, 'binary.JqnabMfPCrzsTaYczTLh', 'nchar.山西省玉市孝南李路i座 331687', 1630000000041)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630615000000, 1054653217, 644785387636789462, 23531, 71, -8451351.603558, -83219909153.306000, 1, 'binary.IPnIZyUyPMYVVcGjGYqM', 'nchar.香港特别行政区济南市秀英巫路v座 383455', 1630000000041)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630615000001, -951785691, -4025456465792961539, -30955, -57, 74739.936019, 23857138186877.699219, 1, 'binary.sKKfLrkShxlGFGromDFZ', 'nchar.江西省宁德市梁平香港街U座 132395', 1630000000042)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630615000001, -1327031516, -9041664297455549079, -6004, -44, -6224.384964, -39.676621, 1, 'binary.amQNMbMfFfKQljlJEvIS', 'nchar.澳门特别行政区兴城县平山张街J座 904644', 1630000000042)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630615000000, -1454876122, -1664191631337826145, 2979, -43, 52777147.724176, 1229194365813.330078, 0, 'binary.IxkIHjiRPnVlBdHQSdLm', 'nchar.广西壮族自治区帆县山亭张街G座 965406', 1630000000041)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630630000000, -1983928412, -7333101339332877122, 3761, 37, -38344519.775088, -65261.718170, 0, 'binary.CaPFXqhDLSOaNmNnOsLh', 'nchar.上海市马鞍山市城北宁德路c座 402187', 1630000000042)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630630000000, -98362180, 3588425829881816071, -9471, -3, -739.807551, -194649953397.838013, 0, 'binary.LQjcSrpQYoEDKvOOlKKI', 'nchar.青海省南宁市孝南宜都路F座 208181', 1630000000042)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630629999999, 826686189, 1712941616400699580, 9250, 116, -92395295608805.296875, -2.541895, 1, 'binary.shGShhPNnWdZQDAdbiHo', 'nchar.辽宁省六盘水市门头沟黄街t座 892638', 1630000000042)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630630000000, 186750484, 8366250288933508150, 15325, 23, -4695500.207880, -64.107600, 1, 'binary.IkdeyRigedSouvHypNnn', 'nchar.福建省倩县怀柔潮州街g座 858358', 1630000000042)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630630000001, -667795153, -2546515577402776696, -6866, -12, -8902370.867967, 90891053949.643005, 1, 'binary.OaUjyowfAGaGXKoVLWZR', 'nchar.河南省长春市城东郑路L座 791122', 1630000000043)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630630000001, -120091173, -4224513340638046957, -14010, -106, 47098343261485.101562, 8407041730175.440430, 1, 'binary.NcaWdgNTvgnnrmOyMSRv', 'nchar.西藏自治区健市南湖北镇路F座 518953', 1630000000043)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630630000000, 1431440158, 8179550750509486230, -9219, -2, 81300.928795, -56.108837, 0, 'binary.EBrNkPXmiwUfZTalWRtd', 'nchar.广西壮族自治区南宁市锡山罗路W座 812195', 1630000000042)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630645000000, -412013965, -7913679721963250806, 10182, 93, -5654939.111803, -846597080580.864990, 0, 'binary.qEwLmiFwCrLZumFHOYjb', 'nchar.西藏自治区合山县滨城贵阳街j座 422034', 1630000000043)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630645000000, 1503721252, -1036748170680619716, 3291, -18, 4.597429, 62188775030.721703, 0, 'binary.XKwsLUCTTzTuOxhWzjkr', 'nchar.新疆维吾尔自治区红市大东邯郸路r座 535699', 1630000000043)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630644999999, 1420828021, 2546086650460850227, 17595, 113, 43.359177, 8941185489440.390625, 1, 'binary.AYcgokdEsFMUOIcUpxnM', 'nchar.云南省桂英县高明张街s座 776931', 1630000000043)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630645000000, 1943840699, 8246807181436486678, 25310, 76, 35456238889763.703125, 642320879.336482, 1, 'binary.RVHWrlKKNsXNwSQINHba', 'nchar.重庆市华市滨城李路l座 828098', 1630000000043)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630645000001, -2009472018, -7027630570263393098, -21696, -67, -8424796538428.150391, -49416.120873, 1, 'binary.JpOZwpoVKCDjiqsFhbaS', 'nchar.安徽省深圳市高明陈街w座 281329', 1630000000044)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630645000001, -177128590, -4886914070408811519, -24807, -23, 170878.982356, -2.287124, 1, 'binary.BMICVHrfygzpAnylwEKr', 'nchar.河北省萍市锡山韩路y座 412263', 1630000000044)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630645000000, 1079184574, 5846412039067228007, -30088, -25, -4212078244.552720, 679329505569.878052, 0, 'binary.gazNmQBKvUAcQQdcOFfh', 'nchar.福建省柳州市翔安石家庄路k座 779547', 1630000000043)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630660000000, -1579595934, -4731667789895780038, 15158, 77, 526489893298.299011, -396.223420, 0, 'binary.xUUoAUpZeJJsquiJOKsO', 'nchar.吉林省兵县闵行乌鲁木齐路Q座 534261', 1630000000044)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630660000000, -223991084, -2619845069313547475, -14196, 65, 9226.169120, 44944134.392266, 0, 'binary.JGVSgWAUVwADbVUvpSiy', 'nchar.天津市太原县六枝特王街g座 766627', 1630000000044)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630659999999, 395324903, 3937705474539369893, 15533, 65, 0.584074, -967418.449977, 1, 'binary.vWVYYMXIWQAEHBTKKWjP', 'nchar.广西壮族自治区上海市南溪淮安路O座 810462', 1630000000044)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630660000000, 762050937, 673715091051246407, 16484, 115, 97643376.627697, 763.832549, 1, 'binary.YaiVXlPHKJSEOWuRsRwS', 'nchar.江苏省东莞县友好彭路R座 578518', 1630000000044)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630660000001, -9754404, -5144925573679303827, -11209, -98, -7.448873, -448115242.603042, 1, 'binary.QNAVDZqImSFgnHpVdWtS', 'nchar.澳门特别行政区沈阳县和平张街H座 176143', 1630000000045)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630660000001, -1866035632, -2427166402076174209, -27881, -22, 7.396055, -698670.261246, 1, 'binary.uZkazUtEEMgPQgeEXPEe', 'nchar.山西省秀珍县房山西宁街g座 249969', 1630000000045)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630660000000, -1051806775, 7747932212903150302, -3555, 33, 136.466491, -14409582572.354000, 0, 'binary.tuGfceNZePXxugZSWrOK', 'nchar.湖南省海燕市朝阳长春街y座 700043', 1630000000044)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630675000000, -98786523, -8235989830378110865, 1159, 12, 539082.659713, -335587651.682570, 0, 'binary.XQfGXHbnwGQPulAcrAPX', 'nchar.澳门特别行政区郑州市沈河张路S座 530101', 1630000000045)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630675000000, -1987879809, 8328244568807450232, -19551, 26, -829281.870267, -2.960861, 0, 'binary.tsChIuGKmCXkiRucjEmc', 'nchar.澳门特别行政区强市长寿银川街V座 830000', 1630000000045)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630674999999, 2068284680, 6162503383056743038, 27733, 48, -35701228967431.296875, -53.652855, 1, 'binary.wNSWJmuVBjpRVVCpksQC', 'nchar.河北省磊县锡山通辽路I座 205908', 1630000000045)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630675000000, 1151350758, 2480050564933255492, 12388, 102, -738078.108110, 37271792.258503, 1, 'binary.HLSVgACKcmyXKMuAaoCr', 'nchar.云南省龙县丰都乌鲁木齐街j座 584869', 1630000000045)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630675000001, -853667209, -5695092490915541765, -2174, -112, 26.293549, 2.242559, 1, 'binary.OgZwwWPZuNLXAAdBcanX', 'nchar.四川省巢湖市门头沟广州路C座 781739', 1630000000046)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630675000001, -1502448521, -720023346921095762, -11220, -47, -4788.467802, -40669738519.174500, 1, 'binary.omeLfsdjbehlFckYepVj', 'nchar.山东省东县城北辛集路e座 898860', 1630000000046)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630675000000, 1648896854, 7223892135855185984, -5070, -5, 1481003953669.600098, 64887.632295, 0, 'binary.TBMjUXWqSdTfxYFZJeBD', 'nchar.北京市博县金平耿路U座 676471', 1630000000045)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630690000000, -287136285, 1361266513134048206, -3947, 6, -94329.350983, 20808636.344494, 0, 'binary.OdsJNUSWeVvTxQdKykmk', 'nchar.陕西省冬梅市魏都王街J座 907521', 1630000000046)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630690000000, -1467609417, -3843627157269427935, -5413, -42, -614746115.104684, 58.540238, 0, 'binary.NtWJpduxaSUSCqzynFYI', 'nchar.内蒙古自治区兰英县梁平合山街l座 424641', 1630000000046)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630689999999, 155983724, 4361653758945923763, 29353, 81, -498.227438, -5998.851784, 1, 'binary.MRJjXYWHBZvzaAjTzrli', 'nchar.安徽省丹丹市江北海门路R座 285157', 1630000000046)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630690000000, 1258506389, 8207419892749460886, 22177, 98, -8132825907524.299805, -80427.275975, 1, 'binary.NKvCFHNYSDBOyLxAkMxr', 'nchar.台湾省荆门市萧山石路c座 773841', 1630000000046)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630690000001, -1760453704, -2551026868258460946, -4549, -24, 80821.980662, 16.963865, 1, 'binary.pARRcJciJQzdzcCZRqZA', 'nchar.江西省关岭县安次阎路W座 514665', 1630000000047)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630690000001, -1840599952, -7633977303259618496, -30566, -98, -54802925.996342, -179096513.804220, 1, 'binary.sXxbWKCIeaTzSmMmYYYY', 'nchar.新疆维吾尔自治区娟市清浦唐街G座 762308', 1630000000047)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630690000000, -1855444632, 2154100108226554299, -9690, 41, -247749.941627, -334792954703.443970, 0, 'binary.zIvNZWbipczRwxcTNujW', 'nchar.黑龙江省帅市和平张路W座 522431', 1630000000046)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630705000000, 1051015641, -2016886488941734520, 25881, 35, -862249.687896, -7581394980.840040, 0, 'binary.HfnUyPAwYxEHQotoCfFT', 'nchar.湖北省郑州县和平哈尔滨街S座 408314', 1630000000047)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630705000000, -510388953, -8064065802993152150, 32481, -96, 7.338368, -983.214115, 0, 'binary.URxUEcHRhmRFuPRLMOfU', 'nchar.北京市海燕县大东安路T座 198225', 1630000000047)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630704999999, 570191599, 5721926649273388978, 16320, 75, 1203.837639, 108022950538.300003, 1, 'binary.VVxoQGPpWrOYSXzSxWlM', 'nchar.重庆市玉梅县城北北镇街j座 561254', 1630000000047)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630705000000, 516742969, 5254851840608530925, 22978, 57, 98.947992, 2.621934, 1, 'binary.qJtFyhoRvILawwiVeHZU', 'nchar.上海市宁德县萧山李路a座 549521', 1630000000047)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630705000001, -1289549437, -4852255743592819912, -21575, -83, 21395693569392.300781, -20107182860.251999, 1, 'binary.mtOMrfEeDMegNqMauPgK', 'nchar.安徽省飞县滨城范街D座 780146', 1630000000048)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630705000001, -530361614, -2932962238369711981, -20256, -101, 1185.387219, -32573778030.118999, 1, 'binary.VFBPpHGESAAGkeRzIbtK', 'nchar.广西壮族自治区金凤县龙潭沈阳路L座 296744', 1630000000048)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630705000000, 726917225, 502661849153334288, -12421, -29, -481018.973281, 685.805197, 0, 'binary.XJeHLodwUXxQGrQUyIzI', 'nchar.河北省彬县萧山海门路s座 748879', 1630000000047)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630720000000, -1161611813, 5817478319014146944, -7727, 6, -378647512.223777, 60632396886.272797, 0, 'binary.jEENQpaYOWcVHUYKPaPn', 'nchar.山东省辽阳市兴山关岭路J座 414020', 1630000000048)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630720000000, 400786735, 1794586209142898723, 29093, -32, 2336.811166, 4.882355, 0, 'binary.xaDzuonslxHsHxVUcTUr', 'nchar.广东省红市南湖兴城路V座 308999', 1630000000048)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630719999999, 1998845359, 296563873066744098, 25562, 88, 543674818246.236023, 589024.805283, 1, 'binary.IROrsLaBEXcULJZvtXDS', 'nchar.河北省婷市南湖奚路F座 111413', 1630000000048)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630720000000, 1792844644, 7461061401252152973, 21237, 38, -26922171.906852, 2426.989224, 1, 'binary.tkXpypBtMzRGUmlEOHzz', 'nchar.上海市齐齐哈尔县涪城呼和浩特路f座 808674', 1630000000048)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630720000001, -1746905381, -2536142587987184204, -30442, -113, -76482733.355534, -14901138135.959801, 1, 'binary.WatcMYFnezOmZtsAGUDK', 'nchar.香港特别行政区杭州县六枝特李路Q座 925367', 1630000000049)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630720000001, -2131665185, -7526157230511290768, -20114, -12, -44.609385, -754455.135801, 1, 'binary.mJjTgToZxAwHTtdlzHCm', 'nchar.湖南省南京市沙湾尤路s座 734395', 1630000000049)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630720000000, 935471992, -1206109439865175665, -13528, 3, 2923999104624.620117, -91367340672254.593750, 0, 'binary.UeJygdaPzMTyLGaYNqVw', 'nchar.上海市佛山县秀英汪街s座 123655', 1630000000048)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630735000000, -1162065386, -3625158475592858359, 15940, -1, -3.678888, 1905.623857, 0, 'binary.ltmEKwzkvDqQIZWAxJtQ', 'nchar.宁夏回族自治区旭县翔安潜江路e座 531775', 1630000000049)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630735000000, 27870294, -1672838010426046661, -271, -67, -36401764.482018, -8776255.622473, 0, 'binary.YpzKzQOXpmQFzkDvKTPZ', 'nchar.贵州省波县和平深圳街H座 685163', 1630000000049)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630734999999, 2123054685, 1637804037910336988, 6220, 95, 70038469196607.601562, 3457393391629.479980, 1, 'binary.cRdRGuitRXDuGAWVDxAE', 'nchar.宁夏回族自治区南昌县高港永安路k座 452965', 1630000000049)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630735000000, 920693621, 6310859245401227510, 303, 79, -60035982.548857, 1543090.996048, 1, 'binary.uTFJycIGGxiHvreuiUWd', 'nchar.黑龙江省重庆市华龙辛集路k座 324574', 1630000000049)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630735000001, -45388705, -6671698571264178710, -1142, -44, 9.623297, -36288182.831700, 1, 'binary.HMSrCuTIHQiHmuAoGqJK', 'nchar.浙江省六盘水市锡山广州街f座 930027', 1630000000050)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630735000001, -1979960230, -5366544476848969489, -9255, -38, 13559526845534.199219, -52598830.800144, 1, 'binary.lwsPLPMBhvmOtIthgnDA', 'nchar.香港特别行政区璐县友好李街I座 225346', 1630000000050)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630735000000, 953316928, 521541905227278512, -7691, 108, 0.929415, 6372.352970, 0, 'binary.FAMTMFGzUQzWeWuWvvCv', 'nchar.青海省浩县吉区李路l座 492272', 1630000000049)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630750000000, 1289374504, -1538126395055742224, -12336, 94, -41.978570, -543035800648.830017, 0, 'binary.SHhbtXRCqxYOqQcWUDtU', 'nchar.台湾省玉英县东丽邵路W座 597939', 1630000000050)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630750000000, 994205413, -8986580971408422193, 23422, -119, -30609466.792492, 844615733.994923, 0, 'binary.cjCTzFJrdzKdAEluupXy', 'nchar.河北省长沙县长寿任街w座 978621', 1630000000050)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630749999999, 422908755, 5457973965996455589, 7028, 70, 40563882.192469, -9235464112.622881, 1, 'binary.xeqmTUMylYPDFkFJGLQY', 'nchar.澳门特别行政区兵县黄浦六安路y座 904155', 1630000000050)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630750000000, 798218536, 664231232740969491, 19717, 18, -63.765636, -5311500109000.320312, 1, 'binary.hiiCPuMjaHFEOXUsUyFs', 'nchar.北京市马鞍山县友好杭州路E座 623925', 1630000000050)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630750000001, -1902203841, -2257246515637886253, -26212, -11, -4386502.688771, 17.396601, 1, 'binary.dEfjmLzekcjvysMUToXa', 'nchar.黑龙江省雪县东城高路T座 703981', 1630000000051)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630750000001, -519516209, -6438472832772944658, -2516, -5, -137.313109, -21.647080, 1, 'binary.XOwaYPgBCUulNhxMuijU', 'nchar.天津市北京市江北宜都路X座 191896', 1630000000051)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630750000000, -1936399460, 5919953889802638962, -6029, 116, -47.500353, 6967.754987, 0, 'binary.TEvmwdpcePbbpPUuCcOL', 'nchar.湖北省帆县西峰邯郸街l座 145632', 1630000000050)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630765000000, -115300759, -5011960136761658883, -1020, 94, 23611.457680, 3419810648081.700195, 0, 'binary.CowDUEaCtAEdafPCLLwM', 'nchar.江苏省沈阳县涪城梧州路g座 537113', 1630000000051)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630765000000, -1632078764, -2531000958759467663, -1282, -120, -44.781583, -983514.899761, 0, 'binary.YgAVHKRKbzDqkjthriDR', 'nchar.宁夏回族自治区桂芳县怀柔辛路h座 764420', 1630000000051)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630764999999, 1335747951, 332059218903242958, 16913, 38, -258359415.692471, -64088.294941, 1, 'binary.ugUzQSUcPjEXMIDrFobB', 'nchar.澳门特别行政区南宁市大东深圳路a座 447398', 1630000000051)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630765000000, 489030717, 421762966583668765, 31429, 94, -93516881483951.796875, -3894133275.320640, 1, 'binary.pMcnfkpKjgIgIQUaEjjG', 'nchar.广西壮族自治区倩县沙市李街q座 504473', 1630000000051)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630765000001, -1552553924, -4861456845426343956, -27864, -39, -47713477064.426201, -5155612964147.650391, 1, 'binary.gZMscjyRXoGyNsFLHpoR', 'nchar.广东省梧州市龙潭蒲路T座 644808', 1630000000052)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630765000001, -915605364, -5868675460644756043, -650, -56, 7433399325297.160156, -52322640586113.500000, 1, 'binary.oopvfRaqoQJMcVsjkWaW', 'nchar.湖北省建平县海陵孙路S座 240665', 1630000000052)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630765000000, -1546478570, 400491964521467219, 12189, -119, -47542.132966, -2297.923262, 0, 'binary.HeXyWwlnWwUQoKSQqlmu', 'nchar.新疆维吾尔自治区拉萨市蓟州李街B座 664953', 1630000000051)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630780000000, -97528818, 8438123946253432917, -2893, 109, -2306175015456.819824, 13390.816579, 0, 'binary.pqfTTMuvVmSCfxCmFTwS', 'nchar.吉林省莉县新城廖街W座 455043', 1630000000052)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630780000000, 1001095126, -3172038662584640805, 31275, -17, -6999060.910240, 2846571851.119890, 0, 'binary.CLiSpqGShMtzcfyVTbqa', 'nchar.四川省长沙县东丽张路i座 936342', 1630000000052)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630779999999, 1092904458, 4456537336906696912, 16315, 4, -13230586.214814, 29375192427.605099, 1, 'binary.IAiAMcGMqeACFMkfgGju', 'nchar.福建省汕尾县丰都海门路R座 239728', 1630000000052)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630780000000, 1949642007, 5173135551369380298, 13811, 118, -59765334.101800, 645915504896.780029, 1, 'binary.KZORRCLafoZgcOOJmook', 'nchar.四川省重庆县新城王街u座 375952', 1630000000052)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630780000001, -437886009, -8023690985345652194, -14116, -22, -41686906094.233299, -2499063538390.600098, 1, 'binary.jWguWEtBAUgcMKtgXieL', 'nchar.台湾省沈阳市山亭牟路X座 126443', 1630000000053)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630780000001, -1594359349, -8856134295356844294, -29431, -58, -5120.713906, 18405344215702.800781, 1, 'binary.eeRKiigMAbjCTZnPHkGn', 'nchar.北京市宁德市合川朱路d座 269386', 1630000000053)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630780000000, -585788677, 3989310696333618920, 29186, 73, -6608.937490, 11.883322, 0, 'binary.lMUQhCelPiyrOIsQUUai', 'nchar.台湾省俊市蓟州淮安街k座 712450', 1630000000052)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630795000000, -738727537, -4908907813359743923, -26386, -105, -841510.274518, -80431949.274122, 0, 'binary.zWAWphEtCuSMACUmkfvN', 'nchar.天津市沈阳县清城杨街X座 500455', 1630000000053)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630795000000, 393352810, -379075072584781331, 9788, 6, 50170524851777.796875, 44763418392.943497, 0, 'binary.lDVSDncOcANkMxzrtUzK', 'nchar.青海省福州市清城蒋路Y座 567996', 1630000000053)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630794999999, 433850999, 5300519222587016898, 9473, 108, -190097.678270, 2904794070549.299805, 1, 'binary.QBIulMvNBsGMfStAKgCQ', 'nchar.重庆市亮市璧山合肥街U座 535443', 1630000000053)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630795000000, 1651231384, 6954565384064882242, 12216, 115, 81102616.344608, 39273.314632, 1, 'binary.mFwsFjuuCpdYWXqSMkgn', 'nchar.西藏自治区武汉市滨城太原路b座 798865', 1630000000053)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630795000001, -1219816305, -7051889644919104807, -28601, -74, 491555.463996, 72211361404034.796875, 1, 'binary.aSnbvFgLheGIahlWoBEf', 'nchar.贵州省倩县大东张街m座 344033', 1630000000054)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630795000001, -2116828534, -5490698997387226161, -12838, -39, 145344193.926365, -463003297.110519, 1, 'binary.wyWFnvMlYxuviQIhtJTb', 'nchar.贵州省鑫县浔阳贵阳街w座 498999', 1630000000054)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630795000000, -717310584, 7305335468389593497, 26992, -11, -33.942630, -4.966136, 0, 'binary.dKiRVdIDouQiceoUupoT', 'nchar.青海省银川市秀英叶路A座 419144', 1630000000053)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630810000000, 1063250813, -5121723296303557180, 3452, -49, 128.722024, 8116.722860, 0, 'binary.XwbrXvujPArWLXYhQRQK', 'nchar.内蒙古自治区玉兰市城东银川街O座 519239', 1630000000054)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630810000000, -433540276, 3265329093020138852, 19837, -63, -8299471626012.559570, -76400754030.522995, 0, 'binary.qUMRRVLolMSqDjJOJGMj', 'nchar.宁夏回族自治区平市高坪广州路D座 168860', 1630000000054)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630809999999, 1771552222, 3276405219271274913, 6311, 75, -9805500055353.400391, -378716.572329, 1, 'binary.YdwgZmlYfeivmxtaWueZ', 'nchar.四川省军市吉区罗街K座 863617', 1630000000054)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630810000000, 1511546600, 5132492378969879970, 21762, 77, -160772007871.860992, 4897001212199.500000, 1, 'binary.HNELJpdiwwcvGwnYiiQt', 'nchar.青海省六安县沈北新李街G座 938908', 1630000000054)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630810000001, -429657495, -726300776399539025, -5426, -71, -906430389.497117, 26283347323689.101562, 1, 'binary.yGIWxOkDtQCQgbPJhHFi', 'nchar.福建省娟市丰都董街I座 482079', 1630000000055)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630810000001, -1918072152, -3958702025992555431, -20124, -100, 325915062.818627, 82697862.465381, 1, 'binary.zNgoJphhqXQOvIxQHzgJ', 'nchar.浙江省秀珍县滨城陈街j座 333330', 1630000000055)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630810000000, 1166488162, 7007631474167031432, -9261, -6, 6932344629.512100, 531610708.460607, 0, 'binary.OwLQNOOkVKVzVtHPJDqC', 'nchar.黑龙江省娜市朝阳李街y座 366075', 1630000000054)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630825000000, 1579885922, 6474658599299815947, 28280, 52, 105208627.487617, 794627163.462063, 0, 'binary.FEXGjykuWeGEfkKImWDw', 'nchar.四川省淮安县长寿海门路T座 448080', 1630000000055)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630825000000, 734586506, 5695938165634694036, -6919, -31, 16653611667.163200, 5495624345.768730, 0, 'binary.YXkQvxGjNZCQDlipuLTw', 'nchar.上海市宜都市滨城田街s座 403680', 1630000000055)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630824999999, 1560886565, 601626973389436785, 25259, 120, -10180.909667, -11157634.936217, 1, 'binary.OlTzMGFNSfSrcixpefQx', 'nchar.四川省太原县沈河申路d座 235380', 1630000000055)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630825000000, 1316061964, 4308983887809754383, 29120, 81, -318524399.639939, -76861841765211.906250, 1, 'binary.BFHoxbNIFRIxSwrutvbT', 'nchar.澳门特别行政区永安市璧山廖街c座 427250', 1630000000055)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630825000001, -2082927804, -1484742345187780268, -6764, -32, -834228179.920564, 182.911517, 1, 'binary.FTNnvEphxFByhzkfmAhb', 'nchar.甘肃省南昌市东丽南昌街v座 686725', 1630000000056)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630825000001, -231751967, -5929805332846772894, -26985, 0, -0.118923, -67637720.762575, 1, 'binary.TYBLuEQrsrxSAJbNcCwe', 'nchar.河南省莹县房山石路a座 319475', 1630000000056)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630825000000, -1500118527, 7742809515581132121, -8755, 59, -47.623223, 4.266104, 0, 'binary.rbQwEMlJmykGPTkOccLp', 'nchar.湖南省辉县和平香港街s座 101830', 1630000000055)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630840000000, -714663691, 5099404450420041478, 27567, 14, 4.945851, -23542476442552.601562, 0, 'binary.tdLqigCoasQnRffCEaDD', 'nchar.江西省太原县华龙何街H座 269392', 1630000000056)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630840000000, -316339515, 6558297131565034564, 10392, 20, -171972.831509, -13296859.696109, 0, 'binary.sBJTgABqvGumXOfORHBt', 'nchar.吉林省昆明县崇文关岭街K座 822911', 1630000000056)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630839999999, 312047911, 6751577130057851873, 24735, 43, 1278880274.608010, -2.634735, 1, 'binary.yaVlkhtTxxgrtZTsFdxa', 'nchar.云南省台北县魏都张路U座 698041', 1630000000056)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630840000000, 364153602, 8378783503904272354, 9837, 123, 617669.431755, 134.118945, 1, 'binary.RPldGyMrCeJhQIJpDaWz', 'nchar.重庆市丽华市淄川昆明路y座 543556', 1630000000056)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630840000001, -1417705247, -1906089795101484200, -3898, -46, 13831918214786.900391, 4384545313581.750000, 1, 'binary.tHnemGaRugDffcmoyWli', 'nchar.青海省建平市清城赵路z座 958026', 1630000000057)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630840000001, -434542286, -3699533697590169746, -21081, -126, 76.275011, -41585.936197, 1, 'binary.XBOSfXHSompdpAuWKgmz', 'nchar.河北省合肥县滨城重庆街k座 644142', 1630000000057)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630840000000, -1785884903, -7685034849720410750, -21661, -99, -3153441121.646620, 502521483208.367004, 0, 'binary.MvIfmuaYUqcNIjJBVEAH', 'nchar.重庆市张家港县东丽孟路X座 295908', 1630000000056)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630855000000, 1437018826, 7400326252504732743, -29033, 112, -53177718.768333, -23404.680449, 0, 'binary.XZcaYfmvLZtxhrmzHWEJ', 'nchar.青海省柳市静安陈路h座 801659', 1630000000057)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630855000000, -869253997, 8132787450326867533, 22924, -123, -902072.768906, 33.572941, 0, 'binary.FjuniySwJpoodedUTAmT', 'nchar.香港特别行政区六盘水县新城太原街W座 631866', 1630000000057)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630854999999, 1667164436, 4152850799576560541, 29745, 75, -450320.964115, 82908.189997, 1, 'binary.NdSzaJxdoQPGYefWPJyi', 'nchar.四川省春梅县璧山刘街a座 898553', 1630000000057)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630855000000, 1667649794, 1088688853312235292, 9455, 97, -3.427779, -66.503636, 1, 'binary.tvFPXcrLOMTMCJFqcwxT', 'nchar.内蒙古自治区柳县璧山齐齐哈尔街T座 380899', 1630000000057)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630855000001, -2145233720, -3109219509113963979, -18812, -108, -981408379333.303955, -684.345037, 1, 'binary.jghuIXEGcUtKikJpzLOh', 'nchar.四川省银川县崇文张街H座 193298', 1630000000058)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630855000001, -1420593489, -5757610227096963226, -16816, -112, 75645.512896, 4840288.738839, 1, 'binary.CCvnOIBQeUgdTvsedOIX', 'nchar.河北省俊市孝南王路V座 596559', 1630000000058)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630855000000, -1569315493, -7039982472137840249, -11039, 43, 396.176284, -6.336007, 0, 'binary.oNPEvbRQaKHZmlKRACCu', 'nchar.江苏省六盘水市锡山杨路Y座 725607', 1630000000057)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630870000000, 1811347552, -8209360489735809987, -12235, -88, 2775714032.345980, -273997.706213, 0, 'binary.riDaPlhuskGRZsNxzooI', 'nchar.辽宁省潮州市东城徐街R座 431820', 1630000000058)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630870000000, 1930606515, -6940503292627563742, 14608, -109, -544236.299151, 18944279791.470501, 0, 'binary.vbFXWFeRlIaxUNtkKXIp', 'nchar.甘肃省红霞市门头沟长沙街A座 350032', 1630000000058)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630869999999, 488072919, 4847709443565724927, 25414, 31, -442295.525846, 654487670258.910034, 1, 'binary.RLQFgpAcQCfhuZExHLkx', 'nchar.江苏省巢湖县秀英翁路z座 536419', 1630000000058)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630870000000, 334934868, 8084257597250217608, 2587, 69, -46356096021.352997, 17315.165561, 1, 'binary.uCbRqUiElxGzVXQYYkmE', 'nchar.云南省太原县黄浦段街q座 675838', 1630000000058)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630870000001, -1943550230, -3503251484175500724, -28810, -67, 4134905907363.700195, -97.850683, 1, 'binary.ozdeiPLKjSgywqNpCBYN', 'nchar.香港特别行政区西宁县城北王街H座 794624', 1630000000059)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630870000001, -970612310, -8553425754362908809, -2144, -72, -99455587437.982300, -97357630030126.500000, 1, 'binary.EcHZioPjEFGIxUhBosdz', 'nchar.辽宁省玉兰市新城拉萨路W座 721716', 1630000000059)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630870000000, -384741102, -7726003315883440776, -2966, -80, -60458.951496, -4975581.359366, 0, 'binary.TZmxmIbpirBCEoTvqUzY', 'nchar.安徽省呼和浩特县六枝特朱路B座 888783', 1630000000058)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630885000000, -2123493348, -4231564094016455736, -31657, 21, 591198324.991950, 794309276463.779053, 0, 'binary.ZKZunxmtjBCRRCZBHTsL', 'nchar.重庆市六盘水市崇文魏街z座 678361', 1630000000059)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630885000000, 766378371, 492367445257699608, -7708, 99, -3.863619, 5004672.630707, 0, 'binary.AmjCwTkBprSgkFpguYng', 'nchar.山东省健市徐汇兴安盟街z座 942291', 1630000000059)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630884999999, 724025582, 7859514857348468608, 2877, 125, -58160.749327, -8273.730579, 1, 'binary.XZmZYlSRZZddcxEOjfEd', 'nchar.新疆维吾尔自治区潜江县清城梧州路m座 660147', 1630000000059)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630885000000, 1075248136, 8396611731005005497, 20388, 107, -7832228.556238, 17032.924250, 1, 'binary.xKdxeqytYTWmncKRyiEZ', 'nchar.吉林省琴县清浦台北路t座 640801', 1630000000059)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630885000001, -2130070769, -5654655793240443105, -30365, -37, 71673.437636, 624958800878.404053, 1, 'binary.qLvSSOSazLTUKvqpCCBL', 'nchar.河南省桂芝市房山韩街x座 576203', 1630000000060)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630885000001, -1953170165, -7336932030848188049, -12255, -67, -1400233209790.189941, 568697943954.404053, 1, 'binary.pQNaWtAWfSUVMWbrhVmk', 'nchar.天津市晶市清河太原路F座 911684', 1630000000060)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630885000000, 281132201, 5123656302965848910, 6859, 69, 6896942.124392, 7059472733608.709961, 0, 'binary.PHwnvqWGOkMIPxdQCkTT', 'nchar.安徽省婷市东城太原街x座 154074', 1630000000059)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630900000000, 2108150002, -3953623224839666424, 26021, -120, -3832507656102.310059, 41875460523837.203125, 0, 'binary.eEaxRyDoGrrYZQfYhDwC', 'nchar.福建省杨市山亭陈街X座 274347', 1630000000060)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630900000000, -95171868, -3159580668189063241, -4074, 47, 82585.410649, -7462199.923442, 0, 'binary.nAHveWPuvkfTISitbBzq', 'nchar.河北省琳县蓟州吴路H座 196552', 1630000000060)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630899999999, 941556630, 7998670511234147715, 15902, 22, -6585.641376, -0.373709, 1, 'binary.WrcojJGdsiZyAaRxYrdw', 'nchar.江苏省东莞市翔安杭州路l座 318171', 1630000000060)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630900000000, 1042736485, 7949270623018065586, 28676, 76, -61426.518584, -889632484.718458, 1, 'binary.MMPKnkKEPUMKxiatOAXs', 'nchar.北京市杭州县永川梧州街L座 989832', 1630000000060)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630900000001, -343938435, -815062415332308272, -26680, -52, 5.498240, 63.248674, 1, 'binary.HoqLkegpacFGqzFWeVXL', 'nchar.山西省辽阳市城北沈阳路u座 992590', 1630000000061)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630900000001, -1788477052, -5188731326210734405, -1324, -57, -2910628601164.899902, 87786.407935, 1, 'binary.LlQNfNmJFPmYIFPwUEzx', 'nchar.重庆市丽华县东丽成都街e座 389910', 1630000000061)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630900000000, 2018195628, 7377417636728033494, 9220, 11, 4207000288325.890137, -618075239621.894043, 0, 'binary.sazxRvzqSdaThdGddmPa', 'nchar.黑龙江省呼和浩特县南长兴安盟路r座 293112', 1630000000060)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630915000000, -1893993866, -4365820244764275930, 31103, 68, -2066117083545.469971, -5037.126794, 0, 'binary.bMuXKucyDJjBggYIIAvd', 'nchar.上海市玉英县合川许街L座 434453', 1630000000061)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630915000000, 176036972, 6462275287504332531, -12663, 9, 7238554.481668, -24.426760, 0, 'binary.nCtwVdottyyUkynsejOK', 'nchar.天津市秀荣市浔阳施路B座 784962', 1630000000061)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630914999999, 462886464, 2194905473450940337, 12828, 98, 3.368086, 9.615829, 1, 'binary.BRImBBEftaaKZeOGRSvW', 'nchar.江西省广州县普陀潜江街J座 122271', 1630000000061)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630915000000, 1703114933, 1832458494225860699, 29866, 118, -3807017124.642100, -912951646003.170044, 1, 'binary.gPctXUuQQLaDZMWiQWzA', 'nchar.陕西省深圳县城北刘路C座 874212', 1630000000061)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630915000001, -674623669, -6380867918231277409, -1843, -119, 435947.604417, 766787648570.537964, 1, 'binary.XevOOqIPqCmaJUcXaaKz', 'nchar.云南省巢湖市丰都柳州街t座 325613', 1630000000062)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630915000001, -1559227469, -7051851803063121646, -27579, -65, -509948.590885, 2838.905231, 1, 'binary.QfqLcJjHqpLoidJREEnX', 'nchar.广西壮族自治区桂芝县清城宋路Q座 713677', 1630000000062)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630915000000, -1205146480, -6555909736905274411, 10866, 11, -917806.522567, 7477874070.574810, 0, 'binary.zDfAtyjfdffSTqbqeobG', 'nchar.北京市春梅县滨城陈路X座 531714', 1630000000061)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630930000000, -1494759877, 7825831993548386780, 3895, -126, -662731502.286191, 540530.198658, 0, 'binary.GGbtLZVCzfmnUdMHExqv', 'nchar.海南省东莞市上街杜街d座 891312', 1630000000062)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630930000000, -1188314039, -5179463148601079589, -727, -86, 3813.243840, 9214.756895, 0, 'binary.uTiBEULehnYGBTLmKywi', 'nchar.河北省勇市东城石街B座 963105', 1630000000062)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630929999999, 1589240480, 6788343323702456012, 17814, 12, -5156431732058.429688, 47771.262506, 1, 'binary.wJyZOXguJRYIbgIuIRye', 'nchar.浙江省桂花市朝阳杨街E座 783650', 1630000000062)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630930000000, 1494846642, 513923720305876961, 29664, 79, 77561611.652663, -742112715702.303955, 1, 'binary.ymFCwOLqQdnnsFafDwkr', 'nchar.上海市六盘水市孝南上海路i座 565965', 1630000000062)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630930000001, -253589870, -4348911754768762667, -11980, -62, -4485569068.944760, 96365.291525, 1, 'binary.CzIxfDKtViliFCPuYkSq', 'nchar.青海省凯县锡山宁德街m座 619840', 1630000000063)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630930000001, -1105799208, -3800641476990756597, -22387, -123, 63290.349790, 171660453137.690002, 1, 'binary.jQRSOxYnBcNuWfRXmBkT', 'nchar.湖北省丽华县城东西安街w座 178969', 1630000000063)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630930000000, -1072430928, -1022000926125289868, 18740, 19, -2339625154017.399902, -5914609622.300700, 0, 'binary.ZbiUjpLNbsjFkdkPPfbM', 'nchar.湖南省杭州县南溪黄路D座 533554', 1630000000062)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630945000000, 2135168110, 1934009776464408479, 6695, 72, 16367931616261.000000, -584177.637447, 0, 'binary.ICfwWUnCEvXrJWGQNYjv', 'nchar.海南省秀梅市丰都重庆路G座 676020', 1630000000063)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630945000000, -1604447094, -5852815210707209114, 17991, 107, 130598712.764873, 289933642.558557, 0, 'binary.yVowiloVQgZDlfKNlbWh', 'nchar.山西省燕市双滦潜江路o座 423049', 1630000000063)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630944999999, 701182235, 666030532933625291, 203, 77, -1.180313, -2809676597535.319824, 1, 'binary.wFVimJeupABVGEvwMiTv', 'nchar.河北省敏市东丽徐街H座 593999', 1630000000063)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630945000000, 1199058915, 6158980904131584989, 19579, 17, 541875543475.742004, -597076.697923, 1, 'binary.FjLluJSaCVzuHbHvJfzo', 'nchar.甘肃省淑兰市南长马路c座 137684', 1630000000063)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630945000001, -1563988450, -9012098528369421000, -14380, -126, -444977.754816, 72.749039, 1, 'binary.OjWhbhEvRlHpqFEHXmfR', 'nchar.浙江省南宁市普陀周街r座 756238', 1630000000064)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630945000001, -135825310, -1557116121902920061, -27295, -123, -9.714553, -1221.739051, 1, 'binary.PoymphzdepeBanAfXkZU', 'nchar.内蒙古自治区颖市花溪吴路Y座 361290', 1630000000064)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630945000000, 220678360, 6988249753807558085, -2581, 63, 44373988506.265701, -776960795071.900024, 0, 'binary.FceJEjYHrZmPNoGcSJYM', 'nchar.香港特别行政区秀华市金平刘街v座 478775', 1630000000063)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630960000000, 15376875, -6664581571745752730, -13465, 6, 16623190903.482500, 48981341.814827, 0, 'binary.lwtmrfdurUFDSkHMZuOe', 'nchar.辽宁省晨市南湖辛集街k座 508374', 1630000000064)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630960000000, 1241969440, -1628740312418081543, 13806, 9, -321375164064.127014, 309512084.295555, 0, 'binary.nxCrfJncdqIIHvFslwbz', 'nchar.重庆市广州县永川太原街S座 529006', 1630000000064)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630959999999, 1631111972, 732723021793120104, 9398, 75, 81.940353, 358.935299, 1, 'binary.QZSHkvsYmRHnGFnyMije', 'nchar.台湾省兴城市孝南兴城街p座 734723', 1630000000064)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630960000000, 1489317687, 5519190191808186905, 7935, 65, -46699875.436843, -3.678916, 1, 'binary.lMozaeGkbfYcTzrxJcoV', 'nchar.广东省辽阳县龙潭侯街q座 888715', 1630000000064)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630960000001, -1081644320, -7901892167113844563, -4059, -78, -12875427.525138, 777315388163.552979, 1, 'binary.TDrOTkriDBpZtXBadhIx', 'nchar.青海省萍县西夏潜江路R座 701876', 1630000000065)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630960000001, -283328976, -5407024306435227732, -18436, -30, -0.910943, 6223301643044.309570, 1, 'binary.bZikexFXrPKScwiyBTtX', 'nchar.台湾省秀华县西峰关岭街Q座 185151', 1630000000065)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630960000000, 667615766, 7516057978804562053, -6908, 82, 8.929773, -91.562693, 0, 'binary.ugfnkKdKlACxwKMCBEbb', 'nchar.海南省成都市沙市孟路Z座 646247', 1630000000064)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630975000000, 1998856189, -4326051996015773570, 17831, 76, -9.863853, -54574439998977.796875, 0, 'binary.JCxsLYijtQSKzNLYAICB', 'nchar.江西省颖县丰都蔡路q座 399088', 1630000000065)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630975000000, -1961052557, -8491150711564049542, 24318, -42, -54126426844007.601562, -2466916.622248, 0, 'binary.XijTExPHHgzfADOYawHs', 'nchar.海南省西宁市丰都齐齐哈尔路m座 331224', 1630000000065)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630974999999, 1021775776, 722158673624256018, 6062, 102, -94606143062457.906250, 35023478748411.500000, 1, 'binary.AstGPfaVxEefogcWzDzq', 'nchar.重庆市潜江市江北李街Y座 700708', 1630000000065)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630975000000, 1449414466, 6336873094660298228, 12438, 74, -790114.216316, -390.420215, 1, 'binary.secMQTXGKVthFXgMEFYl', 'nchar.香港特别行政区上海县西夏谷街n座 477311', 1630000000065)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630975000001, -1715903601, -471065515874324599, -10712, -65, -64736531762581.101562, -96042855.418683, 1, 'binary.lSSCcLFElDWxKZTdHRLN', 'nchar.天津市北镇县淄川禹街B座 570652', 1630000000066)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630975000001, -176995412, -7722430198424559895, -29062, -58, -5301089280.401760, -31441552360.634201, 1, 'binary.ikoqYkfzTrxBhYnfmKUW', 'nchar.新疆维吾尔自治区秀珍县浔阳王路D座 979771', 1630000000066)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630975000000, -65800677, -4755363672240072131, -11875, 34, 67445667.955342, 40434.343296, 0, 'binary.qDCRKCYQnmiDLFcdBAnT', 'nchar.广西壮族自治区北镇县浔阳宜都路W座 236322', 1630000000065)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630990000000, -1190594011, 8137196870545919699, -28040, -9, -68.827873, 52589.409559, 0, 'binary.lTlvbMfWCVccDOCnIDBK', 'nchar.西藏自治区艳市徐汇六安街X座 527915', 1630000000066)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630990000000, 6478249, -2262813485458902331, 28341, 73, -10648.983192, 21.336997, 0, 'binary.vLfZLtfmMGABLIsYcSuG', 'nchar.贵州省倩市朝阳周路Y座 721501', 1630000000066)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630989999999, 1492685945, 2082419493615737887, 1936, 121, 270330631887.458008, 464724.616085, 1, 'binary.esYXdqdUGVFhLCHHciWA', 'nchar.宁夏回族自治区合肥市永川昆明路Z座 321933', 1630000000066)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630990000000, 746488837, 4063291949108736915, 11202, 12, -294206344.943296, -54829.586060, 1, 'binary.ynUOCSQSnPGHiIFNcUlr', 'nchar.河北省慧市朝阳台北街m座 705236', 1630000000066)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630990000001, -457991373, -2036730114286415967, -2136, -105, -5092278572843.200195, 17587773689.161999, 1, 'binary.OCGshioisqgLTUFLRMSc', 'nchar.上海市南宁市沙湾南昌街q座 991276', 1630000000067)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630990000001, -1242447360, -2110971702178298800, -16302, -30, -4130.477367, 8402.793080, 1, 'binary.jHAuHSxbhQHBBbaHCVSe', 'nchar.云南省石家庄县江北程路B座 226188', 1630000000067)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630990000000, 71825038, 5708278874002188089, 4731, -80, 86.310448, -81380752238174.406250, 0, 'binary.WsCFazjKcLJjwrFHPelC', 'nchar.新疆维吾尔自治区永安县白云巢湖路j座 278171', 1630000000066)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631005000000, -925781701, -5868946429108894256, 5048, 19, 6559385936.497240, -9870442107.342270, 0, 'binary.iLESXKenJaWTIeKHFleM', 'nchar.新疆维吾尔自治区浩县高坪田街l座 923007', 1630000000067)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631005000000, -463506149, 8120086957375329125, -6384, 111, -9995250.448939, -736463.811003, 0, 'binary.AcGtBjWRUbXgHysDRMhs', 'nchar.江西省璐县海港惠州街g座 399144', 1630000000067)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631004999999, 306491871, 6281636598236510189, 24086, 67, -639.213146, 35478635363.208801, 1, 'binary.WPbbRgxYIRecrOFSlRYM', 'nchar.西藏自治区宁德市孝南刘路O座 708041', 1630000000067)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631005000000, 1797589305, 162080808454486608, 29046, 34, -54.791716, -71755480989.907196, 1, 'binary.DdiJOQDpLZuOsXbvtmFm', 'nchar.甘肃省台北县长寿高路t座 429482', 1630000000067)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631005000001, -709883988, -950329647663976821, -6274, -38, 64687083.568152, -133880631485.500000, 1, 'binary.cADQdfbZkXfcNPPOPQwp', 'nchar.宁夏回族自治区静市江北覃路u座 523043', 1630000000068)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631005000001, -1863995155, -8908360222222924661, -32472, -104, 82.537309, -89404622537910.703125, 1, 'binary.PBNnqQRGjNfbmaGVYlKD', 'nchar.新疆维吾尔自治区六安县西峰阜新路d座 990661', 1630000000068)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631005000000, -775088092, -151303921680143861, -23850, -72, -89808431301.207199, -646.674122, 0, 'binary.PGaLWNQGDnVSztoNNsSO', 'nchar.贵州省佳市翔安宜都街Q座 487382', 1630000000067)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631020000000, -487874477, 8081110518718693872, 10742, -8, 59233198.443672, -81768146.768733, 0, 'binary.AfIofVnHdvWMCGyeajlC', 'nchar.广东省利市锡山崔街X座 803273', 1630000000068)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631020000000, 822234976, 8508573074776208709, 17893, -96, -82.790723, -288719.773859, 0, 'binary.DoAGpoAKorgskVVmqIgo', 'nchar.河南省淑珍市淄川西宁街S座 903273', 1630000000068)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631019999999, 1639981656, 5471134581418657990, 22743, 76, 6306.116018, -7013891.566539, 1, 'binary.mVTonpWtYFauqLNntbcK', 'nchar.山东省巢湖县门头沟黄路g座 818470', 1630000000068)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631020000000, 1877786780, 1239434978427674122, 3482, 55, 4308520.332734, -7.194133, 1, 'binary.ATBjqErbvWNgzCwEXuVp', 'nchar.辽宁省凯县锡山广州街A座 668121', 1630000000068)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631020000001, -40134567, -5917934079243952151, -8151, -41, -1.160545, -5389824243.111000, 1, 'binary.hSZKVixlyoxpHpdvwYxY', 'nchar.黑龙江省合山市璧山吕路T座 593619', 1630000000069)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631020000001, -2005527285, -6681721755343029867, -25686, -12, -362992022448.400024, -5460308076525.500000, 1, 'binary.oWDnMrJWMhYgnspOVCyk', 'nchar.河南省明市城北阜新路D座 863966', 1630000000069)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631020000000, 24149104, 3385048008347468396, 25731, -51, 8252.733643, -4262287074.133500, 0, 'binary.pwblolLPhJQylCJAhGzn', 'nchar.浙江省柳州县淄川成都路q座 844741', 1630000000068)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631035000000, 212157003, -1467339935461255928, -7186, -34, 7497729511.586900, -98266.367876, 0, 'binary.jDxeCyLAKzKgEHGHAlif', 'nchar.广西壮族自治区桂英县永川张路K座 770340', 1630000000069)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631035000000, -2004196647, -7368122492859630941, 15455, -76, -57468499221.112396, 3727126030.311050, 0, 'binary.FIJWJSaOYEwWZHqhongU', 'nchar.上海市成都市花溪宜都路B座 441493', 1630000000069)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631034999999, 1412945598, 9027606879125343725, 21349, 38, -73686.799340, 3622986817296.899902, 1, 'binary.EPkNzxzaeCAWQetzzzlv', 'nchar.河南省荆门县沙市梧州街X座 162307', 1630000000069)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631035000000, 2125121614, 5109732906858974031, 26301, 81, -52025949738.545998, -0.369444, 1, 'binary.bjABIQwvmToiYBuNZkgo', 'nchar.青海省欢县龙潭荆门路q座 401300', 1630000000069)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631035000001, -517552216, -7052845906951781652, -9720, -120, -259291.640787, -9.749093, 1, 'binary.AmPftzlwURAILqlHhYtv', 'nchar.浙江省凤兰县南湖太原街U座 609741', 1630000000070)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631035000001, -534996716, -6181873037570436300, -10798, -57, -8617907014714.200195, -583022086.598419, 1, 'binary.EhUOJdpDsZWYyyDDATpW', 'nchar.河北省琳市城东六盘水路z座 798939', 1630000000070)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631035000000, 502477321, -3940198187319461410, -590, -122, 11555.763461, 3.474600, 0, 'binary.iufgzJsvjLKSoGJGBiLS', 'nchar.黑龙江省佛山市大东兰州街G座 694559', 1630000000069)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631050000000, -1001974902, 6450834421987986410, 3297, 112, -8092802.381089, -13006311.799326, 0, 'binary.ELgeHxhmjoFyFgkFXZui', 'nchar.贵州省阜新县高明永安路M座 392539', 1630000000070)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631050000000, -1991708456, -864737276653013043, -1035, 104, -621.411878, 358.192500, 0, 'binary.dzcYxzyOZMYgtByHiDTc', 'nchar.重庆市上海县东丽崔街N座 481064', 1630000000070)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631049999999, 1210716085, 5098727175987952663, 14612, 99, 122287663.454320, 1915439.569884, 1, 'binary.xPKETDcCdbWtoYoUCgre', 'nchar.甘肃省晨县上街龚路E座 215933', 1630000000070)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631050000000, 372703643, 5265014540357675070, 11084, 4, 955.287790, -677466762016.219971, 1, 'binary.TyniuJCZqfeldsiTGmnn', 'nchar.四川省济南市六枝特尚路V座 872192', 1630000000070)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631050000001, -1186882087, -6843278012193945664, -21562, -48, -73364783.992379, -8374993150.154530, 1, 'binary.HVKExGKBzdrjjQbKdgim', 'nchar.四川省通辽县西夏秦路J座 522373', 1630000000071)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631050000001, -1513635853, -4943689304527658905, -26256, -17, -157022486.675383, 47593146446.323402, 1, 'binary.xmsafXeSTguAIwMoWsSm', 'nchar.陕西省哈尔滨市大东辛集路G座 719181', 1630000000071)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631050000000, -2096895302, 2527287882879318478, -11906, 39, -2092909636.574400, 832609111532.883057, 0, 'binary.DpAgGGDNkTqeFPziAoZv', 'nchar.重庆市旭县翔安杨街N座 500333', 1630000000070)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631065000000, -607426580, 480106595535647664, -14319, 71, 485500665.146682, -7157.602006, 0, 'binary.NuToHMvRscGxwMENZqOS', 'nchar.贵州省秀荣县璧山申路Y座 936410', 1630000000071)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631065000000, -1266376029, 7399492335983317341, 21772, -21, 3959426326750.160156, 4299575.449645, 0, 'binary.sdhZOekTsIiwOYFEShct', 'nchar.重庆市大冶县友好长沙街T座 325982', 1630000000071)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631064999999, 1209625354, 8611086034262933876, 7159, 57, 66305514.987262, -249576.528082, 1, 'binary.xDqLioOjrUudmVTRaKmP', 'nchar.广东省海门市魏都北京街t座 626317', 1630000000071)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631065000000, 423864530, 7171258036520933259, 5585, 41, -9923192491285.410156, 9.901365, 1, 'binary.JiryCLzxGYRamjkLmMDH', 'nchar.江西省澳门县新城重庆路b座 208813', 1630000000071)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631065000001, -1049869765, -6885754556853202467, -6460, -118, -883111.347273, 654352667.175980, 1, 'binary.ZcJrqAQpBfTRFJupVZDH', 'nchar.陕西省雪县长寿淮安街o座 362739', 1630000000072)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631065000001, -1239790498, -4444805752431922421, -28574, -70, -70077.276971, -76046.699605, 1, 'binary.mCAmZahnycvOkwVDVpgV', 'nchar.辽宁省上海市合川赵路q座 784249', 1630000000072)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631065000000, -1048465520, -2935190573459597102, 21673, 87, -2585566.109742, -2007.606237, 0, 'binary.HNslYqKBtmSuAbsFzwKj', 'nchar.山西省莉市海陵昆明路t座 518218', 1630000000071)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631080000000, 630458653, 711094046704512569, -30447, 17, 18100.228068, -28061872.208530, 0, 'binary.SrRBINnQSvdBTBKNGHFz', 'nchar.吉林省敏市高明尹街t座 876540', 1630000000072)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631080000000, 1587008473, -497820597464002656, -14935, -92, -5426069.379042, -9834.237204, 0, 'binary.qRermCLWFAWnLBFapDsH', 'nchar.陕西省军县璧山石家庄街l座 337307', 1630000000072)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631079999999, 482971664, 4456457973181208346, 29811, 105, -2892240.803873, 28282444.178390, 1, 'binary.ZumfNrRUgHCXCqqjBwgX', 'nchar.安徽省宁德县南湖关岭街K座 983548', 1630000000072)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631080000000, 1255752961, 4617338544672893130, 31384, 45, -3965.493491, -943974751.371172, 1, 'binary.xoiyGCRmWsgMFgfrdONj', 'nchar.内蒙古自治区鹏县南湖石家庄路E座 253529', 1630000000072)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631080000001, -1695698428, -671923235747543675, -26225, -99, -89596996493.199203, 53.155372, 1, 'binary.VzjkizbqoeqlOEtsqzTd', 'nchar.海南省台北市锡山长春街F座 321351', 1630000000073)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631080000001, -470196570, -1577429027012366617, -7829, -48, -39.524752, 51706491389416.898438, 1, 'binary.rOSrdNRJmYutBQkKnIvQ', 'nchar.辽宁省哈尔滨县上街南京街x座 664388', 1630000000073)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631080000000, 752187387, 5814377969189879673, -39, -37, -1240781.998722, -680710584267.488037, 0, 'binary.bFiODlLTjBvFimRGPEZH', 'nchar.上海市桂荣市高明张街U座 936978', 1630000000072)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631095000000, 1299240809, -3372032764175680697, 18966, 71, 284347227.242907, -129822972.670593, 0, 'binary.eqzlYcordfJQQJMBljjF', 'nchar.浙江省台北市大兴邯郸街l座 594497', 1630000000073)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631095000000, -601799850, -773012696448162526, -24255, 69, -85539.695742, 8483599308481.120117, 0, 'binary.fodqpEDidmdgTobhLwVp', 'nchar.江苏省秀荣县大东李路o座 709775', 1630000000073)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631094999999, 369111551, 4483914872861013771, 27035, 120, -61260954640579.898438, 5141985.329967, 1, 'binary.hBVqWjfnRCktdjBpGTYb', 'nchar.宁夏回族自治区六安市秀英哈尔滨街M座 426619', 1630000000073)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631095000000, 1544340960, 8517061698296926738, 16940, 119, -62221249203.984596, 7342075383114.320312, 1, 'binary.EbuTOcRDAAmBGOFvyfZO', 'nchar.宁夏回族自治区武汉县龙潭蔡路M座 210568', 1630000000073)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631095000001, -1324385506, -8127100560658016141, -3090, -49, 1019018870.899400, -86.370275, 1, 'binary.aBZiPEKTvPqdXPsDXgvy', 'nchar.云南省晨县山亭宜都路d座 616893', 1630000000074)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631095000001, -2055898711, -5445111939290776239, -31593, -49, -77345663240.806595, -914.503516, 1, 'binary.BSnRpSkSznADKblDKOiG', 'nchar.河南省凤兰市高明黄路M座 303020', 1630000000074)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631095000000, -943484549, 260761652713813572, -31076, 88, 886963.235930, -38250864.484145, 0, 'binary.qUWErZCBbTHzETMfuRMa', 'nchar.陕西省兴安盟市和平乌鲁木齐街w座 905284', 1630000000073)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631110000000, 1560015019, -6729804845934136099, 352, -115, 9316.130184, 220903.824895, 0, 'binary.fWtsEHhFIrXfqBGYlTNJ', 'nchar.海南省济南县白云李街S座 920870', 1630000000074)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631110000000, 600240137, -4687958607276237783, 31931, 50, 5721.825376, 570753990973.805054, 0, 'binary.utJxZlgPEdcqNAFObuFZ', 'nchar.广东省台北县普陀兴安盟街T座 505320', 1630000000074)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631109999999, 1370148713, 2682283741646434124, 21747, 108, -7336368856593.450195, -86249517.560025, 1, 'binary.ApKvhLsWkpRndmodhPEN', 'nchar.云南省雪梅县安次卢路J座 135754', 1630000000074)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631110000000, 656300451, 4206774300939189669, 24233, 69, 540899878.718127, 5.252156, 1, 'binary.TRvUgqDMiPmpDbTDQJIO', 'nchar.四川省香港县秀英关岭路Y座 839081', 1630000000074)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631110000001, -1440742963, -6981670779291616929, -17863, -118, 9339406547236.320312, -80509839.572484, 1, 'binary.nsuhquhIJtXKINhuhbZm', 'nchar.天津市齐齐哈尔县朝阳西宁路L座 393435', 1630000000075)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631110000001, -1680777424, -3789071286423645612, -29891, -91, -3687858.975654, 28786.604619, 1, 'binary.YfWaoxPFJKJagXhyEOmA', 'nchar.湖北省银川县崇文何路i座 475105', 1630000000075)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631110000000, -1095049119, -2038760289253362613, -4186, -124, 13304836.627667, 77695031812.673798, 0, 'binary.TEZgbSkvAdOPvsssvtOX', 'nchar.四川省鹏市东丽胡路Q座 963536', 1630000000074)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631125000000, -1618319757, 6350170596406303692, -28807, -17, -1265501669.731500, 757504.607632, 0, 'binary.xQXfGqKJJntmbtHJzqHc', 'nchar.吉林省重庆县南湖郑路E座 472846', 1630000000075)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631125000000, -2143880375, -2493359647253702127, 6606, 66, 404444.216865, -7014.675537, 0, 'binary.bPjaavfcVrQcVEYxUtAe', 'nchar.安徽省兰英市涪城王路f座 265166', 1630000000075)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631124999999, 1951461498, 4426546683527923406, 26976, 106, 861491.829361, -5937918253.865560, 1, 'binary.hZDumWlAzVpYfXkEaPlW', 'nchar.云南省重庆县清河陈路P座 522865', 1630000000075)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631125000000, 1219706153, 7536523458446532711, 5155, 120, -675841841.464953, 91755264081.884003, 1, 'binary.KsCtLZBdLUvbbTIKedAw', 'nchar.香港特别行政区永安县高坪南昌街L座 368651', 1630000000075)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631125000001, -1663211983, -3109525234509445099, -21222, -8, -21298956.340869, -9657.318325, 1, 'binary.wBVWJVLFoOdXTzTtRNuT', 'nchar.广西壮族自治区惠州市高明柳州路b座 545369', 1630000000076)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631125000001, -1810125590, -8421898795641820385, -4441, -89, -2734964221.741360, -37493460960.576599, 1, 'binary.yxiYtSxCJYuTTjLuCcLI', 'nchar.辽宁省太原市和平侯街i座 586968', 1630000000076)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631125000000, -1416386308, -7684571277090397009, 22278, -110, -31.162606, -776.531904, 0, 'binary.XYkmkoMHknNtlosmchSh', 'nchar.河北省郑州市花溪周路n座 600734', 1630000000075)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631140000000, 989052773, -5476387551687883714, -11570, 127, -37404372.212454, 42699806.700508, 0, 'binary.SifrNRSsBKcJKDqsnfXN', 'nchar.云南省拉萨县高港张路n座 873322', 1630000000076)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631140000000, 90148890, -9176843805101255090, 7217, -69, 7747.651553, 246706074.781856, 0, 'binary.iuKZktHzJRwchXDkGhbL', 'nchar.河北省娟县高港汕尾路p座 635670', 1630000000076)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631139999999, 2092906691, 5816259342140952837, 27223, 3, -21145.797856, 39546630288.518501, 1, 'binary.hLAAdJsauemmUBKhTtqW', 'nchar.江西省沈阳市海陵周路O座 413415', 1630000000076)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631140000000, 1419713973, 818822617965851283, 32245, 47, 92022664.783976, 3311483.287128, 1, 'binary.yfRnxroKprvGvOyoTlEh', 'nchar.广东省辛集县萧山西安路M座 462571', 1630000000076)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631140000001, -2137737633, -3177008163567761597, -29189, -74, 548693287695.377014, 72371.674202, 1, 'binary.wRfxAxVfEcdfKteGLuaX', 'nchar.内蒙古自治区淮安县普陀杨街A座 734889', 1630000000077)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631140000001, -1023034969, -7642623044713882631, -13299, -59, 3279504.422480, 6716289645.631150, 1, 'binary.vlQPmKsSiqtoGCixEBsV', 'nchar.广西壮族自治区晶市上街宁德街B座 609891', 1630000000077)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631140000000, -30379416, 1958421706180608853, 1892, -29, 4674012.774896, -21398530385.866001, 0, 'binary.pxvGQUheUZkqNsxiAZuT', 'nchar.澳门特别行政区倩县房山杨街U座 114062', 1630000000076)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631155000000, -1117272908, 752098947372767902, -2819, -56, -9085630.993944, -6895774866846.309570, 0, 'binary.HlxBvTuGkChJDLzDaxyu', 'nchar.陕西省昆明县龙潭周街D座 817221', 1630000000077)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631155000000, 2086908228, -5781823409166665769, -3324, 9, -3.671731, 50135208435.527000, 0, 'binary.UWkIRXVdPsZxZFVaWBtS', 'nchar.湖南省郑州县清城古路B座 982490', 1630000000077)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631154999999, 870002160, 4167669984685102889, 30468, 115, 641517032220.147949, 2742.639429, 1, 'binary.JCROwpmJQkPqZkIzrRkU', 'nchar.江西省静市黄浦潜江路B座 249825', 1630000000077)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631155000000, 1396830896, 8133688472099187160, 22272, 75, -4524332410799.129883, -35663261.321445, 1, 'binary.kOEOFvSdtLEFJlgywaSJ', 'nchar.河南省英市长寿汪街E座 114327', 1630000000077)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631155000001, -837033747, -5283148083077494740, -1260, -39, 565321.239314, 275.746366, 1, 'binary.YKouegacQvxoSSgTaPrv', 'nchar.安徽省淑兰市新城李路r座 256922', 1630000000078)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631155000001, -1906954896, -5724786982458123670, -1426, -24, 299922.459639, -1285.219883, 1, 'binary.ZqHGrayEGFBlDHVAObSZ', 'nchar.甘肃省桂芝市金平武街X座 777677', 1630000000078)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631155000000, 745556057, 516983514584470898, -12904, -49, -716006320.556210, -218.138046, 0, 'binary.ZJBaxSKndLZrHVRppYnT', 'nchar.广西壮族自治区梧州县秀英甘街b座 779502', 1630000000077)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631170000000, 2139488417, 6066602372775042425, -1518, 84, -80581942.592809, -37314203203.592003, 0, 'binary.yymGCDKbqZkixkIjIIGT', 'nchar.湖北省荆门市和平淮安路J座 884655', 1630000000078)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631170000000, -374714905, -7736740878366341122, -14276, -79, -64248.970229, -68011884269.584396, 0, 'binary.hMgGBmPqKYkzsArGfbxQ', 'nchar.新疆维吾尔自治区沈阳县城东齐齐哈尔街b座 794676', 1630000000078)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631169999999, 1397781824, 6557161155035163362, 11749, 37, -3.906174, -1.651866, 1, 'binary.hwfhcmJDDTvQaCRmlKCd', 'nchar.江西省玉华市长寿梁路o座 261941', 1630000000078)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631170000000, 1387979869, 2661199544340098486, 9621, 49, 166489205686.515015, -7163639.756925, 1, 'binary.atyzvvxverSTvukiOHRc', 'nchar.澳门特别行政区琳县南长深圳街L座 927560', 1630000000078)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631170000001, -1573224659, -6074816603000302535, -28617, -101, 26624788159.678699, 62.218691, 1, 'binary.QpcyRVyxqqnHSSLxWCdz', 'nchar.台湾省荆门市和平关岭街w座 554552', 1630000000079)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631170000001, -613007188, -2265367830140686655, -30242, -74, 132051246084.664993, 9583075.585374, 1, 'binary.ZwGExMXBaZQnYGUqOZnk', 'nchar.贵州省天津市龙潭吴街P座 514026', 1630000000079)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631170000000, -229811940, -4723474584235949728, -30413, -126, -37.161630, 6566121399.880410, 0, 'binary.QKZaMSaDgNKbuVjVXVZX', 'nchar.湖北省超市友好南宁街b座 387820', 1630000000078)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631185000000, -1462514908, 6815473990049018903, 26765, -60, 8053251416.958930, -94687.447387, 0, 'binary.gMqqCIzQCNGNgbSwolZX', 'nchar.山西省春梅市魏都张街I座 216775', 1630000000079)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631185000000, -1558957537, -3515355671345392932, 32525, 83, -873729.459053, -4948145879174.500000, 0, 'binary.eHTWpPQVAzoRsZXjDHro', 'nchar.内蒙古自治区雪梅市西夏广州路T座 462974', 1630000000079)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631184999999, 1354974803, 4535068117623812145, 15974, 126, -3641262362.474420, -215837.649412, 1, 'binary.mgwWzOiTRoTHUbvzEMSm', 'nchar.宁夏回族自治区欣县东丽南昌街a座 409781', 1630000000079)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631185000000, 909020116, 7166005914556535033, 14716, 26, 50294221.576123, 67159744502236.203125, 1, 'binary.aHafHhYbmjwnjfaDKjaV', 'nchar.天津市马鞍山县南溪宜都路W座 812458', 1630000000079)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631185000001, -1306330996, -7521905733671260635, -1, -23, -651220.365910, 209.745394, 1, 'binary.xazZjdXNjaykPYfDWhos', 'nchar.广西壮族自治区慧县南溪李路n座 595049', 1630000000080)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631185000001, -829584454, -6819959797009261424, -30794, -119, -2545863021084.839844, 48573662965.169998, 1, 'binary.RQfCFnyRhVGAwoZKbfBg', 'nchar.江西省畅市大兴呼和浩特路N座 966282', 1630000000080)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631185000000, 159441443, 3829890051273914786, 22074, -46, -0.126559, -59066.623155, 0, 'binary.sCHQwHDxsHrpfheUsPvS', 'nchar.甘肃省婷婷市和平林街X座 967902', 1630000000079)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631200000000, -1984142506, 918572017792917031, -20270, -51, 7172084337794.419922, -67689618090033.000000, 0, 'binary.pfhSySLjKZbOfIPTmqAB', 'nchar.宁夏回族自治区亮县牧野石家庄路L座 408034', 1630000000080)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631200000000, -1972988665, -3214499273794963694, 3010, 42, -8898.495820, -7139.318526, 0, 'binary.ivdrcXkAeWkscdYzZJEe', 'nchar.浙江省瑞县海港永安街R座 494785', 1630000000080)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631199999999, 1089140558, 5653452583897077783, 8528, 7, -9816158771686.500000, -30.580967, 1, 'binary.tqyfrLsfYAZeSvFCwYXQ', 'nchar.安徽省倩市梁平广州路E座 417291', 1630000000080)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631200000000, 1752602476, 8379644292871336124, 1479, 55, 0.152144, 8340466.258810, 1, 'binary.CaqLbGdnvyvDGmnqAdwA', 'nchar.宁夏回族自治区兰英县滨城董街m座 612134', 1630000000080)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631200000001, -2070609011, -3927356551694226099, -14160, -96, -489939465270.245972, -176.619265, 1, 'binary.HtBovhAaClqmUKDVoPoA', 'nchar.西藏自治区兴城市西峰李街O座 294081', 1630000000081)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631200000001, -507025125, -6769707689103886410, -26670, -62, 7.854748, 91216299872.798492, 1, 'binary.kVTRaSjNPeaJAGYzWqVF', 'nchar.山西省邯郸县六枝特袁街q座 379922', 1630000000081)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631200000000, -91932554, -8911762384559779952, -12134, 114, 8455.888482, 192830806126.287994, 0, 'binary.UmxEgaxmRlsWpDOOjGns', 'nchar.安徽省秀云县金平亢街Z座 413285', 1630000000080)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631215000000, -2106211575, 4343732691913651727, 30695, -65, -1182138766033.570068, -1420143959702.800049, 0, 'binary.lENnnzItGfhHBhuBSKkg', 'nchar.宁夏回族自治区淮安市大东章街Z座 522024', 1630000000081)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631215000000, -1514266386, -1557718503233026241, -10740, -93, -3712.656749, 628.936406, 0, 'binary.KCtPSzikgeatarBhSEXT', 'nchar.安徽省莹县淄川石家庄街i座 428331', 1630000000081)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631214999999, 243460896, 1867078685154320423, 13109, 8, 2562.515572, -8164811617493.440430, 1, 'binary.GwEkcVzBVjqKKvBBYCHl', 'nchar.广西壮族自治区邯郸县门头沟兰州街l座 647959', 1630000000081)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631215000000, 669742703, 3588783546529218834, 10833, 6, -408721127673.909973, 801103747.542088, 1, 'binary.uwFahDnlWanEVobNZkCM', 'nchar.甘肃省张家港县白云罗路f座 137164', 1630000000081)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631215000001, -442417658, -7549129289634916853, -2159, -58, -98911613476.491501, -874.865816, 1, 'binary.aMVpynzemnksvKTWGcDC', 'nchar.吉林省建国市牧野钟街E座 377779', 1630000000082)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631215000001, -1409923779, -8427760215446628026, -18353, -47, -50606823244.468002, -345539443425.945007, 1, 'binary.cugznUahvcPVWaAAckbf', 'nchar.海南省宁市合川宜都街o座 305915', 1630000000082)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631215000000, -1342274642, -8536170080112789400, 30067, 115, 12581450912.783600, -291.972491, 0, 'binary.eOcZzgBgRRIVelvjVeto', 'nchar.澳门特别行政区欢县高坪广州路N座 593765', 1630000000081)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631230000000, -456444500, -653943768841413221, -16654, 115, -74554935958741.203125, 7070430643.419100, 0, 'binary.zrOBefSbOEUgRLtVobMw', 'nchar.北京市璐市东城太原路Z座 165710', 1630000000082)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631230000000, 1112822860, 8809469361686033607, -22243, 84, 718004818.589884, 836.474427, 0, 'binary.cLnnuHpDQIMfMBODNYpW', 'nchar.青海省广州市房山阜新路h座 898094', 1630000000082)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631229999999, 993462171, 7980976104707598817, 22906, 119, -90532577531185.000000, 2883997.534517, 1, 'binary.akkJQvtKVOUMeEJdtLWQ', 'nchar.辽宁省西安市魏都长沙路q座 830446', 1630000000082)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631230000000, 659535811, 6648102677976806439, 14934, 106, 8.999744, -78105262.595693, 1, 'binary.RBWgiabXuPgAeanvsJoE', 'nchar.香港特别行政区倩县长寿王路D座 334836', 1630000000082)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631230000001, -1147877996, -6106212901363852731, -30865, -72, 9390.844553, -7383783838166.849609, 1, 'binary.UsylaHHTjnwUCmWOFRop', 'nchar.新疆维吾尔自治区齐齐哈尔市海港南昌路q座 642772', 1630000000083)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631230000001, -8588768, -7260337404461154045, -21386, -116, 46305041.132235, 6654257695.256100, 1, 'binary.SneLCpCrEbSFqlWIVLwu', 'nchar.陕西省欢县花溪荆门路T座 249790', 1630000000083)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631230000000, 1003481306, 6017067765177374344, -16218, 60, 82.649494, -712.838445, 0, 'binary.qLRxDlzDPjVVbRZHOADr', 'nchar.河北省畅县徐汇大冶街D座 822863', 1630000000082)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631245000000, -294434895, 1924629096591118432, 17988, -44, -63587509.905294, -40747153.224679, 0, 'binary.olALLZcgYbqmLLsQAfvv', 'nchar.云南省强市淄川淮安路c座 262889', 1630000000083)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631245000000, 738129131, 1815388765763244472, -6405, -93, 52.721010, 216.728740, 0, 'binary.yxHFbosrzvHvxaTBUoNe', 'nchar.浙江省福州县梁平嘉禾街i座 170922', 1630000000083)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631244999999, 224419688, 3680471494276801590, 18367, 87, -74383724488933.593750, 0.593016, 1, 'binary.FsYcmhXFIVEwuCmapLFd', 'nchar.青海省合肥市清浦辽阳路I座 165564', 1630000000083)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631245000000, 54658682, 2220380251657075236, 12017, 72, -2409.191361, 16465604801985.900391, 1, 'binary.WVLQqvEMPTlqXptHGBTA', 'nchar.陕西省桂珍县魏都陆路z座 653467', 1630000000083)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631245000001, -111362062, -2079998725394443456, -14043, -83, -8050700.333128, -786.595631, 1, 'binary.wugzMLvmLdkxesgmixOH', 'nchar.安徽省南京县清城柳州路Q座 634494', 1630000000084)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631245000001, -646454407, -7621015329762251852, -32454, -96, 58254987.947866, -86.835965, 1, 'binary.nUZjCcyTPHcAJQjyqiKW', 'nchar.河北省嘉禾市锡山张街F座 809458', 1630000000084)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631245000000, 1630911956, 9008151774496407995, 3422, 97, 1410480123999.199951, -40614657267267.296875, 0, 'binary.kaWyIWeHVgVeCBWtXjos', 'nchar.青海省婷婷市和平黄街G座 571553', 1630000000083)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631260000000, 1229299947, -3222073200735135026, -25377, -68, 76060893505437.296875, 79056750996157.000000, 0, 'binary.TIeullzMmPeJOEDzERVs', 'nchar.内蒙古自治区长春市普陀通辽路R座 260095', 1630000000084)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631260000000, -1829114311, 192679604105772908, 25050, 51, 429584494.456619, -7710.519461, 0, 'binary.sxkAVzIUJGdjrEKTWivG', 'nchar.重庆市帅县南湖长沙街T座 169523', 1630000000084)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631259999999, 2133083813, 7400378387863708089, 15500, 29, 483042885.250631, -4018.354623, 1, 'binary.buDeWRmTqpbBhNKoIWjt', 'nchar.天津市婷县新城呼和浩特街Z座 481098', 1630000000084)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631260000000, 2102189676, 8289210275637280206, 12319, 12, 2360.126260, -0.783536, 1, 'binary.zkocBZqDNaubCdBsYlZK', 'nchar.河南省潜江县吉区邱路R座 988052', 1630000000084)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631260000001, -88622269, -8452436543600128194, -14877, -107, 2.227830, -896.432470, 1, 'binary.fgqZcQmlErAwLKYvDwkA', 'nchar.福建省嘉禾县城北汕尾街S座 162248', 1630000000085)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631260000001, -603730496, -5142204606726944282, -6542, -22, -701.644082, -75915023637.720001, 1, 'binary.JtkfvmLGOlQzBtNKysLt', 'nchar.云南省沈阳县高港李路S座 869342', 1630000000085)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631260000000, 2144837535, 6790333755067852306, 29522, -5, -5292215263549.620117, 9659.226553, 0, 'binary.ZtAHxvlbFZNbmkaMfRDe', 'nchar.台湾省大冶市吉区石家庄街O座 527169', 1630000000084)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631275000000, 1290141025, -2143848369719800423, 7189, 8, -822375762195.462036, 20283811003.944000, 0, 'binary.loHjtwzgyvJksRhJNiQS', 'nchar.河北省佳县徐汇刘路G座 705107', 1630000000085)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631275000000, 801182958, -7572521914234103164, 4604, 107, -859019.418204, -261693048307.811005, 0, 'binary.YAcVYbiCpwSqCdWIgbBo', 'nchar.上海市武汉县闵行杨街a座 347736', 1630000000085)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631274999999, 521859506, 3602105093569999375, 21665, 21, 9019367.551508, 532663957703.719971, 1, 'binary.NgxLslBkOvvWWVcDCUlC', 'nchar.海南省哈尔滨市翔安沈阳路r座 486389', 1630000000085)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631275000000, 1634351470, 1549009549596999035, 26284, 69, -2.961951, -6310787053.197710, 1, 'binary.AERkUPEHGySISfgqulEw', 'nchar.台湾省西安市双滦深圳路R座 706492', 1630000000085)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631275000001, -1577694201, -8482950364465928222, -19741, -105, 351508461253.127991, -28134084860.892399, 1, 'binary.zgfPMXusJGeFxLLImITl', 'nchar.黑龙江省红梅县吉区嘉禾路Y座 870326', 1630000000086)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631275000001, -1817845693, -2298814801521952311, -14645, -83, 223713055.105098, -75532242.499559, 1, 'binary.flJqybaXNWGYZTymaTJG', 'nchar.江西省淑珍市平山周街Y座 953199', 1630000000086)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631275000000, 632609765, 4219748384315739773, -26000, 71, -450862164529.280029, 14560508342304.800781, 0, 'binary.ZHQyoqVmbWNpweggaXDK', 'nchar.宁夏回族自治区辛集市城北辽阳路N座 240842', 1630000000085)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631290000000, -770991939, 5485773724851375125, 23994, -64, 623888353027.609009, -55981854.579338, 0, 'binary.QKhLWpoMPsfwyQYNoPZA', 'nchar.上海市玲县普陀龚街p座 566117', 1630000000086)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631290000000, -793126068, 2224199903529481270, 6843, 106, 99682310978838.093750, 467505798786.474976, 0, 'binary.HmkxkSqBoJJStxYjyvrz', 'nchar.天津市志强市沙市雷路g座 547769', 1630000000086)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631289999999, 896685945, 4158350015273846866, 26730, 53, -6200015073790.200195, 12566017.333818, 1, 'binary.NEkCzZWDoXmvngPlArSM', 'nchar.天津市海口市魏都黄街O座 679546', 1630000000086)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631290000000, 832269201, 4123871706026762985, 13927, 88, 499.930604, -70366099857.880005, 1, 'binary.mADUJbHlBrbziiXvrWeZ', 'nchar.云南省惠州市沙市海口路c座 131412', 1630000000086)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631290000001, -102246541, -3324596924342719960, -3905, -96, -4816.768188, 93915714.748034, 1, 'binary.qiNYdNPnHVKQBpzXATmm', 'nchar.河南省呼和浩特市清浦广州路g座 216441', 1630000000087)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631290000001, -282159644, -7801886583063959893, -15115, -96, -7.823940, -2620.949488, 1, 'binary.GzTEiirhkGvJXbGkiPrq', 'nchar.香港特别行政区玉珍市朝阳刘街r座 939747', 1630000000087)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631290000000, -669399368, 5115472553771177504, 12526, 91, -28942186887.971001, -6.435703, 0, 'binary.XIZFOElmYhlMyQRewDCt', 'nchar.甘肃省乌鲁木齐市西夏贵阳路T座 579190', 1630000000086)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631305000000, 262080824, 8383920960732834455, 13968, -34, -45.211929, -1300.277717, 0, 'binary.hIMCRvrXPAOVpFRmYnUy', 'nchar.江西省北镇市双滦刘街W座 798066', 1630000000087)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631305000000, 2114811730, -1509702226741249727, -13264, 72, 2.150671, 5261316406.519730, 0, 'binary.FtIdbPhtUPjIuTSbpJQq', 'nchar.新疆维吾尔自治区沈阳县锡山林路M座 892641', 1630000000087)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631304999999, 2143525047, 6899945126249773541, 26738, 102, -292.795385, 138207363422.498993, 1, 'binary.eIPLVCyRogBRyzGbrFic', 'nchar.辽宁省福州县长寿杨路z座 941701', 1630000000087)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631305000000, 1073200557, 807774567632061607, 29517, 113, 67931.421812, -1651664.861578, 1, 'binary.UyFmxxBiWhvpNXabSZxN', 'nchar.天津市斌市普陀哈尔滨路P座 502633', 1630000000087)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631305000001, -489200958, -5831038164958816606, -16406, -76, -6392689.485758, 3750666.814934, 1, 'binary.bCLmDBrnkgDcXvcwrBqA', 'nchar.甘肃省哈尔滨市吉区苏街y座 407071', 1630000000088)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631305000001, -2127347634, -4083644771875265580, -20817, -30, -3.785714, 681029605720.448975, 1, 'binary.GcWganOJXCPvFeMQZTYJ', 'nchar.黑龙江省丽丽县高港长春路v座 665579', 1630000000088)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631305000000, 1527621467, -7509002110121261571, -26325, 32, 3.577428, 5008201901846.370117, 0, 'binary.mFsrNoKYqMYXhFEIfVXS', 'nchar.山西省梅市南湖合肥路w座 573200', 1630000000087)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631320000000, -493256377, -8751737434302823435, -27869, -96, 330393978255.500000, 94835651689763.000000, 0, 'binary.PqdASxtDLyeIFzBiaFGV', 'nchar.湖南省关岭市双滦武路o座 527508', 1630000000088)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631320000000, -2087366027, -6303077694275632802, -15261, -89, 43362171018304.000000, -166690.281684, 0, 'binary.zPrdaJdDKwcSHuAxtOlS', 'nchar.澳门特别行政区超市高港刘街N座 420236', 1630000000088)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631319999999, 1129994268, 1552944233564611739, 10242, 108, 562740269.814683, -48475927235174.601562, 1, 'binary.rqJyBwQmuzFVARUJRgKb', 'nchar.山西省巢湖县西夏沈阳街S座 255735', 1630000000088)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631320000000, 2063639050, 2521461528938336565, 2345, 89, -6038709960145.540039, 24451.395881, 1, 'binary.iSmMBujSmVtayVzDoWgG', 'nchar.内蒙古自治区桂香市吉区潘路v座 620160', 1630000000088)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631320000001, -1449143866, -6390394742281036218, -100, -33, -120.220087, 109963155461.466003, 1, 'binary.fPxCvZJXReoaPBmALmPQ', 'nchar.内蒙古自治区婷市兴山冯路h座 570428', 1630000000089)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631320000001, -1895163787, -1867618456195867046, -5071, -105, -1871044419105.310059, 443341008.795331, 1, 'binary.tMibvAJACwmXDYSGDcNH', 'nchar.贵州省兵市静安黄街O座 248304', 1630000000089)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631320000000, -1870734956, 5919383249914923168, 25989, 91, -1.395899, -450627922362.231995, 0, 'binary.rCKozKCilrDzhrinGzhJ', 'nchar.江西省楠县闵行银川路V座 237404', 1630000000088)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631335000000, 408968903, -8367166343934446562, -19738, 15, 854582448833.431030, -9969.161351, 0, 'binary.GLyIXkgZbApVCOoVSqaO', 'nchar.江苏省香港县黄浦淮安街D座 525365', 1630000000089)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631335000000, -1122322355, 8412393600271974612, -23315, 95, -68174012.309009, 15387044.711345, 0, 'binary.MGjySGAlYDujClbZejIQ', 'nchar.安徽省邯郸市城东大冶路G座 360663', 1630000000089)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631334999999, 2026186883, 445653452435547622, 18307, 49, 213343.543679, -4.739704, 1, 'binary.fOSZlDCKdxiQaysyscfx', 'nchar.江西省雷市花溪陈街B座 320678', 1630000000089)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631335000000, 507343304, 7297664243660949047, 12008, 109, -531.925870, 1673114199.854260, 1, 'binary.cTljYtBXIOAsDXqDTlMd', 'nchar.广西壮族自治区帅市崇文王街u座 771522', 1630000000089)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631335000001, -2001883206, -817341878022425389, -2161, -42, 9.708559, -53706854436.695000, 1, 'binary.IUNzHoyqCMDBQckAVEQz', 'nchar.四川省宇县城东淮安街C座 920078', 1630000000090)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631335000001, -1408747533, -4778697158136936011, -27, -101, 47621.171490, 3290528001686.430176, 1, 'binary.vyTXYeaTyzhYwNCQiJrW', 'nchar.广西壮族自治区秀芳县大兴天津路H座 858644', 1630000000090)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631335000000, -1227268662, 4866318703531659868, 8057, -67, 1812966973.800030, 6.738090, 0, 'binary.EMfIIquLTRUJOViCbaXD', 'nchar.河北省长春县翔安禹路f座 740911', 1630000000089)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631350000000, 29364546, 7439453869072957110, 3379, 93, 55692580455.865402, 25451.386688, 0, 'binary.SOQDYnfDtUghMpNrrrvd', 'nchar.吉林省玉梅市清城大冶街w座 657754', 1630000000090)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631350000000, -1512569467, -2170613746493368276, -9226, 86, 94917216138828.500000, 45.344859, 0, 'binary.osdODKgoqrkMjsUQuBxO', 'nchar.湖南省华县山亭赵街o座 343015', 1630000000090)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631349999999, 341004545, 1678452718654659972, 11982, 53, 647268193.946854, -9.321045, 1, 'binary.PlQgjTLDuHwIfIkGmfIq', 'nchar.江苏省长沙市门头沟秦路m座 511307', 1630000000090)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631350000000, 1022164581, 8771702396738754711, 1690, 106, -2055.312055, 363.845057, 1, 'binary.QVYMJaQkcoJnLdYFnwsS', 'nchar.台湾省西宁市南长郑州路f座 795021', 1630000000090)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631350000001, -2139193188, -4423061215457882616, -22771, -31, 492.888005, 22942151260.387100, 1, 'binary.LSzPYUKvvRlNrhoKoYkL', 'nchar.天津市静县浔阳济南路G座 357460', 1630000000091)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631350000001, -1786702987, -5274362232731441209, -16394, -43, 237512315596.303986, -558.984176, 1, 'binary.onBwaQpiFBImnuhjravU', 'nchar.河南省雪市淄川王街Y座 875679', 1630000000091)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631350000000, -1431645289, -6529328401317482981, 9944, -97, 279578438.723650, -79899714.177569, 0, 'binary.ZhbLqmODnxwYrimmIigl', 'nchar.北京市大冶市怀柔刘街t座 692168', 1630000000090)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631365000000, 568429328, -9060392884809893816, 28469, 61, 307437180734.507019, -21602647797.441101, 0, 'binary.IGDZBmjwYmOzJaUbkiKN', 'nchar.福建省兴城市兴山巢湖路P座 743092', 1630000000091)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631365000000, -1550415271, -7558411265476450633, -26998, -58, 1694624.435305, -47865257382.732803, 0, 'binary.JhBLpAPozgqCVUQnONkT', 'nchar.黑龙江省秀云县清城白路q座 555712', 1630000000091)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631364999999, 2050068224, 8869318186483878987, 18135, 56, -324240012.550862, 139208652187.634003, 1, 'binary.fzVzLhgOcjBWiooIbfZy', 'nchar.西藏自治区太原市长寿西宁路h座 862819', 1630000000091)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631365000000, 5839085, 4176280397163068047, 4608, 4, 683.468225, 4674219497.311910, 1, 'binary.YBbSDgHLgpNnIYBAiMnd', 'nchar.广东省欣县沙市惠州街I座 803450', 1630000000091)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631365000001, -1606114187, -8492599008143584602, -21024, -58, 977.510976, -2962257.665275, 1, 'binary.DQLAjLojVsSPrXVqoaFU', 'nchar.江苏省建国市涪城兰州路k座 224858', 1630000000092)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631365000001, -1085786582, -7556940745324501035, -214, 0, 85978773227.734802, 3688136776873.470215, 1, 'binary.LsObIWxTJIHqdNcQniXu', 'nchar.贵州省小红市西夏林路J座 503400', 1630000000092)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631365000000, -486512987, -1848007965784118332, 20641, 39, 7524.727066, 43002.993255, 0, 'binary.JtdWjkAwXwWmNqghjGgR', 'nchar.江苏省福州县萧山李路l座 332686', 1630000000091)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631380000000, 1015255006, -6233642644171313082, -30770, 103, -52871950.517558, -203.631390, 0, 'binary.dcFQefENHBmSbWfsxkNZ', 'nchar.吉林省凤兰县大兴李路j座 146250', 1630000000092)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631380000000, -1944800483, 2662387305551873341, -32553, -7, 49980.865995, -377372259.892400, 0, 'binary.EWzwpfuElPwODlQeSUsg', 'nchar.湖北省荆门县闵行永安街F座 378209', 1630000000092)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631379999999, 869226799, 2896504572324576563, 949, 78, 3067118231.817550, 5.678553, 1, 'binary.thrXsOuUUQEwGkXiZqVD', 'nchar.香港特别行政区荆门市高港天津街K座 250023', 1630000000092)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631380000000, 323027847, 2827349391284089242, 1441, 44, -17278.296831, 78816030.490657, 1, 'binary.YZbYVJvcLQeMTfNdLJhv', 'nchar.贵州省淑珍县永川辽阳街g座 249023', 1630000000092)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631380000001, -2127572203, -4937514564787173908, -63, -76, -123946739.447527, 698804470.690330, 1, 'binary.MtiGMwxbCoGxBrSCfmnb', 'nchar.广东省齐齐哈尔县梁平关岭路t座 761348', 1630000000093)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631380000001, -1306426634, -4986879132935813626, -28229, -23, 39146958063046.203125, 87460954188.246506, 1, 'binary.SZAYkmBTuWBkLyHMsNln', 'nchar.辽宁省荣市浔阳马鞍山街h座 139222', 1630000000093)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631380000000, -24693354, -7179764382339676335, -4810, 42, 49617345.815346, -40955896779.561096, 0, 'binary.AOTdejxaZWZTeWiSQccI', 'nchar.湖南省呼和浩特市南长东莞路Q座 526334', 1630000000092)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631395000000, 110327094, -7171919345203251134, 10385, 50, -88221.924084, -14499963.327697, 0, 'binary.jBWuQXhwybyeWaUeuiCo', 'nchar.河北省深圳县城北昆明路h座 627103', 1630000000093)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631395000000, 1589122997, 4808726552373126087, 13356, -88, -777.180511, -5649.819038, 0, 'binary.PLvgyyFexDVwuXPcgOUJ', 'nchar.广东省璐市梁平石街p座 625686', 1630000000093)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631394999999, 415752279, 5607734497564153556, 28120, 114, -758865393.966240, 2879.590758, 1, 'binary.pNjKEKLwWZUgNZshphnH', 'nchar.新疆维吾尔自治区海门市萧山潜江路N座 450879', 1630000000093)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631395000000, 1725928224, 406982977976053694, 31934, 55, 2365333426653.399902, 52689398.325206, 1, 'binary.tODfUdiMrapEBaRDyjTC', 'nchar.甘肃省旭县六枝特陈路k座 228235', 1630000000093)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631395000001, -1943166726, -461080508996818206, -3563, -103, -984164002.302936, 3328600.852324, 1, 'binary.hSqhMEcJIhZRmIFrQDKl', 'nchar.江苏省桂荣县大东宜都街q座 375062', 1630000000094)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631395000001, -965688414, -4532230014607129716, -32045, -103, -8.400039, -554153162149.166016, 1, 'binary.ZQVXUVCHEOWxAjlnoSEK', 'nchar.吉林省通辽县璧山龙路l座 482360', 1630000000094)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631395000000, -306891211, -3703573295039531167, 6006, 86, 226610060352.789001, -47040211.883215, 0, 'binary.BJYBsZjtDafCchXYbpRr', 'nchar.台湾省哈尔滨县翔安惠州街U座 837583', 1630000000093)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631410000000, 1886325467, -3863586930565866233, 8904, 38, -946832.388950, 55.658536, 0, 'binary.QSJUCkzpJcOiAmsyKmUs', 'nchar.云南省宁德市高港东莞街N座 505827', 1630000000094)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631410000000, 1857465087, -4382581587636034119, 23552, -18, -13.792084, 829139755.682817, 0, 'binary.ljHNBjHCmNlXXVTnknxH', 'nchar.四川省北京县华龙济南街x座 750613', 1630000000094)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631409999999, 1895012884, 2424275775986822787, 22179, 90, -0.394803, -56.575361, 1, 'binary.KBNSdgumJOCWoMmWtLJU', 'nchar.江苏省静县秀英太原路S座 251913', 1630000000094)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631410000000, 1569144121, 5640124213375907439, 8815, 5, 257555.261048, 48254141.237573, 1, 'binary.KKozUPJSMbzhEyunifsH', 'nchar.湖南省伟县萧山萧街k座 474899', 1630000000094)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631410000001, -1384685953, -2320337996509410200, -15795, -10, -98.406889, 42984.674970, 1, 'binary.orjVZeKJsgJsTEqHljkf', 'nchar.海南省建华县沈北新林街g座 586911', 1630000000095)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631410000001, -1012235418, -8338932088221751208, -8979, -127, -25798805.860955, -48091893253761.101562, 1, 'binary.XDTOObdfDUqcGtXWVONC', 'nchar.浙江省柳市双滦鲍路a座 506391', 1630000000095)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631410000000, 726812223, 8571870874380055515, 22786, -83, 942970127.407073, 52.995631, 0, 'binary.xgcSGhNrZOWGKZtOlArO', 'nchar.宁夏回族自治区丽华县蓟州深圳路K座 592080', 1630000000094)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631425000000, 2071766507, 1772340392886850115, -24057, 25, 78.445247, -79637.635931, 0, 'binary.sROneWSFNuXBalpsjyEo', 'nchar.安徽省广州市孝南刘路b座 713687', 1630000000095)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631425000000, 1087816300, 5318228817994763162, -27394, -31, 267905.242469, 8213154732519.320312, 0, 'binary.imZeNtwoLVXCQJPFObQE', 'nchar.山西省沈阳县高港长沙街F座 461428', 1630000000095)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631424999999, 2108909701, 7807744131165483728, 13566, 88, 6.922028, -9.200773, 1, 'binary.vYUalkfScnQyDPeEqCTq', 'nchar.江苏省太原市华龙陈街K座 781219', 1630000000095)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631425000000, 990358809, 3801390332798973674, 31813, 108, 5.517866, 6845.746002, 1, 'binary.DXVimvDjqwoNARHjTXuI', 'nchar.上海市杰市怀柔天津街N座 936198', 1630000000095)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631425000001, -52260486, -2710179305466920974, -16431, -119, -958.607117, -38510688978203.898438, 1, 'binary.erVQJAvbzEahBSRwIflK', 'nchar.江西省桂英县六枝特北镇路A座 788549', 1630000000096)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631425000001, -2093648842, -6938612375617002825, -6697, -12, 4655.180387, 8131192.215618, 1, 'binary.IKRYKZeWgWRTpWFkjzBI', 'nchar.西藏自治区海门市高坪六安路l座 342816', 1630000000096)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631425000000, -1314836279, -7834501286968314160, 15789, -123, 33900643554.123798, -2082127.625636, 0, 'binary.rDTPIDjxHVnobHXBYtXY', 'nchar.北京市昆明市合川董路B座 463423', 1630000000095)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631440000000, -1084304030, 7788263187088917501, -6031, 103, -857.741582, 8339855538945.950195, 0, 'binary.TyxrsRIKPjtzXgKyuAEH', 'nchar.山西省宜都县锡山李路J座 745663', 1630000000096)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631440000000, -1295742585, -5131971234132550528, -2973, 42, 52247.313515, 3926177760.319000, 0, 'binary.vXduhvWXmyRcbKfMRWlx', 'nchar.安徽省兵县高明许街p座 922555', 1630000000096)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631439999999, 20091569, 6781601496202385368, 19493, 34, 430582007336.525024, -2.412247, 1, 'binary.kZJLmYOxtxuJZnMBDufv', 'nchar.内蒙古自治区颖市浔阳王路J座 977519', 1630000000096)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631440000000, 1813973451, 4213442260213870776, 26872, 44, 67809323307135.703125, -88973468.418878, 1, 'binary.LTPEGjxcghSRynZSWwqM', 'nchar.云南省南昌县梁平杨街G座 627457', 1630000000096)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631440000001, -958070866, -691160751112903120, -20406, -41, 2363414966580.939941, 8820731593.777920, 1, 'binary.RZliiclFrDaNImulqJTo', 'nchar.台湾省梧州县沈北新葛街m座 433410', 1630000000097)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631440000001, -1829734602, -7898671321317813289, -22642, -75, 4.101330, 40052.692465, 1, 'binary.KqgvcwBhRIxCNTZaTnLf', 'nchar.福建省金凤市海陵武汉街u座 555612', 1630000000097)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631440000000, 521824259, -8660783264753873829, -29076, -1, 43333588339.556999, -11986.154286, 0, 'binary.GKzBHXETTitGAjMDqHEz', 'nchar.广东省合山县萧山宜都路u座 831011', 1630000000096)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631455000000, 113309185, -5404115880106386219, 26329, -5, 3883055.927579, -91.272110, 0, 'binary.DJuxhSLmckBxGAZrTrdg', 'nchar.黑龙江省丽丽县海陵张路x座 667710', 1630000000097)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631455000000, 1249035464, -1549873989524700903, -1552, 12, 30063970786.732201, 11.623060, 0, 'binary.wGOvxPtoGpoLQkqKqHYd', 'nchar.山东省建国县沙湾南昌路c座 673842', 1630000000097)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631454999999, 420485592, 1118062875777571668, 19952, 48, -13933.583298, -2462443202766.899902, 1, 'binary.PUybFnyERTpaOdCfdUeZ', 'nchar.内蒙古自治区海燕县新城郑州街o座 509072', 1630000000097)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631455000000, 30946503, 7185137291847458233, 19072, 30, -152414.945183, -623547849.652030, 1, 'binary.bPgvDlyyWMQtWdUCTHsf', 'nchar.西藏自治区东莞县上街谢路z座 842631', 1630000000097)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631455000001, -136337675, -6941068854543647771, -30263, -36, 49955993237.460999, 2059.397798, 1, 'binary.AvuVCfvKjWXieILiINcT', 'nchar.台湾省建国县南长刘街Y座 514218', 1630000000098)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631455000001, -1639262737, -5524893491321561875, -14837, -81, 464457921385.609985, -381392530568.276978, 1, 'binary.aDgJHFZdhjaRiLJgewyJ', 'nchar.贵州省建县锡山谢路U座 298544', 1630000000098)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631455000000, 74809487, -2714524097635173095, 20984, 33, 1802605.962684, 77.296856, 0, 'binary.vYEpwfCOmksSKWbrqYDR', 'nchar.香港特别行政区哈尔滨县山亭董街I座 748187', 1630000000097)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631470000000, 222279926, -7912824848487896120, 9941, -30, -29340217214.750000, -19574.321414, 0, 'binary.VsnXIRHAihqVgXYwhHKL', 'nchar.江苏省拉萨县闵行合山街c座 761422', 1630000000098)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631470000000, -557137010, 2878767942030754357, -31366, -47, -4333332356.734030, -1526389473.865540, 0, 'binary.jcMUSJuZgsPjePUNiAdT', 'nchar.海南省东县锡山澳门街q座 268693', 1630000000098)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631469999999, 447814637, 2176882939455175378, 20639, 104, -1356647372.422460, 6.253297, 1, 'binary.fyooIrohcePtHVwJGMuD', 'nchar.河北省关岭市金平香港路p座 886482', 1630000000098)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631470000000, 606893531, 4789787682189143102, 14991, 38, 32272292804541.800781, -943.731667, 1, 'binary.VuPeMxNmIrwVmeDRCPTm', 'nchar.黑龙江省兰州市东城陈街O座 771980', 1630000000098)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631470000001, -994364364, -2126966193042917392, -11261, -99, 7.179842, 9558.331194, 1, 'binary.qpXjZYHYtFFCkEkBnwAb', 'nchar.北京市建国市永川崔路g座 413491', 1630000000099)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631470000001, -1952105143, -3184538982013952257, -9023, -66, 2300257.628379, -83.900999, 1, 'binary.ufpdchwhzdeEoRvElolr', 'nchar.黑龙江省通辽市高明胡街B座 971162', 1630000000099)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631470000000, 1789157808, -942637929512454815, -20552, 22, -2222313287.171970, 295235211456.737000, 0, 'binary.VznwmsilndimcCoBZmyp', 'nchar.澳门特别行政区东莞县静安杭州路m座 966043', 1630000000098)") + tdSql.execute(f"insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631485000000, -1420187501, 5901473154726700075, -22562, 122, -8503751910.519820, 2716.822481, 0, 'binary.SlMenUlCCEwyXsFKefLW', 'nchar.海南省马鞍山市兴山余路y座 674434', 1630000000099)") + tdSql.execute(f"insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631485000000, 325131812, -7109049890424943791, 905, -37, 431982153775.299011, -415064.781722, 0, 'binary.NdxijvQdCvCbloWQQDtL', 'nchar.台湾省济南市大东郭街V座 658918', 1630000000099)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631484999999, 1572668480, 108340538386194628, 29463, 54, -3679552.122859, -769.652903, 1, 'binary.qWPhdNHKBtjYxVZgdXOh', 'nchar.新疆维吾尔自治区华市翔安赵路R座 753468', 1630000000099)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631485000000, 1226876580, 7653003049181344524, 5255, 97, 2751345.497375, 35183472041.125603, 1, 'binary.qAhAfHMuKshsMmbgIOYK', 'nchar.上海市邯郸县兴山张街x座 375209', 1630000000099)") + tdSql.execute(f"insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631485000001, -375162227, -4279732710928397934, -19628, -20, 38774493096.818199, 7390201646933.400391, 1, 'binary.yMDiebwbXIHZiaQKPRut', 'nchar.北京市海燕县南长合山街t座 129283', 1630000000100)") + tdSql.execute(f"insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631485000001, -1188106611, -4426272680272146753, -13643, -51, 40149362212590.203125, 438738.291810, 1, 'binary.CbdGAjuBbirCzXoUVXUG', 'nchar.辽宁省宜都市新城吴街d座 246995', 1630000000100)") + tdSql.execute(f"insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631485000000, 698542868, 244085745806715589, -10836, -29, 23705559.480467, 89055463790.105301, 0, 'binary.TCkhaOCjQbElXSecegWh', 'nchar.新疆维吾尔自治区晶县秀英程街a座 168535', 1630000000099)") + + tdLog.debug("insert data ............ [OK]") + return + + def run(self): + tdSql.prepare() + self.create_tables() + self.insert_data() + tdLog.printNoPrefix("======== test TD-19201 ") + tdSql.execute(f"flush database table_numeric_max_min") + + tdSql.query(f"select MAX(q_float) from stable_1_2") + max_val = tdSql.getData(0, 0) + #print(max_val) + + tdSql.query(f"select ts,q_float from stable_1_2 order by q_float desc limit 5") + first_val = tdSql.getData(0, 1) + #print(first_val) + + tdSql.checkEqual(max_val, first_val) + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 684d6a16e2..a2bf935ebb 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -337,6 +337,8 @@ python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py python3 ./test.py -f 7-tmq/tmq_taosx.py python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py +python3 ./test.py -f 99-TDcase/TD-19201.py + #------------querPolicy 2----------- python3 ./test.py -f 2-query/between.py -Q 2 @@ -517,3 +519,94 @@ python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 python3 ./test.py -f 2-query/sml.py -Q 3 python3 ./test.py -f 2-query/interp.py -Q 3 + +#------------querPolicy 4----------- + +python3 ./test.py -f 2-query/between.py -Q 4 +python3 ./test.py -f 2-query/distinct.py -Q 4 +python3 ./test.py -f 2-query/varchar.py -Q 4 +python3 ./test.py -f 2-query/ltrim.py -Q 4 +python3 ./test.py -f 2-query/rtrim.py -Q 4 +python3 ./test.py -f 2-query/length.py -Q 4 +python3 ./test.py -f 2-query/char_length.py -Q 4 +python3 ./test.py -f 2-query/upper.py -Q 4 +python3 ./test.py -f 2-query/lower.py -Q 4 +python3 ./test.py -f 2-query/join.py -Q 4 +python3 ./test.py -f 2-query/join2.py -Q 4 +python3 ./test.py -f 2-query/cast.py -Q 4 +python3 ./test.py -f 2-query/substr.py -Q 4 +python3 ./test.py -f 2-query/union.py -Q 4 +python3 ./test.py -f 2-query/union1.py -Q 4 +python3 ./test.py -f 2-query/concat.py -Q 4 +python3 ./test.py -f 2-query/concat2.py -Q 4 +python3 ./test.py -f 2-query/concat_ws.py -Q 4 +python3 ./test.py -f 2-query/concat_ws2.py -Q 4 +#python3 ./test.py -f 2-query/check_tsdb.py -Q 4 +python3 ./test.py -f 2-query/spread.py -Q 4 +python3 ./test.py -f 2-query/hyperloglog.py -Q 4 +python3 ./test.py -f 2-query/explain.py -Q 4 +python3 ./test.py -f 2-query/leastsquares.py -Q 4 +python3 ./test.py -f 2-query/timezone.py -Q 4 +python3 ./test.py -f 2-query/Now.py -Q 4 +python3 ./test.py -f 2-query/Today.py -Q 4 +python3 ./test.py -f 2-query/max.py -Q 4 +python3 ./test.py -f 2-query/min.py -Q 4 +python3 ./test.py -f 2-query/count.py -Q 4 +#python3 ./test.py -f 2-query/last.py -Q 4 +python3 ./test.py -f 2-query/first.py -Q 4 +python3 ./test.py -f 2-query/To_iso8601.py -Q 4 +python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 +python3 ./test.py -f 2-query/timetruncate.py -Q 4 +python3 ./test.py -f 2-query/diff.py -Q 4 +python3 ./test.py -f 2-query/Timediff.py -Q 4 +python3 ./test.py -f 2-query/json_tag.py -Q 4 +python3 ./test.py -f 2-query/top.py -Q 4 +python3 ./test.py -f 2-query/bottom.py -Q 4 +python3 ./test.py -f 2-query/percentile.py -Q 4 +python3 ./test.py -f 2-query/apercentile.py -Q 4 +python3 ./test.py -f 2-query/abs.py -Q 4 +python3 ./test.py -f 2-query/ceil.py -Q 4 +python3 ./test.py -f 2-query/floor.py -Q 4 +python3 ./test.py -f 2-query/round.py -Q 4 +python3 ./test.py -f 2-query/log.py -Q 4 +python3 ./test.py -f 2-query/pow.py -Q 4 +python3 ./test.py -f 2-query/sqrt.py -Q 4 +python3 ./test.py -f 2-query/sin.py -Q 4 +python3 ./test.py -f 2-query/cos.py -Q 4 +python3 ./test.py -f 2-query/tan.py -Q 4 +python3 ./test.py -f 2-query/arcsin.py -Q 4 +python3 ./test.py -f 2-query/arccos.py -Q 4 +python3 ./test.py -f 2-query/arctan.py -Q 4 +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 +# python3 ./test.py -f 2-query/nestedQuery.py -Q 4 +# python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 +# python3 ./test.py -f 2-query/avg.py -Q 4 +# python3 ./test.py -f 2-query/elapsed.py -Q 4 +python3 ./test.py -f 2-query/csum.py -Q 4 +#python3 ./test.py -f 2-query/mavg.py -Q 4 +python3 ./test.py -f 2-query/sample.py -Q 4 +python3 ./test.py -f 2-query/function_diff.py -Q 4 +python3 ./test.py -f 2-query/unique.py -Q 4 +python3 ./test.py -f 2-query/stateduration.py -Q 4 +python3 ./test.py -f 2-query/function_stateduration.py -Q 4 +python3 ./test.py -f 2-query/statecount.py -Q 4 +python3 ./test.py -f 2-query/tail.py -Q 4 +python3 ./test.py -f 2-query/ttl_comment.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 +python3 ./test.py -f 2-query/twa.py -Q 4 +python3 ./test.py -f 2-query/irate.py -Q 4 +python3 ./test.py -f 2-query/function_null.py -Q 4 +python3 ./test.py -f 2-query/count_partition.py -Q 4 +python3 ./test.py -f 2-query/max_partition.py -Q 4 +python3 ./test.py -f 2-query/last_row.py -Q 4 +python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 +#python3 ./test.py -f 2-query/sml.py -Q 4 +python3 ./test.py -f 2-query/interp.py -Q 4 + From 9169c90f8da45b6dae8cb81e8604ebf285af37bd Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 26 Sep 2022 18:29:08 +0800 Subject: [PATCH 08/71] fix: migrate logic for multi-tier storage --- source/dnode/mnode/impl/src/mndDb.c | 4 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 45 ++++++++++--------- source/dnode/vnode/src/tsdb/tsdbOpen.c | 1 + .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 26 +++++++++-- source/dnode/vnode/src/tsdb/tsdbRetention2.c | 22 +++++++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 +- 6 files changed, 69 insertions(+), 33 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c5f6d179d8..832a4617b0 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1389,7 +1389,7 @@ static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; - SVTrimDbReq trimReq = {.timestamp = taosGetTimestampMs(), .maxSpeed = 1048576 << 5}; // TODO: use specified maxSpeed + SVTrimDbReq trimReq = {.timestamp = taosGetTimestampMs(), .maxSpeed = 1048576 << 2}; // TODO: use specified maxSpeed int32_t reqLen = tSerializeSVTrimDbReq(NULL, 0, &trimReq); int32_t contLen = reqLen + sizeof(SMsgHead); @@ -1413,7 +1413,7 @@ static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) { if (code != 0) { mError("vgId:%d, failed to send vnode-trim request to vnode since 0x%x", pVgroup->vgId, code); } else { - mInfo("vgId:%d, send vnode-trim request to vnode, time:%d", pVgroup->vgId, trimReq.timestamp); + mInfo("vgId:%d, send vnode-trim request to vnode, time:%" PRIi64, pVgroup->vgId, trimReq.timestamp); } sdbRelease(pSdb, pVgroup); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 53d04cc9c1..81ad1d0e7f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -1101,35 +1101,36 @@ int32_t tsdbFSUpdDel(STsdb *pTsdb, STsdbFS *pFS, STsdbFS *pFSNew, int32_t maxFid ASSERT(pSetOld->pHeadF->commitID == pSetNew->pHeadF->commitID); ASSERT(pSetOld->pHeadF->size == pSetNew->pHeadF->size); ASSERT(pSetOld->pHeadF->offset == pSetNew->pHeadF->offset); - ASSERT(!sameDisk); - // head - *pSetOld->pHeadF = *pSetNew->pHeadF; - pSetOld->pHeadF->nRef = 1; + if (!sameDisk) { + // head + *pSetOld->pHeadF = *pSetNew->pHeadF; + pSetOld->pHeadF->nRef = 1; - // data - ASSERT(pSetOld->pDataF->size == pSetNew->pDataF->size); - *pSetOld->pDataF = *pSetNew->pDataF; - pSetOld->pDataF->nRef = 1; + // data + ASSERT(pSetOld->pDataF->size == pSetNew->pDataF->size); + *pSetOld->pDataF = *pSetNew->pDataF; + pSetOld->pDataF->nRef = 1; - // sma - ASSERT(pSetOld->pSmaF->size == pSetNew->pSmaF->size); - *pSetOld->pSmaF = *pSetNew->pSmaF; - pSetOld->pSmaF->nRef = 1; + // sma + ASSERT(pSetOld->pSmaF->size == pSetNew->pSmaF->size); + *pSetOld->pSmaF = *pSetNew->pSmaF; + pSetOld->pSmaF->nRef = 1; - // stt - ASSERT(pSetOld->nSttF == pSetNew->nSttF); - for (int32_t iStt = 0; iStt < pSetOld->nSttF; ++iStt) { - ASSERT(pSetOld->aSttF[iStt]->size == pSetNew->aSttF[iStt]->size); - ASSERT(pSetOld->aSttF[iStt]->offset == pSetNew->aSttF[iStt]->offset); + // stt + ASSERT(pSetOld->nSttF == pSetNew->nSttF); + for (int32_t iStt = 0; iStt < pSetOld->nSttF; ++iStt) { + ASSERT(pSetOld->aSttF[iStt]->size == pSetNew->aSttF[iStt]->size); + ASSERT(pSetOld->aSttF[iStt]->offset == pSetNew->aSttF[iStt]->offset); - *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; - pSetOld->aSttF[iStt]->nRef = 1; + *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; + pSetOld->aSttF[iStt]->nRef = 1; + } + + // set diskId + pSetOld->diskId = pSetNew->diskId; } - // set diskId - pSetOld->diskId = pSetNew->diskId; - iOld++; iNew++; continue; diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 71d745972e..5072d1a15e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -73,6 +73,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee pTsdb->trimHdl.maxRetentFid = INT32_MIN; pTsdb->trimHdl.minCommitFid = INT32_MAX; + pTsdb->trimHdl.limitSpeed = 1; tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d", TD_VID(pVnode), pTsdb->path, pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index db39d98b9e..7020e52cfa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -634,10 +634,16 @@ static int64_t tsdbFSendFile(TdFilePtr pOutFD, TdFilePtr pInFD, int64_t size, in cost = taosGetTimestampMs() - startMs; tBytes += nBytes; + int64_t nSleep = 0; if (cost < 0) { - taosMsleep(1000); + nSleep = 1000; } else if (cost < 1000) { - taosMsleep(1000 - cost); + nSleep = 1000 - cost; + } + if (nSleep > 0) { + taosMsleep(nSleep); + tsdbDebug("sendFile from %p to %p, fSize:%" PRIi64 ", maxSpeed:%d, msleep:%" PRIi64, pOutFD, pInFD, size, speed, + nSleep); } } if (offset < size) { @@ -659,9 +665,9 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i char fNameFrom[TSDB_FILENAME_LEN]; char fNameTo[TSDB_FILENAME_LEN]; int32_t speed = 0; + int64_t fStatSize = 0; if (atomic_load_8(&pTsdb->trimHdl.limitSpeed)) { - ASSERT(maxSpeed > 0); speed = maxSpeed; } @@ -678,6 +684,9 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i code = TAOS_SYSTEM_ERROR(errno); goto _err; } + fStatSize = 0; + taosStatFile(fNameFrom, &fStatSize, 0); + ASSERT(fStatSize == tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage)); n = tsdbFSendFile(pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage), speed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); @@ -699,7 +708,10 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i code = TAOS_SYSTEM_ERROR(errno); goto _err; } - n = tsdbFSendFile(pOutFD, PInFD, LOGIC_TO_FILE_OFFSET(pSetFrom->pDataF->size, szPage), speed); + fStatSize = 0; + taosStatFile(fNameFrom, &fStatSize, 0); + ASSERT(fStatSize == tsdbLogicToFileSize(pSetFrom->pDataF->size, szPage)); + n = tsdbFSendFile(pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pDataF->size, szPage), speed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -720,6 +732,9 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i code = TAOS_SYSTEM_ERROR(errno); goto _err; } + fStatSize = 0; + taosStatFile(fNameFrom, &fStatSize, 0); + ASSERT(fStatSize == tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage)); n = tsdbFSendFile(pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage), speed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); @@ -742,6 +757,9 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i code = TAOS_SYSTEM_ERROR(errno); goto _err; } + fStatSize = 0; + taosStatFile(fNameFrom, &fStatSize, 0); + ASSERT(fStatSize == tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage)); n = tsdbFSendFile(pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage), speed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention2.c b/source/dnode/vnode/src/tsdb/tsdbRetention2.c index 47fe3c5384..9594334808 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention2.c @@ -17,9 +17,15 @@ enum { RETENTION_NO = 0, RETENTION_EXPIRED = 1, RETENTION_MIGRATE = 2 }; +#if 1 #define MIGRATE_MIN_FSIZE (1048576 << 9) // 512 MB #define MIGRATE_MAX_SPEED (1048576 << 5) // 32 MB #define MIGRATE_MIN_COST (5) // second +#else +#define MIGRATE_MIN_FSIZE (1048576 << 5) // 32 MB +#define MIGRATE_MAX_SPEED (1048576 << 2) // 4 MB +#define MIGRATE_MIN_COST (5) // second +#endif static bool tsdbShouldDoMigrate(STsdb *pTsdb); static int32_t tsdbShouldDoRetention(STsdb *pTsdb, int64_t now); @@ -103,7 +109,6 @@ _wait_commit_end: if (++nLoops > 1000) { nLoops = 0; sched_yield(); - printf("%s:%d sche_yield() minCommitFid:%d maxFid:%d\n", __func__, __LINE__, pTsdb->trimHdl.minCommitFid, maxFid); } } if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { @@ -160,6 +165,7 @@ _exit: */ static int32_t tsdbProcessMigrate(STsdb *pTsdb, int64_t now, int32_t maxSpeed, int32_t retention) { int32_t code = 0; + int32_t nBatch = 0; int32_t nLoops = 0; int32_t maxFid = INT32_MIN; int64_t fSize = 0; @@ -186,6 +192,8 @@ _migrate_loop: int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); SDiskID did; + if (pSet->diskId.level == expLevel) continue; + if (expLevel > 0) { ASSERT(pSet->fid > maxFid); maxFid = pSet->fid; @@ -197,6 +205,7 @@ _migrate_loop: fSize += pSet->aSttF[iStt]->size; } if (fSize / MIGRATE_MAX_SPEED > MIGRATE_MIN_COST) { + tsdbInfo("vgId:%d migrate loop[%d] with maxFid:%d", TD_VID(pTsdb->pVnode), nBatch, maxFid); break; } } @@ -209,7 +218,6 @@ _wait_commit_end: if (++nLoops > 1000) { nLoops = 0; sched_yield(); - printf("%s:%d sche_yield()\n", __func__, __LINE__); } } if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { @@ -231,9 +239,14 @@ _wait_commit_end: if (pSet->fid > maxFid) break; + tsdbInfo("vgId:%d migrate loop[%d] with maxFid:%d, fid:%d, did:%d, level:%d, expLevel:%d", TD_VID(pTsdb->pVnode), + nBatch, maxFid, pSet->fid, pSet->diskId.id, pSet->diskId.level, expLevel); + if (expLevel < 0) { SET_DFSET_EXPIRED(pSet); - } else if (expLevel > 0) { + } else { + if (expLevel == pSet->diskId.level) continue; + if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { code = terrno; goto _exit; @@ -277,6 +290,9 @@ _merge_fs: } taosThreadRwlockUnlock(&pTsdb->rwLock); + ++nBatch; + goto _migrate_loop; + _exit: tsdbFSDestroy(&fs); tsdbFSDestroy(&fsLatest); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 0263740e4a..a80211f402 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -389,7 +389,7 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, goto _exit; } - vInfo("vgId:%d, trim vnode request will be processed, time:%d", pVnode->config.vgId, trimReq.timestamp); + vInfo("vgId:%d, trim vnode request will be processed, time:%" PRIi64, pVnode->config.vgId, trimReq.timestamp); // process code = tsdbDoRetention(pVnode->pTsdb, trimReq.timestamp); @@ -461,7 +461,7 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, goto _exit; } - vInfo("vgId:%d, trim vnode request will be processed, time:%d", TD_VID(pVnode), pVndTrimReq->trimReq.timestamp); + vInfo("vgId:%d, trim vnode request will be processed, time:%" PRIi64, TD_VID(pVnode), pVndTrimReq->trimReq.timestamp); TdThreadAttr thAttr = {0}; taosThreadAttrInit(&thAttr); From aa055abd7cda13e627bef0d756cc2fb8a44a0abd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Sep 2022 10:52:55 +0800 Subject: [PATCH 09/71] refactor:do some internal refactor. --- include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 3 +- source/libs/executor/src/executor.c | 3 +- source/libs/qworker/src/qworker.c | 38 ++++++++++++++++---------- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 8c1d957381..81882ca1ac 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -136,7 +136,7 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table * @param handle * @return */ -int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, SLocalFetch *pLocal); +int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bool* hasMore, SLocalFetch *pLocal); int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds); /** diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 8d1525e081..aad141a404 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -690,7 +690,8 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma while (1) { uint64_t ts; - int32_t code = qExecTaskOpt(taskInfo, pResList, &ts, NULL); + bool hasMore = false; + int32_t code = qExecTaskOpt(taskInfo, pResList, &ts, &hasMore, NULL); if (code < 0) { if (code == TSDB_CODE_QRY_IN_EXEC) { break; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 373cb451f4..0fbbc25873 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -479,7 +479,7 @@ static void freeBlock(void* param) { blockDataDestroy(pBlock); } -int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, SLocalFetch* pLocal) { +int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bool* hasMore, SLocalFetch* pLocal) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; int64_t threadId = taosGetSelfPthreadId(); @@ -536,6 +536,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, SL } } + *hasMore = (pRes != NULL); uint64_t el = (taosGetTimestampUs() - st); pTaskInfo->cost.elapsedTime += el; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 3c102938d0..ae9dd82a58 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -107,9 +107,12 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { QW_TASK_DLOG("start to execTask, loopIdx:%d", i++); // if *taskHandle is NULL, it's killed right now + bool hasMore = false; + if (taskHandle) { qwDbgSimulateSleep(); - code = qExecTaskOpt(taskHandle, pResList, &useconds, &localFetch); + + code = qExecTaskOpt(taskHandle, pResList, &useconds, &hasMore, &localFetch); if (code) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) { QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -122,20 +125,8 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { ++execNum; - if (taosArrayGetSize(pResList) == 0) { - QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds); - dsEndPut(sinkHandle, useconds); - - QW_ERR_JRET(qwHandleTaskComplete(QW_FPARAMS(), ctx)); - - if (queryStop) { - *queryStop = true; - } - - break; - } - - for (int32_t j = 0; j < taosArrayGetSize(pResList); ++j) { + size_t numOfResBlock = taosArrayGetSize(pResList); + for (int32_t j = 0; j < numOfResBlock; ++j) { SSDataBlock *pRes = taosArrayGetP(pResList, j); ASSERT(pRes->info.rows > 0); @@ -149,6 +140,23 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue); } + if (numOfResBlock == 0 || (hasMore == false)) { + if (numOfResBlock == 0) { + QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds); + } else { + QW_TASK_DLOG("qExecTask done", ""); + } + + dsEndPut(sinkHandle, useconds); + QW_ERR_JRET(qwHandleTaskComplete(QW_FPARAMS(), ctx)); + + if (queryStop) { + *queryStop = true; + } + + break; + } + if (!qcontinue) { if (queryStop) { *queryStop = true; From 43c954b6e7eddf3dc7af3d338f6d4d54316748a9 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 27 Sep 2022 14:27:18 +0800 Subject: [PATCH 10/71] chore: data migration support specify max speed --- include/common/tmsg.h | 2 +- source/common/src/tmsg.c | 4 +- source/dnode/mnode/impl/src/mndDb.c | 17 ++++++-- source/dnode/vnode/src/inc/tsdb.h | 6 +-- source/dnode/vnode/src/inc/vnodeInt.h | 4 +- source/dnode/vnode/src/sma/smaRollup.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 5 ++- source/dnode/vnode/src/tsdb/tsdbOpen.c | 2 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 42 +++++++++++-------- source/dnode/vnode/src/tsdb/tsdbRetention2.c | 12 ++++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 11 ++--- 11 files changed, 65 insertions(+), 42 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a6de4f8863..30d38d1177 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -875,7 +875,7 @@ int32_t tDeserializeSTrimDbReq(void* buf, int32_t bufLen, STrimDbReq* pReq); typedef struct { int64_t timestamp; // unit: millisecond - int32_t maxSpeed; // 0 no limit, unit: bit/s + int64_t maxSpeed; // 0 no limit, unit: Byte/s } SVTrimDbReq; int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 3793b0188d..04aed7ca36 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2709,7 +2709,7 @@ int32_t tSerializeSVTrimDbReq(void *buf, int32_t bufLen, SVTrimDbReq *pReq) { if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI64(&encoder, pReq->timestamp) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxSpeed) < 0) return -1; + if (tEncodeI64(&encoder, pReq->maxSpeed) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2723,7 +2723,7 @@ int32_t tDeserializeSVTrimDbReq(void *buf, int32_t bufLen, SVTrimDbReq *pReq) { if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI64(&decoder, &pReq->timestamp) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxSpeed) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->maxSpeed) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 832a4617b0..cb47e9dac1 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1385,11 +1385,19 @@ _OVER: return code; } -static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) { +/** + * @brief trim database + * + * @param pMnode + * @param pDb + * @param maxSpeed MB/s + * @return int32_t + */ +static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb, int32_t maxSpeed) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; - SVTrimDbReq trimReq = {.timestamp = taosGetTimestampMs(), .maxSpeed = 1048576 << 2}; // TODO: use specified maxSpeed + SVTrimDbReq trimReq = {.timestamp = taosGetTimestampMs(), .maxSpeed = maxSpeed << 20}; int32_t reqLen = tSerializeSVTrimDbReq(NULL, 0, &trimReq); int32_t contLen = reqLen + sizeof(SMsgHead); @@ -1413,7 +1421,8 @@ static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) { if (code != 0) { mError("vgId:%d, failed to send vnode-trim request to vnode since 0x%x", pVgroup->vgId, code); } else { - mInfo("vgId:%d, send vnode-trim request to vnode, time:%" PRIi64, pVgroup->vgId, trimReq.timestamp); + mInfo("vgId:%d, send vnode-trim request to vnode, time:%" PRIi64 ", max speed:%" PRIi64, pVgroup->vgId, + trimReq.timestamp, trimReq.maxSpeed); } sdbRelease(pSdb, pVgroup); } @@ -1443,7 +1452,7 @@ static int32_t mndProcessTrimDbReq(SRpcMsg *pReq) { goto _OVER; } - code = mndTrimDb(pMnode, pDb); + code = mndTrimDb(pMnode, pDb, trimReq.maxSpeed); _OVER: if (code != 0) { diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index f379a7c528..251e364497 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -264,7 +264,7 @@ int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast); -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, int32_t maxSpeed); +int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, int64_t maxSpeed); // SDataFReader int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); @@ -320,8 +320,8 @@ struct STsdbFS { }; struct STsdbTrimHdl { - volatile int8_t state; // 0 idle 1 in use - volatile int8_t limitSpeed; // 0 no limit, 1 with limit + volatile int8_t state; // 0 idle 1 in use + volatile int8_t commitInWait; // 0 not in wait, 1 in wait volatile int32_t maxRetentFid; volatile int32_t minCommitFid; }; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index d9acb6da21..8757dc88e6 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -145,7 +145,7 @@ int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepC int tsdbClose(STsdb** pTsdb); int32_t tsdbBegin(STsdb* pTsdb); int32_t tsdbCommit(STsdb* pTsdb); -int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now, int32_t maxSpeed); +int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now, int64_t maxSpeed); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, @@ -200,7 +200,7 @@ int32_t smaSyncPostCommit(SSma* pSma); int32_t smaAsyncPreCommit(SSma* pSma); int32_t smaAsyncCommit(SSma* pSma); int32_t smaAsyncPostCommit(SSma* pSma); -int32_t smaDoRetention(SSma* pSma, int64_t now, int32_t maxSpeed); +int32_t smaDoRetention(SSma* pSma, int64_t now, int64_t maxSpeed); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 4f5f741b9e..e149abbace 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -664,7 +664,7 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { * @param maxSpeed * @return int32_t */ -int32_t smaDoRetention(SSma *pSma, int64_t now, int32_t maxSpeed) { +int32_t smaDoRetention(SSma *pSma, int64_t now, int64_t maxSpeed) { int32_t code = TSDB_CODE_SUCCESS; if (!VND_IS_RSMA(pSma->pVnode)) { return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index bc9c37fc76..7e8aeafe8a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -763,10 +763,11 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { _wait_retention_end: while (atomic_load_32(&pTsdb->trimHdl.maxRetentFid) >= minCommitFid) { - atomic_val_compare_exchange_8(&pTsdb->trimHdl.limitSpeed, 1, 0); + atomic_val_compare_exchange_8(&pTsdb->trimHdl.commitInWait, 0, 1); if (++nLoops > 1000) { nLoops = 0; sched_yield(); + printf("%s:%d wait retention to finish\n", __func__, __LINE__); } } if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { @@ -779,7 +780,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { } else { goto _wait_retention_end; } - atomic_store_8(&pTsdb->trimHdl.limitSpeed, 1); + atomic_store_8(&pTsdb->trimHdl.commitInWait, 0); } code = tsdbFSCopy(pTsdb, &pCommitter->fs); diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 5072d1a15e..8ce61d49b9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -73,7 +73,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee pTsdb->trimHdl.maxRetentFid = INT32_MIN; pTsdb->trimHdl.minCommitFid = INT32_MAX; - pTsdb->trimHdl.limitSpeed = 1; + pTsdb->trimHdl.commitInWait = 0; tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d", TD_VID(pVnode), pTsdb->path, pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 7020e52cfa..31fe940848 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -610,13 +610,14 @@ _err: /** * @brief send file with limited speed(rough control) * + * @param pTsdb * @param pFileOut * @param pFileIn * @param size * @param speed 0 no limit, unit: B/s * @return int64_t */ -static int64_t tsdbFSendFile(TdFilePtr pOutFD, TdFilePtr pInFD, int64_t size, int32_t speed) { +static int64_t tsdbFSendFile(STsdb *pTsdb, TdFilePtr pOutFD, TdFilePtr pInFD, int64_t size, int64_t speed) { if (speed <= 0) { return taosFSendFile(pOutFD, pInFD, 0, size); } @@ -626,9 +627,15 @@ static int64_t tsdbFSendFile(TdFilePtr pOutFD, TdFilePtr pInFD, int64_t size, in int64_t nBytes = 0; int64_t startMs = 0; int64_t cost = 0; - while ((offset + speed) < size) { + + while ((offset + nBytes) < size) { + if (atomic_load_8(&pTsdb->trimHdl.commitInWait) == 1) { + tsdbInfo("vgId:%d sendFile without limit since conflicts, fSize:%" PRIi64 ", maxSpeed:%" PRIi64, + TD_VID(pTsdb->pVnode), size, speed); + goto _send_remain; + } startMs = taosGetTimestampMs(); - if ((nBytes = taosFSendFile(pOutFD, pInFD, &offset, speed)) < 0) { + if ((nBytes = taosFSendFile(pOutFD, pInFD, &offset, speed / 10)) < 0) { return nBytes; } cost = taosGetTimestampMs() - startMs; @@ -642,20 +649,24 @@ static int64_t tsdbFSendFile(TdFilePtr pOutFD, TdFilePtr pInFD, int64_t size, in } if (nSleep > 0) { taosMsleep(nSleep); - tsdbDebug("sendFile from %p to %p, fSize:%" PRIi64 ", maxSpeed:%d, msleep:%" PRIi64, pOutFD, pInFD, size, speed, - nSleep); + tsdbInfo("vgId:%d sendFile and msleep:%" PRIi64 ", fSize:%" PRIi64 ", tBytes:%" PRIi64 " maxSpeed:%" PRIi64, + TD_VID(pTsdb->pVnode), nSleep, size, tBytes, speed); } } + +_send_remain: if (offset < size) { if ((nBytes = taosFSendFile(pOutFD, pInFD, &offset, size - offset)) < 0) { return nBytes; } tBytes += nBytes; + tsdbInfo("vgId:%d sendFile remain, fSize:%" PRIi64 ", tBytes:%" PRIi64 " maxSpeed:%" PRIi64, TD_VID(pTsdb->pVnode), + size, tBytes, speed); } return tBytes; } -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, int32_t maxSpeed) { +int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, int64_t maxSpeed) { int32_t code = 0; int64_t n; int64_t size; @@ -664,13 +675,8 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i int32_t szPage = pTsdb->pVnode->config.szPage; char fNameFrom[TSDB_FILENAME_LEN]; char fNameTo[TSDB_FILENAME_LEN]; - int32_t speed = 0; int64_t fStatSize = 0; - if (atomic_load_8(&pTsdb->trimHdl.limitSpeed)) { - speed = maxSpeed; - } - // head tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo); @@ -687,7 +693,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i fStatSize = 0; taosStatFile(fNameFrom, &fStatSize, 0); ASSERT(fStatSize == tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage)); - n = tsdbFSendFile(pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage), speed); + n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage), maxSpeed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -711,7 +717,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i fStatSize = 0; taosStatFile(fNameFrom, &fStatSize, 0); ASSERT(fStatSize == tsdbLogicToFileSize(pSetFrom->pDataF->size, szPage)); - n = tsdbFSendFile(pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pDataF->size, szPage), speed); + n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pDataF->size, szPage), maxSpeed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -735,7 +741,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i fStatSize = 0; taosStatFile(fNameFrom, &fStatSize, 0); ASSERT(fStatSize == tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage)); - n = tsdbFSendFile(pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage), speed); + n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage), maxSpeed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -757,10 +763,10 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i code = TAOS_SYSTEM_ERROR(errno); goto _err; } - fStatSize = 0; - taosStatFile(fNameFrom, &fStatSize, 0); - ASSERT(fStatSize == tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage)); - n = tsdbFSendFile(pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage), speed); + fStatSize = 0; + taosStatFile(fNameFrom, &fStatSize, 0); + ASSERT(fStatSize == tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage)); + n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage), maxSpeed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention2.c b/source/dnode/vnode/src/tsdb/tsdbRetention2.c index 9594334808..acd5a95649 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention2.c @@ -30,7 +30,7 @@ enum { RETENTION_NO = 0, RETENTION_EXPIRED = 1, RETENTION_MIGRATE = 2 }; static bool tsdbShouldDoMigrate(STsdb *pTsdb); static int32_t tsdbShouldDoRetention(STsdb *pTsdb, int64_t now); static int32_t tsdbProcessExpire(STsdb *pTsdb, int64_t now, int32_t retention); -static int32_t tsdbProcessMigrate(STsdb *pTsdb, int64_t now, int32_t maxSpeed, int32_t retention); +static int32_t tsdbProcessMigrate(STsdb *pTsdb, int64_t now, int64_t maxSpeed, int32_t retention); static bool tsdbShouldDoMigrate(STsdb *pTsdb) { if (tfsGetLevel(pTsdb->pVnode->pTfs) < 2) { @@ -109,6 +109,7 @@ _wait_commit_end: if (++nLoops > 1000) { nLoops = 0; sched_yield(); + printf("%s:%d wait commit finished\n", __func__, __LINE__); } } if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { @@ -163,7 +164,7 @@ _exit: * @param retention * @return int32_t */ -static int32_t tsdbProcessMigrate(STsdb *pTsdb, int64_t now, int32_t maxSpeed, int32_t retention) { +static int32_t tsdbProcessMigrate(STsdb *pTsdb, int64_t now, int64_t maxSpeed, int32_t retention) { int32_t code = 0; int32_t nBatch = 0; int32_t nLoops = 0; @@ -183,6 +184,11 @@ _migrate_loop: tsdbFSDestroy(&fs); tsdbFSDestroy(&fsLatest); + if (atomic_load_8(&pTsdb->trimHdl.commitInWait) == 1) { + atomic_store_32(&pTsdb->trimHdl.maxRetentFid, INT32_MIN); + taosMsleep(10); + } + code = tsdbFSCopy(pTsdb, &fs); if (code) goto _exit; @@ -315,7 +321,7 @@ _exit: * @param maxSpeed * @return int32_t */ -int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now, int32_t maxSpeed) { +int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed) { int32_t code = 0; int32_t retention = RETENTION_NO; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index a80211f402..4139a8ca2b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -455,13 +455,14 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, } if (atomic_val_compare_exchange_8(&pHandle->state, 0, 1) != 0) { - vInfo("vgId:%d, trim vnode request will not be processed since duplicated req, time:%" PRIi64, TD_VID(pVnode), - pVndTrimReq->trimReq.timestamp); + vInfo("vgId:%d, trim vnode request ignored since duplicated req, time:%" PRIi64 ", max speed:%" PRIi64, + TD_VID(pVnode), pVndTrimReq->trimReq.timestamp, pVndTrimReq->trimReq.maxSpeed); taosMemoryFree(pVndTrimReq); goto _exit; } - vInfo("vgId:%d, trim vnode request will be processed, time:%" PRIi64, TD_VID(pVnode), pVndTrimReq->trimReq.timestamp); + vInfo("vgId:%d, trim vnode request will be processed, time:%" PRIi64 ", max speed:%" PRIi64, TD_VID(pVnode), + pVndTrimReq->trimReq.timestamp, pVndTrimReq->trimReq.maxSpeed); TdThreadAttr thAttr = {0}; taosThreadAttrInit(&thAttr); @@ -474,10 +475,10 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, taosThreadAttrDestroy(&thAttr); int8_t oldVal = atomic_val_compare_exchange_8(&pHandle->state, 1, 0); ASSERT(oldVal == 1); - vError("vgId:%d, failed to create pthread for trim vnode since %s", TD_VID(pVnode), tstrerror(code)); + vError("vgId:%d, failed to create pthread to trim vnode since %s", TD_VID(pVnode), tstrerror(code)); goto _exit; } - vDebug("vgId:%d, success to create pthread for trim vnode", TD_VID(pVnode)); + vDebug("vgId:%d, success to create pthread to trim vnode", TD_VID(pVnode)); taosThreadAttrDestroy(&thAttr); From 024f6a2149bb0422903ada4beeed42b5507de9c8 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 27 Sep 2022 14:30:53 +0800 Subject: [PATCH 11/71] chore: send file with specified speed --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 31fe940848..6350acb000 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -635,7 +635,7 @@ static int64_t tsdbFSendFile(STsdb *pTsdb, TdFilePtr pOutFD, TdFilePtr pInFD, in goto _send_remain; } startMs = taosGetTimestampMs(); - if ((nBytes = taosFSendFile(pOutFD, pInFD, &offset, speed / 10)) < 0) { + if ((nBytes = taosFSendFile(pOutFD, pInFD, &offset, speed)) < 0) { return nBytes; } cost = taosGetTimestampMs() - startMs; From 3d92b6c6f91382ef180b94329d1dd546ea7bb38b Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 27 Sep 2022 14:33:17 +0800 Subject: [PATCH 12/71] chore: remove debug logs when migrate --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 13 ------------- source/dnode/vnode/src/tsdb/tsdbRetention2.c | 2 +- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 7e8aeafe8a..02284fb6d9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -767,7 +767,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { if (++nLoops > 1000) { nLoops = 0; sched_yield(); - printf("%s:%d wait retention to finish\n", __func__, __LINE__); + // printf("%s:%d wait retention to finish\n", __func__, __LINE__); } } if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 6350acb000..b1bc487fc3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -675,7 +675,6 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i int32_t szPage = pTsdb->pVnode->config.szPage; char fNameFrom[TSDB_FILENAME_LEN]; char fNameTo[TSDB_FILENAME_LEN]; - int64_t fStatSize = 0; // head tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); @@ -690,9 +689,6 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i code = TAOS_SYSTEM_ERROR(errno); goto _err; } - fStatSize = 0; - taosStatFile(fNameFrom, &fStatSize, 0); - ASSERT(fStatSize == tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage)); n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage), maxSpeed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); @@ -714,9 +710,6 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i code = TAOS_SYSTEM_ERROR(errno); goto _err; } - fStatSize = 0; - taosStatFile(fNameFrom, &fStatSize, 0); - ASSERT(fStatSize == tsdbLogicToFileSize(pSetFrom->pDataF->size, szPage)); n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pDataF->size, szPage), maxSpeed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); @@ -738,9 +731,6 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i code = TAOS_SYSTEM_ERROR(errno); goto _err; } - fStatSize = 0; - taosStatFile(fNameFrom, &fStatSize, 0); - ASSERT(fStatSize == tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage)); n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage), maxSpeed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); @@ -763,9 +753,6 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i code = TAOS_SYSTEM_ERROR(errno); goto _err; } - fStatSize = 0; - taosStatFile(fNameFrom, &fStatSize, 0); - ASSERT(fStatSize == tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage)); n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage), maxSpeed); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention2.c b/source/dnode/vnode/src/tsdb/tsdbRetention2.c index acd5a95649..ed45c6bc2a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention2.c @@ -109,7 +109,7 @@ _wait_commit_end: if (++nLoops > 1000) { nLoops = 0; sched_yield(); - printf("%s:%d wait commit finished\n", __func__, __LINE__); + // printf("%s:%d wait commit finished\n", __func__, __LINE__); } } if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { From b6563e4920d7e60b9735fb7fa2afff38f8dea8eb Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 27 Sep 2022 15:13:03 +0800 Subject: [PATCH 13/71] build: release ver-3.0.1.3 --- cmake/cmake.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index fcb31adc39..2d86335b3c 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.1.2") + SET(TD_VER_NUMBER "3.0.1.3") ENDIF () IF (DEFINED VERCOMPATIBLE) From eead619f1a6f231abd4df09166c0cacad8d396cc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 27 Sep 2022 16:02:40 +0800 Subject: [PATCH 14/71] fix: coverity issues --- include/common/tmsg.h | 9 ++-- source/common/src/tglobal.c | 19 +++++-- source/dnode/mgmt/mgmt_qnode/src/qmWorker.c | 1 + source/dnode/mgmt/mgmt_vnode/src/vmFile.c | 15 ++++-- source/util/src/tqueue.c | 8 +-- source/util/src/tuuid.c | 4 +- source/util/src/tworker.c | 8 +-- tools/shell/src/shellArguments.c | 2 +- utils/tsim/src/simExe.c | 59 ++++++++++++++------- utils/tsim/src/simMain.c | 2 +- utils/tsim/src/simParse.c | 13 +++-- 11 files changed, 93 insertions(+), 47 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 49db88d703..7aec00c7c1 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -55,11 +55,10 @@ extern int32_t tMsgDict[]; #define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8) #define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff) -#define TMSG_INFO(TYPE) \ - ((TYPE) >= 0 && ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG || \ - (TYPE) < TDMT_SCH_MAX_MSG || (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || \ - (TYPE) < TDMT_SYNC_MAX_MSG)) \ - ? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] \ +#define TMSG_INFO(TYPE) \ + ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG || (TYPE) < TDMT_SCH_MAX_MSG || \ + (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || (TYPE) < TDMT_SYNC_MAX_MSG) \ + ? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] \ : 0 #define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index e1d7569e29..41d170686e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -205,7 +205,9 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input tstrncpy(cfgFile, cfgDir, sizeof(cfgDir)); } - if (apolloUrl == NULL || apolloUrl[0] == '\0') cfgGetApollUrl(envCmd, envFile, apolloUrl); + if (apolloUrl != NULL && apolloUrl[0] == '\0') { + cfgGetApollUrl(envCmd, envFile, apolloUrl); + } if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { uError("failed to load from apollo url:%s since %s", apolloUrl, terrstr()); @@ -1126,11 +1128,20 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi if (tsc) { tsLogEmbedded = 0; - if (taosAddClientLogCfg(pCfg) != 0) return -1; + if (taosAddClientLogCfg(pCfg) != 0) { + cfgCleanup(pCfg); + return -1; + } } else { tsLogEmbedded = 1; - if (taosAddClientLogCfg(pCfg) != 0) return -1; - if (taosAddServerLogCfg(pCfg) != 0) return -1; + if (taosAddClientLogCfg(pCfg) != 0) { + cfgCleanup(pCfg); + return -1; + } + if (taosAddServerLogCfg(pCfg) != 0) { + cfgCleanup(pCfg); + return -1; + } } if (taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) { diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index 1c7edbe6be..66386b0ee0 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -87,6 +87,7 @@ int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { return 0; default: terrno = TSDB_CODE_INVALID_PARA; + taosFreeQitem(pMsg); return -1; } } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index cbcb541200..2d8ff5f436 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -141,6 +141,7 @@ _OVER: } int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { + int32_t ret = 0; char file[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0}; snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP); @@ -159,15 +160,18 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { int32_t len = 0; int32_t maxLen = MAX_CONTENT_LEN; char *content = taosMemoryCalloc(1, maxLen + 1); - if (content == NULL) { + if (content != NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + ret = -1; + goto _OVER; } len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); for (int32_t i = 0; i < numOfVnodes; ++i) { SVnodeObj *pVnode = pVnodes[i]; + if (pVnode == NULL) continue; + len += snprintf(content + len, maxLen - len, " {\n"); len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId); len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped); @@ -181,11 +185,14 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { len += snprintf(content + len, maxLen - len, " ]\n"); len += snprintf(content + len, maxLen - len, "}\n"); + terrno = 0; + +_OVER: + taosWriteFile(pFile, content, len); taosFsyncFile(pFile); taosCloseFile(&pFile); taosMemoryFree(content); - terrno = 0; for (int32_t i = 0; i < numOfVnodes; ++i) { SVnodeObj *pVnode = pVnodes[i]; @@ -196,6 +203,8 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { taosMemoryFree(pVnodes); } + if (ret != 0) return -1; + dDebug("successed to write %s, numOfVnodes:%d", realfile, numOfVnodes); return taosRenameFile(file, realfile); } \ No newline at end of file diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index eb70002680..eb62e12a19 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -151,15 +151,15 @@ int64_t taosQueueMemorySize(STaosQueue *queue) { void *taosAllocateQitem(int32_t size, EQItype itype) { STaosQnode *pNode = taosMemoryCalloc(1, sizeof(STaosQnode) + size); - pNode->size = size; - pNode->itype = itype; - pNode->timestamp = taosGetTimestampUs(); - if (pNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pNode->size = size; + pNode->itype = itype; + pNode->timestamp = taosGetTimestampUs(); + if (itype == RPC_QITEM) { int64_t alloced = atomic_add_fetch_64(&tsRpcQueueMemoryUsed, size); if (alloced > tsRpcQueueMemoryAllowed) { diff --git a/source/util/src/tuuid.c b/source/util/src/tuuid.c index 7460ccbc82..d192b1229d 100644 --- a/source/util/src/tuuid.c +++ b/source/util/src/tuuid.c @@ -20,8 +20,8 @@ static int32_t tUUIDSerialNo = 0; int32_t tGenIdPI32(void) { if (tUUIDHashId == 0) { - char uid[64]; - int32_t code = taosGetSystemUUID(uid, tListLen(uid)); + char uid[65] = {0}; + int32_t code = taosGetSystemUUID(uid, sizeof(uid)); if (code != TSDB_CODE_SUCCESS) { terrno = TAOS_SYSTEM_ERROR(errno); } else { diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 1f0731812c..d9ded20070 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -46,7 +46,7 @@ int32_t tQWorkerInit(SQWorkerPool *pool) { void tQWorkerCleanup(SQWorkerPool *pool) { for (int32_t i = 0; i < pool->max; ++i) { SQWorker *worker = pool->workers + i; - if (worker == NULL) continue; + // if (worker == NULL) continue; if (taosCheckPthreadValid(worker->thread)) { taosQsetThreadResume(pool->qset); } @@ -54,7 +54,7 @@ void tQWorkerCleanup(SQWorkerPool *pool) { for (int32_t i = 0; i < pool->max; ++i) { SQWorker *worker = pool->workers + i; - if (worker == NULL) continue; + // if (worker == NULL) continue; if (taosCheckPthreadValid(worker->thread)) { taosThreadJoin(worker->thread, NULL); taosThreadClear(&worker->thread); @@ -138,8 +138,8 @@ STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { } void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue) { - taosCloseQueue(queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue); + taosCloseQueue(queue); } int32_t tWWorkerInit(SWWorkerPool *pool) { @@ -283,8 +283,8 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { } void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue) { - taosCloseQueue(queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue); + taosCloseQueue(queue); } int32_t tSingleWorkerInit(SSingleWorker *pWorker, const SSingleWorkerCfg *pCfg) { diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 93f8b8a23b..eedb6d7295 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -358,7 +358,7 @@ static int32_t shellCheckArgs() { return -1; } - if (pArgs->password != NULL && (strlen(pArgs->password) <= 0)) { + if (/*pArgs->password != NULL && */ (strlen(pArgs->password) <= 0)) { printf("Invalid password\r\n"); return -1; } diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c index 16732ff9a1..5bb53287db 100644 --- a/utils/tsim/src/simExe.c +++ b/utils/tsim/src/simExe.c @@ -224,10 +224,24 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { } int32_t simExecuteExpression(SScript *script, char *exp) { - char *op1, *op2, *var1, *var2, *var3, *rest; - int32_t op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1; - char t0[1024], t1[1024], t2[1024], t3[2048]; - int32_t result; + char *op1 = NULL; + char *op2 = NULL; + char *var1 = NULL; + char *var2 = NULL; + char *var3 = NULL; + char *rest = NULL; + int32_t op1Len = 0; + int32_t op2Len = 0; + int32_t var1Len = 0; + int32_t var2Len = 0; + int32_t var3Len = 0; + int32_t val0 = 0; + int32_t val1 = 0; + char t0[1024] = {0}; + char t1[1024] = {0}; + char t2[1024] = {0}; + char t3[2048] = {0}; + int32_t result = 0; rest = paGetToken(exp, &var1, &var1Len); rest = paGetToken(rest, &op1, &op1Len); @@ -241,9 +255,9 @@ int32_t simExecuteExpression(SScript *script, char *exp) { t0[var1Len] = 0; } - if (var2[0] == '$') - strcpy(t1, simGetVariable(script, var2 + 1, var2Len - 1)); - else { + if (var2[0] == '$') { + tstrncpy(t1, simGetVariable(script, var2 + 1, var2Len - 1), 1024); + } else { memcpy(t1, var2, var2Len); t1[var2Len] = 0; } @@ -258,14 +272,21 @@ int32_t simExecuteExpression(SScript *script, char *exp) { t2[var3Len] = 0; } + int64_t t1l = atoll(t1); + int64_t t2l = atoll(t2); + if (op2[0] == '+') { - sprintf(t3, "%lld", atoll(t1) + atoll(t2)); + sprintf(t3, "%" PRId64, t1l + t2l); } else if (op2[0] == '-') { - sprintf(t3, "%lld", atoll(t1) - atoll(t2)); + sprintf(t3, "%" PRId64, t1l - t2l); } else if (op2[0] == '*') { - sprintf(t3, "%lld", atoll(t1) * atoll(t2)); + sprintf(t3, "%" PRId64, t1l * t2l); } else if (op2[0] == '/') { - sprintf(t3, "%lld", atoll(t1) / atoll(t2)); + if (t2l == 0) { + sprintf(t3, "%" PRId64, INT64_MAX); + } else { + sprintf(t3, "%" PRId64, t1l / t2l); + } } else if (op2[0] == '.') { sprintf(t3, "%s%s", t1, t2); } @@ -636,7 +657,7 @@ bool simCreateTaosdConnect(SScript *script, char *rest) { } bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { - char timeStr[30] = {0}; + char timeStr[80] = {0}; time_t tt; struct tm tp; SCmdLine *line = &script->lines[script->linePos]; @@ -943,7 +964,7 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { return true; } - TAOS_RES *pSql = pSql = taos_query(script->taos, rest); + TAOS_RES *pSql = taos_query(script->taos, rest); int32_t ret = taos_errno(pSql); taos_free_result(pSql); @@ -961,7 +982,7 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { } bool simExecuteLineInsertCmd(SScript *script, char *rest) { - char buf[TSDB_MAX_BINARY_LEN]; + char buf[TSDB_MAX_BINARY_LEN] = {0}; simVisuallizeOption(script, rest, buf); rest = buf; @@ -973,10 +994,7 @@ bool simExecuteLineInsertCmd(SScript *script, char *rest) { char *lines[] = {rest}; #if 0 int32_t ret = taos_insert_lines(script->taos, lines, 1); -#else - int32_t ret = 0; -#endif - if (ret == TSDB_CODE_SUCCESS) { + if (ret == TSDB_CODE_SUCCESS) { simDebug("script:%s, taos:%p, %s executed. success.", script->fileName, script->taos, rest); script->linePos++; return true; @@ -985,6 +1003,11 @@ bool simExecuteLineInsertCmd(SScript *script, char *rest) { tstrerror(ret)); return false; } +#else + simDebug("script:%s, taos:%p, %s executed. success.", script->fileName, script->taos, rest); + script->linePos++; + return true; +#endif } bool simExecuteLineInsertErrorCmd(SScript *script, char *rest) { diff --git a/utils/tsim/src/simMain.c b/utils/tsim/src/simMain.c index 713e46df58..e58a22cf68 100644 --- a/utils/tsim/src/simMain.c +++ b/utils/tsim/src/simMain.c @@ -33,7 +33,7 @@ int32_t main(int32_t argc, char *argv[]) { if (strcmp(argv[i], "-c") == 0 && i < argc - 1) { tstrncpy(configDir, argv[++i], 128); } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) { - strcpy(scriptFile, argv[++i]); + tstrncpy(scriptFile, argv[++i], MAX_FILE_NAME_LEN); } else if (strcmp(argv[i], "-m") == 0) { useMultiProcess = true; } else if (strcmp(argv[i], "-v") == 0) { diff --git a/utils/tsim/src/simParse.c b/utils/tsim/src/simParse.c index b9f7610be8..14be10bc8a 100644 --- a/utils/tsim/src/simParse.c +++ b/utils/tsim/src/simParse.c @@ -175,14 +175,17 @@ SScript *simBuildScriptObj(char *fileName) { SScript *simParseScript(char *fileName) { TdFilePtr pFile; int32_t tokenLen, lineNum = 0; - char buffer[10*1024], name[128], *token, *rest; - SCommand *pCmd; - SScript *script; + char buffer[10 * 1024] = {0}; + char name[PATH_MAX] = {9}; + char *token = NULL; + char *rest = NULL; + SCommand *pCmd = NULL; + SScript *script = NULL; if ((fileName[0] == '.') || (fileName[0] == '/')) { - strcpy(name, fileName); + tstrncpy(name, fileName, PATH_MAX); } else { - sprintf(name, "%s" TD_DIRSEP "%s", simScriptDir, fileName); + snprintf(name, PATH_MAX, "%s" TD_DIRSEP "%s", simScriptDir, fileName); taosRealPath(name, NULL, sizeof(name)); } From 6c19c062866fcef29d046cfb9f50088a7e4bfa2b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 27 Sep 2022 17:16:12 +0800 Subject: [PATCH 15/71] fix: coverity issues --- source/dnode/mgmt/mgmt_vnode/src/vmFile.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index 2d8ff5f436..25e85227e3 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -160,7 +160,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { int32_t len = 0; int32_t maxLen = MAX_CONTENT_LEN; char *content = taosMemoryCalloc(1, maxLen + 1); - if (content != NULL) { + if (content == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; ret = -1; goto _OVER; @@ -184,11 +184,9 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { } len += snprintf(content + len, maxLen - len, " ]\n"); len += snprintf(content + len, maxLen - len, "}\n"); - terrno = 0; _OVER: - taosWriteFile(pFile, content, len); taosFsyncFile(pFile); taosCloseFile(&pFile); From f02d5b2a9d40dbc7de94664e1ce5fd861b53bbf3 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 27 Sep 2022 17:32:39 +0800 Subject: [PATCH 16/71] docs: 3.0.1.3 --- docs/en/28-releases/01-tdengine.md | 4 ++++ docs/en/28-releases/02-tools.md | 4 ++++ docs/zh/28-releases/01-tdengine.md | 4 ++++ docs/zh/28-releases/02-tools.md | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index 414986d107..a65a2fff63 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -6,6 +6,10 @@ description: TDengine release history, Release Notes and download links. import Release from "/components/ReleaseV3"; +## 3.0.1.3 + + + ## 3.0.1.2 diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index 086d3adea2..a83723bff7 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -6,6 +6,10 @@ description: taosTools release history, Release Notes, download links. import Release from "/components/ReleaseV3"; +## 2.2.3 + + + ## 2.2.2 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index b05cf7a942..597e98238b 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -6,6 +6,10 @@ description: TDengine 发布历史、Release Notes 及下载链接 import Release from "/components/ReleaseV3"; +## 3.0.1.3 + + + ## 3.0.1.2 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index f793981d06..8604885d3c 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -6,6 +6,10 @@ description: taosTools 的发布历史、Release Notes 和下载链接 import Release from "/components/ReleaseV3"; +## 2.2.3 + + + ## 2.2.2 From c426fa8bb18683ffaf9e660615cc6029ca38afcf Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 27 Sep 2022 17:43:33 +0800 Subject: [PATCH 17/71] feat: code optimization for data migrate --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 3 +- source/dnode/vnode/src/tsdb/tsdbRetention2.c | 230 +++++++------------ 2 files changed, 82 insertions(+), 151 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 02284fb6d9..866a9b3aaa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -767,7 +767,6 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { if (++nLoops > 1000) { nLoops = 0; sched_yield(); - // printf("%s:%d wait retention to finish\n", __func__, __LINE__); } } if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { @@ -780,7 +779,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { } else { goto _wait_retention_end; } - atomic_store_8(&pTsdb->trimHdl.commitInWait, 0); + atomic_val_compare_exchange_8(&pTsdb->trimHdl.commitInWait, 1, 0); } code = tsdbFSCopy(pTsdb, &pCommitter->fs); diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention2.c b/source/dnode/vnode/src/tsdb/tsdbRetention2.c index ed45c6bc2a..d72211047f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention2.c @@ -17,20 +17,12 @@ enum { RETENTION_NO = 0, RETENTION_EXPIRED = 1, RETENTION_MIGRATE = 2 }; -#if 1 -#define MIGRATE_MIN_FSIZE (1048576 << 9) // 512 MB #define MIGRATE_MAX_SPEED (1048576 << 5) // 32 MB #define MIGRATE_MIN_COST (5) // second -#else -#define MIGRATE_MIN_FSIZE (1048576 << 5) // 32 MB -#define MIGRATE_MAX_SPEED (1048576 << 2) // 4 MB -#define MIGRATE_MIN_COST (5) // second -#endif static bool tsdbShouldDoMigrate(STsdb *pTsdb); static int32_t tsdbShouldDoRetention(STsdb *pTsdb, int64_t now); -static int32_t tsdbProcessExpire(STsdb *pTsdb, int64_t now, int32_t retention); -static int32_t tsdbProcessMigrate(STsdb *pTsdb, int64_t now, int64_t maxSpeed, int32_t retention); +static int32_t tsdbProcessRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed, int32_t retention, int8_t type); static bool tsdbShouldDoMigrate(STsdb *pTsdb) { if (tfsGetLevel(pTsdb->pVnode->pTfs) < 2) { @@ -74,110 +66,31 @@ static int32_t tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { return retention; } -static int32_t tsdbProcessExpire(STsdb *pTsdb, int64_t now, int32_t retention) { - int32_t code = 0; - int32_t nLoops = 0; - int32_t maxFid = INT32_MIN; - STsdbFS fs = {0}; - STsdbFS fsLatest = {0}; - - if (!(retention & RETENTION_EXPIRED)) { - goto _exit; - } - - code = tsdbFSCopy(pTsdb, &fs); - if (code) goto _exit; - - int32_t fsSize = taosArrayGetSize(fs.aDFileSet); - for (int32_t iSet = 0; iSet < fsSize; iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - SDiskID did; - - if (expLevel < 0) { - SET_DFSET_EXPIRED(pSet); - if (pSet->fid > maxFid) maxFid = pSet->fid; - } else { - break; - } - } - - if (maxFid == INT32_MIN) goto _exit; - -_wait_commit_end: - while (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) { - if (++nLoops > 1000) { - nLoops = 0; - sched_yield(); - // printf("%s:%d wait commit finished\n", __func__, __LINE__); - } - } - if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { - if (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) { - atomic_store_8(&pTsdb->trimHdl.state, 0); - goto _wait_commit_end; - } - atomic_store_32(&pTsdb->trimHdl.maxRetentFid, maxFid); - atomic_store_8(&pTsdb->trimHdl.state, 0); - } else { - goto _wait_commit_end; - } - -_merge_fs: - taosThreadRwlockWrlock(&pTsdb->rwLock); - if ((code = tsdbFSCopy(pTsdb, &fsLatest))) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; - } - // 1) merge tsdbFSNew and pTsdb->fs - if ((code = tsdbFSUpdDel(pTsdb, &fsLatest, &fs, maxFid))) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; - } - // 2) save CURRENT - if ((code = tsdbFSCommit1(pTsdb, &fsLatest))) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; - } - // 3) apply the tsdbFS to pTsdb->fs - if ((code = tsdbFSCommit2(pTsdb, &fsLatest))) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; - } - taosThreadRwlockUnlock(&pTsdb->rwLock); - -_exit: - tsdbFSDestroy(&fs); - tsdbFSDestroy(&fsLatest); - if (code != 0) { - tsdbError("vgId:%d, tsdb do retention(expire) failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - ASSERT(0); - } - return code; -} - /** - * @brief + * @brief process retention * * @param pTsdb * @param now + * @param maxSpeed * @param retention + * @param type 0 RETENTION_EXPIRED, 1 RETENTION_MIGRATE * @return int32_t */ -static int32_t tsdbProcessMigrate(STsdb *pTsdb, int64_t now, int64_t maxSpeed, int32_t retention) { +static int32_t tsdbProcessRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed, int32_t retention, int8_t type) { int32_t code = 0; int32_t nBatch = 0; int32_t nLoops = 0; - int32_t maxFid = INT32_MIN; + int32_t maxFid = 0; int64_t fSize = 0; + int64_t speed = maxSpeed > 0 ? maxSpeed : MIGRATE_MAX_SPEED; STsdbFS fs = {0}; STsdbFS fsLatest = {0}; - if (!(retention & RETENTION_MIGRATE)) { + if (!(retention & type)) { goto _exit; } -_migrate_loop: +_retention_loop: // reset maxFid = INT32_MIN; fSize = 0; @@ -186,32 +99,47 @@ _migrate_loop: if (atomic_load_8(&pTsdb->trimHdl.commitInWait) == 1) { atomic_store_32(&pTsdb->trimHdl.maxRetentFid, INT32_MIN); - taosMsleep(10); + taosMsleep(50); } code = tsdbFSCopy(pTsdb, &fs); if (code) goto _exit; int32_t fsSize = taosArrayGetSize(fs.aDFileSet); - for (int32_t iSet = 0; iSet < fsSize; ++iSet) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - SDiskID did; + if (type == RETENTION_MIGRATE) { + for (int32_t iSet = 0; iSet < fsSize; ++iSet) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); + int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); + SDiskID did; - if (pSet->diskId.level == expLevel) continue; + if (pSet->diskId.level == expLevel) continue; - if (expLevel > 0) { - ASSERT(pSet->fid > maxFid); - maxFid = pSet->fid; - fSize += (pSet->pDataF->size + pSet->pHeadF->size + pSet->pSmaF->size); - if (fSize / MIGRATE_MAX_SPEED > MIGRATE_MIN_COST) { - break; + if (expLevel > 0) { + ASSERT(pSet->fid > maxFid); + maxFid = pSet->fid; + fSize += (pSet->pDataF->size + pSet->pHeadF->size + pSet->pSmaF->size); + if (fSize / speed > MIGRATE_MIN_COST) { + break; + } + for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) { + fSize += pSet->aSttF[iStt]->size; + } + if (fSize / speed > MIGRATE_MIN_COST) { + tsdbInfo("vgId:%d migrate loop[%d] with maxFid:%d", TD_VID(pTsdb->pVnode), nBatch, maxFid); + break; + } } - for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) { - fSize += pSet->aSttF[iStt]->size; - } - if (fSize / MIGRATE_MAX_SPEED > MIGRATE_MIN_COST) { - tsdbInfo("vgId:%d migrate loop[%d] with maxFid:%d", TD_VID(pTsdb->pVnode), nBatch, maxFid); + } + } else if (type == RETENTION_EXPIRED) { + for (int32_t iSet = 0; iSet < fsSize; ++iSet) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); + int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); + SDiskID did; + + if (expLevel < 0) { + SET_DFSET_EXPIRED(pSet); + if (pSet->fid > maxFid) maxFid = pSet->fid; + } else { break; } } @@ -219,7 +147,7 @@ _migrate_loop: if (maxFid == INT32_MIN) goto _exit; -_wait_commit_end: +_commit_conflict_check: while (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) { if (++nLoops > 1000) { nLoops = 0; @@ -229,46 +157,48 @@ _wait_commit_end: if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { if (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) { atomic_store_8(&pTsdb->trimHdl.state, 0); - goto _wait_commit_end; + goto _commit_conflict_check; } atomic_store_32(&pTsdb->trimHdl.maxRetentFid, maxFid); atomic_store_8(&pTsdb->trimHdl.state, 0); } else { - goto _wait_commit_end; + goto _commit_conflict_check; } // migrate - for (int32_t iSet = 0; iSet < fsSize; ++iSet) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - SDiskID did; + if (type == RETENTION_MIGRATE) { + for (int32_t iSet = 0; iSet < fsSize; ++iSet) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); + int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); + SDiskID did; - if (pSet->fid > maxFid) break; + if (pSet->fid > maxFid) break; - tsdbInfo("vgId:%d migrate loop[%d] with maxFid:%d, fid:%d, did:%d, level:%d, expLevel:%d", TD_VID(pTsdb->pVnode), - nBatch, maxFid, pSet->fid, pSet->diskId.id, pSet->diskId.level, expLevel); + tsdbInfo("vgId:%d migrate loop[%d] with maxFid:%d, fid:%d, did:%d, level:%d, expLevel:%d", TD_VID(pTsdb->pVnode), + nBatch, maxFid, pSet->fid, pSet->diskId.id, pSet->diskId.level, expLevel); - if (expLevel < 0) { - SET_DFSET_EXPIRED(pSet); - } else { - if (expLevel == pSet->diskId.level) continue; + if (expLevel < 0) { + SET_DFSET_EXPIRED(pSet); + } else { + if (expLevel == pSet->diskId.level) continue; - if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { - code = terrno; - goto _exit; + if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { + code = terrno; + goto _exit; + } + + if (did.level == pSet->diskId.level) continue; + + // copy file to new disk + SDFileSet fSet = *pSet; + fSet.diskId = did; + + code = tsdbDFileSetCopy(pTsdb, pSet, &fSet, maxSpeed); + if (code) goto _exit; + + code = tsdbFSUpsertFSet(&fs, &fSet); + if (code) goto _exit; } - - if (did.level == pSet->diskId.level) continue; - - // copy file to new disk - SDFileSet fSet = *pSet; - fSet.diskId = did; - - code = tsdbDFileSetCopy(pTsdb, pSet, &fSet, maxSpeed); - if (code) goto _exit; - - code = tsdbFSUpsertFSet(&fs, &fSet); - if (code) goto _exit; } } @@ -296,14 +226,16 @@ _merge_fs: } taosThreadRwlockUnlock(&pTsdb->rwLock); - ++nBatch; - goto _migrate_loop; + if (type == RETENTION_MIGRATE) { + ++nBatch; + goto _retention_loop; + } _exit: tsdbFSDestroy(&fs); tsdbFSDestroy(&fsLatest); if (code != 0) { - tsdbError("vgId:%d, tsdb do retention(migrate) failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, tsdb do retention %" PRIi8 " failed since %s", TD_VID(pTsdb->pVnode), type, tstrerror(code)); ASSERT(0); } return code; @@ -331,21 +263,21 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed) { } // step 1: process expire - code = tsdbProcessExpire(pTsdb, now, retention); + code = tsdbProcessRetention(pTsdb, now, maxSpeed, retention, RETENTION_EXPIRED); if (code < 0) goto _exit; // step 2: process multi-tier migration - code = tsdbProcessMigrate(pTsdb, now, maxSpeed, retention); + code = tsdbProcessRetention(pTsdb, now, maxSpeed, retention, RETENTION_MIGRATE); if (code < 0) goto _exit; _exit: pTsdb->trimHdl.maxRetentFid = INT32_MIN; if (code != 0) { - tsdbError("vgId:%d, tsdb do retention:%d failed since %s", TD_VID(pTsdb->pVnode), retention, tstrerror(code)); + tsdbError("vgId:%d, tsdb do retention %d failed since %s", TD_VID(pTsdb->pVnode), retention, tstrerror(code)); ASSERT(0); // tsdbFSRollback(pTsdb->pFS); } else { - tsdbInfo("vgId:%d, tsdb do retention:%d succeed", TD_VID(pTsdb->pVnode), retention); + tsdbInfo("vgId:%d, tsdb do retention %d succeed", TD_VID(pTsdb->pVnode), retention); } return code; } \ No newline at end of file From 6536ad72a64ac97eadb0b587cb6f2c4e8b587ee4 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 27 Sep 2022 17:48:09 +0800 Subject: [PATCH 18/71] chore: update file naming for tsdb retention --- source/dnode/vnode/CMakeLists.txt | 2 +- source/dnode/vnode/src/tsdb/tsdbRetention.c | 291 ++++++++++++++----- source/dnode/vnode/src/tsdb/tsdbRetention2.c | 283 ------------------ 3 files changed, 223 insertions(+), 353 deletions(-) delete mode 100644 source/dnode/vnode/src/tsdb/tsdbRetention2.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index d304f553ef..1f7a059ffc 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -49,7 +49,7 @@ target_sources( "src/tsdb/tsdbUtil.c" "src/tsdb/tsdbSnapshot.c" "src/tsdb/tsdbCacheRead.c" - "src/tsdb/tsdbRetention2.c" + "src/tsdb/tsdbRetention.c" "src/tsdb/tsdbDiskData.c" "src/tsdb/tsdbCompact.c" "src/tsdb/tsdbMergeTree.c" diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 02d0d12ea2..d72211047f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -15,16 +15,16 @@ #include "tsdb.h" -static bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { - if (taosArrayGetSize(pTsdb->fs.aDFileSet) == 0) { - return false; - } +enum { RETENTION_NO = 0, RETENTION_EXPIRED = 1, RETENTION_MIGRATE = 2 }; - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, 0); - if (tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now) < 0) { - return true; - } +#define MIGRATE_MAX_SPEED (1048576 << 5) // 32 MB +#define MIGRATE_MIN_COST (5) // second +static bool tsdbShouldDoMigrate(STsdb *pTsdb); +static int32_t tsdbShouldDoRetention(STsdb *pTsdb, int64_t now); +static int32_t tsdbProcessRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed, int32_t retention, int8_t type); + +static bool tsdbShouldDoMigrate(STsdb *pTsdb) { if (tfsGetLevel(pTsdb->pVnode->pTfs) < 2) { return false; } @@ -33,98 +33,251 @@ static bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { if (keepCfg->keep0 == keepCfg->keep1 && keepCfg->keep1 == keepCfg->keep2) { return false; } + return true; +} - for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { +static int32_t tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { + int32_t retention = RETENTION_NO; + if (taosArrayGetSize(pTsdb->fs.aDFileSet) == 0) { + return retention; + } + + SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, 0); + if (tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now) < 0) { + retention |= RETENTION_EXPIRED; + } + + if (!tsdbShouldDoMigrate(pTsdb)) { + return retention; + } + + for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); ++iSet) { pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, keepCfg, now); - SDiskID did; + int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); if (expLevel == pSet->diskId.level) continue; - if (expLevel < 0) { - return true; - } else { - if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { - return false; - } - - if (did.level == pSet->diskId.level) continue; - - return true; + if (expLevel > 0) { + retention |= RETENTION_MIGRATE; + break; } } - return false; + return retention; } -int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { +/** + * @brief process retention + * + * @param pTsdb + * @param now + * @param maxSpeed + * @param retention + * @param type 0 RETENTION_EXPIRED, 1 RETENTION_MIGRATE + * @return int32_t + */ +static int32_t tsdbProcessRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed, int32_t retention, int8_t type) { int32_t code = 0; + int32_t nBatch = 0; + int32_t nLoops = 0; + int32_t maxFid = 0; + int64_t fSize = 0; + int64_t speed = maxSpeed > 0 ? maxSpeed : MIGRATE_MAX_SPEED; + STsdbFS fs = {0}; + STsdbFS fsLatest = {0}; - if (!tsdbShouldDoRetention(pTsdb, now)) { - return code; + if (!(retention & type)) { + goto _exit; } - // do retention - STsdbFS fs; +_retention_loop: + // reset + maxFid = INT32_MIN; + fSize = 0; + tsdbFSDestroy(&fs); + tsdbFSDestroy(&fsLatest); + + if (atomic_load_8(&pTsdb->trimHdl.commitInWait) == 1) { + atomic_store_32(&pTsdb->trimHdl.maxRetentFid, INT32_MIN); + taosMsleep(50); + } code = tsdbFSCopy(pTsdb, &fs); - if (code) goto _err; + if (code) goto _exit; - for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - SDiskID did; + int32_t fsSize = taosArrayGetSize(fs.aDFileSet); + if (type == RETENTION_MIGRATE) { + for (int32_t iSet = 0; iSet < fsSize; ++iSet) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); + int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); + SDiskID did; - if (expLevel < 0) { - taosMemoryFree(pSet->pHeadF); - taosMemoryFree(pSet->pDataF); - taosMemoryFree(pSet->aSttF[0]); - taosMemoryFree(pSet->pSmaF); - taosArrayRemove(fs.aDFileSet, iSet); - iSet--; - } else { - if (expLevel == 0) continue; - if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { - code = terrno; - goto _exit; + if (pSet->diskId.level == expLevel) continue; + + if (expLevel > 0) { + ASSERT(pSet->fid > maxFid); + maxFid = pSet->fid; + fSize += (pSet->pDataF->size + pSet->pHeadF->size + pSet->pSmaF->size); + if (fSize / speed > MIGRATE_MIN_COST) { + break; + } + for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) { + fSize += pSet->aSttF[iStt]->size; + } + if (fSize / speed > MIGRATE_MIN_COST) { + tsdbInfo("vgId:%d migrate loop[%d] with maxFid:%d", TD_VID(pTsdb->pVnode), nBatch, maxFid); + break; + } } + } + } else if (type == RETENTION_EXPIRED) { + for (int32_t iSet = 0; iSet < fsSize; ++iSet) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); + int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); + SDiskID did; - if (did.level == pSet->diskId.level) continue; - - // copy file to new disk (todo) - SDFileSet fSet = *pSet; - fSet.diskId = did; - - code = tsdbDFileSetCopy(pTsdb, pSet, &fSet); - if (code) goto _err; - - code = tsdbFSUpsertFSet(&fs, &fSet); - if (code) goto _err; + if (expLevel < 0) { + SET_DFSET_EXPIRED(pSet); + if (pSet->fid > maxFid) maxFid = pSet->fid; + } else { + break; + } } } - // do change fs - code = tsdbFSCommit1(pTsdb, &fs); - if (code) goto _err; + if (maxFid == INT32_MIN) goto _exit; - taosThreadRwlockWrlock(&pTsdb->rwLock); - - code = tsdbFSCommit2(pTsdb, &fs); - if (code) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; +_commit_conflict_check: + while (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) { + if (++nLoops > 1000) { + nLoops = 0; + sched_yield(); + } + } + if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { + if (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) { + atomic_store_8(&pTsdb->trimHdl.state, 0); + goto _commit_conflict_check; + } + atomic_store_32(&pTsdb->trimHdl.maxRetentFid, maxFid); + atomic_store_8(&pTsdb->trimHdl.state, 0); + } else { + goto _commit_conflict_check; } + // migrate + if (type == RETENTION_MIGRATE) { + for (int32_t iSet = 0; iSet < fsSize; ++iSet) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); + int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); + SDiskID did; + + if (pSet->fid > maxFid) break; + + tsdbInfo("vgId:%d migrate loop[%d] with maxFid:%d, fid:%d, did:%d, level:%d, expLevel:%d", TD_VID(pTsdb->pVnode), + nBatch, maxFid, pSet->fid, pSet->diskId.id, pSet->diskId.level, expLevel); + + if (expLevel < 0) { + SET_DFSET_EXPIRED(pSet); + } else { + if (expLevel == pSet->diskId.level) continue; + + if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { + code = terrno; + goto _exit; + } + + if (did.level == pSet->diskId.level) continue; + + // copy file to new disk + SDFileSet fSet = *pSet; + fSet.diskId = did; + + code = tsdbDFileSetCopy(pTsdb, pSet, &fSet, maxSpeed); + if (code) goto _exit; + + code = tsdbFSUpsertFSet(&fs, &fSet); + if (code) goto _exit; + } + } + } + +_merge_fs: + taosThreadRwlockWrlock(&pTsdb->rwLock); + + if ((code = tsdbFSCopy(pTsdb, &fsLatest))) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _exit; + } + // 1) merge tsdbFSNew and pTsdb->fs + if ((code = tsdbFSUpdDel(pTsdb, &fsLatest, &fs, maxFid))) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _exit; + } + // 2) save CURRENT + if ((code = tsdbFSCommit1(pTsdb, &fsLatest))) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _exit; + } + // 3) apply the tsdbFS to pTsdb->fs + if ((code = tsdbFSCommit2(pTsdb, &fsLatest))) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _exit; + } taosThreadRwlockUnlock(&pTsdb->rwLock); - tsdbFSDestroy(&fs); + if (type == RETENTION_MIGRATE) { + ++nBatch; + goto _retention_loop; + } _exit: + tsdbFSDestroy(&fs); + tsdbFSDestroy(&fsLatest); + if (code != 0) { + tsdbError("vgId:%d, tsdb do retention %" PRIi8 " failed since %s", TD_VID(pTsdb->pVnode), type, tstrerror(code)); + ASSERT(0); + } return code; +} -_err: - tsdbError("vgId:%d, tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - ASSERT(0); - // tsdbFSRollback(pTsdb->pFS); +/** + * @brief Data migration between multi-tier storage, including remove expired data. + * 1) firstly, remove expired DFileSet; + * 2) partition the tsdbFS by the expLevel and the estimated cost(e.g. 5s) to copy, and migrate + * DFileSet groups between multi-tier storage; + * 3) update the tsdbFS and CURRENT in the same transaction; + * 4) finish the migration + * @param pTsdb + * @param now + * @param maxSpeed + * @return int32_t + */ +int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed) { + int32_t code = 0; + int32_t retention = RETENTION_NO; + + retention = tsdbShouldDoRetention(pTsdb, now); + if (retention == RETENTION_NO) { + goto _exit; + } + + // step 1: process expire + code = tsdbProcessRetention(pTsdb, now, maxSpeed, retention, RETENTION_EXPIRED); + if (code < 0) goto _exit; + + // step 2: process multi-tier migration + code = tsdbProcessRetention(pTsdb, now, maxSpeed, retention, RETENTION_MIGRATE); + if (code < 0) goto _exit; + +_exit: + pTsdb->trimHdl.maxRetentFid = INT32_MIN; + if (code != 0) { + tsdbError("vgId:%d, tsdb do retention %d failed since %s", TD_VID(pTsdb->pVnode), retention, tstrerror(code)); + ASSERT(0); + // tsdbFSRollback(pTsdb->pFS); + } else { + tsdbInfo("vgId:%d, tsdb do retention %d succeed", TD_VID(pTsdb->pVnode), retention); + } return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention2.c b/source/dnode/vnode/src/tsdb/tsdbRetention2.c deleted file mode 100644 index d72211047f..0000000000 --- a/source/dnode/vnode/src/tsdb/tsdbRetention2.c +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tsdb.h" - -enum { RETENTION_NO = 0, RETENTION_EXPIRED = 1, RETENTION_MIGRATE = 2 }; - -#define MIGRATE_MAX_SPEED (1048576 << 5) // 32 MB -#define MIGRATE_MIN_COST (5) // second - -static bool tsdbShouldDoMigrate(STsdb *pTsdb); -static int32_t tsdbShouldDoRetention(STsdb *pTsdb, int64_t now); -static int32_t tsdbProcessRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed, int32_t retention, int8_t type); - -static bool tsdbShouldDoMigrate(STsdb *pTsdb) { - if (tfsGetLevel(pTsdb->pVnode->pTfs) < 2) { - return false; - } - - STsdbKeepCfg *keepCfg = &pTsdb->keepCfg; - if (keepCfg->keep0 == keepCfg->keep1 && keepCfg->keep1 == keepCfg->keep2) { - return false; - } - return true; -} - -static int32_t tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { - int32_t retention = RETENTION_NO; - if (taosArrayGetSize(pTsdb->fs.aDFileSet) == 0) { - return retention; - } - - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, 0); - if (tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now) < 0) { - retention |= RETENTION_EXPIRED; - } - - if (!tsdbShouldDoMigrate(pTsdb)) { - return retention; - } - - for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); ++iSet) { - pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - - if (expLevel == pSet->diskId.level) continue; - - if (expLevel > 0) { - retention |= RETENTION_MIGRATE; - break; - } - } - - return retention; -} - -/** - * @brief process retention - * - * @param pTsdb - * @param now - * @param maxSpeed - * @param retention - * @param type 0 RETENTION_EXPIRED, 1 RETENTION_MIGRATE - * @return int32_t - */ -static int32_t tsdbProcessRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed, int32_t retention, int8_t type) { - int32_t code = 0; - int32_t nBatch = 0; - int32_t nLoops = 0; - int32_t maxFid = 0; - int64_t fSize = 0; - int64_t speed = maxSpeed > 0 ? maxSpeed : MIGRATE_MAX_SPEED; - STsdbFS fs = {0}; - STsdbFS fsLatest = {0}; - - if (!(retention & type)) { - goto _exit; - } - -_retention_loop: - // reset - maxFid = INT32_MIN; - fSize = 0; - tsdbFSDestroy(&fs); - tsdbFSDestroy(&fsLatest); - - if (atomic_load_8(&pTsdb->trimHdl.commitInWait) == 1) { - atomic_store_32(&pTsdb->trimHdl.maxRetentFid, INT32_MIN); - taosMsleep(50); - } - - code = tsdbFSCopy(pTsdb, &fs); - if (code) goto _exit; - - int32_t fsSize = taosArrayGetSize(fs.aDFileSet); - if (type == RETENTION_MIGRATE) { - for (int32_t iSet = 0; iSet < fsSize; ++iSet) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - SDiskID did; - - if (pSet->diskId.level == expLevel) continue; - - if (expLevel > 0) { - ASSERT(pSet->fid > maxFid); - maxFid = pSet->fid; - fSize += (pSet->pDataF->size + pSet->pHeadF->size + pSet->pSmaF->size); - if (fSize / speed > MIGRATE_MIN_COST) { - break; - } - for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) { - fSize += pSet->aSttF[iStt]->size; - } - if (fSize / speed > MIGRATE_MIN_COST) { - tsdbInfo("vgId:%d migrate loop[%d] with maxFid:%d", TD_VID(pTsdb->pVnode), nBatch, maxFid); - break; - } - } - } - } else if (type == RETENTION_EXPIRED) { - for (int32_t iSet = 0; iSet < fsSize; ++iSet) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - SDiskID did; - - if (expLevel < 0) { - SET_DFSET_EXPIRED(pSet); - if (pSet->fid > maxFid) maxFid = pSet->fid; - } else { - break; - } - } - } - - if (maxFid == INT32_MIN) goto _exit; - -_commit_conflict_check: - while (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) { - if (++nLoops > 1000) { - nLoops = 0; - sched_yield(); - } - } - if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { - if (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) { - atomic_store_8(&pTsdb->trimHdl.state, 0); - goto _commit_conflict_check; - } - atomic_store_32(&pTsdb->trimHdl.maxRetentFid, maxFid); - atomic_store_8(&pTsdb->trimHdl.state, 0); - } else { - goto _commit_conflict_check; - } - - // migrate - if (type == RETENTION_MIGRATE) { - for (int32_t iSet = 0; iSet < fsSize; ++iSet) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - SDiskID did; - - if (pSet->fid > maxFid) break; - - tsdbInfo("vgId:%d migrate loop[%d] with maxFid:%d, fid:%d, did:%d, level:%d, expLevel:%d", TD_VID(pTsdb->pVnode), - nBatch, maxFid, pSet->fid, pSet->diskId.id, pSet->diskId.level, expLevel); - - if (expLevel < 0) { - SET_DFSET_EXPIRED(pSet); - } else { - if (expLevel == pSet->diskId.level) continue; - - if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { - code = terrno; - goto _exit; - } - - if (did.level == pSet->diskId.level) continue; - - // copy file to new disk - SDFileSet fSet = *pSet; - fSet.diskId = did; - - code = tsdbDFileSetCopy(pTsdb, pSet, &fSet, maxSpeed); - if (code) goto _exit; - - code = tsdbFSUpsertFSet(&fs, &fSet); - if (code) goto _exit; - } - } - } - -_merge_fs: - taosThreadRwlockWrlock(&pTsdb->rwLock); - - if ((code = tsdbFSCopy(pTsdb, &fsLatest))) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; - } - // 1) merge tsdbFSNew and pTsdb->fs - if ((code = tsdbFSUpdDel(pTsdb, &fsLatest, &fs, maxFid))) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; - } - // 2) save CURRENT - if ((code = tsdbFSCommit1(pTsdb, &fsLatest))) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; - } - // 3) apply the tsdbFS to pTsdb->fs - if ((code = tsdbFSCommit2(pTsdb, &fsLatest))) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; - } - taosThreadRwlockUnlock(&pTsdb->rwLock); - - if (type == RETENTION_MIGRATE) { - ++nBatch; - goto _retention_loop; - } - -_exit: - tsdbFSDestroy(&fs); - tsdbFSDestroy(&fsLatest); - if (code != 0) { - tsdbError("vgId:%d, tsdb do retention %" PRIi8 " failed since %s", TD_VID(pTsdb->pVnode), type, tstrerror(code)); - ASSERT(0); - } - return code; -} - -/** - * @brief Data migration between multi-tier storage, including remove expired data. - * 1) firstly, remove expired DFileSet; - * 2) partition the tsdbFS by the expLevel and the estimated cost(e.g. 5s) to copy, and migrate - * DFileSet groups between multi-tier storage; - * 3) update the tsdbFS and CURRENT in the same transaction; - * 4) finish the migration - * @param pTsdb - * @param now - * @param maxSpeed - * @return int32_t - */ -int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed) { - int32_t code = 0; - int32_t retention = RETENTION_NO; - - retention = tsdbShouldDoRetention(pTsdb, now); - if (retention == RETENTION_NO) { - goto _exit; - } - - // step 1: process expire - code = tsdbProcessRetention(pTsdb, now, maxSpeed, retention, RETENTION_EXPIRED); - if (code < 0) goto _exit; - - // step 2: process multi-tier migration - code = tsdbProcessRetention(pTsdb, now, maxSpeed, retention, RETENTION_MIGRATE); - if (code < 0) goto _exit; - -_exit: - pTsdb->trimHdl.maxRetentFid = INT32_MIN; - if (code != 0) { - tsdbError("vgId:%d, tsdb do retention %d failed since %s", TD_VID(pTsdb->pVnode), retention, tstrerror(code)); - ASSERT(0); - // tsdbFSRollback(pTsdb->pFS); - } else { - tsdbInfo("vgId:%d, tsdb do retention %d succeed", TD_VID(pTsdb->pVnode), retention); - } - return code; -} \ No newline at end of file From b7253d881411152bd79d0615838c815423e3a10b Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 27 Sep 2022 17:55:01 +0800 Subject: [PATCH 19/71] chore: code optimization for tsdb commit --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 - source/dnode/vnode/src/tsdb/tsdbFS.c | 339 ----------------------- 2 files changed, 343 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 866a9b3aaa..1c63487583 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -994,14 +994,11 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { if(inTrim) taosThreadRwlockWrlock(&pTsdb->rwLock); - int64_t startTime = taosGetTimestampMs(); code = tsdbFSCommit1(pTsdb, &pCommitter->fs); if (code) { if(inTrim) taosThreadRwlockUnlock(&pTsdb->rwLock); goto _err; } - int64_t endTime = taosGetTimestampMs(); - tsdbInfo("vgId:%d, tsdb end commit - commit1 fsSize:%d cost: %" PRIi64 " ms", TD_VID(pTsdb->pVnode), (int32_t)taosArrayGetSize(pCommitter->fs.aDFileSet), endTime - startTime); // lock if(!inTrim) taosThreadRwlockWrlock(&pTsdb->rwLock); @@ -1012,7 +1009,6 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { taosThreadRwlockUnlock(&pTsdb->rwLock); goto _err; } - tsdbInfo("vgId:%d, tsdb end commit - commit2 cost: %" PRIi64 " ms", TD_VID(pTsdb->pVnode), taosGetTimestampMs() - endTime); pTsdb->imem = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 81ad1d0e7f..188810600f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -714,343 +714,6 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { _exit: return code; } -#if 0 -int32_t tsdbFSCommit1(STsdb *pTsdb, STsdbFS *pFSNew) { - int32_t code = 0; - char tfname[TSDB_FILENAME_LEN]; - char fname[TSDB_FILENAME_LEN]; - - snprintf(tfname, TSDB_FILENAME_LEN - 1, "%s%s%s%sCURRENT.t", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), TD_DIRSEP, - pTsdb->path, TD_DIRSEP); - snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sCURRENT", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), TD_DIRSEP, - pTsdb->path, TD_DIRSEP); - - // gnrt CURRENT.t - code = tsdbGnrtCurrent(pTsdb, pFSNew, tfname); - if (code) goto _err; - - // rename - code = taosRenameFile(tfname, fname); - if (code) { - code = TAOS_SYSTEM_ERROR(code); - goto _err; - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb fs commit phase 1 failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { - int32_t code = 0; - int32_t nRef; - char fname[TSDB_FILENAME_LEN]; - - // del - if (pFSNew->pDelFile) { - SDelFile *pDelFile = pTsdb->fs.pDelFile; - - if (pDelFile == NULL || (pDelFile->commitID != pFSNew->pDelFile->commitID)) { - pTsdb->fs.pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile)); - if (pTsdb->fs.pDelFile == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - *pTsdb->fs.pDelFile = *pFSNew->pDelFile; - pTsdb->fs.pDelFile->nRef = 1; - - if (pDelFile) { - nRef = atomic_sub_fetch_32(&pDelFile->nRef, 1); - if (nRef == 0) { - tsdbDelFileName(pTsdb, pDelFile, fname); - taosRemoveFile(fname); - taosMemoryFree(pDelFile); - } - } - } - } else { - ASSERT(pTsdb->fs.pDelFile == NULL); - } - - // data - int32_t iOld = 0; - int32_t iNew = 0; - while (true) { - int32_t nOld = taosArrayGetSize(pTsdb->fs.aDFileSet); - int32_t nNew = taosArrayGetSize(pFSNew->aDFileSet); - SDFileSet fSet; - int8_t sameDisk; - - if (iOld >= nOld && iNew >= nNew) break; - - SDFileSet *pSetOld = (iOld < nOld) ? taosArrayGet(pTsdb->fs.aDFileSet, iOld) : NULL; - SDFileSet *pSetNew = (iNew < nNew) ? taosArrayGet(pFSNew->aDFileSet, iNew) : NULL; - - if (pSetOld && pSetNew) { - if (pSetOld->fid == pSetNew->fid) { - goto _merge_old_and_new; - } else if (pSetOld->fid < pSetNew->fid) { - goto _remove_old; - } else { - goto _add_new; - } - } else if (pSetOld) { - goto _remove_old; - } else { - goto _add_new; - } - - _merge_old_and_new: - sameDisk = ((pSetOld->diskId.level == pSetNew->diskId.level) && (pSetOld->diskId.id == pSetNew->diskId.id)); - - // head - fSet.pHeadF = pSetOld->pHeadF; - if ((!sameDisk) || (pSetOld->pHeadF->commitID != pSetNew->pHeadF->commitID)) { - pSetOld->pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); - if (pSetOld->pHeadF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->pHeadF = *pSetNew->pHeadF; - pSetOld->pHeadF->nRef = 1; - - nRef = atomic_sub_fetch_32(&fSet.pHeadF->nRef, 1); - if (nRef == 0) { - tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pHeadF, fname); - taosRemoveFile(fname); - taosMemoryFree(fSet.pHeadF); - } - } else { - ASSERT(fSet.pHeadF->size == pSetNew->pHeadF->size); - ASSERT(fSet.pHeadF->offset == pSetNew->pHeadF->offset); - } - - // data - fSet.pDataF = pSetOld->pDataF; - if ((!sameDisk) || (pSetOld->pDataF->commitID != pSetNew->pDataF->commitID)) { - pSetOld->pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile)); - if (pSetOld->pDataF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->pDataF = *pSetNew->pDataF; - pSetOld->pDataF->nRef = 1; - - nRef = atomic_sub_fetch_32(&fSet.pDataF->nRef, 1); - if (nRef == 0) { - tsdbDataFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pDataF, fname); - taosRemoveFile(fname); - taosMemoryFree(fSet.pDataF); - } - } else { - ASSERT(pSetOld->pDataF->size <= pSetNew->pDataF->size); - pSetOld->pDataF->size = pSetNew->pDataF->size; - } - - // sma - fSet.pSmaF = pSetOld->pSmaF; - if ((!sameDisk) || (pSetOld->pSmaF->commitID != pSetNew->pSmaF->commitID)) { - pSetOld->pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); - if (pSetOld->pSmaF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->pSmaF = *pSetNew->pSmaF; - pSetOld->pSmaF->nRef = 1; - - nRef = atomic_sub_fetch_32(&fSet.pSmaF->nRef, 1); - if (nRef == 0) { - tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pSmaF, fname); - taosRemoveFile(fname); - taosMemoryFree(fSet.pSmaF); - } - } else { - ASSERT(pSetOld->pSmaF->size <= pSetNew->pSmaF->size); - pSetOld->pSmaF->size = pSetNew->pSmaF->size; - } - - // stt - if (sameDisk) { - if (pSetNew->nSttF > pSetOld->nSttF) { - ASSERT(pSetNew->nSttF = pSetOld->nSttF + 1); - pSetOld->aSttF[pSetOld->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (pSetOld->aSttF[pSetOld->nSttF] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->aSttF[pSetOld->nSttF] = *pSetNew->aSttF[pSetOld->nSttF]; - pSetOld->aSttF[pSetOld->nSttF]->nRef = 1; - pSetOld->nSttF++; - } else if (pSetNew->nSttF < pSetOld->nSttF) { - ASSERT(pSetNew->nSttF == 1); - for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { - SSttFile *pSttFile = pSetOld->aSttF[iStt]; - nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); - if (nRef == 0) { - tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); - taosRemoveFile(fname); - taosMemoryFree(pSttFile); - } - pSetOld->aSttF[iStt] = NULL; - } - - pSetOld->nSttF = 1; - pSetOld->aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (pSetOld->aSttF[0] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->aSttF[0] = *pSetNew->aSttF[0]; - pSetOld->aSttF[0]->nRef = 1; - } else { - for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { - if (pSetOld->aSttF[iStt]->commitID != pSetNew->aSttF[iStt]->commitID) { - SSttFile *pSttFile = pSetOld->aSttF[iStt]; - nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); - if (nRef == 0) { - tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); - taosRemoveFile(fname); - taosMemoryFree(pSttFile); - } - - pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (pSetOld->aSttF[iStt] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; - pSetOld->aSttF[iStt]->nRef = 1; - } else { - ASSERT(pSetOld->aSttF[iStt]->size == pSetOld->aSttF[iStt]->size); - ASSERT(pSetOld->aSttF[iStt]->offset == pSetOld->aSttF[iStt]->offset); - } - } - } - } else { - ASSERT(pSetOld->nSttF == pSetNew->nSttF); - for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { - SSttFile *pSttFile = pSetOld->aSttF[iStt]; - nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); - if (nRef == 0) { - tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); - taosRemoveFile(fname); - taosMemoryFree(pSttFile); - } - - pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (pSetOld->aSttF[iStt] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; - pSetOld->aSttF[iStt]->nRef = 1; - } - } - - if (!sameDisk) { - pSetOld->diskId = pSetNew->diskId; - } - - iOld++; - iNew++; - continue; - - _remove_old: - nRef = atomic_sub_fetch_32(&pSetOld->pHeadF->nRef, 1); - if (nRef == 0) { - tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pHeadF, fname); - taosRemoveFile(fname); - taosMemoryFree(pSetOld->pHeadF); - } - - nRef = atomic_sub_fetch_32(&pSetOld->pDataF->nRef, 1); - if (nRef == 0) { - tsdbDataFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pDataF, fname); - taosRemoveFile(fname); - taosMemoryFree(pSetOld->pDataF); - } - - nRef = atomic_sub_fetch_32(&pSetOld->pSmaF->nRef, 1); - if (nRef == 0) { - tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pSmaF, fname); - taosRemoveFile(fname); - taosMemoryFree(pSetOld->pSmaF); - } - - for (int8_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { - nRef = atomic_sub_fetch_32(&pSetOld->aSttF[iStt]->nRef, 1); - if (nRef == 0) { - tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aSttF[iStt], fname); - taosRemoveFile(fname); - taosMemoryFree(pSetOld->aSttF[iStt]); - } - } - - taosArrayRemove(pTsdb->fs.aDFileSet, iOld); - continue; - - _add_new: - fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nSttF = 1}; - - // head - fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); - if (fSet.pHeadF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *fSet.pHeadF = *pSetNew->pHeadF; - fSet.pHeadF->nRef = 1; - - // data - fSet.pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile)); - if (fSet.pDataF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *fSet.pDataF = *pSetNew->pDataF; - fSet.pDataF->nRef = 1; - - // sma - fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); - if (fSet.pSmaF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *fSet.pSmaF = *pSetNew->pSmaF; - fSet.pSmaF->nRef = 1; - - // stt - ASSERT(pSetNew->nSttF == 1); - fSet.aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (fSet.aSttF[0] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *fSet.aSttF[0] = *pSetNew->aSttF[0]; - fSet.aSttF[0]->nRef = 1; - - if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - iOld++; - iNew++; - continue; - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb fs commit phase 2 failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -#endif - -#if 1 /** * @brief Update or delete DFileSet in pFS according to DFileSet (fid <= maxFid) in pFSNew. @@ -1488,8 +1151,6 @@ _err: return code; } -#endif - int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) { int32_t code = 0; int32_t nRef; From 70f2c685704675a1e4b64770089a352b0c23232b Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 27 Sep 2022 17:57:44 +0800 Subject: [PATCH 20/71] chore: code format --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 6 +++--- source/dnode/vnode/src/tsdb/tsdbFS.c | 13 ++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 1c63487583..4613201237 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -992,16 +992,16 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { ASSERT(eno == 0); - if(inTrim) taosThreadRwlockWrlock(&pTsdb->rwLock); + if (inTrim) taosThreadRwlockWrlock(&pTsdb->rwLock); code = tsdbFSCommit1(pTsdb, &pCommitter->fs); if (code) { - if(inTrim) taosThreadRwlockUnlock(&pTsdb->rwLock); + if (inTrim) taosThreadRwlockUnlock(&pTsdb->rwLock); goto _err; } // lock - if(!inTrim) taosThreadRwlockWrlock(&pTsdb->rwLock); + if (!inTrim) taosThreadRwlockWrlock(&pTsdb->rwLock); // commit or rollback code = tsdbFSCommit2(pTsdb, &pCommitter->fs); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 188810600f..d02cbdc089 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -717,12 +717,12 @@ _exit: /** * @brief Update or delete DFileSet in pFS according to DFileSet (fid <= maxFid) in pFSNew. - * - * @param pTsdb - * @param pFS - * @param pFSNew + * + * @param pTsdb + * @param pFS + * @param pFSNew * @param maxFid - * @return int32_t + * @return int32_t */ int32_t tsdbFSUpdDel(STsdb *pTsdb, STsdbFS *pFS, STsdbFS *pFSNew, int32_t maxFid) { int32_t code = 0; @@ -817,7 +817,6 @@ _err: return code; } - int32_t tsdbFSCommit1(STsdb *pTsdb, STsdbFS *pFSNew) { int32_t code = 0; char tfname[TSDB_FILENAME_LEN]; @@ -1151,7 +1150,7 @@ _err: return code; } -int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) { +int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) { int32_t code = 0; int32_t nRef; From 747297bd28c48aaea7a18a7ae1d8ad5a7ff698aa Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 27 Sep 2022 18:11:44 +0800 Subject: [PATCH 21/71] feat(stream): stream fill --- include/libs/nodes/nodes.h | 1 + include/libs/nodes/plannodes.h | 2 + include/libs/stream/streamState.h | 12 +- source/libs/executor/inc/executorimpl.h | 20 + source/libs/executor/inc/tfill.h | 113 +- source/libs/executor/src/executorimpl.c | 2 + source/libs/executor/src/scanoperator.c | 84 +- source/libs/executor/src/tfill.c | 1113 ++++++++++++++++- source/libs/executor/src/timewindowoperator.c | 77 +- source/libs/nodes/src/nodesCodeFuncs.c | 3 + source/libs/nodes/src/nodesMsgFuncs.c | 2 + source/libs/nodes/src/nodesUtilFuncs.c | 4 +- source/libs/planner/src/planPhysiCreater.c | 4 +- source/libs/stream/src/streamState.c | 67 +- tests/script/tsim/stream/deleteInterval.sim | 11 +- .../tsim/stream/fillIntervalDelete0.sim | 375 ++++++ .../tsim/stream/fillIntervalDelete1.sim | 379 ++++++ .../script/tsim/stream/fillIntervalLinear.sim | 695 ++++++++++ .../tsim/stream/fillIntervalPartitionBy.sim | 171 +++ .../tsim/stream/fillIntervalPrevNext.sim | 1036 +++++++++++++++ .../script/tsim/stream/fillIntervalValue.sim | 488 ++++++++ tests/script/tsim/stream/partitionby.sim | 28 +- .../tsim/stream/partitionbyColumnInterval.sim | 47 + 23 files changed, 4595 insertions(+), 139 deletions(-) create mode 100644 tests/script/tsim/stream/fillIntervalDelete0.sim create mode 100644 tests/script/tsim/stream/fillIntervalDelete1.sim create mode 100644 tests/script/tsim/stream/fillIntervalLinear.sim create mode 100644 tests/script/tsim/stream/fillIntervalPartitionBy.sim create mode 100644 tests/script/tsim/stream/fillIntervalPrevNext.sim create mode 100644 tests/script/tsim/stream/fillIntervalValue.sim diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 32249e3d8b..560832cd74 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -239,6 +239,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_FILL, + QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index e6e34648f9..25ff18a8fc 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -464,6 +464,8 @@ typedef struct SFillPhysiNode { EOrder inputTsOrder; } SFillPhysiNode; +typedef SFillPhysiNode SStreamFillPhysiNode; + typedef struct SMultiTableIntervalPhysiNode { SIntervalPhysiNode interval; SNodeList* pPartitionKeys; diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 849d83a58b..0adcf976f0 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -31,6 +31,7 @@ typedef struct { TDB* db; TTB* pStateDb; TTB* pFuncStateDb; + TTB* pFillStateDb; // todo refactor TXN txn; } SStreamState; @@ -51,15 +52,22 @@ int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key); int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateDel(SStreamState* pState, const SWinKey* key); + +int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); +int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); +int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key); + int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal); void streamFreeVal(void* val); SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key); -SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key); -SStreamStateCur* streamStateSeekKeyPrev(SStreamState* pState, const SWinKey* key); +SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key); +SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key); +SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* key); void streamStateFreeCur(SStreamStateCur* pCur); +int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 12e00a0a06..a9a0267fbf 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -34,6 +34,7 @@ extern "C" { #include "scalar.h" #include "taosdef.h" #include "tarray.h" +#include "tfill.h" #include "thash.h" #include "tlockfree.h" #include "tmsg.h" @@ -798,6 +799,22 @@ typedef struct SStreamPartitionOperatorInfo { SSDataBlock* pDelRes; } SStreamPartitionOperatorInfo; +typedef struct SStreamFillOperatorInfo { + SStreamFillSupporter* pFillSup; + SSDataBlock* pRes; + SSDataBlock* pSrcBlock; + int32_t srcRowIndex; + SSDataBlock* pPrevSrcBlock; + SSDataBlock* pSrcDelBlock; + int32_t srcDelRowIndex; + SSDataBlock* pDelRes; + SNode* pCondition; + SArray* pColMatchColInfo; + int32_t primaryTsCol; + int32_t primarySrcSlotId; + SStreamFillInfo* pFillInfo; +} SStreamFillOperatorInfo; + typedef struct STimeSliceOperatorInfo { SSDataBlock* pRes; STimeWindow win; @@ -1006,6 +1023,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, + SExecTaskInfo* pTaskInfo); int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, int32_t numOfOutput, SArray* pPseudoList); @@ -1094,6 +1113,7 @@ int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupI SExecTaskInfo* pTaskInfo); int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult); int32_t saveOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize); +void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); #ifdef __cplusplus } diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index 63abfc019d..ed019be767 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -23,12 +23,13 @@ extern "C" { #include "os.h" #include "taosdef.h" #include "tcommon.h" +#include "tsimplehash.h" struct SSDataBlock; typedef struct SFillColInfo { - SExprInfo *pExpr; - bool notFillCol; // denote if this column needs fill operation + SExprInfo* pExpr; + bool notFillCol; // denote if this column needs fill operation SVariant fillVal; } SFillColInfo; @@ -51,46 +52,96 @@ typedef struct { } SRowVal; typedef struct SFillInfo { - TSKEY start; // start timestamp - TSKEY end; // endKey for fill - TSKEY currentKey; // current active timestamp, the value may be changed during the fill procedure. - int32_t tsSlotId; // primary time stamp slot id - int32_t srcTsSlotId; // timestamp column id in the source data block. - int32_t order; // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC] - int32_t type; // fill type - int32_t numOfRows; // number of rows in the input data block - int32_t index; // active row index - int32_t numOfTotal; // number of filled rows in one round - int32_t numOfCurrent; // number of filled rows in current results - int32_t numOfCols; // number of columns, including the tags columns - SInterval interval; - SRowVal prev; - SRowVal next; - SSDataBlock *pSrcBlock; - int32_t alloc; // data buffer size in rows + TSKEY start; // start timestamp + TSKEY end; // endKey for fill + TSKEY currentKey; // current active timestamp, the value may be changed during the fill procedure. + int32_t tsSlotId; // primary time stamp slot id + int32_t srcTsSlotId; // timestamp column id in the source data block. + int32_t order; // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC] + int32_t type; // fill type + int32_t numOfRows; // number of rows in the input data block + int32_t index; // active row index + int32_t numOfTotal; // number of filled rows in one round + int32_t numOfCurrent; // number of filled rows in current results + int32_t numOfCols; // number of columns, including the tags columns + SInterval interval; + SRowVal prev; + SRowVal next; + SSDataBlock* pSrcBlock; + int32_t alloc; // data buffer size in rows - SFillColInfo* pFillCol; // column info for fill operations - SFillTagColInfo* pTags; // tags value for filling gap - const char* id; + SFillColInfo* pFillCol; // column info for fill operations + SFillTagColInfo* pTags; // tags value for filling gap + const char* id; } SFillInfo; +typedef struct SResultCellData { + bool isNull; + int8_t type; + int32_t bytes; + char pData[]; +} SResultCellData; + +typedef struct SResultRowData { + TSKEY key; + SResultCellData* pRowVal; +} SResultRowData; + +typedef struct SStreamFillLinearInfo { + TSKEY nextEnd; + SArray* pDeltaVal; // double. value for Fill(linear). + SArray* pNextDeltaVal; // double. value for Fill(linear). + int64_t winIndex; + bool hasNext; +} SStreamFillLinearInfo; + +typedef struct SStreamFillInfo { + TSKEY start; // startKey for fill + TSKEY end; // endKey for fill + TSKEY current; // current Key for fill + TSKEY preRowKey; + TSKEY nextRowKey; + SResultRowData* pResRow; + SStreamFillLinearInfo* pLinearInfo; + bool needFill; + int32_t type; // fill type + int32_t pos; + SArray* delRanges; + int32_t delIndex; +} SStreamFillInfo; + +typedef struct SStreamFillSupporter { + int32_t type; // fill type + SInterval interval; + SResultRowData prev; + SResultRowData cur; + SResultRowData next; + SResultRowData nextNext; + SFillColInfo* pAllColInfo; // fill exprs and not fill exprs + int32_t numOfAllCols; // number of all exprs, including the tags columns + int32_t numOfFillCols; + int32_t numOfNotFillCols; + int32_t rowSize; + SSHashObj* pResMap; + bool hasDelete; +} SStreamFillSupporter; + int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); - -void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); -void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); -void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); -struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr, int32_t numOfNotFillCols, const struct SNodeListNode* val); -bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); +void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); +void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); +void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); +struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr, + int32_t numOfNotFillCols, const struct SNodeListNode* val); +bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); SFillInfo* taosCreateFillInfo(TSKEY skey, int32_t numOfFillCols, int32_t numOfNotFillCols, int32_t capacity, SInterval* pInterval, int32_t fillType, struct SFillColInfo* pCol, int32_t slotId, int32_t order, const char* id); -void* taosDestroyFillInfo(struct SFillInfo *pFillInfo); +void* taosDestroyFillInfo(struct SFillInfo* pFillInfo); int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity); -int64_t getFillInfoStart(struct SFillInfo *pFillInfo); - +int64_t getFillInfoStart(struct SFillInfo* pFillInfo); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a0729bdf3b..9917791312 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3742,6 +3742,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createMergeJoinOperatorInfo(ops, size, (SSortMergeJoinPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) { pOptr = createFillOperatorInfo(ops[0], (SFillPhysiNode*)pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL == type) { + pOptr = createStreamFillOperatorInfo(ops[0], (SStreamFillPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC == type) { pOptr = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC == type) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6c5c33ae29..5e1ec29a75 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1022,11 +1022,7 @@ static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, return calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, 0); } -static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) { - if (pInfo->partitionSup.needCalc) { - return getGroupIdByCol(pInfo, uid, ts, maxVersion); - } - +static uint64_t getGroupIdByUid(SStreamScanInfo* pInfo, uint64_t uid) { SHashObj* map = pInfo->pTableScanOp->pTaskInfo->tableqinfoList.map; uint64_t* groupId = taosHashGet(map, &uid, sizeof(int64_t)); if (groupId) { @@ -1035,6 +1031,14 @@ static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, return 0; } +static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) { + if (pInfo->partitionSup.needCalc) { + return getGroupIdByCol(pInfo, uid, ts, maxVersion); + } + + return getGroupIdByUid(pInfo, uid); +} + static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) { if ((*pRowIndex) == pBlock->info.rows) { return false; @@ -1081,26 +1085,32 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ return true; } -static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, SInterval* pInterval, +static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval, SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC); STimeWindow endWin = win; STimeWindow preWin = win; + uint64_t groupId = gpIdCol[*pRowIndex]; while (1) { if (hasGroup) { (*pRowIndex) += 1; } else { - (*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, startTsCol, *pRowIndex, endWin.ekey, binarySearchForKey, - NULL, TSDB_ORDER_ASC); + while ((groupId == gpIdCol[(*pRowIndex)] && startTsCol[*pRowIndex] < endWin.ekey)) { + (*pRowIndex) += 1; + if ((*pRowIndex) == pDataBlockInfo->rows) { + break; + } + } } + do { preWin = endWin; getNextTimeWindow(pInterval, &endWin, TSDB_ORDER_ASC); } while (endTsCol[(*pRowIndex) - 1] >= endWin.skey); endWin = preWin; - if (win.ekey == endWin.ekey || (*pRowIndex) == pDataBlockInfo->rows) { + if (win.ekey == endWin.ekey || (*pRowIndex) == pDataBlockInfo->rows || groupId != gpIdCol[*pRowIndex]) { win.ekey = endWin.ekey; return win; } @@ -1235,11 +1245,13 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS int64_t version = pSrcBlock->info.version - 1; for (int32_t i = 0; i < rows;) { uint64_t srcUid = srcUidData[i]; - uint64_t groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); - uint64_t srcGpId = srcGp[i]; - TSKEY calStartTs = srcStartTsCol[i]; + uint64_t groupId = srcGp[i]; + if (groupId == 0) { + groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); + } + TSKEY calStartTs = srcStartTsCol[i]; colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false); - STimeWindow win = getSlidingWindow(srcStartTsCol, srcEndTsCol, &pInfo->interval, &pSrcBlock->info, &i, + STimeWindow win = getSlidingWindow(srcStartTsCol, srcEndTsCol, srcGp, &pInfo->interval, &pSrcBlock->info, &i, pInfo->partitionSup.needCalc); TSKEY calEndTs = srcStartTsCol[i - 1]; colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false); @@ -1248,15 +1260,6 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false); colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&groupId), false); pDestBlock->info.rows++; - if (pInfo->partitionSup.needCalc && srcGpId != 0 && groupId != srcGpId) { - colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false); - colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false); - colDataAppend(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false); - colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false); - colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false); - colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&srcGpId), false); - pDestBlock->info.rows++; - } } return TSDB_CODE_SUCCESS; } @@ -1331,7 +1334,7 @@ void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) { if (out) { blockDataCleanup(pInfo->pUpdateDataRes); - blockDataEnsureCapacity(pInfo->pUpdateDataRes, pBlock->info.rows); + blockDataEnsureCapacity(pInfo->pUpdateDataRes, pBlock->info.rows * 2); } SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); @@ -1352,10 +1355,12 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock isDeletedStreamWindow(&win, pBlock->info.groupId, pInfo->pTableScanOp, &pInfo->twAggSup); if ((update || closedWin) && out) { qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin); - uint64_t gpId = closedWin && pInfo->partitionSup.needCalc - ? calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId) - : 0; + uint64_t gpId = 0; appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid, &gpId); + if (closedWin && pInfo->partitionSup.needCalc) { + gpId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId); + appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid, &gpId); + } } } if (out && pInfo->pUpdateDataRes->info.rows > 0) { @@ -1532,6 +1537,30 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S return 0; } +// for partition by tag +static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { + SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + TSKEY* startTsCol = (TSKEY*)pStartTsCol->pData; + SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + uint64_t* gpCol = (uint64_t*)pGpCol->pData; + SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); + uint64_t* uidCol = (uint64_t*)pUidCol->pData; + int32_t rows = pBlock->info.rows; + if (!pInfo->partitionSup.needCalc) { + for (int32_t i = 0; i < rows; i++) { + uint64_t groupId = getGroupIdByUid(pInfo, uidCol[i]); + colDataAppend(pGpCol, i, (const char*)&groupId, false); + } + } else { + // SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, uidCol[i], startTsCol, ts, maxVersion); + // if (!pPreRes || pPreRes->info.rows == 0) { + // return 0; + // } + // ASSERT(pPreRes->info.rows == 1); + // return calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, 0); + } +} + static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -1628,7 +1657,8 @@ FETCH_NEXT_BLOCK: } else { pDelBlock = pBlock; } - printDataBlock(pBlock, "stream scan delete recv filtered"); + setBlockGroupIdByUid(pInfo, pDelBlock); + printDataBlock(pDelBlock, "stream scan delete recv filtered"); if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) { generateDeleteResultBlock(pInfo, pDelBlock, pInfo->pDeleteDataRes); pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT; diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index f23552c5a7..ea0d26f4de 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -19,6 +19,7 @@ #include "tmsg.h" #include "ttypes.h" +#include "executorimpl.h" #include "tcommon.h" #include "thash.h" #include "ttime.h" @@ -35,18 +36,30 @@ #define GET_DEST_SLOT_ID(_p) ((_p)->pExpr->base.resSchema.slotId) +#define FILL_POS_INVALID 0 +#define FILL_POS_START 1 +#define FILL_POS_MID 2 +#define FILL_POS_END 3 + +typedef struct STimeRange { + TSKEY skey; + TSKEY ekey; + uint64_t groupId; +} STimeRange; + static void doSetVal(SColumnInfoData* pDstColInfoData, int32_t rowIndex, const SGroupKeys* pKey); -static bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnInfoData* pDstColInfoData, int32_t rowIndex); +static bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnInfoData* pDstColInfoData, + int32_t rowIndex); static void setNullRow(SSDataBlock* pBlock, SFillInfo* pFillInfo, int32_t rowIndex) { - for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - int32_t dstSlotId = GET_DEST_SLOT_ID(pCol); + for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { + SFillColInfo* pCol = &pFillInfo->pFillCol[i]; + int32_t dstSlotId = GET_DEST_SLOT_ID(pCol); SColumnInfoData* pDstColInfo = taosArrayGet(pBlock->pDataBlock, dstSlotId); if (pCol->notFillCol) { bool filled = fillIfWindowPseudoColumn(pFillInfo, pCol, pDstColInfo, rowIndex); if (!filled) { - SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; + SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; SGroupKeys* pKey = taosArrayGet(p, i); doSetVal(pDstColInfo, rowIndex, pKey); } @@ -76,8 +89,9 @@ static void doSetUserSpecifiedValue(SColumnInfoData* pDst, SVariant* pVar, int32 } } -//fill windows pseudo column, _wstart, _wend, _wduration and return true, otherwise return false -static bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnInfoData* pDstColInfoData, int32_t rowIndex) { +// fill windows pseudo column, _wstart, _wend, _wduration and return true, otherwise return false +static bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnInfoData* pDstColInfoData, + int32_t rowIndex) { if (!pCol->notFillCol) { return false; } @@ -89,15 +103,15 @@ static bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, S colDataAppend(pDstColInfoData, rowIndex, (const char*)&pFillInfo->currentKey, false); return true; } else if (pCol->pExpr->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_END) { - //TODO: include endpoint + // TODO: include endpoint SInterval* pInterval = &pFillInfo->interval; - int32_t step = (pFillInfo->order == TSDB_ORDER_ASC) ? 1 : -1; - int64_t windowEnd = + int32_t step = (pFillInfo->order == TSDB_ORDER_ASC) ? 1 : -1; + int64_t windowEnd = taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision); colDataAppend(pDstColInfoData, rowIndex, (const char*)&windowEnd, false); return true; } else if (pCol->pExpr->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_DURATION) { - //TODO: include endpoint + // TODO: include endpoint colDataAppend(pDstColInfoData, rowIndex, (const char*)&pFillInfo->interval.sliding, false); return true; } @@ -115,13 +129,13 @@ static void doFillOneRow(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* // set the other values if (pFillInfo->type == TSDB_FILL_PREV) { - SArray* p = FILL_IS_ASC_FILL(pFillInfo)? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; + SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; SColumnInfoData* pDstColInfoData = taosArrayGet(pBlock->pDataBlock, GET_DEST_SLOT_ID(pCol)); - bool filled = fillIfWindowPseudoColumn(pFillInfo, pCol, pDstColInfoData, index); + bool filled = fillIfWindowPseudoColumn(pFillInfo, pCol, pDstColInfoData, index); if (!filled) { SGroupKeys* pKey = taosArrayGet(p, i); doSetVal(pDstColInfoData, index, pKey); @@ -131,9 +145,9 @@ static void doFillOneRow(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->next.pRowVal : pFillInfo->prev.pRowVal; // todo refactor: start from 0 not 1 for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; + SFillColInfo* pCol = &pFillInfo->pFillCol[i]; SColumnInfoData* pDstColInfoData = taosArrayGet(pBlock->pDataBlock, GET_DEST_SLOT_ID(pCol)); - bool filled = fillIfWindowPseudoColumn(pFillInfo, pCol, pDstColInfoData, index); + bool filled = fillIfWindowPseudoColumn(pFillInfo, pCol, pDstColInfoData, index); if (!filled) { SGroupKeys* pKey = taosArrayGet(p, i); doSetVal(pDstColInfoData, index, pKey); @@ -154,7 +168,7 @@ static void doFillOneRow(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* if (pCol->notFillCol) { bool filled = fillIfWindowPseudoColumn(pFillInfo, pCol, pDstCol, index); if (!filled) { - SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; + SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; SGroupKeys* pKey = taosArrayGet(p, i); doSetVal(pDstCol, index, pKey); } @@ -190,13 +204,13 @@ static void doFillOneRow(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - int32_t slotId = GET_DEST_SLOT_ID(pCol); + int32_t slotId = GET_DEST_SLOT_ID(pCol); SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, slotId); if (pCol->notFillCol) { bool filled = fillIfWindowPseudoColumn(pFillInfo, pCol, pDst, index); if (!filled) { - SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; + SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; SGroupKeys* pKey = taosArrayGet(p, i); doSetVal(pDst, index, pKey); } @@ -261,8 +275,8 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray } else if (type == QUERY_NODE_OPERATOR) { SColumnInfoData* pSrcCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, i); - bool isNull = colDataIsNull_s(pSrcCol, rowIndex); - char* p = colDataGetData(pSrcCol, rowIndex); + bool isNull = colDataIsNull_s(pSrcCol, rowIndex); + char* p = colDataGetData(pSrcCol, rowIndex); saveColData(pRow, i, p, isNull); } else { ASSERT(0); @@ -425,9 +439,9 @@ struct SFillInfo* taosCreateFillInfo(TSKEY skey, int32_t numOfFillCols, int32_t pFillInfo->order = order; pFillInfo->srcTsSlotId = primaryTsSlotId; - for(int32_t i = 0; i < numOfNotFillCols; ++i) { + for (int32_t i = 0; i < numOfNotFillCols; ++i) { SFillColInfo* p = &pCol[i + numOfFillCols]; - int32_t srcSlotId = GET_DEST_SLOT_ID(p); + int32_t srcSlotId = GET_DEST_SLOT_ID(p); if (srcSlotId == primaryTsSlotId) { pFillInfo->tsSlotId = i + numOfFillCols; break; @@ -499,9 +513,9 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) { } taosArrayDestroy(pFillInfo->next.pRowVal); -// for (int32_t i = 0; i < pFillInfo->numOfTags; ++i) { -// taosMemoryFreeClear(pFillInfo->pTags[i].tagVal); -// } + // for (int32_t i = 0; i < pFillInfo->numOfTags; ++i) { + // taosMemoryFreeClear(pFillInfo->pTags[i].tagVal); + // } taosMemoryFreeClear(pFillInfo->pTags); taosMemoryFreeClear(pFillInfo->pFillCol); @@ -640,7 +654,7 @@ SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprIn } } - for(int32_t i = 0; i < numOfNotFillExpr; ++i) { + for (int32_t i = 0; i < numOfNotFillExpr; ++i) { SExprInfo* pExprInfo = &pNotFillExpr[i]; pFillCol[i + numOfFillExpr].pExpr = pExprInfo; pFillCol[i + numOfFillExpr].notFillCol = true; @@ -648,3 +662,1050 @@ SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprIn return pFillCol; } + +TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) { + STimeWindow win = {.skey = ts, .ekey = ts}; + getNextIntervalWindow(pInterval, &win, TSDB_ORDER_ASC); + return win.skey; +} + +TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { + STimeWindow win = {.skey = ts, .ekey = ts}; + getNextIntervalWindow(pInterval, &win, TSDB_ORDER_DESC); + return win.skey; +} + +void setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) { + colDataAppend(pCol, rowId, pCell->pData, pCell->isNull); +} + +SResultCellData* getResultCell(SResultRowData* pRaw, int32_t index) { + if (!pRaw || !pRaw->pRowVal) { + return NULL; + } + char* pData = (char*)pRaw->pRowVal; + SResultCellData* pCell = pRaw->pRowVal; + for (int32_t i = 0; i < index; i++) { + pData += (pCell->bytes + sizeof(SResultCellData)); + pCell = (SResultCellData*)pData; + } + return pCell; +} + +void* destroyFillColumnInfo(SFillColInfo* pFillCol, int32_t start, int32_t end) { + for (int32_t i = start; i < end; i++) { + destroyExprInfo(pFillCol[i].pExpr, 1); + taosMemoryFreeClear(pFillCol[i].pExpr); + taosVariantDestroy(&pFillCol[i].fillVal); + } + taosMemoryFree(pFillCol); + return NULL; +} + +void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { + pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); + tSimpleHashCleanup(pFillSup->pResMap); + pFillSup->pResMap = NULL; + taosMemoryFree(pFillSup); + return NULL; +} + +void* destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) { + taosArrayDestroy(pFillLinear->pDeltaVal); + taosArrayDestroy(pFillLinear->pNextDeltaVal); + taosMemoryFree(pFillLinear); + return NULL; +} +void* destroyStreamFillInfo(SStreamFillInfo* pFillInfo) { + if (pFillInfo->type == TSDB_FILL_SET_VALUE || pFillInfo->type == TSDB_FILL_NULL) { + taosMemoryFreeClear(pFillInfo->pResRow->pRowVal); + taosMemoryFreeClear(pFillInfo->pResRow); + } + pFillInfo->pLinearInfo = destroyStreamFillLinearInfo(pFillInfo->pLinearInfo); + taosMemoryFree(pFillInfo); + return NULL; +} + +void destroyStreamFillOperatorInfo(void* param) { + SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param; + pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo); + pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup); + pInfo->pRes = blockDataDestroy(pInfo->pRes); + pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); + pInfo->pColMatchColInfo = taosArrayDestroy(pInfo->pColMatchColInfo); + taosMemoryFree(pInfo); +} + +static void resetFillWindow(SResultRowData* pRowData) { + pRowData->key = INT64_MIN; + pRowData->pRowVal = NULL; +} + +void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, SStreamState* pState) { + resetFillWindow(&pFillSup->prev); + resetFillWindow(&pFillSup->cur); + resetFillWindow(&pFillSup->next); + resetFillWindow(&pFillSup->nextNext); +} + +void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { + SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState; + resetPrevAndNextWindow(pFillSup, pState); + + SWinKey key = {.ts = ts, .groupId = groupId}; + void* curVal = NULL; + int32_t curVLen = 0; + int32_t code = streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); + ASSERT(code == TSDB_CODE_SUCCESS); + pFillSup->cur.key = key.ts; + pFillSup->cur.pRowVal = curVal; +} + +void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { + SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState; + resetPrevAndNextWindow(pFillSup, pState); + + SWinKey key = {.ts = ts, .groupId = groupId}; + void* curVal = NULL; + int32_t curVLen = 0; + int32_t code = streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); + ASSERT(code == TSDB_CODE_SUCCESS); + pFillSup->cur.key = key.ts; + pFillSup->cur.pRowVal = curVal; + + SStreamStateCur* pCur = streamStateFillSeekKeyPrev(pState, &key); + SWinKey preKey = {.groupId = groupId}; + void* preVal = NULL; + int32_t preVLen = 0; + if (pCur) { + code = streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); + } + + if (pCur && code == TSDB_CODE_SUCCESS) { + pFillSup->prev.key = preKey.ts; + pFillSup->prev.pRowVal = preVal; + + code = streamStateCurNext(pState, pCur); + ASSERT(code == TSDB_CODE_SUCCESS); + + code = streamStateCurNext(pState, pCur); + if (code != TSDB_CODE_SUCCESS) { + pCur = NULL; + } + } else { + pCur = streamStateFillSeekKeyNext(pState, &key); + } + + if (pCur) { + SWinKey nextKey = {.groupId = groupId}; + void* nextVal = NULL; + int32_t nextVLen = 0; + code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); + if (code == TSDB_CODE_SUCCESS) { + pFillSup->next.key = nextKey.ts; + pFillSup->next.pRowVal = nextVal; + if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { + code = streamStateCurNext(pState, pCur); + if (code == TSDB_CODE_SUCCESS) { + SWinKey nextNextKey = {.groupId = groupId}; + void* nextNextVal = NULL; + int32_t nextNextVLen = 0; + code = streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); + if (code == TSDB_CODE_SUCCESS) { + pFillSup->nextNext.key = nextNextKey.ts; + pFillSup->nextNext.pRowVal = nextNextVal; + } + } + } + } + } +} + +static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; } +static bool hasNextWindow(SStreamFillSupporter* pFillSup) { return pFillSup->next.key != INT64_MIN; } +static bool hasNextNextWindow(SStreamFillSupporter* pFillSup) { + return pFillSup->nextNext.key != INT64_MIN; + return false; +} + +static void transBlockToResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SResultRowData* pRowVal) { + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); + SResultCellData* pCell = getResultCell(pRowVal, i); + if (!colDataIsNull_s(pColData, rowId)) { + pCell->isNull = false; + pCell->type = pColData->info.type; + pCell->bytes = pColData->info.bytes; + char* val = colDataGetData(pColData, rowId); + if (IS_VAR_DATA_TYPE(pCell->type)) { + memcpy(pCell->pData, val, varDataTLen(val)); + } else { + memcpy(pCell->pData, val, pCell->bytes); + } + } else { + pCell->isNull = true; + } + } + pRowVal->key = ts; +} + +static void calcDeltaData(SSDataBlock* pBlock, int32_t rowId, SResultRowData* pRowVal, SArray* pDelta, + SFillColInfo* pFillCol, int32_t numOfCol, int32_t winCount, int32_t order) { + for (int32_t i = 0; i < numOfCol; i++) { + if (!pFillCol[i].notFillCol) { + int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + char* var = colDataGetData(pCol, rowId); + double start = 0; + GET_TYPED_DATA(start, double, pCol->info.type, var); + SResultCellData* pCell = getResultCell(pRowVal, slotId); + double end = 0; + GET_TYPED_DATA(end, double, pCell->type, pCell->pData); + double delta = 0; + if (order == TSDB_ORDER_ASC) { + delta = (end - start) / winCount; + } else { + delta = (start - end) / winCount; + } + taosArraySet(pDelta, slotId, &delta); + } + } +} + +static void calcRowDeltaData(SResultRowData* pStartRow, SResultRowData* pEndRow, SArray* pDelta, SFillColInfo* pFillCol, + int32_t numOfCol, int32_t winCount) { + for (int32_t i = 0; i < numOfCol; i++) { + if (!pFillCol[i].notFillCol) { + int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); + SResultCellData* pSCell = getResultCell(pStartRow, slotId); + double start = 0.0; + GET_TYPED_DATA(start, double, pSCell->type, pSCell->pData); + SResultCellData* pECell = getResultCell(pEndRow, slotId); + double end = 0.0; + GET_TYPED_DATA(end, double, pECell->type, pECell->pData); + double delta = (end - start) / winCount; + taosArraySet(pDelta, slotId, &delta); + } + } +} + +static void setFillInfoStart(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { + ts = taosTimeAdd(ts, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + pFillInfo->start = ts; +} + +static void setFillInfoEnd(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { + ts = taosTimeAdd(ts, pInterval->sliding * -1, pInterval->slidingUnit, pInterval->precision); + pFillInfo->end = ts; +} + +static void setFillKeyInfo(TSKEY start, TSKEY end, SInterval* pInterval, SStreamFillInfo* pFillInfo) { + setFillInfoStart(start, pInterval, pFillInfo); + pFillInfo->current = pFillInfo->start; + setFillInfoEnd(end, pInterval, pFillInfo); +} + +void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) { + if (!hasPrevWindow(pFillSup) || !hasNextWindow(pFillSup)) { + pFillInfo->needFill = false; + return; + } + + pFillInfo->needFill = true; + pFillInfo->start = start; + pFillInfo->current = pFillInfo->start; + pFillInfo->end = end; + pFillInfo->pos = FILL_POS_INVALID; + switch (pFillInfo->type) { + case TSDB_FILL_NULL: + case TSDB_FILL_SET_VALUE: + break; + case TSDB_FILL_PREV: + pFillInfo->pResRow = &pFillSup->prev; + break; + case TSDB_FILL_NEXT: + pFillInfo->pResRow = &pFillSup->next; + break; + case TSDB_FILL_LINEAR: { + setFillKeyInfo(pFillSup->prev.key, pFillSup->next.key, &pFillSup->interval, pFillInfo); + pFillInfo->pLinearInfo->hasNext = false; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + int32_t numOfWins = taosTimeCountInterval(pFillSup->prev.key, pFillSup->next.key, pFillSup->interval.sliding, + pFillSup->interval.slidingUnit, pFillSup->interval.precision); + calcRowDeltaData(&pFillSup->prev, &pFillSup->next, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, + pFillSup->numOfAllCols, numOfWins); + pFillInfo->pResRow = &pFillSup->prev; + pFillInfo->pLinearInfo->winIndex = 0; + } break; + default: + ASSERT(0); + break; + } +} + +void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillSupporter* pFillSup, + SStreamFillInfo* pFillInfo) { + pFillInfo->preRowKey = pFillSup->cur.key; + if (!hasPrevWindow(pFillSup) && !hasNextWindow(pFillSup)) { + pFillInfo->needFill = false; + pFillInfo->pos = FILL_POS_START; + return; + } + TSKEY prevWKey = INT64_MIN; + TSKEY nextWKey = INT64_MIN; + if (hasPrevWindow(pFillSup)) { + prevWKey = pFillSup->prev.key; + } + if (hasNextWindow(pFillSup)) { + nextWKey = pFillSup->next.key; + } + + pFillInfo->needFill = true; + pFillInfo->pos = FILL_POS_INVALID; + switch (pFillInfo->type) { + case TSDB_FILL_NULL: + case TSDB_FILL_SET_VALUE: { + if (pFillSup->prev.key == pFillInfo->preRowKey) { + resetFillWindow(&pFillSup->prev); + } + if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) { + if (pFillSup->next.key == pFillInfo->nextRowKey) { + pFillInfo->preRowKey = INT64_MIN; + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + } else { + pFillInfo->needFill = false; + pFillInfo->pos = FILL_POS_START; + } + } else if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + } else { + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + } + } break; + case TSDB_FILL_PREV: { + if (hasNextWindow(pFillSup) && ((pFillSup->next.key != pFillInfo->nextRowKey) || + (pFillSup->next.key == pFillInfo->nextRowKey && hasNextNextWindow(pFillSup)) || + (pFillSup->next.key == pFillInfo->nextRowKey && !hasPrevWindow(pFillSup)))) { + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + pFillSup->prev.key = pFillSup->cur.key; + pFillSup->prev.pRowVal = pFillSup->cur.pRowVal; + } else if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + pFillInfo->preRowKey = INT64_MIN; + } + pFillInfo->pResRow = &pFillSup->prev; + } break; + case TSDB_FILL_NEXT: { + if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + pFillSup->next.key = pFillSup->cur.key; + pFillSup->next.pRowVal = pFillSup->cur.pRowVal; + pFillInfo->preRowKey = INT64_MIN; + } else { + ASSERT(hasNextWindow(pFillSup)); + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + } + pFillInfo->pResRow = &pFillSup->next; + } break; + case TSDB_FILL_LINEAR: { + pFillInfo->pLinearInfo->winIndex = 0; + if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_MID; + pFillInfo->pLinearInfo->nextEnd = nextWKey; + int32_t numOfWins = taosTimeCountInterval(prevWKey, ts, pFillSup->interval.sliding, + pFillSup->interval.slidingUnit, pFillSup->interval.precision); + calcRowDeltaData(&pFillSup->prev, &pFillSup->cur, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, + pFillSup->numOfAllCols, numOfWins); + pFillInfo->pResRow = &pFillSup->prev; + + numOfWins = taosTimeCountInterval(ts, nextWKey, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + calcRowDeltaData(&pFillSup->cur, &pFillSup->next, pFillInfo->pLinearInfo->pNextDeltaVal, pFillSup->pAllColInfo, + pFillSup->numOfAllCols, numOfWins); + pFillInfo->pLinearInfo->hasNext = true; + } else if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + int32_t numOfWins = taosTimeCountInterval(prevWKey, ts, pFillSup->interval.sliding, + pFillSup->interval.slidingUnit, pFillSup->interval.precision); + calcRowDeltaData(&pFillSup->prev, &pFillSup->cur, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, + pFillSup->numOfAllCols, numOfWins); + pFillInfo->pResRow = &pFillSup->prev; + pFillInfo->pLinearInfo->hasNext = false; + } else { + ASSERT(hasNextWindow(pFillSup)); + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + int32_t numOfWins = taosTimeCountInterval(ts, nextWKey, pFillSup->interval.sliding, + pFillSup->interval.slidingUnit, pFillSup->interval.precision); + calcRowDeltaData(&pFillSup->cur, &pFillSup->next, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, + pFillSup->numOfAllCols, numOfWins); + pFillInfo->pResRow = &pFillSup->cur; + pFillInfo->pLinearInfo->hasNext = false; + } + } break; + default: + ASSERT(0); + break; + } + ASSERT(pFillInfo->pos != FILL_POS_INVALID); +} + +static bool checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t groupId) { + SWinKey key = {.groupId = groupId, .ts = ts}; + if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) { + return false; + } + tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); + return true; +} + +static void buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFillSup, TSKEY ts, SSDataBlock* pBlock) { + uint64_t groupId = pBlock->info.groupId; + if (pFillSup->hasDelete && !checkResult(pFillSup, ts, groupId)) { + return; + } + for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId); + SFillInfo tmpInfo = { + .currentKey = ts, + .order = TSDB_ORDER_ASC, + .interval = pFillSup->interval, + }; + bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pColData, pBlock->info.rows); + if (!filled) { + SResultCellData* pCell = getResultCell(pResRow, slotId); + setRowCell(pColData, pBlock->info.rows, pCell); + } + } + pBlock->info.rows++; +} + +static bool hasRemainCalc(SStreamFillInfo* pFillInfo) { + if (pFillInfo->current != INT64_MIN && pFillInfo->current <= pFillInfo->end) { + return true; + } + return false; +} + +static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { + while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { + buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); + pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + } +} + +static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { + while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { + uint64_t groupId = pBlock->info.groupId; + SWinKey key = {.groupId = groupId, .ts = pFillInfo->current}; + if (pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId)) { + pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + pFillInfo->pLinearInfo->winIndex++; + continue; + } + pFillInfo->pLinearInfo->winIndex++; + for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; + SFillInfo tmp = { + .currentKey = pFillInfo->current, + .order = TSDB_ORDER_ASC, + .interval = pFillSup->interval, + }; + + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId); + int16_t type = pColData->info.type; + SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId); + int32_t index = pBlock->info.rows; + if (pFillCol->notFillCol) { + bool filled = fillIfWindowPseudoColumn(&tmp, pFillCol, pColData, index); + if (!filled) { + setRowCell(pColData, index, pCell); + } + } else { + if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) { + colDataAppendNULL(pColData, index); + continue; + } + double* pDelta = taosArrayGet(pFillInfo->pLinearInfo->pDeltaVal, slotId); + double vCell = 0; + GET_TYPED_DATA(vCell, double, pCell->type, pCell->pData); + vCell += (*pDelta) * pFillInfo->pLinearInfo->winIndex; + int64_t result = 0; + SET_TYPED_DATA(&result, pCell->type, vCell); + colDataAppend(pColData, index, (const char*)&result, false); + } + } + pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + pBlock->info.rows++; + } +} + +static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) { + SWinKey key = {.groupId = groupId, .ts = pRow->key}; + int32_t code = streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len); + ASSERT(code == TSDB_CODE_SUCCESS); +} + +static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { + if (pFillInfo->needFill == false) { + buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); + return; + } + + if (pFillInfo->pos == FILL_POS_START) { + buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); + } + if (pFillInfo->type != TSDB_FILL_LINEAR) { + doStreamFillNormal(pFillSup, pFillInfo, pRes); + } else { + doStreamFillLinear(pFillSup, pFillInfo, pRes); + + if (pFillInfo->pos == FILL_POS_MID) { + buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); + } + + if (pFillInfo->current > pFillInfo->end && pFillInfo->pLinearInfo->hasNext) { + pFillInfo->pLinearInfo->hasNext = false; + pFillInfo->pLinearInfo->winIndex = 0; + taosArrayClear(pFillInfo->pLinearInfo->pDeltaVal); + taosArrayAddAll(pFillInfo->pLinearInfo->pDeltaVal, pFillInfo->pLinearInfo->pNextDeltaVal); + pFillInfo->pResRow = &pFillSup->cur; + setFillKeyInfo(pFillSup->cur.key, pFillInfo->pLinearInfo->nextEnd, &pFillSup->interval, pFillInfo); + doStreamFillLinear(pFillSup, pFillInfo, pRes); + } + } + if (pFillInfo->pos == FILL_POS_END) { + buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); + } +} + +void keepBlockRowInDiscBuf(SOperatorInfo* pOperator, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, TSKEY* tsCol, + int32_t rowId, uint64_t groupId, int32_t rowSize) { + TSKEY ts = tsCol[rowId]; + pFillInfo->nextRowKey = ts; + SResultRowData tmpNextRow = {.key = ts}; + tmpNextRow.pRowVal = taosMemoryCalloc(1, rowSize); + transBlockToResultRow(pBlock, rowId, ts, &tmpNextRow); + keepResultInDiscBuf(pOperator, groupId, &tmpNextRow, rowSize); + taosMemoryFreeClear(tmpNextRow.pRowVal); +} + +static void doFillResults(SOperatorInfo* pOperator, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, + SSDataBlock* pBlock, TSKEY* tsCol, int32_t rowId, SSDataBlock* pRes) { + uint64_t groupId = pBlock->info.groupId; + getWindowFromDiscBuf(pOperator, tsCol[rowId], groupId, pFillSup); + if (pFillSup->prev.key == pFillInfo->preRowKey) { + resetFillWindow(&pFillSup->prev); + } + setFillValueInfo(pBlock, tsCol[rowId], rowId, pFillSup, pFillInfo); + doStreamFillRange(pFillInfo, pFillSup, pRes); +} + +static void doStreamFillImpl(SOperatorInfo* pOperator) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStreamFillSupporter* pFillSup = pInfo->pFillSup; + SStreamFillInfo* pFillInfo = pInfo->pFillInfo; + SSDataBlock* pBlock = pInfo->pSrcBlock; + uint64_t groupId = pBlock->info.groupId; + SSDataBlock* pRes = pInfo->pRes; + pRes->info.groupId = groupId; + if (hasRemainCalc(pFillInfo)) { + doStreamFillRange(pFillInfo, pFillSup, pRes); + } + + SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol); + TSKEY* tsCol = (TSKEY*)pTsCol->pData; + + if (pInfo->srcRowIndex == 0) { + keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); + SSDataBlock* preBlock = pInfo->pPrevSrcBlock; + if (preBlock->info.rows > 0) { + int preRowId = preBlock->info.rows - 1; + SColumnInfoData* pPreTsCol = taosArrayGet(preBlock->pDataBlock, pInfo->primaryTsCol); + doFillResults(pOperator, pFillSup, pFillInfo, preBlock, (TSKEY*)pPreTsCol->pData, preRowId, pRes); + } + pInfo->srcRowIndex++; + } + + while (pInfo->srcRowIndex < pBlock->info.rows) { + TSKEY ts = tsCol[pInfo->srcRowIndex]; + keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); + doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); + if (pInfo->pRes->info.rows == pInfo->pRes->info.capacity) { + blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + return; + } + pInfo->srcRowIndex++; + } + blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + blockDataCleanup(pInfo->pPrevSrcBlock); + copyDataBlock(pInfo->pPrevSrcBlock, pInfo->pSrcBlock); + blockDataCleanup(pInfo->pSrcBlock); +} + +static void buildDeleteRange(TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { + SSDataBlock* pBlock = delRes; + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + colDataAppend(pStartCol, pBlock->info.rows, (const char*)&start, false); + colDataAppend(pEndCol, pBlock->info.rows, (const char*)&end, false); + colDataAppendNULL(pUidCol, pBlock->info.rows); + colDataAppend(pGroupCol, pBlock->info.rows, (const char*)&groupId, false); + colDataAppendNULL(pCalStartCol, pBlock->info.rows); + colDataAppendNULL(pCalEndCol, pBlock->info.rows); + pBlock->info.rows++; +} + +static void buildDeleteResult(SStreamFillSupporter* pFillSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, + SSDataBlock* delRes) { + if (hasPrevWindow(pFillSup)) { + TSKEY start = getNextWindowTs(pFillSup->prev.key, &pFillSup->interval); + buildDeleteRange(start, endTs, groupId, delRes); + } else if (hasNextWindow(pFillSup)) { + TSKEY end = getPrevWindowTs(pFillSup->next.key, &pFillSup->interval); + buildDeleteRange(startTs, end, groupId, delRes); + } else { + buildDeleteRange(startTs, endTs, groupId, delRes); + } +} + +static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup); + setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo); + SWinKey key = {.ts = startTs, .groupId = groupId}; + if (!pInfo->pFillInfo->needFill) { + streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); + buildDeleteResult(pInfo->pFillSup, startTs, endTs, groupId, pInfo->pDelRes); + } else { + STimeRange tw = { + .skey = startTs, + .ekey = endTs, + .groupId = groupId, + }; + taosArrayPush(pInfo->pFillInfo->delRanges, &tw); + while (key.ts <= endTs) { + key.ts = taosTimeAdd(key.ts, pInfo->pFillSup->interval.sliding, pInfo->pFillSup->interval.slidingUnit, + pInfo->pFillSup->interval.precision); + tSimpleHashPut(pInfo->pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); + } + } +} + +static void doDeleteFillFinalize(SOperatorInfo* pOperator) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SStreamFillInfo* pFillInfo = pInfo->pFillInfo; + int32_t size = taosArrayGetSize(pFillInfo->delRanges); + tSimpleHashClear(pInfo->pFillSup->pResMap); + for (; pFillInfo->delIndex < size; pFillInfo->delIndex++) { + STimeRange* range = taosArrayGet(pFillInfo->delRanges, pFillInfo->delIndex); + if (pInfo->pRes->info.groupId != 0 && pInfo->pRes->info.groupId != range->groupId) { + return; + } + getWindowFromDiscBuf(pOperator, range->skey, range->groupId, pInfo->pFillSup); + setDeleteFillValueInfo(range->skey, range->ekey, pInfo->pFillSup, pInfo->pFillInfo); + if (pInfo->pFillInfo->needFill) { + doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes); + pInfo->pRes->info.groupId = range->groupId; + } + SWinKey key = {.ts = range->skey, .groupId = range->groupId}; + streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); + } +} + +static void doDeleteFillResult(SOperatorInfo* pOperator) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SStreamFillSupporter* pFillSup = pInfo->pFillSup; + SStreamFillInfo* pFillInfo = pInfo->pFillInfo; + SSDataBlock* pBlock = pInfo->pSrcDelBlock; + SSDataBlock* pRes = pInfo->pRes; + SSDataBlock* pDelRes = pInfo->pDelRes; + + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + TSKEY* tsStarts = (TSKEY*)pStartCol->pData; + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + uint64_t* groupIds = (uint64_t*)pGroupCol->pData; + while (pInfo->srcDelRowIndex < pBlock->info.rows) { + TSKEY ts = tsStarts[pInfo->srcDelRowIndex]; + TSKEY endTs = ts; + uint64_t groupId = groupIds[pInfo->srcDelRowIndex]; + SWinKey key = {.ts = ts, .groupId = groupId}; + SStreamStateCur* pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); + if (!pCur) { + pInfo->srcDelRowIndex++; + continue; + } + + SWinKey nextKey = {.groupId = groupId, .ts = ts}; + while (pInfo->srcDelRowIndex < pBlock->info.rows) { + void* nextVal = NULL; + int32_t nextLen = 0; + TSKEY delTs = tsStarts[pInfo->srcDelRowIndex]; + uint64_t delGroupId = groupIds[pInfo->srcDelRowIndex]; + int32_t code = TSDB_CODE_SUCCESS; + if (groupId != delGroupId) { + break; + } + if (delTs > nextKey.ts) { + break; + } + endTs = delTs; + SWinKey delKey = {.groupId = delGroupId, .ts = delTs}; + if (delTs == nextKey.ts) { + code = streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); + if (code == TSDB_CODE_SUCCESS) { + code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen); + } + if (delTs != ts) { + streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); + } + if (code != TSDB_CODE_SUCCESS) { + break; + } + } + pInfo->srcDelRowIndex++; + } + doDeleteFillResultImpl(pOperator, ts, endTs, groupId); + } + pFillInfo->current = pFillInfo->end + 1; +} + +static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) { + blockDataCleanup(pInfo->pPrevSrcBlock); + tSimpleHashClear(pInfo->pFillSup->pResMap); + pInfo->pFillSup->hasDelete = false; + taosArrayClear(pInfo->pFillInfo->delRanges); + pInfo->pFillInfo->delIndex = 0; +} + +static void doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock, SSDataBlock* pDstBlock) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SExprSupp* pSup = &pOperator->exprSupp; + + blockDataCleanup(pDstBlock); + blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows); + setInputDataBlock(pOperator, pSup->pCtx, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); + projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + pDstBlock->info.groupId = pSrcBlock->info.groupId; + + SColumnInfoData* pDst = taosArrayGet(pDstBlock->pDataBlock, pInfo->primaryTsCol); + SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, pInfo->primarySrcSlotId); + colDataAssign(pDst, pSrc, pDstBlock->info.rows, &pDstBlock->info); + + int32_t numOfNotFill = pInfo->pFillSup->numOfAllCols - pInfo->pFillSup->numOfFillCols; + for (int32_t i = 0; i < numOfNotFill; ++i) { + SFillColInfo* pCol = &pInfo->pFillSup->pAllColInfo[i + pInfo->pFillSup->numOfFillCols]; + ASSERT(pCol->notFillCol); + + SExprInfo* pExpr = pCol->pExpr; + int32_t srcSlotId = pExpr->base.pParam[0].pCol->slotId; + int32_t dstSlotId = pExpr->base.resSchema.slotId; + + SColumnInfoData* pDst1 = taosArrayGet(pDstBlock->pDataBlock, dstSlotId); + SColumnInfoData* pSrc1 = taosArrayGet(pSrcBlock->pDataBlock, srcSlotId); + colDataAssign(pDst1, pSrc1, pDstBlock->info.rows, &pDstBlock->info); + } + blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol); +} + +static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + blockDataCleanup(pInfo->pRes); + if (pOperator->status == OP_RES_TO_RETURN) { + if (hasRemainCalc(pInfo->pFillInfo)) { + doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes); + if (pInfo->pRes->info.rows > 0) { + return pInfo->pRes; + } + } + doDeleteFillFinalize(pOperator); + if (pInfo->pRes->info.rows > 0) { + printDataBlock(pInfo->pRes, "stream fill"); + return pInfo->pRes; + } + doSetOperatorCompleted(pOperator); + resetStreamFillInfo(pInfo); + return NULL; + } + + SSDataBlock* fillResult = NULL; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + while (1) { + if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows) { + // If there are delete datablocks, we receive them first. + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + pOperator->status = OP_RES_TO_RETURN; + SSDataBlock* preBlock = pInfo->pPrevSrcBlock; + if (preBlock->info.rows > 0) { + int preRowId = preBlock->info.rows - 1; + SColumnInfoData* pPreTsCol = taosArrayGet(preBlock->pDataBlock, pInfo->primaryTsCol); + doFillResults(pOperator, pInfo->pFillSup, pInfo->pFillInfo, preBlock, (TSKEY*)pPreTsCol->pData, preRowId, + pInfo->pRes); + } + pInfo->pFillInfo->preRowKey = INT64_MIN; + if (pInfo->pRes->info.rows > 0) { + printDataBlock(pInfo->pRes, "stream fill"); + return pInfo->pRes; + } + break; + } + printDataBlock(pBlock, "stream fill recv"); + + switch (pBlock->info.type) { + case STREAM_RETRIEVE: + return pBlock; + case STREAM_DELETE_RESULT: { + pInfo->pSrcDelBlock = pBlock; + pInfo->srcDelRowIndex = 0; + blockDataCleanup(pInfo->pDelRes); + pInfo->pFillSup->hasDelete = true; + doDeleteFillResult(pOperator); + if (pInfo->pDelRes->info.rows > 0) { + printDataBlock(pInfo->pDelRes, "stream fill delete"); + return pInfo->pDelRes; + } + continue; + } break; + case STREAM_NORMAL: + case STREAM_INVALID: { + doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock); + pInfo->srcRowIndex = 0; + } break; + default: + ASSERT(0); + break; + } + } + + doStreamFillImpl(pOperator); + doFilter(pInfo->pCondition, pInfo->pRes, pInfo->pColMatchColInfo); + pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; + if (pInfo->pRes->info.rows > 0) { + break; + } + } + if (pOperator->status == OP_RES_TO_RETURN) { + doDeleteFillFinalize(pOperator); + } + + if (pInfo->pRes->info.rows == 0) { + doSetOperatorCompleted(pOperator); + resetStreamFillInfo(pInfo); + return NULL; + } + + pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; + printDataBlock(pInfo->pRes, "stream fill"); + return pInfo->pRes; +} + +static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { + pFillSup->rowSize = sizeof(SResultCellData) * pFillSup->numOfAllCols; + for (int i = 0; i < pFillSup->numOfAllCols; i++) { + SFillColInfo* pCol = &pFillSup->pAllColInfo[i]; + SResSchema* pSchema = &pCol->pExpr->base.resSchema; + pFillSup->rowSize += pSchema->bytes; + } + pFillSup->next.key = INT64_MIN; + pFillSup->nextNext.key = INT64_MIN; + pFillSup->prev.key = INT64_MIN; + pFillSup->next.pRowVal = NULL; + pFillSup->nextNext.pRowVal = NULL; + pFillSup->prev.pRowVal = NULL; + return TSDB_CODE_SUCCESS; +} + +static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, + SExprInfo* pFillExprInfo, int32_t numOfFillCols) { + SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); + if (!pFillSup) { + return NULL; + } + pFillSup->numOfFillCols = numOfFillCols; + int32_t numOfNotFillCols = 0; + SExprInfo* pNotFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); + pFillSup->pAllColInfo = createFillColInfo(pFillExprInfo, pFillSup->numOfFillCols, pNotFillExprInfo, numOfNotFillCols, + (const SNodeListNode*)(pPhyFillNode->pValues)); + pFillSup->type = convertFillType(pPhyFillNode->mode); + pFillSup->numOfAllCols = pFillSup->numOfFillCols + numOfNotFillCols; + pFillSup->interval = *pInterval; + + int32_t code = initResultBuf(pFillSup); + if (code != TSDB_CODE_SUCCESS) { + destroyStreamFillSupporter(pFillSup); + return NULL; + } + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pFillSup->pResMap = tSimpleHashInit(16, hashFn); + pFillSup->hasDelete = false; + return pFillSup; +} + +SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { + SStreamFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SStreamFillInfo)); + pFillInfo->start = INT64_MIN; + pFillInfo->current = INT64_MIN; + pFillInfo->end = INT64_MIN; + pFillInfo->preRowKey = INT64_MIN; + pFillInfo->needFill = false; + pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo)); + pFillInfo->pLinearInfo->hasNext = false; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + pFillInfo->pLinearInfo->pDeltaVal = NULL; + pFillInfo->pLinearInfo->pNextDeltaVal = NULL; + if (pFillSup->type == TSDB_FILL_LINEAR) { + pFillInfo->pLinearInfo->pDeltaVal = taosArrayInit(pFillSup->numOfAllCols, sizeof(double)); + pFillInfo->pLinearInfo->pNextDeltaVal = taosArrayInit(pFillSup->numOfAllCols, sizeof(double)); + for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) { + double value = 0.0; + taosArrayPush(pFillInfo->pLinearInfo->pDeltaVal, &value); + taosArrayPush(pFillInfo->pLinearInfo->pNextDeltaVal, &value); + } + } + pFillInfo->pLinearInfo->winIndex = 0; + + pFillInfo->pResRow = NULL; + if (pFillSup->type == TSDB_FILL_SET_VALUE || pFillSup->type == TSDB_FILL_NULL) { + pFillInfo->pResRow = taosMemoryCalloc(1, sizeof(SResultRowData)); + pFillInfo->pResRow->key = INT64_MIN; + pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize); + for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { + SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); + SResultCellData* pCell = getResultCell(pFillInfo->pResRow, i); + pCell->bytes = pColData->info.bytes; + pCell->type = pColData->info.type; + } + } + + pFillInfo->type = pFillSup->type; + pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeRange)); + pFillInfo->delIndex = 0; + return pFillInfo; +} + +SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, + SExecTaskInfo* pTaskInfo) { + SStreamFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFillOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + SInterval* pInterval = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL == downstream->operatorType + ? &((SStreamFinalIntervalOperatorInfo*)downstream->info)->interval + : &((SStreamIntervalOperatorInfo*)downstream->info)->interval; + int32_t numOfFillCols = 0; + SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); + pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols); + if (!pInfo->pFillSup) { + goto _error; + } + + SResultInfo* pResultInfo = &pOperator->resultInfo; + initResultSizeInfo(&pOperator->resultInfo, 4096); + pInfo->pRes = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc); + pInfo->pSrcBlock = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc); + pInfo->pPrevSrcBlock = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity); + blockDataEnsureCapacity(pInfo->pPrevSrcBlock, pOperator->resultInfo.capacity); + + pInfo->pFillInfo = initStreamFillInfo(pInfo->pFillSup, pInfo->pRes); + if (!pInfo->pFillInfo) { + goto _error; + } + + if (pInfo->pFillInfo->type == TSDB_FILL_SET_VALUE) { + for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pInfo->pFillSup->pAllColInfo + i; + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId); + SVariant* pVar = &(pFillCol->fillVal); + if (pCell->type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); + SET_TYPED_DATA(pCell->pData, pCell->type, v); + } else if (pCell->type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); + SET_TYPED_DATA(pCell->pData, pCell->type, v); + } else if (IS_SIGNED_NUMERIC_TYPE(pCell->type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); + SET_TYPED_DATA(pCell->pData, pCell->type, v); + } else { + pCell->isNull = true; + } + } + } else if (pInfo->pFillInfo->type == TSDB_FILL_NULL) { + for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pInfo->pFillSup->pAllColInfo + i; + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId); + pCell->isNull = true; + } + } + + pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); + blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity); + + pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId; + pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId; + + int32_t numOfOutputCols = 0; + SArray* pColMatchColInfo = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, + &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); + pInfo->pCondition = pPhyFillNode->node.pConditions; + pInfo->pColMatchColInfo = pColMatchColInfo; + initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); + pInfo->srcRowIndex = 0; + + pOperator->name = "FillOperator"; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, NULL, destroyStreamFillOperatorInfo, + NULL, NULL, NULL); + + int32_t code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + return pOperator; + +_error: + destroyStreamFillOperatorInfo(pInfo); + taosMemoryFreeClear(pOperator); + return NULL; +} diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 4671570802..12d35be0b7 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -271,6 +271,10 @@ static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t o tw->ekey -= 1; } +void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) { + getNextTimeWindow(pInterval, pInterval->precision, order, tw); +} + void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) { SqlFunctionCtx* pCtx = pSup->pCtx; @@ -3146,20 +3150,21 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - if (pInfo->binfo.pRes->info.rows == 0) { - pOperator->status = OP_EXEC_DONE; - if (!IS_FINAL_OP(pInfo)) { - clearFunctionContext(&pOperator->exprSupp); - // semi interval operator clear disk buffer - clearStreamIntervalOperator(pInfo); - qDebug("===stream===clear semi operator"); - } else { - freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); - } - return NULL; + if (pInfo->binfo.pRes->info.rows != 0) { + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); + return pInfo->binfo.pRes; } - printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); - return pInfo->binfo.pRes; + + doSetOperatorCompleted(pOperator); + if (!IS_FINAL_OP(pInfo)) { + clearFunctionContext(&pOperator->exprSupp); + // semi interval operator clear disk buffer + clearStreamIntervalOperator(pInfo); + qDebug("===stream===clear semi operator"); + } else { + freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); + } + return NULL; } else { if (!IS_FINAL_OP(pInfo)) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); @@ -3316,7 +3321,13 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pPullDataRes; } - // we should send result first. + doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); + if (pInfo->pDelRes->info.rows != 0) { + // process the rest of the data + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); + return pInfo->pDelRes; + } + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows != 0) { printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); @@ -3330,13 +3341,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { // process the rest of the data return pInfo->pUpdateRes; } - - doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); - if (pInfo->pDelRes->info.rows != 0) { - // process the rest of the data - printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); - return pInfo->pDelRes; - } return NULL; } @@ -5744,19 +5748,18 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, "single interval"); + printDataBlock(pInfo->pDelRes, "single interval delete"); return pInfo->pDelRes; } doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); - // doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - qDebug("===stream===single interval is done"); - freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); + if (pInfo->binfo.pRes->info.rows > 0) { + printDataBlock(pInfo->binfo.pRes, "single interval"); + return pInfo->binfo.pRes; } - printDataBlock(pInfo->binfo.pRes, "single interval"); - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; + + doSetOperatorCompleted(pOperator); + return NULL; } SOperatorInfo* downstream = pOperator->pDownstream[0]; @@ -5823,24 +5826,24 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } taosArraySort(pUpdated, resultrowComparAsc); - // new disc buf - // finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, - // pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); removeDeleteResults(pUpdatedMap, pInfo->pDelWins); taosHashCleanup(pUpdatedMap); + doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, "single interval"); + printDataBlock(pInfo->pDelRes, "single interval delete"); return pInfo->pDelRes; } - // doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - // new disc buf doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); - printDataBlock(pInfo->binfo.pRes, "single interval"); - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; + if (pInfo->binfo.pRes->info.rows > 0) { + printDataBlock(pInfo->binfo.pRes, "single interval"); + return pInfo->binfo.pRes; + } + + return NULL; } void destroyStreamIntervalOperatorInfo(void* param) { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 78afadb75c..e401a3da7f 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -254,6 +254,7 @@ const char* nodesNodeName(ENodeType type) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: return "PhysiStreamSemiInterval"; case QUERY_NODE_PHYSICAL_PLAN_FILL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: return "PhysiFill"; case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: return "PhysiSessionWindow"; @@ -4635,6 +4636,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: return physiIntervalNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_FILL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: return physiFillNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: @@ -4788,6 +4790,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: return jsonToPhysiIntervalNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_FILL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: return jsonToPhysiFillNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 989a4f5925..8d1d332ac7 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -3633,6 +3633,7 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { code = physiIntervalNodeToMsg(pObj, pEncoder); break; case QUERY_NODE_PHYSICAL_PLAN_FILL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: code = physiFillNodeToMsg(pObj, pEncoder); break; case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: @@ -3770,6 +3771,7 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) { code = msgToPhysiIntervalNode(pDecoder, pObj); break; case QUERY_NODE_PHYSICAL_PLAN_FILL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: code = msgToPhysiFillNode(pDecoder, pObj); break; case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 71e53bd9be..f8dda501e9 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -511,6 +511,7 @@ SNode* nodesMakeNode(ENodeType type) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: return makeNode(type, sizeof(SStreamSemiIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_FILL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: return makeNode(type, sizeof(SFillPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: return makeNode(type, sizeof(SSessionWinodwPhysiNode)); @@ -1156,7 +1157,8 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); break; - case QUERY_NODE_PHYSICAL_PLAN_FILL: { + case QUERY_NODE_PHYSICAL_PLAN_FILL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: { SFillPhysiNode* pPhyNode = (SFillPhysiNode*)pNode; destroyPhysiNode((SPhysiNode*)pPhyNode); nodesDestroyList(pPhyNode->pFillExprs); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 421da8f110..810b82b9fc 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1409,7 +1409,9 @@ static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi static int32_t createFillPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SFillLogicNode* pFillNode, SPhysiNode** pPhyNode) { - SFillPhysiNode* pFill = (SFillPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pFillNode, QUERY_NODE_PHYSICAL_PLAN_FILL); + SFillPhysiNode* pFill = (SFillPhysiNode*)makePhysiNode( + pCxt, (SLogicNode*)pFillNode, + pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL : QUERY_NODE_PHYSICAL_PLAN_FILL); if (NULL == pFill) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index fde8bca77b..7f3e155a70 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -40,6 +40,11 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath) { goto _err; } + // todo refactor + if (tdbTbOpen("func.state.db", sizeof(SWinKey), -1, SWinKeyCmpr, pState->db, &pState->pFillStateDb) < 0) { + goto _err; + } + if (tdbTbOpen("func.state.db", sizeof(STupleKey), -1, STupleKeyCmpr, pState->db, &pState->pFuncStateDb) < 0) { goto _err; } @@ -55,6 +60,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath) { _err: tdbTbClose(pState->pStateDb); tdbTbClose(pState->pFuncStateDb); + tdbTbClose(pState->pFillStateDb); tdbClose(pState->db); taosMemoryFree(pState); return NULL; @@ -64,6 +70,7 @@ void streamStateClose(SStreamState* pState) { tdbCommit(pState->db, &pState->txn); tdbTbClose(pState->pStateDb); tdbTbClose(pState->pFuncStateDb); + tdbTbClose(pState->pFillStateDb); tdbClose(pState->db); taosMemoryFree(pState); @@ -126,14 +133,30 @@ int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key) { int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { return tdbTbUpsert(pState->pStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn); } + +// todo refactor +int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { + return tdbTbUpsert(pState->pFillStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn); +} + int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { return tdbTbGet(pState->pStateDb, key, sizeof(SWinKey), pVal, pVLen); } +// todo refactor +int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { + return tdbTbGet(pState->pFillStateDb, key, sizeof(SWinKey), pVal, pVLen); +} + int32_t streamStateDel(SStreamState* pState, const SWinKey* key) { return tdbTbDelete(pState->pStateDb, key, sizeof(SWinKey), &pState->txn); } +// todo refactor +int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key) { + return tdbTbDelete(pState->pFillStateDb, key, sizeof(SWinKey), &pState->txn); +} + int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { // todo refactor int32_t size = *pVLen; @@ -165,6 +188,31 @@ SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) { return pCur; } +SStreamStateCur* streamStateFillGetCur(SStreamState* pState, const SWinKey* key) { + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + if (pCur == NULL) return NULL; + tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL); + + int32_t c; + tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c); + if (c != 0) { + taosMemoryFree(pCur); + return NULL; + } + return pCur; +} + +SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key) { + SStreamStateCur* pCur = streamStateFillGetCur(pState, key); + if (pCur) { + int32_t code = streamStateGetGroupKVByCur(pCur, key, NULL, 0); + if (code == 0) { + return pCur; + } + } + return NULL; +} + int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { const SWinKey* pKTmp = NULL; int32_t kLen; @@ -175,6 +223,17 @@ int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** return 0; } +int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { + uint64_t groupId = pKey->groupId; + int32_t code = streamStateGetKVByCur(pCur, pKey, pVal, pVLen); + if (code == 0) { + if (pKey->groupId == groupId) { + return 0; + } + } + return -1; +} + int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur) { // return tdbTbcMoveToFirst(pCur->pCur); @@ -185,12 +244,12 @@ int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur) { return tdbTbcMoveToLast(pCur->pCur); } -SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key) { +SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key) { SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } - if (tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL) < 0) { + if (tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL) < 0) { taosMemoryFree(pCur); return NULL; } @@ -211,12 +270,12 @@ SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key return pCur; } -SStreamStateCur* streamStateSeekKeyPrev(SStreamState* pState, const SWinKey* key) { +SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* key) { SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } - if (tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL) < 0) { + if (tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL) < 0) { taosMemoryFree(pCur); return NULL; } diff --git a/tests/script/tsim/stream/deleteInterval.sim b/tests/script/tsim/stream/deleteInterval.sim index 00d10afad9..7532b2d5de 100644 --- a/tests/script/tsim/stream/deleteInterval.sim +++ b/tests/script/tsim/stream/deleteInterval.sim @@ -413,13 +413,8 @@ if $data12 != 3 then goto loop14 endi -return 1 -sql drop stream if exists streams3; -sql drop database if exists test3; -sql drop database if exists test; sql create database test3 vgroups 4; -sql create database test vgroups 1; sql use test3; sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); @@ -435,7 +430,7 @@ sql delete from t1; loop15: sleep 200 -sql select * from test.streamt2 order by c1, c2, c3; +sql select * from test.streamt3 order by c1, c2, c3; $loop_count = $loop_count + 1 if $loop_count == 10 then @@ -453,7 +448,7 @@ sql delete from t1 where ts > 100; loop16: sleep 200 -sql select * from test.streamt2 order by c1, c2, c3; +sql select * from test.streamt3 order by c1, c2, c3; $loop_count = $loop_count + 1 if $loop_count == 10 then @@ -471,7 +466,7 @@ sql delete from st; loop17: sleep 200 -sql select * from test.streamt2 order by c1, c2, c3; +sql select * from test.streamt3 order by c1, c2, c3; $loop_count = $loop_count + 1 if $loop_count == 10 then diff --git a/tests/script/tsim/stream/fillIntervalDelete0.sim b/tests/script/tsim/stream/fillIntervalDelete0.sim new file mode 100644 index 0000000000..77d09d5ae8 --- /dev/null +++ b/tests/script/tsim/stream/fillIntervalDelete0.sim @@ -0,0 +1,375 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 200 +sql connect + +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop stream if exists streams5; +sql drop database if exists test1; +sql create database test1 vgroups 1; +sql use test1; +sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); +sql create stream streams1 trigger at_once into streamt1 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(NULL); +sql create stream streams2 trigger at_once into streamt2 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value,100,200,300); +sql create stream streams3 trigger at_once into streamt3 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); +sql create stream streams4 trigger at_once into streamt4 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); +sql create stream streams5 trigger at_once into streamt5 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); +sql insert into t1 values(1648791213000,1,1,1,1.0,'aaa'); +sleep 200 + +$loop_count = 0 + +loop0: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1 order by ts; + +if $rows != 1 then + print =====rows=$rows + goto loop0 +endi + +sql delete from t1; + +$loop_count = 0 + +loop1: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1 order by ts; + +if $rows != 0 then + print =====rows1=$rows + goto loop1 +endi + +sql select * from streamt2 order by ts; + +if $rows != 0 then + print =====rows2=$rows + goto loop1 +endi + +sql select * from streamt3 order by ts; + +if $rows != 0 then + print =====rows3=$rows + goto loop1 +endi + +sql select * from streamt4 order by ts; + +if $rows != 0 then + print =====rows4=$rows + goto loop1 +endi + +sql select * from streamt5 order by ts; + +if $rows != 0 then + print =====rows5=$rows + goto loop1 +endi + +sql insert into t1 values(1648791210000,4,4,4,4.0,'ddd'); +sql insert into t1 values(1648791215000,2,2,2,2.0,'bbb'); +sql insert into t1 values(1648791217000,3,3,3,3.0,'ccc'); +sql insert into t1 values(1648791219000,5,5,5,5.0,'eee'); + +$loop_count = 0 + +loop2: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1 order by ts; + +if $rows != 10 then + print =====rows=$rows + goto loop2 +endi + +#temp +system sh/stop_dnodes.sh +return 1 + +sql delete from t1 where ts >= 1648791214000; + +$loop_count = 0 + +loop3: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1 order by ts; + +if $rows != 1 then + print =====rows1=$rows + goto loop3 +endi + +sql select * from streamt2 order by ts; + +if $rows != 1 then + print =====rows2=$rows + goto loop3 +endi + +sql select * from streamt3 order by ts; + +if $rows != 1 then + print =====rows3=$rows + goto loop3 +endi + +sql select * from streamt4 order by ts; + +if $rows != 1 then + print =====rows4=$rows + goto loop3 +endi + +sql select * from streamt5 order by ts; + +if $rows != 1 then + print =====rows5=$rows + goto loop3 +endi + +if $data01 != 4 then + print =====data01=$data01 + return -1 +endi + + + +sql insert into t1 values(1648791213000,5,5,5,5.0,'eee'); +sql insert into t1 values(1648791215000,5,5,5,5.0,'eee'); +sql insert into t1 values(1648791219000,6,6,6,6.0,'fff'); + +$loop_count = 0 + +loop4: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1 order by ts; + +if $rows != 4 then + print =====rows=$rows + goto loop4 +endi + + +sql delete from t1 where ts <= 1648791216000; + +$loop_count = 0 + +loop5: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1 order by ts; + +if $rows != 1 then + print =====rows1=$rows + goto loop5 +endi + +sql select * from streamt2 order by ts; + +if $rows != 1 then + print =====rows2=$rows + goto loop5 +endi + +sql select * from streamt3 order by ts; + +if $rows != 1 then + print =====rows3=$rows + goto loop5 +endi + +sql select * from streamt4 order by ts; + +if $rows != 1 then + print =====rows4=$rows + goto loop5 +endi + +sql select * from streamt5 order by ts; + +if $rows != 1 then + print =====rows5=$rows + goto loop5 +endi + +if $data01 != 6 then + print =====data01=$data01 + return -1 +endi + + + + +sql drop stream if exists streams6; +sql drop stream if exists streams7; +sql drop stream if exists streams8; +sql drop stream if exists streams9; +sql drop stream if exists streams10; +sql drop database if exists test6; +sql create database test6 vgroups 1; +sql use test6; +sql create stable st(ts timestamp, a int, b int , c int, d double, s varchar(20)) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(1,1,1); +sql create stream streams6 trigger at_once into streamt6 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(NULL); +sql create stream streams7 trigger at_once into streamt7 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value,100,200,300); +sql create stream streams8 trigger at_once into streamt8 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); +sql create stream streams9 trigger at_once into streamt9 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); +sql create stream streams10 trigger at_once into streamt10 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); + +sql insert into t1 values(1648791210000,1,1,1,1.0,'aaa'); +sql insert into t1 values(1648791217000,1,1,1,1.0,'aaa'); + +sql insert into t2 values(1648791215000,1,1,1,1.0,'aaa'); + +sleep 200 + +$loop_count = 0 + +loop7: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt6 order by ts; + +if $rows != 8 then + print =====rows=$rows + goto loop7 +endi + +sql delete from t1; + +$loop_count = 0 + +loop8: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt6 order by ts; + +if $rows != 0 then + print =====rows6=$rows + goto loop8 +endi + +sql select * from streamt7 order by ts; + +if $rows != 0 then + print =====rows7=$rows + goto loop8 +endi + +sql select * from streamt8 order by ts; + +if $rows != 0 then + print =====rows8=$rows + goto loop8 +endi + +sql select * from streamt9 order by ts; + +if $rows != 0 then + print =====rows9=$rows + goto loop8 +endi + +sql select * from streamt10 order by ts; + +if $rows != 0 then + print =====rows10=$rows + goto loop8 +endi + + + + + + + + + + + + + + + + + + + + + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop stream if exists streams5; +sql drop stream if exists streams6; +sql drop stream if exists streams7; +sql drop stream if exists streams8; + +sql use test1; +sql select * from t1; +print $data00 + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/fillIntervalDelete1.sim b/tests/script/tsim/stream/fillIntervalDelete1.sim new file mode 100644 index 0000000000..8e6972975e --- /dev/null +++ b/tests/script/tsim/stream/fillIntervalDelete1.sim @@ -0,0 +1,379 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 200 +sql connect + +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop stream if exists streams5; +sql drop database if exists test1; +sql create database test1 vgroups 1; +sql use test1; +sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); +sql create stream streams1 trigger at_once into streamt1 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(NULL); +sql create stream streams2 trigger at_once into streamt2 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value,100,200,300); +sql create stream streams3 trigger at_once into streamt3 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); +sql create stream streams4 trigger at_once into streamt4 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); +sql create stream streams5 trigger at_once into streamt5 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); + +sql insert into t1 values(1648791210000,0,0,0,0.0,'aaa'); +sql insert into t1 values(1648791213000,1,1,1,1.0,'bbb'); +sql insert into t1 values(1648791215000,5,5,5,5.0,'ccc'); +sql insert into t1 values(1648791217000,6,6,6,6.0,'ddd'); + +$loop_count = 0 + +loop0: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1 order by ts; + +if $rows != 8 then + print =====rows=$rows + goto loop0 +endi + + +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop2: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + + +sql select * from streamt1 order by ts; + +if $rows != 8 then + print ====streamt1=rows1=$rows + goto loop2 +endi +if $data31 != NULL then + print ====streamt1=data31=$data31 + goto loop2 +endi + +sql select * from streamt2 order by ts; + +if $rows != 8 then + print ====streamt2=rows2=$rows + goto loop2 +endi +if $data31 != 100 then + print ====streamt2=data31=$data31 + goto loop2 +endi + +sql select * from streamt3 order by ts; + +if $rows != 8 then + print ====streamt3=rows3=$rows + goto loop2 +endi +if $data31 != 5 then + print ====streamt3=data31=$data31 + goto loop2 +endi + +sql select * from streamt4 order by ts; + +if $rows != 8 then + print ====streamt4=rows4=$rows + goto loop2 +endi +if $data31 != 0 then + print ====streamt4=data31=$data31 + goto loop2 +endi + +sql select * from streamt5 order by ts; + +if $rows != 8 then + print ====streamt5=rows5=$rows + goto loop2 +endi +if $data31 != 3 then + print ====streamt5=data31=$data31 + goto loop2 +endi + + +sql insert into t1 values(1648791212000,5,5,5,5.0,'eee'); +sql insert into t1 values(1648791213000,6,6,6,6.0,'fff'); + +$loop_count = 0 + +loop3: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1 order by ts; + +if $data21 != 5 then + print ====133=rows=$rows + goto loop3 +endi +if $data31 != 6 then + print ====137=rows=$rows + goto loop3 +endi + + +sql delete from t1 where ts >= 1648791211000 and ts <= 1648791214000; + +$loop_count = 0 + +loop4: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1 order by ts; + +if $rows != 8 then + print ====streamt1=rows1=$rows + goto loop4 +endi +if $data31 != NULL then + print ====streamt1=data31=$data31 + goto loop4 +endi + +sql select * from streamt2 order by ts; + +if $rows != 8 then + print ====streamt2=rows2=$rows + goto loop4 +endi +if $data31 != 100 then + print ====streamt2=data31=$data31 + goto loop4 +endi + +sql select * from streamt3 order by ts; + +if $rows != 8 then + print ====streamt3=rows3=$rows + goto loop4 +endi +if $data31 != 5 then + print ====streamt3=data31=$data31 + goto loop4 +endi + +sql select * from streamt4 order by ts; + +if $rows != 8 then + print ====streamt4=rows4=$rows + goto loop4 +endi +if $data31 != 0 then + print ====streamt4=data31=$data31 + goto loop4 +endi + +sql select * from streamt5 order by ts; + +if $rows != 8 then + print ====streamt5=rows5=$rows + goto loop4 +endi +if $data31 != 3 then + print ====streamt5=data31=$data31 + goto loop4 +endi + + + +sql drop stream if exists streams6; +sql drop stream if exists streams7; +sql drop stream if exists streams8; +sql drop stream if exists streams9; +sql drop stream if exists streams10; +sql drop database if exists test6; +sql create database test6 vgroups 1; +sql use test6; +sql create stable st(ts timestamp, a int, b int , c int, d double, s varchar(20)) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(1,1,1); +sql create stream streams6 trigger at_once into streamt6 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(NULL); +sql create stream streams7 trigger at_once into streamt7 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value,100,200,300); +sql create stream streams8 trigger at_once into streamt8 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); +sql create stream streams9 trigger at_once into streamt9 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); +sql create stream streams10 trigger at_once into streamt10 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); + +sql insert into t1 values(1648791210000,1,1,1,1.0,'aaa'); +sql insert into t1 values(1648791215000,6,8,8,8.0,'bbb'); +sql insert into t1 values(1648791220000,11,10,10,10.0,'ccc'); +sql insert into t1 values(1648791221000,6,6,6,6.0,'fff'); + +sql insert into t2 values(1648791212000,4,4,4,4.0,'ddd'); +sql insert into t2 values(1648791214000,5,5,5,5.0,'eee'); +sql insert into t2 values(1648791216000,2,2,2,2.0,'bbb'); +sql insert into t2 values(1648791222000,6,6,6,6.0,'fff'); + +$loop_count = 0 + +loop5: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt6 order by ts; + +if $rows != 13 then + print ====streamt6=rows1=$rows + goto loop5 +endi +if $data21 != 4 then + print ====streamt6=data21=$data21 + goto loop5 +endi + +sql delete from t2; +print delete from t2; + +$loop_count = 0 + +loop6: + +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt6 order by ts; + +if $rows != 12 then + print ====streamt6=rows2=$rows + goto loop6 +endi +if $data31 != NULL then + print ====streamt6=data31=$data31 + goto loop6 +endi + + +sql select * from streamt7 order by ts; + +if $rows != 12 then + print ====streamt7=rows2=$rows + goto loop6 +endi +if $data31 != 100 then + print ====streamt7=data31=$data31 + goto loop6 +endi + +sql select * from streamt8 order by ts; + +if $rows != 12 then + print ====streamt8=rows3=$rows + goto loop6 +endi +if $data31 != 6 then + print ====streamt8=data31=$data31 + goto loop6 +endi + +sql select * from streamt9 order by ts; + +if $rows != 12 then + print ====streamt9=rows4=$rows + goto loop6 +endi +if $data31 != 1 then + print ====streamt9=data31=$data31 + goto loop6 +endi + +sql select * from streamt10 order by ts; + +if $rows != 12 then + print ====streamt10=rows5=$rows + goto loop6 +endi +if $data21 != 3 then + print ====streamt10=data21=$data21 + return -1 +endi +if $data31 != 4 then + print ====streamt10=data31=$data31 + return -1 +endi +if $data71 != 8 then + print ====streamt10=data71=$data71 + return -1 +endi +if $data91 != 10 then + print ====streamt10=data91=$data91 + return -1 +endi + + + + + + + + + + + + + + + + + + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop stream if exists streams5; +sql drop stream if exists streams6; +sql drop stream if exists streams7; +sql drop stream if exists streams8; +sql drop stream if exists streams9; +sql drop stream if exists streams10; + +sql use test1; +sql select * from t1; +print $data00 + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/fillIntervalLinear.sim b/tests/script/tsim/stream/fillIntervalLinear.sim new file mode 100644 index 0000000000..46ff785fd3 --- /dev/null +++ b/tests/script/tsim/stream/fillIntervalLinear.sim @@ -0,0 +1,695 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 200 +sql connect + +sql drop stream if exists streams1; +sql drop database if exists test1; +sql create database test1 vgroups 1; +sql use test1; +sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); +sql create stream streams1 trigger at_once into streamt1 as select _wstart as ts, max(a)+sum(c), avg(b), first(s), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); +sql insert into t1 values(1648791213000,4,4,4,4.0,'aaa') (1648791216000,5,5,5,5.0,'bbb'); +sql insert into t1 values(1648791210000,1,1,1,1.0,'ccc') (1648791219000,2,2,2,2.0,'ddd') (1648791222000,3,3,3,3.0,'eee'); + + +$loop_count = 0 + +loop1: +sleep 200 +sql use test1; +sql select * from streamt1 order by ts; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 13 then + print =====rows=$rows + goto loop1 +endi + +if $data01 != 2.000000000 then + print =====data01=$data01 + return -1 +endi +if $data02 != 1.000000000 then + print =====data02=$data02 + return -1 +endi +if $data03 != ccc then + print =====data03=$data03 + return -1 +endi +if $data04 != 1 then + print =====data04=$data04 + return -1 +endi + + +if $data11 != 4.000000000 then + print =====data11=$data11 + return -1 +endi +if $data12 != 2.000000000 then + print =====data12=$data12 + return -1 +endi +if $data13 != NULL then + print =====data13=$data13 + return -1 +endi + + +if $data21 != 6.000000000 then + print =====data21=$data21 + return -1 +endi +if $data22 != 3.000000000 then + print =====data22=$data22 + return -1 +endi +if $data23 != NULL then + print =====data23=$data23 + return -1 +endi + + +if $data31 != 8.000000000 then + print =====data31=$data31 + return -1 +endi +if $data32 != 4.000000000 then + print =====data32=$data32 + return -1 +endi +if $data33 != aaa then + print =====data33=$data33 + return -1 +endi + + +if $data41 != 8.666666667 then + print =====data41=$data41 + return -1 +endi +if $data42 != 4.333333333 then + print =====data42=$data42 + return -1 +endi +if $data43 != NULL then + print =====data43=$data43 + return -1 +endi + + +if $data51 != 9.333333333 then + print =====data01=$data01 + return -1 +endi +if $data52 != 4.666666667 then + print =====data52=$data52 + return -1 +endi +if $data53 != NULL then + print =====data53=$data53 + return -1 +endi + + +if $data61 != 10.000000000 then + print =====data61=$data61 + return -1 +endi +if $data62 != 5.000000000 then + print =====data62=$data62 + return -1 +endi + + +if $data71 != 8.000000000 then + print =====data71=$data71 + return -1 +endi +if $data72 != 4.000000000 then + print =====data72=$data72 + return -1 +endi + + +if $data81 != 6.000000000 then + print =====data81=$data81 + return -1 +endi +if $data82 != 3.000000000 then + print =====data82=$data82 + return -1 +endi + + +if $data91 != 4.000000000 then + print =====data91=$data91 + return -1 +endi +if $data92 != 2.000000000 then + print =====data92=$data92 + return -1 +endi + +if $data[10][1] != 4.666666667 then + print =====data[10][1]=$data[10][1] + return -1 +endi +if $data[10][2] != 2.333333333 then + print =====data[10][2]=$data[10][2] + return -1 +endi + + +if $data[11][1] != 5.333333333 then + print =====data[11][1]=$data[11][1] + return -1 +endi +if $data[11][2] != 2.666666667 then + print =====data[11][2]=$data[11][2] + return -1 +endi + + +if $data[12][1] != 6.000000000 then + print =====data[12][1]=$data[12][1] + return -1 +endi +if $data[12][2] != 3.000000000 then + print =====data[12][2]=$data[12][2] + return -1 +endi + + + +sql drop stream if exists streams2; +sql drop database if exists test2; +sql create database test2 vgroups 1; +sql use test2; +sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); +sql create stream streams2 trigger at_once into streamt2 as select _wstart as ts, max(a)+sum(c), avg(b), first(s), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); +sql insert into t1 values(1648791210000,1,1,1,1.0,'ccc') (1648791219000,2,2,2,2.0,'ddd') (1648791222000,3,3,3,3.0,'eee'); +sql insert into t1 values(1648791213000,4,4,4,4.0,'aaa') (1648791216000,5,5,5,5.0,'bbb'); + + +$loop_count = 0 + +loop2: + +sleep 200 + +sql select * from streamt2 order by ts; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 13 then + print =====rows=$rows + goto loop2 +endi + +if $data01 != 2.000000000 then + print =====data01=$data01 + return -1 +endi +if $data02 != 1.000000000 then + print =====data02=$data02 + return -1 +endi +if $data03 != ccc then + print =====data03=$data03 + return -1 +endi +if $data04 != 1 then + print =====data04=$data04 + return -1 +endi + + +if $data11 != 4.000000000 then + print =====data11=$data11 + return -1 +endi +if $data12 != 2.000000000 then + print =====data12=$data12 + return -1 +endi +if $data13 != NULL then + print =====data13=$data13 + return -1 +endi + + +if $data21 != 6.000000000 then + print =====data21=$data21 + return -1 +endi +if $data22 != 3.000000000 then + print =====data22=$data22 + return -1 +endi +if $data23 != NULL then + print =====data23=$data23 + return -1 +endi + + +if $data31 != 8.000000000 then + print =====data31=$data31 + return -1 +endi +if $data32 != 4.000000000 then + print =====data32=$data32 + return -1 +endi +if $data33 != aaa then + print =====data33=$data33 + return -1 +endi + + +if $data41 != 8.666666667 then + print =====data41=$data41 + return -1 +endi +if $data42 != 4.333333333 then + print =====data42=$data42 + return -1 +endi +if $data43 != NULL then + print =====data43=$data43 + return -1 +endi + + +if $data51 != 9.333333333 then + print =====data01=$data01 + return -1 +endi +if $data52 != 4.666666667 then + print =====data52=$data52 + return -1 +endi +if $data53 != NULL then + print =====data53=$data53 + return -1 +endi + + +if $data61 != 10.000000000 then + print =====data61=$data61 + return -1 +endi +if $data62 != 5.000000000 then + print =====data62=$data62 + return -1 +endi + + +if $data71 != 8.000000000 then + print =====data71=$data71 + return -1 +endi +if $data72 != 4.000000000 then + print =====data72=$data72 + return -1 +endi + + +if $data81 != 6.000000000 then + print =====data81=$data81 + return -1 +endi +if $data82 != 3.000000000 then + print =====data82=$data82 + return -1 +endi + + +if $data91 != 4.000000000 then + print =====data91=$data91 + return -1 +endi +if $data92 != 2.000000000 then + print =====data92=$data92 + return -1 +endi + +if $data[10][1] != 4.666666667 then + print =====data[10][1]=$data[10][1] + return -1 +endi +if $data[10][2] != 2.333333333 then + print =====data[10][2]=$data[10][2] + return -1 +endi + + +if $data[11][1] != 5.333333333 then + print =====data[11][1]=$data[11][1] + return -1 +endi +if $data[11][2] != 2.666666667 then + print =====data[11][2]=$data[11][2] + return -1 +endi + + +if $data[12][1] != 6.000000000 then + print =====data[12][1]=$data[12][1] + return -1 +endi +if $data[12][2] != 3.000000000 then + print =====data[12][2]=$data[12][2] + return -1 +endi + + + +sql drop stream if exists streams3; +sql drop database if exists test3; +sql create database test3 vgroups 1; +sql use test3; +sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); +sql create stream streams3 trigger at_once into streamt3 as select _wstart as ts, max(a), b+c, s, b+1, 1 from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(linear); +sql insert into t1 values(1648791215000,1,1,1,1.0,'aaa'); +sql insert into t1 values(1648791217000,2,2,2,2.0,'bbb'); +sql insert into t1 values(1648791211000,3,3,3,3.0,'ccc'); +sql insert into t1 values(1648791213000,4,4,4,4.0,'ddd'); + + +$loop_count = 0 + +loop3: +sleep 300 +sql select * from streamt3 order by ts; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + + +if $rows != 7 then + print =====rows=$rows + goto loop3 +endi + + +if $data01 != 3 then + print =====data01=$data01 + return -1 +endi +if $data02 != 6.000000000 then + print =====data02=$data02 + return -1 +endi +if $data03 != ccc then + print =====data03=$data03 + return -1 +endi + +if $data11 != 3 then + print =====data11=$data11 + return -1 +endi +if $data12 != 7.000000000 then + print =====data12=$data12 + return -1 +endi +if $data13 != NULL then + print =====data13=$data13 + return -1 +endi + + +if $data21 != 4 then + print =====data21=$data21 + return -1 +endi +if $data22 != 8.000000000 then + print =====data22=$data22 + return -1 +endi +if $data23 != ddd then + print =====data23=$data23 + return -1 +endi + + +if $data31 != 2 then + print =====data31=$data31 + return -1 +endi +if $data32 != 5.000000000 then + print =====data32=$data32 + return -1 +endi +if $data33 != NULL then + print =====data33=$data33 + return -1 +endi + + +if $data41 != 1 then + print =====data41=$data41 + return -1 +endi +if $data42 != 2.000000000 then + print =====data42=$data42 + return -1 +endi +if $data43 != aaa then + print =====data43=$data43 + return -1 +endi + + +if $data51 != 1 then + print =====data51=$data51 + return -1 +endi +if $data52 != 3.000000000 then + print =====data52=$data52 + return -1 +endi +if $data53 != NULL then + print =====data53=$data53 + return -1 +endi + + +if $data61 != 2 then + print =====data61=$data61 + return -1 +endi +if $data62 != 4.000000000 then + print =====data62=$data62 + return -1 +endi +if $data63 != bbb then + print =====data63=$data63 + return -1 +endi + + +sql insert into t1 values(1648791212000,5,5,5,5.0,'eee'); +sql insert into t1 values(1648791207000,6,6,6,6.0,'fff') (1648791209000,7,7,7,7.0,'ggg') (1648791219000,8,8,8,8.0,'hhh') (1648791221000,9,9,9,9.0,'iii'); + + + +$loop_count = 0 + +loop4: + +sleep 200 + +sql select * from test3.streamt3 order by ts; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + + +if $rows != 15 then + print =====rows=$rows + goto loop4 +endi + + +if $data01 != 6 then + print =====data01=$data01 + return -1 +endi +if $data02 != 12.000000000 then + print =====data02=$data02 + return -1 +endi +if $data03 != fff then + print =====data03=$data03 + return -1 +endi + +if $data11 != 6 then + print =====data11=$data11 + return -1 +endi +if $data12 != 13.000000000 then + print =====data12=$data12 + return -1 +endi +if $data13 != NULL then + print =====data13=$data13 + return -1 +endi + + +if $data21 != 7 then + print =====data21=$data21 + return -1 +endi +if $data22 != 14.000000000 then + print =====data22=$data22 + return -1 +endi +if $data23 != ggg then + print =====data23=$data23 + return -1 +endi + + +if $data31 != 5 then + print =====data31=$data31 + return -1 +endi +if $data32 != 10.000000000 then + print =====data32=$data32 + return -1 +endi +if $data33 != NULL then + print =====data33=$data33 + return -1 +endi + +if $data51 != 5 then + print =====data51=$data51 + return -1 +endi +if $data52 != 10.000000000 then + print =====data52=$data52 + return -1 +endi +if $data53 != eee then + print =====data53=$data53 + return -1 +endi + + +if $data[11][1] != 5 then + print =====data[11][1]=$data[11][1] + return -1 +endi +if $data[11][2] != 10.000000000 then + print =====data[11][2]=$data[11][2] + return -1 +endi +if $data[11][3] != NULL then + print =====data[11][3]=$data[11][3] + return -1 +endi + +if $data[12][1] != 8 then + print =====data[12][1]=$data[12][1] + return -1 +endi +if $data[12][2] != 16.000000000 then + print =====data[12][2]=$data[12][2] + return -1 +endi +if $data[12][3] != hhh then + print =====data[12][3]=$data[12][3] + return -1 +endi + +if $data[13][1] != 8 then + print =====data[13][1]=$data[13][1] + return -1 +endi +if $data[13][2] != 17.000000000 then + print =====data[13][2]=$data[13][2] + return -1 +endi +if $data[13][3] != NULL then + print =====data[13][3]=$data[13][3] + return -1 +endi + +if $data[14][1] != 9 then + print =====data[14][1]=$data[14][1] + return -1 +endi +if $data[14][2] != 18.000000000 then + print =====data[14][2]=$data[14][2] + return -1 +endi +if $data[14][3] != iii then + print =====data[14][3]=$data[14][3] + return -1 +endi + + + + + + + + + + + + + + + + + + + + + + + + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop stream if exists streams5; +sql drop stream if exists streams6; +sql drop stream if exists streams7; +sql drop stream if exists streams8; + +sql use test1; +sql select * from t1; +print $data00 + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/fillIntervalPartitionBy.sim b/tests/script/tsim/stream/fillIntervalPartitionBy.sim new file mode 100644 index 0000000000..384aa2c8e4 --- /dev/null +++ b/tests/script/tsim/stream/fillIntervalPartitionBy.sim @@ -0,0 +1,171 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 200 +sql connect + +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop stream if exists streams5; +sql drop database if exists test1; +sql create database test1 vgroups 1; +sql use test1; +sql create stable st(ts timestamp, a int, b int , c int, d double, s varchar(20)) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams1 trigger at_once into streamt1 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(NULL); +sql create stream streams2 trigger at_once into streamt2 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(value,100,200,300); +sql create stream streams3 trigger at_once into streamt3 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(next); +sql create stream streams4 trigger at_once into streamt4 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(prev); +sql create stream streams5 trigger at_once into streamt5 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(linear); + +sql insert into t1 values(1648791210000,0,0,0,0.0,'aaa'); +sql insert into t1 values(1648791213000,1,1,1,1.0,'bbb'); +sql insert into t1 values(1648791215000,5,5,5,5.0,'ccc'); +sql insert into t1 values(1648791216000,6,6,6,6.0,'ddd'); +sql insert into t2 values(1648791210000,7,0,0,0.0,'aaa'); +sql insert into t2 values(1648791213000,8,1,1,1.0,'bbb'); +sql insert into t2 values(1648791215000,9,5,5,5.0,'ccc'); +sql insert into t2 values(1648791216000,10,6,6,6.0,'ddd'); + +$loop_count = 0 + +loop2: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + + +sql select * from streamt1 order by group_id, ts; + +if $rows != 14 then + print ====streamt1=rows1=$rows + goto loop2 +endi + +sql select * from streamt2 order by group_id, ts; + +if $rows != 14 then + print ====streamt2=rows2=$rows + goto loop2 +endi + +sql select * from streamt3 order by group_id, ts; + +if $rows != 14 then + print ====streamt3=rows3=$rows + goto loop2 +endi + +sql select * from streamt4 order by group_id, ts; + +if $rows != 14 then + print ====streamt4=rows4=$rows + goto loop2 +endi + +sql select * from streamt5 order by group_id, ts; + +if $rows != 14 then + print ====streamt5=rows5=$rows + goto loop2 +endi + +sql delete from t1 where ts = 1648791216000; +print ======delete from t1 where ts = 1648791216000; + +$loop_count = 0 + +loop3: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1 order by group_id, ts; + +if $rows != 13 then + print ====streamt1=rows1=$rows + goto loop3 +endi + +sql select * from streamt2 order by group_id, ts; + +if $rows != 13 then + print ====streamt2=rows2=$rows + goto loop3 +endi + +sql select * from streamt3 order by group_id, ts; + +if $rows != 13 then + print ====streamt3=rows3=$rows + goto loop3 +endi + +sql select * from streamt4 order by group_id, ts; + +if $rows != 13 then + print ====streamt4=rows4=$rows + goto loop3 +endi + +sql select * from streamt5 order by group_id, ts; + +if $rows != 13 then + print ====streamt5=rows5=$rows + goto loop3 +endi + + + + + + + + + + + + + + + + + + + + + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop stream if exists streams5; +sql drop stream if exists streams6; +sql drop stream if exists streams7; +sql drop stream if exists streams8; +sql drop stream if exists streams9; +sql drop stream if exists streams10; + +sql use test1; +sql select * from t1; +print $data00 + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/fillIntervalPrevNext.sim b/tests/script/tsim/stream/fillIntervalPrevNext.sim new file mode 100644 index 0000000000..5eab5fdac1 --- /dev/null +++ b/tests/script/tsim/stream/fillIntervalPrevNext.sim @@ -0,0 +1,1036 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 200 +sql connect + +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop database if exists test1; +sql create database test1 vgroups 1; +sql use test1; +sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); +sql create stream streams1 trigger at_once into streamt1 as select _wstart as ts, count(*) c1, max(b)+sum(a) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); +sql create stream streams2 trigger at_once into streamt2 as select _wstart as ts, count(*) c1, max(a)+min(c), avg(b) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); +sql insert into t1 values(1648791213000,4,4,4,4.0,'aaa') (1648791215000,5,5,5,5.0,'aaa'); +sql insert into t1 values(1648791211000,1,1,1,1.0,'aaa') (1648791217000,2,2,2,2.0,'aaa') (1648791220000,3,3,3,3.0,'aaa'); + + +$loop_count = 0 + +loop1: +sleep 200 +sql use test1; +sql select * from streamt1 order by ts; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 10 then + print =====rows=$rows + goto loop1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != 2.000000000 then + print =====data02=$data02 + goto loop1 +endi + + +if $data11 != 1 then + print =====data11=$data11 + goto loop1 +endi + +if $data12 != 2.000000000 then + print =====data12=$data12 + goto loop1 +endi + + +if $data21 != 1 then + print =====data21=$data21 + goto loop1 +endi + +if $data22 != 8.000000000 then + print =====data22=$data22 + goto loop1 +endi + + +if $data31 != 1 then + print =====data31=$data31 + goto loop1 +endi + +if $data32 != 8.000000000 then + print =====data32=$data32 + goto loop1 +endi + + +if $data41 != 1 then + print =====data41=$data41 + goto loop1 +endi + +if $data42 != 10.000000000 then + print =====data42=$data42 + goto loop1 +endi + + +if $data51 != 1 then + print =====data01=$data01 + goto loop1 +endi + +if $data52 != 10.000000000 then + print =====data52=$data52 + goto loop1 +endi + + +if $data61 != 1 then + print =====data61=$data61 + goto loop1 +endi + +if $data62 != 4.000000000 then + print =====data62=$data62 + goto loop1 +endi + + +if $data71 != 1 then + print =====data71=$data71 + goto loop1 +endi + +if $data72 != 4.000000000 then + print =====data72=$data72 + goto loop1 +endi + + +if $data81 != 1 then + print =====data81=$data81 + goto loop1 +endi + +if $data82 != 4.000000000 then + print =====data82=$data82 + goto loop1 +endi + + +if $data91 != 1 then + print =====data91=$data91 + goto loop1 +endi + +if $data92 != 6.000000000 then + print =====data92=$data92 + goto loop1 +endi + +sql use test1; +sql select * from streamt2 order by ts; + +print next----------------------151 + +if $rows != 10 then + print =====rows=$rows + goto loop1 +endi + +if $data02 != 2.000000000 then + print =====data02=$data02 + goto loop1 +endi +if $data03 != 1.000000000 then + print =====data03=$data03 + goto loop1 +endi + +if $data12 != 8.000000000 then + print =====data12=$data12 + goto loop1 +endi +if $data13 != 4.000000000 then + print =====data13=$data13 + goto loop1 +endi + + +if $data22 != 8.000000000 then + print =====data22=$data22 + goto loop1 +endi +if $data23 != 4.000000000 then + print =====data23=$data23 + goto loop1 +endi + + +if $data32 != 10.000000000 then + print =====data32=$data32 + goto loop1 +endi +if $data33 != 5.000000000 then + print =====data33=$data33 + goto loop1 +endi + + +if $data42 != 10.000000000 then + print =====data42=$data42 + goto loop1 +endi +if $data43 != 5.000000000 then + print =====data43=$data43 + goto loop1 +endi + + +if $data52 != 4.000000000 then + print =====data52=$data52 + goto loop1 +endi +if $data53 != 2.000000000 then + print =====data53=$data53 + goto loop1 +endi + + +if $data62 != 4.000000000 then + print =====data62=$data62 + goto loop1 +endi +if $data63 != 2.000000000 then + print =====data63=$data63 + goto loop1 +endi + + +if $data72 != 6.000000000 then + print =====data72=$data72 + return -1 +endi +if $data73 != 3.000000000 then + print =====data73=$data73 + return -1 +endi + + +if $data82 != 6.000000000 then + print =====data82=$data82 + return -1 +endi +if $data83 != 3.000000000 then + print =====data83=$data83 + return -1 +endi + + +if $data92 != 6.000000000 then + print =====data92=$data92 + return -1 +endi +if $data93 != 3.000000000 then + print =====data93=$data93 + return -1 +endi + + + +sql drop stream if exists streams5; +sql drop stream if exists streams6; +sql drop database if exists test5; +sql create database test5 vgroups 1; +sql use test5; +sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); +sql create stream streams5 trigger at_once into streamt5 as select _wstart as ts, count(*) c1, max(b)+sum(a) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); +sql create stream streams6 trigger at_once into streamt6 as select _wstart as ts, count(*) c1, max(a)+min(c), avg(b) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); +sql insert into t1 values(1648791211000,1,1,1,1.0,'aaa') (1648791217000,2,2,2,2.0,'aaa') (1648791220000,3,3,3,3.0,'aaa'); +sql insert into t1 values(1648791213000,4,4,4,4.0,'aaa') (1648791215000,5,5,5,5.0,'aaa'); + +$loop_count = 0 + +loop5: +sleep 200 +sql select * from streamt5 order by ts; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 10 then + print =====rows=$rows + goto loop5 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop5 +endi + +if $data02 != 2.000000000 then + print =====data02=$data02 + goto loop5 +endi + + +if $data11 != 1 then + print =====data11=$data11 + goto loop5 +endi + +if $data12 != 2.000000000 then + print =====data12=$data12 + goto loop5 +endi + + +if $data21 != 1 then + print =====data21=$data21 + goto loop5 +endi + +if $data22 != 8.000000000 then + print =====data22=$data22 + goto loop5 +endi + + +if $data31 != 1 then + print =====data31=$data31 + goto loop5 +endi + +if $data32 != 8.000000000 then + print =====data32=$data32 + goto loop5 +endi + + +if $data41 != 1 then + print =====data41=$data41 + goto loop5 +endi + +if $data42 != 10.000000000 then + print =====data42=$data42 + goto loop5 +endi + + +if $data51 != 1 then + print =====data01=$data01 + goto loop5 +endi + +if $data52 != 10.000000000 then + print =====data52=$data52 + goto loop5 +endi + + +if $data61 != 1 then + print =====data61=$data61 + goto loop5 +endi + +if $data62 != 4.000000000 then + print =====data62=$data62 + goto loop5 +endi + + +if $data71 != 1 then + print =====data71=$data71 + goto loop5 +endi + +if $data72 != 4.000000000 then + print =====data72=$data72 + goto loop5 +endi + + +if $data81 != 1 then + print =====data81=$data81 + goto loop5 +endi + +if $data82 != 4.000000000 then + print =====data82=$data82 + goto loop5 +endi + + +if $data91 != 1 then + print =====data91=$data91 + goto loop5 +endi + +if $data92 != 6.000000000 then + print =====data92=$data92 + goto loop5 +endi + + +$loop_count = 0 + +loop6: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 200 + +sql select * from streamt6 order by ts; + +if $rows != 10 then + print =====rows=$rows + goto loop6 +endi + +if $data02 != 2.000000000 then + print =====data02=$data02 + goto loop6 +endi +if $data03 != 1.000000000 then + print =====data03=$data03 + goto loop6 +endi + +if $data12 != 8.000000000 then + print =====data12=$data12 + goto loop6 +endi +if $data13 != 4.000000000 then + print =====data13=$data13 + goto loop6 +endi + + +if $data22 != 8.000000000 then + print =====data22=$data22 + goto loop6 +endi +if $data23 != 4.000000000 then + print =====data23=$data23 + goto loop6 +endi + + +if $data32 != 10.000000000 then + print =====data32=$data32 + goto loop6 +endi +if $data33 != 5.000000000 then + print =====data33=$data33 + goto loop6 +endi + + +if $data42 != 10.000000000 then + print =====data42=$data42 + goto loop6 +endi +if $data43 != 5.000000000 then + print =====data43=$data43 + goto loop6 +endi + + +if $data52 != 4.000000000 then + print =====data52=$data52 + goto loop6 +endi +if $data53 != 2.000000000 then + print =====data53=$data53 + goto loop6 +endi + + +if $data62 != 4.000000000 then + print =====data62=$data62 + goto loop6 +endi +if $data63 != 2.000000000 then + print =====data63=$data63 + goto loop6 +endi + + +if $data72 != 6.000000000 then + print =====data72=$data72 + return -1 +endi +if $data73 != 3.000000000 then + print =====data73=$data73 + return -1 +endi + + +if $data82 != 6.000000000 then + print =====data82=$data82 + return -1 +endi +if $data83 != 3.000000000 then + print =====data83=$data83 + return -1 +endi + + +if $data92 != 6.000000000 then + print =====data92=$data92 + return -1 +endi +if $data93 != 3.000000000 then + print =====data93=$data93 + return -1 +endi + + + +sql drop stream if exists streams7; +sql drop stream if exists streams8; +sql drop database if exists test7; +sql create database test7 vgroups 1; +sql use test7; +sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); +sql create stream streams7 trigger at_once into streamt7 as select _wstart as ts, max(a), b+c, s from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(prev); +sql create stream streams8 trigger at_once into streamt8 as select _wstart as ts, max(a), 1, b+1 from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(next); +sql insert into t1 values(1648791215000,1,1,1,1.0,'aaa'); +sql insert into t1 values(1648791217000,2,2,2,2.0,'bbb'); +sql insert into t1 values(1648791211000,3,3,3,3.0,'ccc'); +sql insert into t1 values(1648791213000,4,4,4,4.0,'ddd'); + + +$loop_count = 0 + +loop7: +sleep 300 +sql select * from streamt7 order by ts; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + + +if $rows != 7 then + print =====rows=$rows + goto loop7 +endi + + +if $data01 != 3 then + print =====data01=$data01 + return -1 +endi +if $data02 != 6.000000000 then + print =====data02=$data02 + return -1 +endi +if $data03 != ccc then + print =====data03=$data03 + return -1 +endi + +if $data11 != 3 then + print =====data11=$data11 + return -1 +endi +if $data12 != 6.000000000 then + print =====data12=$data12 + return -1 +endi +if $data13 != ccc then + print =====data13=$data13 + return -1 +endi + + +if $data21 != 4 then + print =====data21=$data21 + return -1 +endi +if $data22 != 8.000000000 then + print =====data22=$data22 + return -1 +endi +if $data23 != ddd then + print =====data23=$data23 + return -1 +endi + + +if $data31 != 4 then + print =====data31=$data31 + return -1 +endi +if $data32 != 8.000000000 then + print =====data32=$data32 + return -1 +endi +if $data33 != ddd then + print =====data33=$data33 + return -1 +endi + + +if $data41 != 1 then + print =====data41=$data41 + return -1 +endi +if $data42 != 2.000000000 then + print =====data42=$data42 + return -1 +endi +if $data43 != aaa then + print =====data43=$data43 + return -1 +endi + + +if $data51 != 1 then + print =====data51=$data51 + return -1 +endi +if $data52 != 2.000000000 then + print =====data52=$data52 + return -1 +endi +if $data53 != aaa then + print =====data53=$data53 + return -1 +endi + + +if $data61 != 2 then + print =====data61=$data61 + return -1 +endi +if $data62 != 4.000000000 then + print =====data62=$data62 + return -1 +endi +if $data63 != bbb then + print =====data63=$data63 + return -1 +endi + +#-------------- + +sleep 200 +sql select * from streamt8 order by ts; + + +if $rows != 7 then + print =====rows=$rows + return -1 +endi + + +if $data01 != 3 then + print =====data01=$data01 + return -1 +endi +if $data02 != 1 then + print =====data02=$data02 + return -1 +endi +if $data03 != 4.000000000 then + print =====data03=$data03 + return -1 +endi + +if $data11 != 4 then + print =====data11=$data11 + return -1 +endi +if $data12 != 1 then + print =====data12=$data12 + return -1 +endi +if $data13 != 5.000000000 then + print =====data13=$data13 + return -1 +endi + + +if $data21 != 4 then + print =====data21=$data21 + return -1 +endi +if $data22 != 1 then + print =====data22=$data22 + return -1 +endi +if $data23 != 5.000000000 then + print =====data23=$data23 + return -1 +endi + + +if $data31 != 1 then + print =====data31=$data31 + return -1 +endi +if $data32 != 1 then + print =====data32=$data32 + return -1 +endi +if $data33 != 2.000000000 then + print =====data33=$data33 + return -1 +endi + + +if $data41 != 1 then + print =====data41=$data41 + return -1 +endi +if $data42 != 1 then + print =====data42=$data42 + return -1 +endi +if $data43 != 2.000000000 then + print =====data43=$data43 + return -1 +endi + + +if $data51 != 2 then + print =====data51=$data51 + return -1 +endi +if $data52 != 1 then + print =====data52=$data52 + return -1 +endi +if $data53 != 3.000000000 then + print =====data53=$data53 + return -1 +endi + + +if $data61 != 2 then + print =====data61=$data61 + return -1 +endi +if $data62 != 1 then + print =====data62=$data62 + return -1 +endi +if $data63 != 3.000000000 then + print =====data63=$data63 + return -1 +endi + +sql insert into t1 values(1648791212000,5,5,5,5.0,'eee'); +sql insert into t1 values(1648791207000,6,6,6,6.0,'fff') (1648791209000,7,7,7,7.0,'ggg') (1648791219000,8,8,8,8.0,'hhh') (1648791221000,9,9,9,9.0,'iii'); + + + +$loop_count = 0 + +loop8: +sleep 200 +sql select * from streamt7 order by ts; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + + +if $rows != 15 then + print =====rows=$rows + goto loop8 +endi + + +if $data01 != 6 then + print =====data01=$data01 + return -1 +endi +if $data02 != 12.000000000 then + print =====data02=$data02 + return -1 +endi +if $data03 != fff then + print =====data03=$data03 + return -1 +endi + +if $data11 != 6 then + print =====data11=$data11 + return -1 +endi +if $data12 != 12.000000000 then + print =====data12=$data12 + return -1 +endi +if $data13 != fff then + print =====data13=$data13 + return -1 +endi + + +if $data21 != 7 then + print =====data21=$data21 + return -1 +endi +if $data22 != 14.000000000 then + print =====data22=$data22 + return -1 +endi +if $data23 != ggg then + print =====data23=$data23 + return -1 +endi + + +if $data31 != 7 then + print =====data31=$data31 + return -1 +endi +if $data32 != 14.000000000 then + print =====data32=$data32 + return -1 +endi +if $data33 != ggg then + print =====data33=$data33 + return -1 +endi + +if $data51 != 5 then + print =====data51=$data51 + return -1 +endi +if $data52 != 10.000000000 then + print =====data52=$data52 + return -1 +endi +if $data53 != eee then + print =====data53=$data53 + return -1 +endi + + +if $data[11][1] != 2 then + print =====data[11][1]=$data[11][1] + return -1 +endi +if $data[11][2] != 4.000000000 then + print =====data[11][2]=$data[11][2] + return -1 +endi +if $data[11][3] != bbb then + print =====data[11][3]=$data[11][3] + return -1 +endi + +if $data[12][1] != 8 then + print =====data[12][1]=$data[12][1] + return -1 +endi +if $data[12][2] != 16.000000000 then + print =====data[12][2]=$data[12][2] + return -1 +endi +if $data[12][3] != hhh then + print =====data[12][3]=$data[12][3] + return -1 +endi + +if $data[13][1] != 8 then + print =====data[13][1]=$data[13][1] + return -1 +endi +if $data[13][2] != 16.000000000 then + print =====data[13][2]=$data[13][2] + return -1 +endi +if $data[13][3] != hhh then + print =====data[13][3]=$data[13][3] + return -1 +endi + +if $data[14][1] != 9 then + print =====data[14][1]=$data[14][1] + return -1 +endi +if $data[14][2] != 18.000000000 then + print =====data[14][2]=$data[14][2] + return -1 +endi +if $data[14][3] != iii then + print =====data[14][3]=$data[14][3] + return -1 +endi + +print fill next-----------------890 +sql use test7; +sql select * from streamt8 order by ts; + +if $rows != 15 then + print =====rows=$rows + goto loop8 +endi + + +if $data01 != 6 then + print =====data01=$data01 + return -1 +endi +if $data02 != 1 then + print =====data02=$data02 + return -1 +endi +if $data03 != 7.000000000 then + print =====data03=$data03 + return -1 +endi + +if $data11 != 7 then + print =====data11=$data11 + return -1 +endi +if $data13 != 8.000000000 then + print =====data13=$data13 + return -1 +endi + + +if $data21 != 7 then + print =====data21=$data21 + return -1 +endi +if $data23 != 8.000000000 then + print =====data23=$data23 + return -1 +endi + + +if $data31 != 3 then + print =====data31=$data31 + return -1 +endi +if $data33 != 4.000000000 then + print =====data33=$data33 + return -1 +endi + +if $data51 != 5 then + print =====data51=$data51 + return -1 +endi +if $data53 != 6.000000000 then + print =====data53=$data53 + return -1 +endi + + +if $data[11][1] != 8 then + print =====data[11][1]=$data[11][1] + return -1 +endi +if $data[11][2] != 1 then + print =====data[11][2]=$data[11][2] + return -1 +endi +if $data[11][3] != 9.000000000 then + print =====data[11][3]=$data[11][3] + return -1 +endi + +if $data[12][1] != 8 then + print =====data[12][1]=$data[12][1] + return -1 +endi +if $data[12][3] != 9.000000000 then + print =====data[12][3]=$data[12][3] + return -1 +endi + +if $data[13][1] != 9 then + print =====data[13][1]=$data[13][1] + return -1 +endi +if $data[13][3] != 10.000000000 then + print =====data[13][3]=$data[13][3] + return -1 +endi + +if $data[14][1] != 9 then + print =====data[14][1]=$data[14][1] + return -1 +endi +if $data[14][3] != 10.000000000 then + print =====data[14][3]=$data[14][3] + return -1 +endi + + + + + + + + + + + + + + + + + + + + + + + + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop stream if exists streams5; +sql drop stream if exists streams6; +sql drop stream if exists streams7; +sql drop stream if exists streams8; + +sql use test1; +sql select * from t1; +print $data00 + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/fillIntervalValue.sim b/tests/script/tsim/stream/fillIntervalValue.sim new file mode 100644 index 0000000000..113eae9270 --- /dev/null +++ b/tests/script/tsim/stream/fillIntervalValue.sim @@ -0,0 +1,488 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 200 +sql connect + +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; + +sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20));; +sql create stream streams1 trigger at_once into streamt as select _wstart ts, count(*) c1 from t1 where ts > 1648791210000 and ts < 1648791413000 interval(10s) fill(value, 100); +sql insert into t1 values(1648791213000,1,2,3,1.0,'aaa'); +sleep 100 +sql insert into t1 values(1648791233000,1,2,3,1.0,'aaa'); +sql insert into t1 values(1648791223000,1,2,3,1.0,'aaa'); +sql insert into t1 values(1648791283000,1,2,3,1.0,'aaa'); +sql insert into t1 values(1648791253000,1,2,3,1.0,'aaa'); + +$loop_count = 0 + +loop0: +sleep 200 +sql select * from streamt order by ts; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 8 then + print =====rows=$rows + goto loop0 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop0 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop0 +endi + +if $data21 != 1 then + print =====data21=$data21 + goto loop0 +endi + +if $data31 != 100 then + print =====data31=$data31 + goto loop0 +endi + +if $data41 != 1 then + print =====data41=$data41 + goto loop0 +endi + +if $data51 != 100 then + print =====data01=$data01 + goto loop0 +endi + +if $data61 != 100 then + print =====data61=$data61 + goto loop0 +endi + +if $data71 != 1 then + print =====data71=$data71 + goto loop0 +endi + +sql drop stream if exists streams2; +sql drop database if exists test2; +sql create database test2 vgroups 1; +sql use test2; +sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); +sql create stream streams2 trigger at_once into streamt2 as select _wstart as ts, count(*) c1, max(b)+sum(a) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value, 100,200); +sql insert into t1 values(1648791211000,1,1,1,1.0,'aaa') (1648791217000,2,2,2,2.0,'aaa') (1648791220000,3,3,3,3.0,'aaa'); +sql insert into t1 values(1648791213000,4,4,4,4.0,'aaa') (1648791215000,5,5,5,5.0,'aaa'); + +$loop_count = 0 + +loop1: +sleep 200 +sql select * from streamt2 order by ts; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 10 then + print =====rows=$rows + goto loop1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != 2.000000000 then + print =====data02=$data02 + goto loop1 +endi + + +if $data11 != 100 then + print =====data11=$data11 + goto loop1 +endi + +if $data12 != 200.000000000 then + print =====data12=$data12 + goto loop1 +endi + + +if $data21 != 1 then + print =====data21=$data21 + goto loop1 +endi + +if $data22 != 8.000000000 then + print =====data22=$data22 + goto loop1 +endi + + +if $data31 != 100 then + print =====data31=$data31 + goto loop1 +endi + +if $data32 != 200.000000000 then + print =====data32=$data32 + goto loop1 +endi + + +if $data41 != 1 then + print =====data41=$data41 + goto loop1 +endi + +if $data42 != 10.000000000 then + print =====data42=$data42 + goto loop1 +endi + + +if $data51 != 100 then + print =====data01=$data01 + goto loop1 +endi + +if $data52 != 200.000000000 then + print =====data52=$data52 + goto loop1 +endi + + +if $data61 != 1 then + print =====data61=$data61 + goto loop1 +endi + +if $data62 != 4.000000000 then + print =====data62=$data62 + goto loop1 +endi + + +if $data71 != 100 then + print =====data71=$data71 + goto loop1 +endi + +if $data72 != 200.000000000 then + print =====data72=$data72 + goto loop1 +endi + + +if $data81 != 100 then + print =====data81=$data81 + goto loop1 +endi + +if $data82 != 200.000000000 then + print =====data82=$data82 + goto loop1 +endi + + +if $data91 != 1 then + print =====data91=$data91 + goto loop1 +endi + +if $data92 != 6.000000000 then + print =====data92=$data92 + goto loop1 +endi + +sql drop stream if exists streams3; +sql drop database if exists test3; +sql create database test3 vgroups 1; +sql use test3; +sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); +sql create stream streams3 trigger at_once into streamt3 as select _wstart as ts, max(b), a+b, c from t1 where ts >= 1648791200000 and ts < 1648791261000 interval(10s) sliding(3s) fill(value, 100,200,300); + +sql insert into t1 values(1648791220000,1,1,1,1.0,'aaa'); +sleep 100 +sql insert into t1 values(1648791260000,1,1,1,1.0,'aaa'); +sleep 100 +sql insert into t1 values(1648791200000,1,1,1,1.0,'aaa'); + +$loop_count = 0 + +loop3: +sleep 200 +sql select * from streamt3 order by ts; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 23 then + print =====rows=$rows + goto loop3 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 2.000000000 then + print =====data02=$data02 + goto loop3 +endi + +if $data03 != 1 then + print =====data03=$data03 + goto loop3 +endi + + +if $data21 != 1 then + print =====data21=$data21 + goto loop3 +endi + +if $data22 != 2.000000000 then + print =====data22=$data22 + goto loop3 +endi + +if $data23 != 1 then + print =====data23=$data23 + goto loop3 +endi + + +if $data31 != 100 then + print =====data31=$data31 + goto loop3 +endi + +if $data32 != 200.000000000 then + print =====data32=$data32 + goto loop3 +endi + +if $data33 != 300 then + print =====data33=$data33 + goto loop3 +endi + +if $data61 != 100 then + print =====data61=$data61 + goto loop3 +endi + +if $data62 != 200.000000000 then + print =====data62=$data62 + goto loop3 +endi + +if $data63 != 300 then + print =====data63=$data63 + goto loop3 +endi + + +if $data71 != 1 then + print =====data71=$data71 + goto loop3 +endi + +if $data72 != 2.000000000 then + print =====data72=$data72 + goto loop3 +endi + +if $data73 != 1 then + print =====data73=$data73 + goto loop3 +endi + + +if $data91 != 1 then + print =====data91=$data91 + goto loop3 +endi + +if $data92 != 2.000000000 then + print =====data92=$data92 + goto loop3 +endi + +if $data93 != 1 then + print =====data93=$data93 + goto loop3 +endi + + +if $data[10][1] != 100 then + print =====data[10][1]=$data[10][1] + goto loop3 +endi + +if $data[10][2] != 200.000000000 then + print =====data[10][2]=$data[10][2] + goto loop3 +endi + +if $data[10][3] != 300 then + print =====data[10][3]=$data[10][3] + goto loop3 +endi + +if $data[19][1] != 100 then + print =====data[19][1]=$data[19][1] + goto loop3 +endi + +if $data[19][2] != 200.000000000 then + print =====data[19][2]=$data[19][2] + goto loop3 +endi + +if $data[19][3] != 300 then + print =====data[19][3]=$data[19][3] + goto loop3 +endi + + +if $data[20][1] != 1 then + print =====data[20][1]=$data[20][1] + goto loop3 +endi + +if $data[20][2] != 2.000000000 then + print =====data[20][2]=$data[20][2] + goto loop3 +endi + +if $data[20][3] != 1 then + print =====data[20][3]=$data[20][3] + goto loop3 +endi + + +if $data[22][1] != 1 then + print =====data[22][1]=$data[22][1] + goto loop3 +endi + +if $data[22][2] != 2.000000000 then + print =====data[22][2]=$data[22][2] + goto loop3 +endi + +if $data[22][3] != 1 then + print =====data[22][3]=$data[22][3] + goto loop3 +endi + + +sql drop stream if exists streams4; +sql drop database if exists test4; +sql create database test4 vgroups 1; +sql use test4; + +sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20));; +sql create stream streams4 trigger at_once into streamt4 as select _wstart ts, count(*) c1 from t1 where ts > 1648791210000 and ts < 1648791413000 interval(10s) fill(NULL); +sql insert into t1 values(1648791213000,1,2,3,1.0,'aaa'); +sql insert into t1 values(1648791233000,1,2,3,1.0,'aaa'); + +$loop_count = 0 + +loop4: +sleep 200 +sql select * from streamt4 order by ts; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 3 then + print =====rows=$rows + goto loop4 +endi + +if $data11 != NULL then + print =====data11=$data11 + goto loop4 +endi + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop stream if exists streams5; +sql drop stream if exists streams6; +sql drop stream if exists streams7; +sql drop stream if exists streams8; + +sql use test; +sql select * from t1; +print $data00 + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/partitionby.sim b/tests/script/tsim/stream/partitionby.sim index e5e02c3873..bc2c07b951 100644 --- a/tests/script/tsim/stream/partitionby.sim +++ b/tests/script/tsim/stream/partitionby.sim @@ -5,13 +5,14 @@ sleep 50 sql connect sql create database test vgroups 4; +sql create database test0 vgroups 1; sql use test; sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); -sql create stream stream_t1 trigger at_once into streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st partition by ta,tb,tc interval(10s); +sql create stream stream_t1 trigger at_once into test0.streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st partition by ta,tb,tc interval(10s); sql insert into ts1 values(1648791213001,1,12,3,1.0); sql insert into ts2 values(1648791213001,1,12,3,1.0); @@ -22,7 +23,7 @@ $loop_count = 0 loop0: sleep 300 -sql select * from streamtST1; +sql select * from test0.streamtST1; $loop_count = $loop_count + 1 if $loop_count == 10 then @@ -34,6 +35,29 @@ print =====rows=$rows goto loop0 endi +sql insert into ts1 values(1648791223001,1,12,3,1.0); +sql insert into ts2 values(1648791223001,1,12,3,1.0); + +sql insert into ts3 values(1648791223001,1,12,3,1.0); +sql insert into ts4 values(1648791223001,1,12,3,1.0); +sleep 300 +sql delete from st where ts = 1648791223001; + +loop00: +sleep 300 +sql select * from test0.streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 4 then + print =====rows=$rows + goto loop00 +endi + + print =====loop0 sql create database test1 vgroups 1; diff --git a/tests/script/tsim/stream/partitionbyColumnInterval.sim b/tests/script/tsim/stream/partitionbyColumnInterval.sim index 24fdb9c994..fd1d796fdb 100644 --- a/tests/script/tsim/stream/partitionbyColumnInterval.sim +++ b/tests/script/tsim/stream/partitionbyColumnInterval.sim @@ -562,6 +562,53 @@ if $data21 != 1 then goto loop14 endi +sql drop stream if exists streams5; +sql drop database if exists test5; +sql create database test5 vgroups 4; +sql use test5; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(2,2,2); +sql create table t4 using st tags(2,2,2); +sql create stream streams5 trigger at_once into test.streamt5 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t2 values(1648791213000,2,2,3,1.0); +sql insert into t3 values(1648791213000,3,2,3,1.0); +sql insert into t4 values(1648791213000,4,2,3,1.0); + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t2 values(1648791223000,2,2,3,1.0); +sql insert into t3 values(1648791223000,3,2,3,1.0); +sql insert into t4 values(1648791223000,4,2,3,1.0); + +sleep 300 + +sql delete from st where ts = 1648791223000; + +sql select * from test.streamt5; + +$loop_count = 0 + +loop15: +sleep 50 +sql select * from test.streamt5 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $rows != 4 then + print =====rows=$rows + print =====rows=$rows + print =====rows=$rows +# goto loop15 +endi + + + $loop_all = $loop_all + 1 print ============loop_all=$loop_all From 8943fd2fc2b14695e05e9c2b45d2839f57de98c2 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 27 Sep 2022 18:16:48 +0800 Subject: [PATCH 22/71] chore: log optimization for data migrate --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 12 ++++++------ source/dnode/vnode/src/tsdb/tsdbRetention.c | 12 +++++++----- source/dnode/vnode/src/vnd/vnodeSvr.c | 10 +++++----- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index b1bc487fc3..b998ba38a0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -630,8 +630,8 @@ static int64_t tsdbFSendFile(STsdb *pTsdb, TdFilePtr pOutFD, TdFilePtr pInFD, in while ((offset + nBytes) < size) { if (atomic_load_8(&pTsdb->trimHdl.commitInWait) == 1) { - tsdbInfo("vgId:%d sendFile without limit since conflicts, fSize:%" PRIi64 ", maxSpeed:%" PRIi64, - TD_VID(pTsdb->pVnode), size, speed); + tsdbDebug("vgId:%d sendFile without limit since conflicts, fSize:%" PRIi64 ", maxSpeed:%" PRIi64, + TD_VID(pTsdb->pVnode), size, speed); goto _send_remain; } startMs = taosGetTimestampMs(); @@ -649,8 +649,8 @@ static int64_t tsdbFSendFile(STsdb *pTsdb, TdFilePtr pOutFD, TdFilePtr pInFD, in } if (nSleep > 0) { taosMsleep(nSleep); - tsdbInfo("vgId:%d sendFile and msleep:%" PRIi64 ", fSize:%" PRIi64 ", tBytes:%" PRIi64 " maxSpeed:%" PRIi64, - TD_VID(pTsdb->pVnode), nSleep, size, tBytes, speed); + tsdbDebug("vgId:%d sendFile and msleep:%" PRIi64 ", fSize:%" PRIi64 ", tBytes:%" PRIi64 " maxSpeed:%" PRIi64, + TD_VID(pTsdb->pVnode), nSleep, size, tBytes, speed); } } @@ -660,8 +660,8 @@ _send_remain: return nBytes; } tBytes += nBytes; - tsdbInfo("vgId:%d sendFile remain, fSize:%" PRIi64 ", tBytes:%" PRIi64 " maxSpeed:%" PRIi64, TD_VID(pTsdb->pVnode), - size, tBytes, speed); + tsdbDebug("vgId:%d sendFile remain, fSize:%" PRIi64 ", tBytes:%" PRIi64 " maxSpeed:%" PRIi64, TD_VID(pTsdb->pVnode), + size, tBytes, speed); } return tBytes; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index d72211047f..3b96cc20dc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -125,7 +125,7 @@ _retention_loop: fSize += pSet->aSttF[iStt]->size; } if (fSize / speed > MIGRATE_MIN_COST) { - tsdbInfo("vgId:%d migrate loop[%d] with maxFid:%d", TD_VID(pTsdb->pVnode), nBatch, maxFid); + tsdbDebug("vgId:%d migrate loop[%d] with maxFid:%d", TD_VID(pTsdb->pVnode), nBatch, maxFid); break; } } @@ -174,8 +174,8 @@ _commit_conflict_check: if (pSet->fid > maxFid) break; - tsdbInfo("vgId:%d migrate loop[%d] with maxFid:%d, fid:%d, did:%d, level:%d, expLevel:%d", TD_VID(pTsdb->pVnode), - nBatch, maxFid, pSet->fid, pSet->diskId.id, pSet->diskId.level, expLevel); + tsdbDebug("vgId:%d migrate loop[%d] with maxFid:%d, fid:%d, did:%d, level:%d, expLevel:%d", TD_VID(pTsdb->pVnode), + nBatch, maxFid, pSet->fid, pSet->diskId.id, pSet->diskId.level, expLevel); if (expLevel < 0) { SET_DFSET_EXPIRED(pSet); @@ -273,11 +273,13 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed) { _exit: pTsdb->trimHdl.maxRetentFid = INT32_MIN; if (code != 0) { - tsdbError("vgId:%d, tsdb do retention %d failed since %s", TD_VID(pTsdb->pVnode), retention, tstrerror(code)); + tsdbError("vgId:%d, tsdb do retention %d failed since %s, time:%" PRIi64 ", max speed:%" PRIi64, + TD_VID(pTsdb->pVnode), retention, tstrerror(code), now, maxSpeed); ASSERT(0); // tsdbFSRollback(pTsdb->pFS); } else { - tsdbInfo("vgId:%d, tsdb do retention %d succeed", TD_VID(pTsdb->pVnode), retention); + tsdbInfo("vgId:%d, tsdb do retention %d succeed, time:%" PRIi64 "max speed:%" PRIi64, TD_VID(pTsdb->pVnode), + retention, now, maxSpeed); } return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 4139a8ca2b..464fd50ab0 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -425,14 +425,14 @@ void *vnodeProcessTrimReqFunc(void *param) { _exit: oldVal = atomic_val_compare_exchange_8(&pVnode->trimDbH.state, 1, 0); ASSERT(oldVal == 1); - taosMemoryFree(pReq); if (code) { - vError("vgId:%d, trim vnode thread failed since %s, time:%" PRIi64, TD_VID(pVnode), tstrerror(code), - pReq->trimReq.timestamp); + vError("vgId:%d, trim vnode thread failed since %s, time:%" PRIi64 ", max speed:%" PRIi64, TD_VID(pVnode), + tstrerror(code), pReq->trimReq.timestamp, pReq->trimReq.maxSpeed); } else { - vInfo("vgId:%d, trim vnode thread finished, time:%" PRIi64, TD_VID(pVnode), pReq->trimReq.timestamp); + vInfo("vgId:%d, trim vnode thread finish, time:%" PRIi64 ", max speed:%" PRIi64, TD_VID(pVnode), + pReq->trimReq.timestamp, pReq->trimReq.maxSpeed); } - + taosMemoryFree(pReq); return NULL; } From 398c23edecb8aafa3feb30d0a1ead1253343376c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Sep 2022 18:37:02 +0800 Subject: [PATCH 23/71] enh(query): only load the required columns. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 39 ++++++++------------------ 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b76b6b6280..c6fb817f6d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -951,15 +951,22 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn return TSDB_CODE_SUCCESS; } -static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData) { +static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData, uint64_t uid) { int64_t st = taosGetTimestampUs(); + tBlockDataReset(pBlockData); + TABLEID tid = {.suid = pReader->suid, .uid = uid}; + int32_t code = tBlockDataInit(pBlockData, &tid, pReader->pSchema, pReader->suppInfo.colIds, pReader->suppInfo.numOfCols); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; ASSERT(pBlockInfo != NULL); SDataBlk* pBlock = getCurrentBlock(pBlockIter); - int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); + code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, code:%s %s", @@ -2455,14 +2462,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { ASSERT(pBlockIter->numOfBlocks == 0); code = buildComposedDataBlock(pReader); } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { - tBlockDataReset(&pStatus->fileBlockData); - TABLEID tid = {.suid = pReader->suid, .uid = pScanInfo->uid}; - code = tBlockDataInit(&pStatus->fileBlockData, &tid, pReader->pSchema, NULL, 0); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2936,14 +2936,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn setFileBlockActiveInBlockIter(pBlockIter, neighborIndex, step); // 3. load the neighbor block, and set it to be the currently accessed file data block - tBlockDataReset(&pStatus->fileBlockData); - TABLEID tid = {.suid = pReader->suid, .uid = pFBlock->uid}; - int32_t code = tBlockDataInit(&pStatus->fileBlockData, &tid, pReader->pSchema, NULL, 0); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); + int32_t code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pFBlock->uid); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3701,15 +3694,7 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pStatus->blockIter); STableBlockScanInfo* pBlockScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); - tBlockDataReset(&pStatus->fileBlockData); - TABLEID tid = {.suid = pReader->suid, .uid = pBlockScanInfo->uid}; - int32_t code = tBlockDataInit(&pStatus->fileBlockData, &tid, pReader->pSchema, NULL, 0); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return NULL; - } - - code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData); + int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { tBlockDataDestroy(&pStatus->fileBlockData, 1); terrno = code; From ddf121496f24428b7361ad2b47052fd827a3fc78 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 27 Sep 2022 18:42:42 +0800 Subject: [PATCH 24/71] chore: adjust debug log --- source/dnode/vnode/src/tsdb/tsdbRetention.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 3b96cc20dc..971b950975 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -278,7 +278,7 @@ _exit: ASSERT(0); // tsdbFSRollback(pTsdb->pFS); } else { - tsdbInfo("vgId:%d, tsdb do retention %d succeed, time:%" PRIi64 "max speed:%" PRIi64, TD_VID(pTsdb->pVnode), + tsdbInfo("vgId:%d, tsdb do retention %d succeed, time:%" PRIi64 ", max speed:%" PRIi64, TD_VID(pTsdb->pVnode), retention, now, maxSpeed); } return code; From c565c4cb636cd3eef7003e9db178bce5f0435244 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Sep 2022 18:50:51 +0800 Subject: [PATCH 25/71] fix(query): remove the primary timestamp column in the required column list. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c6fb817f6d..f9b65b6527 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -956,7 +956,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI tBlockDataReset(pBlockData); TABLEID tid = {.suid = pReader->suid, .uid = uid}; - int32_t code = tBlockDataInit(pBlockData, &tid, pReader->pSchema, pReader->suppInfo.colIds, pReader->suppInfo.numOfCols); + int32_t code = tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols-1); if (code != TSDB_CODE_SUCCESS) { return code; } From 7dfba33a65905649a3d082f8e0fd782ca3e9957c Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 27 Sep 2022 18:54:23 +0800 Subject: [PATCH 26/71] docs: add keepColumnName description --- docs/en/14-reference/12-config/index.md | 11 ++++++++++- docs/zh/14-reference/12-config/index.md | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 5ab6f59454..726a1ccd69 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -177,12 +177,21 @@ The parameters described in this document by the effect that they have on the sy ### maxNumOfDistinctRes | Attribute | Description | -| -------- | -------------------------------- | --- | +| -------- | -------------------------------- | | Applicable | Server Only | | Meaning | The maximum number of distinct rows returned | | Value Range | [100,000 - 100,000,000] | | Default Value | 100,000 | +### keepColumnName + +| Attribute | Description | +| -------- | -------------------------------- | +| Applicable | Client only | +| Meaning | When the Last, First, LastRow function is queried, whether the returned column name contains the function name. | +| Value Range | 0 means including the function name, 1 means not including the function name. | +| Default Value | 0 | + ## Locale Parameters ### timezone diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 179a3c6df0..6f26878cdd 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -177,12 +177,21 @@ taos --dump-config ### maxNumOfDistinctRes | 属性 | 说明 | -| -------- | -------------------------------- | --- | +| -------- | -------------------------------- | | 适用范围 | 仅服务端适用 | | 含义 | 允许返回的 distinct 结果最大行数 | | 取值范围 | 默认值为 10 万，最大值 1 亿 | | 缺省值 | 10 万 | +### keepColumnName + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 仅客户端适用 | +| 含义 | Last、First、LastRow 函数查询时，返回的列名是否包含函数名。 | +| 取值范围 | 0 表示包含函数名，1 表示不包含函数名。 | +| 缺省值 | 0 | + ## 区域相关 ### timezone From 3db1920ae587c28b9ec3c8e49985bf1bc8715cef Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Tue, 27 Sep 2022 19:56:51 +0800 Subject: [PATCH 27/71] chore: install new web service instead of nginxd (#17090) Partial of [TD-19267](https://jira.taosdata.com:18080/browse/TD-19268) --- packaging/cfg/nginxd.service | 22 --------- packaging/check_package.sh | 10 +--- packaging/tools/install.sh | 81 +++++---------------------------- packaging/tools/make_install.sh | 13 ++++++ packaging/tools/makepkg.sh | 23 ++++------ packaging/tools/remove.sh | 15 ------ 6 files changed, 34 insertions(+), 130 deletions(-) delete mode 100644 packaging/cfg/nginxd.service diff --git a/packaging/cfg/nginxd.service b/packaging/cfg/nginxd.service deleted file mode 100644 index 50bbc1a21d..0000000000 --- a/packaging/cfg/nginxd.service +++ /dev/null @@ -1,22 +0,0 @@ -[Unit] -Description=Nginx For TDengine Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=forking -PIDFile=/usr/local/nginxd/logs/nginx.pid -ExecStart=/usr/local/nginxd/sbin/nginx -ExecStop=/usr/local/nginxd/sbin/nginx -s stop -TimeoutStopSec=1000000s -LimitNOFILE=infinity -LimitNPROC=infinity -LimitCORE=infinity -TimeoutStartSec=0 -StandardOutput=null -Restart=always -StartLimitBurst=3 -StartLimitInterval=60s - -[Install] -WantedBy=multi-user.target diff --git a/packaging/check_package.sh b/packaging/check_package.sh index e728c6455a..5c3a2f9267 100644 --- a/packaging/check_package.sh +++ b/packaging/check_package.sh @@ -38,8 +38,6 @@ temp_version="" fin_result="" service_config_dir="/etc/systemd/system" -nginx_port=6060 -nginx_dir="/usr/local/nginxd" # Color setting RED='\033[0;31m' @@ -132,10 +130,7 @@ function check_main_path() { check_file ${install_main_dir} $i done if [ "$verMode" == "cluster" ]; then - nginx_main_dir=("admin" "conf" "html" "sbin" "logs") - for i in "${nginx_main_dir[@]}";do - check_file ${nginx_dir} $i - done + check_file ${install_main_dir} "share/admin" fi echo -e "Check main path:\033[32mOK\033[0m!" } @@ -150,9 +145,6 @@ function check_bin_path() { for i in "${lbin_dir[@]}";do check_link ${bin_link_dir}/$i done - if [ "$verMode" == "cluster" ]; then - check_file ${nginx_dir}/sbin nginx - fi echo -e "Check bin path:\033[32mOK\033[0m!" } diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 5088e9bdac..19abda8793 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -50,8 +50,7 @@ install_main_dir=${installDir} bin_dir="${installDir}/bin" service_config_dir="/etc/systemd/system" -nginx_port=6060 -nginx_dir="/usr/local/nginxd" +web_port=6041 # Color setting RED='\033[0;31m' @@ -182,7 +181,7 @@ function install_main_path() { ${csudo}mkdir -p ${install_main_dir}/include # ${csudo}mkdir -p ${install_main_dir}/init.d if [ "$verMode" == "cluster" ]; then - ${csudo}mkdir -p ${nginx_dir} + ${csudo}mkdir -p ${install_main_dir}/share fi if [[ -e ${script_dir}/email ]]; then @@ -218,12 +217,6 @@ function install_bin() { [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : - - if [ "$verMode" == "cluster" ]; then - ${csudo}cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo}chmod 0555 ${nginx_dir}/* - ${csudo}mkdir -p ${nginx_dir}/logs - ${csudo}chmod 777 ${nginx_dir}/sbin/nginx - fi } function install_lib() { @@ -574,6 +567,13 @@ function install_examples() { fi } +function install_web() { + if [ -d "${script_dir}/share" ]; then + ${csudo}cp -rf ${binary_dir}/share/* ${install_main_dir}/share + fi +} + + function clean_service_on_sysvinit() { if pidof ${serverName} &>/dev/null; then ${csudo}service ${serverName} stop || : @@ -654,16 +654,6 @@ function clean_service_on_systemd() { fi ${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null ${csudo}rm -f ${tarbitratord_service_config} - - if [ "$verMode" == "cluster" ]; then - nginx_service_config="${service_config_dir}/nginxd.service" - if systemctl is-active --quiet nginxd; then - echo "Nginx for ${productName} is running, stopping it..." - ${csudo}systemctl stop nginxd &>/dev/null || echo &>/dev/null - fi - ${csudo}systemctl disable nginxd &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${nginx_service_config} - fi } function install_service_on_systemd() { @@ -677,19 +667,6 @@ function install_service_on_systemd() { ${csudo}systemctl enable ${serverName} ${csudo}systemctl daemon-reload - - if [ "$verMode" == "cluster" ]; then - [ -f ${script_dir}/cfg/nginxd.service ] && - ${csudo}cp ${script_dir}/cfg/nginxd.service \ - ${service_config_dir}/ || : - ${csudo}systemctl daemon-reload - - if ! ${csudo}systemctl enable nginxd &>/dev/null; then - ${csudo}systemctl daemon-reexec - ${csudo}systemctl enable nginxd - fi - ${csudo}systemctl start nginxd - fi } function install_adapter_service() { @@ -793,19 +770,6 @@ function updateProduct() { sleep 1 fi - if [ "$verMode" == "cluster" ]; then - if pidof nginx &>/dev/null; then - if ((${service_mod} == 0)); then - ${csudo}systemctl stop nginxd || : - elif ((${service_mod} == 1)); then - ${csudo}service nginxd stop || : - else - kill_process nginx - fi - sleep 1 - fi - fi - install_main_path install_log @@ -817,6 +781,7 @@ function updateProduct() { fi install_examples + install_web if [ -z $1 ]; then install_bin install_service @@ -825,18 +790,6 @@ function updateProduct() { install_adapter_config openresty_work=false - if [ "$verMode" == "cluster" ]; then - # Check if openresty is installed - # Check if nginx is installed successfully - if type curl &>/dev/null; then - if curl -sSf http://127.0.0.1:${nginx_port} &>/dev/null; then - echo -e "\033[44;32;1mNginx for ${productName} is updated successfully!${NC}" - openresty_work=true - else - echo -e "\033[44;31;5mNginx for ${productName} does not work! Please try again!\033[0m" - fi - fi - fi echo echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}" @@ -857,7 +810,7 @@ function updateProduct() { fi if [ ${openresty_work} = 'true' ]; then - echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}" else echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell${NC}" fi @@ -906,6 +859,7 @@ function installProduct() { install_connector fi install_examples + install_web if [ -z $1 ]; then # install service and client # For installing new @@ -915,17 +869,6 @@ function installProduct() { install_adapter_config openresty_work=false - if [ "$verMode" == "cluster" ]; then - # Check if nginx is installed successfully - if type curl &>/dev/null; then - if curl -sSf http://127.0.0.1:${nginx_port} &>/dev/null; then - echo -e "\033[44;32;1mNginx for ${productName} is installed successfully!${NC}" - openresty_work=true - else - echo -e "\033[44;31;5mNginx for ${productName} does not work! Please try again!\033[0m" - fi - fi - fi install_config diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index d1e7a222cc..58b4385001 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -151,6 +151,7 @@ function install_main_path() { ${csudo}mkdir -p ${install_main_dir}/driver ${csudo}mkdir -p ${install_main_dir}/examples ${csudo}mkdir -p ${install_main_dir}/include + ${csudo}mkdir -p ${install_main_dir}/share # ${csudo}mkdir -p ${install_main_dir}/init.d else ${csudo}rm -rf ${install_main_dir} || ${csudo}rm -rf ${install_main_2_dir} || : @@ -161,6 +162,7 @@ function install_main_path() { ${csudo}mkdir -p ${install_main_dir}/driver || ${csudo}mkdir -p ${install_main_2_dir}/driver ${csudo}mkdir -p ${install_main_dir}/examples || ${csudo}mkdir -p ${install_main_2_dir}/examples ${csudo}mkdir -p ${install_main_dir}/include || ${csudo}mkdir -p ${install_main_2_dir}/include + ${csudo}mkdir -p ${install_main_dir}/share || ${csudo}mkdir -p ${install_main_2_dir}/share fi } @@ -469,6 +471,16 @@ function install_examples() { fi } +function install_web() { + if [ -d "${binary_dir}/build/share" ]; then + if [ "$osType" != "Darwin" ]; then + ${csudo}cp -rf ${binary_dir}/build/share/* ${install_main_dir}/share || : + else + ${csudo}cp -rf ${binary_dir}/build/share/* ${install_main_dir}/share || ${csudo}cp -rf ${binary_dir}/build/share/* ${install_main_2_dir}/share || : + fi + fi +} + function clean_service_on_sysvinit() { if pidof ${serverName} &>/dev/null; then ${csudo}service ${serverName} stop || : @@ -596,6 +608,7 @@ function update_TDengine() { install_lib # install_connector install_examples + install_web install_bin install_service diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 2305b96b36..fb0ef4f9a3 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -107,7 +107,7 @@ else fi install_files="${script_dir}/install.sh" -nginx_dir="${top_dir}/../enterprise/src/plugins/web" +web_dir="${top_dir}/../enterprise/src/plugins/web" init_file_deb=${script_dir}/../deb/taosd init_file_rpm=${script_dir}/../rpm/taosd @@ -132,10 +132,6 @@ if [ -f "${cfg_dir}/${serverName}.service" ]; then cp ${cfg_dir}/${serverName}.service ${install_dir}/cfg || : fi -if [ -f "${top_dir}/packaging/cfg/nginxd.service" ]; then - cp ${top_dir}/packaging/cfg/nginxd.service ${install_dir}/cfg || : -fi - mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm @@ -222,16 +218,6 @@ if [ "$verMode" == "cluster" ]; then sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh mv remove_temp.sh ${install_dir}/bin/remove.sh - mkdir -p ${install_dir}/nginxd && cp -r ${nginx_dir}/* ${install_dir}/nginxd - cp ${nginx_dir}/png/taos.png ${install_dir}/nginxd/admin/images/taos.png - rm -rf ${install_dir}/nginxd/png - - if [ "$cpuType" == "aarch64" ]; then - cp -f ${install_dir}/nginxd/sbin/arm/64bit/nginx ${install_dir}/nginxd/sbin/ - elif [ "$cpuType" == "aarch32" ]; then - cp -f ${install_dir}/nginxd/sbin/arm/32bit/nginx ${install_dir}/nginxd/sbin/ - fi - rm -rf ${install_dir}/nginxd/sbin/arm fi cd ${install_dir} @@ -288,6 +274,13 @@ if [[ $dbName == "taos" ]]; then cp -r ${examples_dir}/C# ${install_dir}/examples mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json fi + + # Add web files + if [ -d "${web_dir}/admin" ]; then + mkdir -p ${install_dir}/share/ + cp ${web_dir}/admin ${install_dir}/share/ -r + cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png + fi fi # Copy driver diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index bf4ff564f5..a648750904 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -27,13 +27,11 @@ local_bin_link_dir="/usr/local/bin" lib_link_dir="/usr/lib" lib64_link_dir="/usr/lib64" inc_link_dir="/usr/include" -install_nginxd_dir="/usr/local/nginxd" service_config_dir="/etc/systemd/system" taos_service_name=${serverName} taosadapter_service_name="taosadapter" tarbitrator_service_name="tarbitratord" -nginx_service_name="nginxd" csudo="" if command -v sudo >/dev/null; then csudo="sudo " @@ -153,18 +151,6 @@ function clean_service_on_systemd() { fi ${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null ${csudo}rm -f ${tarbitratord_service_config} - - if [ "$verMode" == "cluster" ]; then - nginx_service_config="${service_config_dir}/${nginx_service_name}.service" - if [ -d ${install_nginxd_dir} ]; then - if systemctl is-active --quiet ${nginx_service_name}; then - echo "Nginx for ${productName} is running, stopping it..." - ${csudo}systemctl stop ${nginx_service_name} &>/dev/null || echo &>/dev/null - fi - ${csudo}systemctl disable ${nginx_service_name} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${nginx_service_config} - fi - fi } function clean_service_on_sysvinit() { @@ -239,7 +225,6 @@ clean_config ${csudo}rm -rf ${data_link_dir} || : ${csudo}rm -rf ${install_main_dir} -${csudo}rm -rf ${install_nginxd_dir} if [[ -e /etc/os-release ]]; then osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) else From 636004199b0e592817c5e3d0eb3f295d16a1e171 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 27 Sep 2022 21:32:18 +0800 Subject: [PATCH 28/71] fix(idx): fix index read crash --- source/libs/index/src/indexCache.c | 1 + source/libs/index/src/indexFstFile.c | 42 +++++++++++++++++----------- source/libs/index/test/jsonUT.cc | 4 +-- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 7e867db755..39bba4e269 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -302,6 +302,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR char* p = taosMemoryCalloc(1, strlen(c->colVal) + 1); memcpy(p, c->colVal, strlen(c->colVal)); cond = cmpFn(p + skip, term->colVal, dType); + taosMemoryFree(p); } } if (cond == MATCH) { diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index 2a33ddd477..7021fdfae3 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -69,6 +69,8 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of int32_t blkOffset = offset % kBlockSize; int32_t blkLeft = kBlockSize - blkOffset; + if (offset >= ctx->file.size) return 0; + do { char key[128] = {0}; idxGenLRUKey(key, ctx->file.buf, blkId); @@ -80,24 +82,34 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of memcpy(buf + total, blk->buf + blkOffset, nread); taosLRUCacheRelease(ctx->lru, h, false); } else { - int32_t cacheMemSize = sizeof(SDataBlock) + kBlockSize; + int32_t left = ctx->file.size - offset; + if (left < kBlockSize) { + nread = TMIN(left, len); + int32_t bytes = taosPReadFile(ctx->file.pFile, buf + total, nread, offset); + assert(bytes == nread); - SDataBlock* blk = taosMemoryCalloc(1, cacheMemSize); - blk->blockId = blkId; - blk->nread = taosPReadFile(ctx->file.pFile, blk->buf, kBlockSize, blkId * kBlockSize); - assert(blk->nread <= kBlockSize); + total += bytes; + return total; + } else { + int32_t cacheMemSize = sizeof(SDataBlock) + kBlockSize; - if (blk->nread < kBlockSize && blk->nread < len) { - break; - } + SDataBlock* blk = taosMemoryCalloc(1, cacheMemSize); + blk->blockId = blkId; + blk->nread = taosPReadFile(ctx->file.pFile, blk->buf, kBlockSize, blkId * kBlockSize); + assert(blk->nread <= kBlockSize); - nread = TMIN(blkLeft, len); - memcpy(buf + total, blk->buf + blkOffset, nread); + if (blk->nread < kBlockSize && blk->nread < len) { + break; + } - LRUStatus s = taosLRUCacheInsert(ctx->lru, key, strlen(key), blk, cacheMemSize, deleteDataBlockFromLRU, NULL, - TAOS_LRU_PRIORITY_LOW); - if (s != TAOS_LRU_STATUS_OK) { - return -1; + nread = TMIN(blkLeft, len); + memcpy(buf + total, blk->buf + blkOffset, nread); + + LRUStatus s = taosLRUCacheInsert(ctx->lru, key, strlen(key), blk, cacheMemSize, deleteDataBlockFromLRU, NULL, + TAOS_LRU_PRIORITY_LOW); + if (s != TAOS_LRU_STATUS_OK) { + return -1; + } } } total += nread; @@ -146,9 +158,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int } else { ctx->file.pFile = taosOpenFile(path, TD_FILE_READ); - int64_t size = 0; taosFStatFile(ctx->file.pFile, &ctx->file.size, NULL); - ctx->file.size = (int)size; #ifdef USE_MMAP ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size); #endif diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 1911514d97..8ae3fd4135 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -172,9 +172,9 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx"); - for (int i = 0; i < 10; i++) { + for (int i = 0; i < 10000; i++) { colVal[i % colVal.size()] = '0' + i % 128; - for (size_t i = 0; i < 100; i++) { + for (size_t i = 0; i < 10; i++) { SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); From 6f08407def9e32d6a2ca5d1e39cef8e77aad5257 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 28 Sep 2022 01:10:09 +0800 Subject: [PATCH 29/71] feat: taosbenchmark support ttl (#17091) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 448c2b99c0..5d2fcf27b2 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG f03c09a + GIT_TAG 70f5a1c SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 0841442db68ea8de4316764e365deaf100eee237 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 28 Sep 2022 07:41:38 +0800 Subject: [PATCH 30/71] feat: logic optimization for data migration --- source/dnode/vnode/src/inc/tsdb.h | 4 +-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 24 ++++++++++++---- source/dnode/vnode/src/tsdb/tsdbFS.c | 18 +++++++----- source/dnode/vnode/src/tsdb/tsdbRetention.c | 32 +++++++++++++++------ 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ab940b2107..180be0ff6e 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -322,6 +322,7 @@ int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSc // structs ======================= struct STsdbFS { + int64_t version; SDelFile *pDelFile; SArray *aDFileSet; // SArray }; @@ -575,9 +576,6 @@ struct SDFileSet { SSttFile *aSttF[TSDB_MAX_STT_TRIGGER]; }; -#define SET_DFSET_EXPIRED(d) ((d)->diskId.id = -1) -#define IS_DFSET_EXPIRED(d) ((d)->diskId.id == -1) - struct SRowIter { TSDBROW *pRow; STSchema *pTSchema; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 6fa8d503b7..447efcebf9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -988,21 +988,33 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { int32_t code = 0; STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; - bool inTrim = atomic_load_8(&pTsdb->pVnode->trimDbH.state); + STsdbFS fsLatest = {0}; ASSERT(eno == 0); - if (inTrim) taosThreadRwlockWrlock(&pTsdb->rwLock); + // lock + taosThreadRwlockWrlock(&pTsdb->rwLock); + + ASSERT(pCommitter->fs.version <= pTsdb->fs.version); + + if (pCommitter->fs.version < pTsdb->fs.version) { + if ((code = tsdbFSCopy(pTsdb, &fsLatest))) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _err; + } + + if ((code = tsdbFSUpdDel(pTsdb, &pCommitter->fs, &fsLatest, pTsdb->trimHdl.minCommitFid - 1))) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _err; + } + } code = tsdbFSCommit1(pTsdb, &pCommitter->fs); if (code) { - if (inTrim) taosThreadRwlockUnlock(&pTsdb->rwLock); + taosThreadRwlockUnlock(&pTsdb->rwLock); goto _err; } - // lock - if (!inTrim) taosThreadRwlockWrlock(&pTsdb->rwLock); - // commit or rollback code = tsdbFSCommit2(pTsdb, &pCommitter->fs); if (code) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index d02cbdc089..ef4f6930be 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -419,6 +419,7 @@ int32_t tsdbFSOpen(STsdb *pTsdb) { int32_t code = 0; // open handle + pTsdb->fs.version = 0; pTsdb->fs.pDelFile = NULL; pTsdb->fs.aDFileSet = taosArrayInit(0, sizeof(SDFileSet)); if (pTsdb->fs.aDFileSet == NULL) { @@ -534,6 +535,7 @@ int32_t tsdbFSClose(STsdb *pTsdb) { int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { int32_t code = 0; + pFS->version = pTsdb->fs.version; pFS->pDelFile = NULL; pFS->aDFileSet = taosArrayInit(taosArrayGetSize(pTsdb->fs.aDFileSet), sizeof(SDFileSet)); if (pFS->aDFileSet == NULL) { @@ -746,12 +748,12 @@ int32_t tsdbFSUpdDel(STsdb *pTsdb, STsdbFS *pFS, STsdbFS *pFSNew, int32_t maxFid if (pSetOld && pSetNew) { if (pSetOld->fid == pSetNew->fid) { - if (IS_DFSET_EXPIRED(pSetNew)) goto _remove_old; goto _merge_migrate; - } else if (pSetOld->fid < pSetNew->fid) { - ++iOld; + } else if (pSetOld->fid > pSetNew->fid) { + goto _remove_old; } else { - ++iNew; + ++iOld; + ASSERT(0); } continue; } else { @@ -794,8 +796,8 @@ int32_t tsdbFSUpdDel(STsdb *pTsdb, STsdbFS *pFS, STsdbFS *pFSNew, int32_t maxFid pSetOld->diskId = pSetNew->diskId; } - iOld++; - iNew++; + ++iOld; + ++iNew; continue; _remove_old: @@ -806,7 +808,7 @@ int32_t tsdbFSUpdDel(STsdb *pTsdb, STsdbFS *pFS, STsdbFS *pFSNew, int32_t maxFid } taosMemoryFree(pSetOld->pSmaF); taosArrayRemove(pFS->aDFileSet, iOld); - iNew++; + ++iNew; continue; } @@ -850,6 +852,8 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { int32_t nRef; char fname[TSDB_FILENAME_LEN]; + ++pTsdb->fs.version; + // del if (pFSNew->pDelFile) { SDelFile *pDelFile = pTsdb->fs.pDelFile; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 971b950975..fbe16b6a42 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -17,7 +17,7 @@ enum { RETENTION_NO = 0, RETENTION_EXPIRED = 1, RETENTION_MIGRATE = 2 }; -#define MIGRATE_MAX_SPEED (1048576 << 5) // 32 MB +#define MIGRATE_MAX_SPEED (1048576 << 4) // 16 MB, vnode level #define MIGRATE_MIN_COST (5) // second static bool tsdbShouldDoMigrate(STsdb *pTsdb); @@ -105,8 +105,8 @@ _retention_loop: code = tsdbFSCopy(pTsdb, &fs); if (code) goto _exit; - int32_t fsSize = taosArrayGetSize(fs.aDFileSet); if (type == RETENTION_MIGRATE) { + int32_t fsSize = taosArrayGetSize(fs.aDFileSet); for (int32_t iSet = 0; iSet < fsSize; ++iSet) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); @@ -119,26 +119,35 @@ _retention_loop: maxFid = pSet->fid; fSize += (pSet->pDataF->size + pSet->pHeadF->size + pSet->pSmaF->size); if (fSize / speed > MIGRATE_MIN_COST) { + tsdbDebug("vgId:%d migrate loop %d with maxFid:%d", TD_VID(pTsdb->pVnode), nBatch, maxFid); break; } for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) { fSize += pSet->aSttF[iStt]->size; } if (fSize / speed > MIGRATE_MIN_COST) { - tsdbDebug("vgId:%d migrate loop[%d] with maxFid:%d", TD_VID(pTsdb->pVnode), nBatch, maxFid); + tsdbDebug("vgId:%d migrate loop %d with maxFid:%d", TD_VID(pTsdb->pVnode), nBatch, maxFid); break; } } } } else if (type == RETENTION_EXPIRED) { - for (int32_t iSet = 0; iSet < fsSize; ++iSet) { + for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); ++iSet) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); SDiskID did; if (expLevel < 0) { - SET_DFSET_EXPIRED(pSet); + ASSERT(pSet->fid > maxFid); if (pSet->fid > maxFid) maxFid = pSet->fid; + taosMemoryFree(pSet->pHeadF); + taosMemoryFree(pSet->pDataF); + taosMemoryFree(pSet->pSmaF); + for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) { + taosMemoryFree(pSet->aSttF[iStt]); + } + taosArrayRemove(fs.aDFileSet, iSet); + --iSet; } else { break; } @@ -167,18 +176,25 @@ _commit_conflict_check: // migrate if (type == RETENTION_MIGRATE) { - for (int32_t iSet = 0; iSet < fsSize; ++iSet) { + for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); ++iSet) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); SDiskID did; if (pSet->fid > maxFid) break; - tsdbDebug("vgId:%d migrate loop[%d] with maxFid:%d, fid:%d, did:%d, level:%d, expLevel:%d", TD_VID(pTsdb->pVnode), + tsdbDebug("vgId:%d migrate loop %d with maxFid:%d, fid:%d, did:%d, level:%d, expLevel:%d", TD_VID(pTsdb->pVnode), nBatch, maxFid, pSet->fid, pSet->diskId.id, pSet->diskId.level, expLevel); if (expLevel < 0) { - SET_DFSET_EXPIRED(pSet); + taosMemoryFree(pSet->pHeadF); + taosMemoryFree(pSet->pDataF); + taosMemoryFree(pSet->pSmaF); + for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) { + taosMemoryFree(pSet->aSttF[iStt]); + } + taosArrayRemove(fs.aDFileSet, iSet); + --iSet; } else { if (expLevel == pSet->diskId.level) continue; From e5abe59bb1547b5c27256aa8f64d5c105b64aa3a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 28 Sep 2022 09:29:54 +0800 Subject: [PATCH 31/71] fix: coverity issues --- source/dnode/mgmt/mgmt_vnode/src/vmFile.c | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 12 +++++++++--- source/dnode/mnode/sdb/src/sdbFile.c | 6 ++++-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index 25e85227e3..82fc286a94 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -135,6 +135,7 @@ _OVER: if (content != NULL) taosMemoryFree(content); if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); + if (*ppCfgs == NULL && pCfgs != NULL) taosMemoryFree(pCfgs); terrno = code; return code; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index e3ad1b9389..4047bc2340 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -175,7 +175,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->hashSuffix = pCreate->hashSuffix; pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024; - pCfg->standby = pCfg->standby; + pCfg->standby = 0; pCfg->syncCfg.myIndex = pCreate->selfIndex; pCfg->syncCfg.replicaNum = pCreate->replica; memset(&pCfg->syncCfg.nodeInfo, 0, sizeof(pCfg->syncCfg.nodeInfo)); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index f0c43d8b36..19ed2cbc88 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -58,11 +58,14 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { if (pVnode->path == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pVnode); return -1; } if (vmAllocQueue(pMgmt, pVnode) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pVnode->path); + taosMemoryFree(pVnode); return -1; } @@ -221,6 +224,7 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) { SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); for (int32_t i = 0; i < numOfVnodes; ++i) { + if (ppVnodes == NULL || ppVnodes[i] == NULL) continue; vmCloseVnode(pMgmt, ppVnodes[i]); } @@ -380,7 +384,9 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) { for (int32_t v = 0; v < numOfVnodes; ++v) { int32_t t = v % threadNum; SVnodeThread *pThread = &threads[t]; - pThread->ppVnodes[pThread->vnodeNum++] = ppVnodes[v]; + if (pThread->ppVnodes != NULL) { + pThread->ppVnodes[pThread->vnodeNum++] = ppVnodes[v]; + } } pMgmt->state.openVnodes = 0; @@ -411,8 +417,8 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) { taosMemoryFree(threads); for (int32_t i = 0; i < numOfVnodes; ++i) { - SVnodeObj *pVnode = ppVnodes[i]; - vmReleaseVnode(pMgmt, pVnode); + if (ppVnodes == NULL || ppVnodes[i] == NULL) continue; + vmReleaseVnode(pMgmt, ppVnodes[i]); } if (ppVnodes != NULL) { diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index ff0c4b26ef..b5cfa7b0f6 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -514,7 +514,7 @@ static void sdbCloseIter(SSdbIter *pIter) { } if (pIter->name != NULL) { - taosRemoveFile(pIter->name); + (void)taosRemoveFile(pIter->name); taosMemoryFree(pIter->name); pIter->name = NULL; } @@ -606,6 +606,7 @@ int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter) { if (pIter->file == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to open %s since %s", pIter->name, terrstr()); + sdbCloseIter(pIter); return -1; } @@ -636,9 +637,9 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i return -1; } - sdbCloseIter(pIter); if (sdbReadFile(pSdb) != 0) { mError("sdbiter:%p, failed to read from %s since %s", pIter, datafile, terrstr()); + sdbCloseIter(pIter); return -1; } @@ -656,6 +657,7 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i } mInfo("sdbiter:%p, success applyed to sdb", pIter); + sdbCloseIter(pIter); return 0; } From b78a39ced100ea70586505838a981b41e20a5941 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 28 Sep 2022 09:47:51 +0800 Subject: [PATCH 32/71] fix: resource release when end commit --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 447efcebf9..0bfb90a374 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1000,26 +1000,26 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { if (pCommitter->fs.version < pTsdb->fs.version) { if ((code = tsdbFSCopy(pTsdb, &fsLatest))) { taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; + goto _exit; } if ((code = tsdbFSUpdDel(pTsdb, &pCommitter->fs, &fsLatest, pTsdb->trimHdl.minCommitFid - 1))) { taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; + goto _exit; } } code = tsdbFSCommit1(pTsdb, &pCommitter->fs); if (code) { taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; + goto _exit; } // commit or rollback code = tsdbFSCommit2(pTsdb, &pCommitter->fs); if (code) { taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; + goto _exit; } pTsdb->imem = NULL; @@ -1027,8 +1027,10 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { // unlock taosThreadRwlockUnlock(&pTsdb->rwLock); +_exit: tsdbUnrefMemTable(pMemTable); tsdbFSDestroy(&pCommitter->fs); + tsdbFSDestroy(&fsLatest); taosArrayDestroy(pCommitter->aTbDataP); atomic_store_32(&pTsdb->trimHdl.minCommitFid, INT32_MAX); @@ -1036,13 +1038,12 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { // code = tsdbMerge(pTsdb); // if (code) goto _err; // } - - tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); - return code; - -_err: - atomic_store_32(&pTsdb->trimHdl.minCommitFid, INT32_MAX); - tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if(code == 0) { + tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); + } else { + tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + } + return code; } From 14412f28118e1b6626b984f8ca9ba9282dba5f16 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 28 Sep 2022 11:16:52 +0800 Subject: [PATCH 33/71] refactor(query): do some internal refactor. --- source/dnode/vnode/src/inc/tsdb.h | 4 ++-- source/dnode/vnode/src/tsdb/tsdbCache.c | 5 ++++- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 20 +++++--------------- source/dnode/vnode/src/tsdb/tsdbRead.c | 3 +-- 4 files changed, 12 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 916311bbee..75b2f74096 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -662,8 +662,8 @@ typedef struct { } SSkmInfo; int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, - STimeWindow *pTimeWindow, SVersionRange *pVerRange, void *pBlockLoadInfo, STSchema *pSchema, - int16_t *pCols, int32_t numOfCols, const char *idStr); + STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, + bool destroyLoadInfo, const char *idStr); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); bool tMergeTreeNext(SMergeTree *pMTree); TSDBROW tMergeTreeGetRow(SMergeTree *pMTree); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 775f452864..3a921349e6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -420,6 +420,7 @@ typedef enum { typedef struct { SFSLASTNEXTROWSTATES state; // [input] STsdb *pTsdb; // [input] + STSchema *pTSchema;// [input] tb_uid_t suid; tb_uid_t uid; int32_t nFileSet; @@ -455,9 +456,10 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { code = tsdbDataFReaderOpen(&state->pDataFReader, state->pTsdb, pFileSet); if (code) goto _err; + SSttBlockLoadInfo* pLoadInfo = tCreateLastBlockLoadInfo(state->pTSchema, NULL, 0); tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->suid, state->uid, &(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX}, - &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, NULL, NULL, NULL, 0, NULL); + &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, pLoadInfo,true, NULL); bool hasVal = tMergeTreeNext(&state->mergeTree); if (!hasVal) { state->state = SFSLASTNEXTROW_FILESET; @@ -892,6 +894,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->fsLastState.state = (SFSLASTNEXTROWSTATES)SFSNEXTROW_FS; pIter->fsLastState.pTsdb = pTsdb; pIter->fsLastState.aDFileSet = pIter->pReadSnap->fs.aDFileSet; + pIter->fsLastState.pTSchema = pTSchema; pIter->fsLastState.suid = suid; pIter->fsLastState.uid = uid; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 126d2d729b..7e6a0d04ff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -474,8 +474,8 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { } int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, - STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pBlockLoadInfo, STSchema* pSchema, - int16_t* pCols, int32_t numOfCols, const char* idStr) { + STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, + bool destroyLoadInfo, const char *idStr) { pMTree->backward = backward; pMTree->pIter = NULL; pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); @@ -488,22 +488,12 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); int32_t code = TSDB_CODE_SUCCESS; - SSttBlockLoadInfo* pLoadInfo = NULL; - if (pBlockLoadInfo == NULL) { - ASSERT(0); - if (pMTree->pLoadInfo == NULL) { - pMTree->destroyLoadInfo = true; - pMTree->pLoadInfo = tCreateLastBlockLoadInfo(pSchema, pCols, numOfCols); - } - - pLoadInfo = pMTree->pLoadInfo; - } else { - pLoadInfo = pBlockLoadInfo; - } + pMTree->pLoadInfo = pBlockLoadInfo; + pMTree->destroyLoadInfo = destroyLoadInfo; for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file struct SLDataIter* pIter = NULL; - code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, &pLoadInfo[i]); + code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, &pMTree->pLoadInfo[i]); if (code != TSDB_CODE_SUCCESS) { goto _end; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f9b65b6527..2ea6a57b9e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2005,8 +2005,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan int32_t code = tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader, - pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo, - pReader->pSchema, pReader->suppInfo.colIds, pReader->suppInfo.numOfCols, pReader->idStr); + pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo, false, pReader->idStr); if (code != TSDB_CODE_SUCCESS) { return false; } From 395c80b2ade02a350157f849b63ca58e3edc56c7 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Wed, 28 Sep 2022 11:19:02 +0800 Subject: [PATCH 34/71] chore: fix install.sh for enterprise --- packaging/tools/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 19abda8793..9694a89a8f 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -569,7 +569,7 @@ function install_examples() { function install_web() { if [ -d "${script_dir}/share" ]; then - ${csudo}cp -rf ${binary_dir}/share/* ${install_main_dir}/share + ${csudo}cp -rf ${script_dir}/share/* ${install_main_dir}/share fi } From c2d67ae183371fdd1584d8fea474834907c48c81 Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Wed, 28 Sep 2022 11:46:54 +0800 Subject: [PATCH 35/71] Update subscribeDb3.py test: fix TD-19276 --- tests/system-test/7-tmq/subscribeDb3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/subscribeDb3.py b/tests/system-test/7-tmq/subscribeDb3.py index e8e475456c..0ff609ab6b 100644 --- a/tests/system-test/7-tmq/subscribeDb3.py +++ b/tests/system-test/7-tmq/subscribeDb3.py @@ -295,7 +295,7 @@ class TDTestCase: ifManualCommit = 1 keyList = 'group.id:cgrp1,\ enable.auto.commit:true,\ - auto.commit.interval.ms:1000,\ + auto.commit.interval.ms:200,\ auto.offset.reset:earliest' self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) From 1c02d5733a9f2513afe8c500e9bcdce748df24d1 Mon Sep 17 00:00:00 2001 From: WANG MINGMING Date: Wed, 28 Sep 2022 11:47:09 +0800 Subject: [PATCH 36/71] Update 13-schemaless.md --- docs/zh/14-reference/13-schemaless/13-schemaless.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md index a33abafaf8..07d5b0692b 100644 --- a/docs/zh/14-reference/13-schemaless/13-schemaless.md +++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md @@ -67,9 +67,12 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 "measurement,tag_key1=tag_value1,tag_key2=tag_value2" ``` +:::tip 需要注意的是，这里的 tag_key1, tag_key2 并不是用户输入的标签的原始顺序，而是使用了标签名称按照字符串升序排列后的结果。所以，tag_key1 并不是在行协议中输入的第一个标签。 排列完成以后计算该字符串的 MD5 散列值 "md5_val"。然后将计算的结果与字符串组合生成表名：“t_md5_val”。其中的 “t_” 是固定的前缀，每个通过该映射关系自动生成的表都具有该前缀。 -为了让用户可以指定生成的表名，可以通过配置 smlChildTableName 来指定（比如 配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一次自动建表时指定的 tag_set，其他的会忽略）。 +:::tip +为了让用户可以指定生成的表名，可以通过在taos.cfg里配置 smlChildTableName 参数来指定。 +举例如下：配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 2. 如果解析行协议获得的超级表不存在，则会创建这个超级表（不建议手动创建超级表，不然插入数据可能异常）。 3. 如果解析行协议获得子表不存在，则 Schemaless 会按照步骤 1 或 2 确定的子表名来创建子表。 From 71127891eb8e89b88de481d8ad36421160237634 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 28 Sep 2022 11:51:31 +0800 Subject: [PATCH 37/71] fix:smlDataFormat to false --- source/common/src/tglobal.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index e4128c2474..c0203dadb8 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -85,8 +85,7 @@ uint16_t tsTelemPort = 80; char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null"; char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table name can be specified in tag value. // If set to empty system will generate table name using MD5 hash. -bool tsSmlDataFormat = - true; // true means that the name and order of cols in each line are the same(only for influx protocol) +bool tsSmlDataFormat = false; // true means that the name and order of cols in each line are the same(only for influx protocol) // query int32_t tsQueryPolicy = 1; From d3c335a06e1bf2d23590f1d3f8e9de93f1590949 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 28 Sep 2022 12:56:17 +0800 Subject: [PATCH 38/71] chore: fs merge optimization for data migrate --- source/dnode/vnode/src/tsdb/tsdbRetention.c | 29 ++++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index fbe16b6a42..f3890b7b53 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -221,22 +221,31 @@ _commit_conflict_check: _merge_fs: taosThreadRwlockWrlock(&pTsdb->rwLock); - if ((code = tsdbFSCopy(pTsdb, &fsLatest))) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; - } - // 1) merge tsdbFSNew and pTsdb->fs - if ((code = tsdbFSUpdDel(pTsdb, &fsLatest, &fs, maxFid))) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; + // 1) prepare fs, merge tsdbFSNew and pTsdb->fs if needed + STsdbFS *pTsdbFS = &fs; + ASSERT(fs.version <= pTsdb->fs.version); + + if (fs.version < pTsdb->fs.version) { + if ((code = tsdbFSCopy(pTsdb, &fsLatest))) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _exit; + } + + if ((code = tsdbFSUpdDel(pTsdb, &fsLatest, &fs, maxFid))) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _exit; + } + pTsdbFS = &fsLatest; } + // 2) save CURRENT - if ((code = tsdbFSCommit1(pTsdb, &fsLatest))) { + if ((code = tsdbFSCommit1(pTsdb, pTsdbFS))) { taosThreadRwlockUnlock(&pTsdb->rwLock); goto _exit; } + // 3) apply the tsdbFS to pTsdb->fs - if ((code = tsdbFSCommit2(pTsdb, &fsLatest))) { + if ((code = tsdbFSCommit2(pTsdb, pTsdbFS))) { taosThreadRwlockUnlock(&pTsdb->rwLock); goto _exit; } From b71a9e34831b607c283fe8eec08f32fcad624b1d Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 27 Sep 2022 19:58:19 +0800 Subject: [PATCH 39/71] enh: improve error handling in syncNodeOpen --- source/dnode/vnode/src/vnd/vnodeOpen.c | 1 - source/libs/sync/inc/syncUtil.h | 2 +- source/libs/sync/src/syncIndexMgr.c | 7 +- source/libs/sync/src/syncMain.c | 141 +++++++++++++++++-------- source/libs/sync/src/syncRaftCfg.c | 2 - source/libs/sync/src/syncRaftStore.c | 6 +- source/libs/sync/src/syncRespMgr.c | 4 + source/libs/sync/src/syncSnapshot.c | 5 +- source/libs/sync/src/syncUtil.c | 14 ++- source/libs/sync/src/syncVoteMgr.c | 5 +- 10 files changed, 131 insertions(+), 56 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 4ccfea4051..69785570f7 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -161,7 +161,6 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // open sync if (vnodeSyncOpen(pVnode, dir)) { vError("vgId:%d, failed to open sync since %s", TD_VID(pVnode), tstrerror(terrno)); - terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; } diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 7ecff7ae97..96e22720e8 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -32,7 +32,7 @@ uint64_t syncUtilAddr2U64(const char* host, uint16_t port); void syncUtilU642Addr(uint64_t u64, char* host, size_t len, uint16_t* port); void syncUtilnodeInfo2EpSet(const SNodeInfo* pNodeInfo, SEpSet* pEpSet); void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); -void syncUtilnodeInfo2raftId(const SNodeInfo* pNodeInfo, SyncGroupId vgId, SRaftId* raftId); +bool syncUtilnodeInfo2raftId(const SNodeInfo* pNodeInfo, SyncGroupId vgId, SRaftId* raftId); bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2); bool syncUtilEmptyId(const SRaftId* pId); diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 3bda9bcd51..28b5313ac5 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -20,7 +20,10 @@ SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) { SSyncIndexMgr *pSyncIndexMgr = taosMemoryMalloc(sizeof(SSyncIndexMgr)); - ASSERT(pSyncIndexMgr != NULL); + if (pSyncIndexMgr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } memset(pSyncIndexMgr, 0, sizeof(SSyncIndexMgr)); pSyncIndexMgr->replicas = &(pSyncNode->replicasId); @@ -248,4 +251,4 @@ SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftI } ASSERT(0); return -1; -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 17157fbd23..3d79c6c464 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -51,15 +51,17 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths); int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); -// life cycle -static void syncFreeNode(void* param); // --------------------------------- +static void syncNodeFreeCb(void *param) { + syncNodeClose(param); + param = NULL; +} int32_t syncInit() { int32_t ret = 0; if (!syncEnvIsStart()) { - tsNodeRefId = taosOpenRef(200, syncFreeNode); + tsNodeRefId = taosOpenRef(200, syncNodeFreeCb); if (tsNodeRefId < 0) { sError("failed to init node ref"); syncCleanUp(); @@ -86,11 +88,15 @@ void syncCleanUp() { int64_t syncOpen(const SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); - ASSERT(pSyncNode != NULL); + if (pSyncNode == NULL) { + sError("failed to open sync node. vgId:%d", pSyncInfo->vgId); + return -1; + } pSyncNode->rid = taosAddRef(tsNodeRefId, pSyncNode); if (pSyncNode->rid < 0) { - syncFreeNode(pSyncNode); + syncNodeClose(pSyncNode); + pSyncNode = NULL; return -1; } @@ -136,11 +142,9 @@ void syncStartStandBy(int64_t rid) { void syncStop(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) return; - int32_t vgId = pSyncNode->vgId; - syncNodeClose(pSyncNode); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + taosRemoveRef(tsNodeRefId, rid); sDebug("vgId:%d, sync rid:%" PRId64 " is removed from rsetId:%" PRId64, vgId, rid, tsNodeRefId); } @@ -210,7 +214,7 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR; - sError("syncNodeCheckNewConfig error"); + sError("invalid new config. vgId:%d", pSyncNode->vgId); return -1; } @@ -237,7 +241,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR; - sError("syncNodeCheckNewConfig error"); + sError("invalid new config. vgId:%d", pSyncNode->vgId); return -1; } @@ -941,16 +945,18 @@ _END: SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { SSyncInfo* pSyncInfo = (SSyncInfo*)pOldSyncInfo; - SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); - ASSERT(pSyncNode != NULL); - memset(pSyncNode, 0, sizeof(SSyncNode)); + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryCalloc(1, sizeof(SSyncNode)); + if (pSyncNode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } int32_t ret = 0; if (!taosDirExist((char*)(pSyncInfo->path))) { if (taosMkDir(pSyncInfo->path) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); sError("failed to create dir:%s since %s", pSyncInfo->path, terrstr()); - return NULL; + goto _error; } } @@ -963,15 +969,21 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { meta.lastConfigIndex = SYNC_INDEX_INVALID; meta.batchSize = pSyncInfo->batchSize; ret = raftCfgCreateFile((SSyncCfg*)&(pSyncInfo->syncCfg), meta, pSyncNode->configPath); - ASSERT(ret == 0); - + if (ret != 0) { + sError("failed to create raft cfg file. configPath: %s", pSyncNode->configPath); + goto _error; + } } else { // update syncCfg by raft_config.json pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath); - ASSERT(pSyncNode->pRaftCfg != NULL); + if (pSyncNode->pRaftCfg == NULL) { + sError("failed to open raft cfg file. path:%s", pSyncNode->configPath); + goto _error; + } pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg; raftCfgClose(pSyncNode->pRaftCfg); + pSyncNode->pRaftCfg = NULL; } // init by SSyncInfo @@ -988,11 +1000,17 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { // init raft config pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath); - ASSERT(pSyncNode->pRaftCfg != NULL); + if (pSyncNode->pRaftCfg == NULL) { + sError("failed to open raft cfg file. path:%s", pSyncNode->configPath); + goto _error; + } // init internal pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; - syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); + if (!syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId)) { + sError("failed to determine my raft member id. vgId:%d", pSyncNode->vgId); + goto _error; + } // init peersNum, peers, peersId pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1; @@ -1004,17 +1022,24 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { } } for (int i = 0; i < pSyncNode->peersNum; ++i) { - syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i]); + if (!syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i])) { + sError("failed to determine raft member id. vgId:%d, peer:%d", pSyncNode->vgId, i); + goto _error; + } } // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum; for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); + if(!syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) { + sError("failed to determine raft member id. vgId:%d, replica:%d", pSyncNode->vgId, i); + goto _error; + } } // init raft algorithm pSyncNode->pFsm = pSyncInfo->pFsm; + pSyncInfo->pFsm = NULL; pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum); pSyncNode->leaderCache = EMPTY_RAFT_ID; @@ -1047,29 +1072,50 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { // init TLA+ server vars pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; pSyncNode->pRaftStore = raftStoreOpen(pSyncNode->raftStorePath); - ASSERT(pSyncNode->pRaftStore != NULL); + if (pSyncNode->pRaftStore == NULL) { + sError("failed to open raft store. path: %s", pSyncNode->raftStorePath); + goto _error; + } // init TLA+ candidate vars pSyncNode->pVotesGranted = voteGrantedCreate(pSyncNode); - ASSERT(pSyncNode->pVotesGranted != NULL); + if (pSyncNode->pVotesGranted == NULL) { + sError("failed to create VotesGranted. vgId:%d", pSyncNode->vgId); + goto _error; + } pSyncNode->pVotesRespond = votesRespondCreate(pSyncNode); - ASSERT(pSyncNode->pVotesRespond != NULL); + if (pSyncNode->pVotesRespond == NULL) { + sError("failed to create VotesRespond. vgId:%d", pSyncNode->vgId); + goto _error; + } // init TLA+ leader vars pSyncNode->pNextIndex = syncIndexMgrCreate(pSyncNode); - ASSERT(pSyncNode->pNextIndex != NULL); + if (pSyncNode->pNextIndex == NULL) { + sError("failed to create SyncIndexMgr. vgId:%d", pSyncNode->vgId); + goto _error; + } pSyncNode->pMatchIndex = syncIndexMgrCreate(pSyncNode); - ASSERT(pSyncNode->pMatchIndex != NULL); + if (pSyncNode->pMatchIndex == NULL) { + sError("failed to create SyncIndexMgr. vgId:%d", pSyncNode->vgId); + goto _error; + } // init TLA+ log vars pSyncNode->pLogStore = logStoreCreate(pSyncNode); - ASSERT(pSyncNode->pLogStore != NULL); + if (pSyncNode->pLogStore == NULL) { + sError("failed to create SyncLogStore. vgId:%d", pSyncNode->vgId); + goto _error; + } SyncIndex commitIndex = SYNC_INDEX_INVALID; if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { SSnapshot snapshot = {0}; int32_t code = pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); - ASSERT(code == 0); + if (code != 0) { + sError("failed to get snapshot info. vgId:%d, code:%d", pSyncNode->vgId, code); + goto _error; + } if (snapshot.lastApplyIndex > commitIndex) { commitIndex = snapshot.lastApplyIndex; syncNodeEventLog(pSyncNode, "reset commit index by snapshot"); @@ -1132,7 +1178,10 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { // tools pSyncNode->pSyncRespMgr = syncRespMgrCreate(pSyncNode, SYNC_RESP_TTL_MS); - ASSERT(pSyncNode->pSyncRespMgr != NULL); + if (pSyncNode->pSyncRespMgr == NULL) { + sError("failed to create SyncRespMgr. vgId:%d", pSyncNode->vgId); + goto _error; + } // restore state pSyncNode->restoreFinish = false; @@ -1162,6 +1211,15 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { syncNodeEventLog(pSyncNode, "sync open"); return pSyncNode; + +_error: + if (pSyncInfo->pFsm) { + taosMemoryFree(pSyncInfo->pFsm); + pSyncInfo->pFsm = NULL; + } + syncNodeClose(pSyncNode); + pSyncNode = NULL; + return NULL; } void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode) { @@ -1214,20 +1272,28 @@ void syncNodeStartStandBy(SSyncNode* pSyncNode) { void syncNodeClose(SSyncNode* pSyncNode) { syncNodeEventLog(pSyncNode, "sync close"); - + if (pSyncNode == NULL) { + return; + } int32_t ret; - ASSERT(pSyncNode != NULL); ret = raftStoreClose(pSyncNode->pRaftStore); ASSERT(ret == 0); syncRespMgrDestroy(pSyncNode->pSyncRespMgr); + pSyncNode->pSyncRespMgr = NULL; voteGrantedDestroy(pSyncNode->pVotesGranted); + pSyncNode->pVotesGranted = NULL; votesRespondDestory(pSyncNode->pVotesRespond); + pSyncNode->pVotesRespond = NULL; syncIndexMgrDestroy(pSyncNode->pNextIndex); + pSyncNode->pNextIndex = NULL; syncIndexMgrDestroy(pSyncNode->pMatchIndex); + pSyncNode->pMatchIndex = NULL; logStoreDestory(pSyncNode->pLogStore); + pSyncNode->pLogStore = NULL; raftCfgClose(pSyncNode->pRaftCfg); + pSyncNode->pRaftCfg = NULL; syncNodeStopPingTimer(pSyncNode); syncNodeStopElectTimer(pSyncNode); @@ -1249,8 +1315,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { pSyncNode->pNewNodeReceiver = NULL; } - // free memory in syncFreeNode - // taosMemoryFree(pSyncNode); + taosMemoryFree(pSyncNode); } // option @@ -2534,7 +2599,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { return; } } else { - sError("syncNodeEqHeartbeatTimer FpEqMsg is NULL"); + sError("vgId:%d, enqueue msg cb ptr (i.e. FpEqMsg) not set.", pSyncNode->vgId); } syncTimeoutDestroy(pSyncMsg); @@ -2774,14 +2839,6 @@ int32_t syncNodeOnClientRequestBatchCb(SSyncNode* ths, SyncClientRequestBatch* p return 0; } -static void syncFreeNode(void* param) { - SSyncNode* pNode = param; - // inner object already free - // syncNodePrint2((char*)"==syncFreeNode==", pNode); - - taosMemoryFree(pNode); -} - const char* syncStr(ESyncState state) { switch (state) { case TAOS_SYNC_STATE_FOLLOWER: diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index ab404d1b9a..57126d0871 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -364,8 +364,6 @@ int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) { int32_t sysErr = errno; const char *sysErrStr = strerror(errno); sError("create raft cfg file error, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", err, err, errStr, sysErr, sysErrStr); - ASSERT(0); - return -1; } diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 29f78b582f..22b47a2c45 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -28,7 +28,7 @@ SRaftStore *raftStoreOpen(const char *path) { SRaftStore *pRaftStore = taosMemoryMalloc(sizeof(SRaftStore)); if (pRaftStore == NULL) { - sError("raftStoreOpen malloc error"); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } memset(pRaftStore, 0, sizeof(*pRaftStore)); @@ -72,7 +72,9 @@ static int32_t raftStoreInit(SRaftStore *pRaftStore) { } int32_t raftStoreClose(SRaftStore *pRaftStore) { - ASSERT(pRaftStore != NULL); + if (pRaftStore == NULL) { + return 0; + } taosCloseFile(&pRaftStore->pFile); taosMemoryFree(pRaftStore); diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index d7ed864180..103c225476 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -19,6 +19,10 @@ SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) { SSyncRespMgr *pObj = (SSyncRespMgr *)taosMemoryMalloc(sizeof(SSyncRespMgr)); + if (pObj == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } memset(pObj, 0, sizeof(SSyncRespMgr)); pObj->pRespHash = diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 0be3392a9a..68d81813ac 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -35,7 +35,10 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI SSyncSnapshotSender *pSender = NULL; if (condition) { pSender = taosMemoryMalloc(sizeof(SSyncSnapshotSender)); - ASSERT(pSender != NULL); + if (pSender == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } memset(pSender, 0, sizeof(*pSender)); pSender->start = false; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 325073f366..6f234631da 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -26,7 +26,8 @@ uint64_t syncUtilAddr2U64(const char* host, uint16_t port) { uint32_t hostU32 = taosGetIpv4FromFqdn(host); if (hostU32 == (uint32_t)-1) { - sError("Get IP address error"); + sError("failed to resolve ipv4 addr. host:%s", host); + terrno = TSDB_CODE_TSC_INVALID_FQDN; return -1; } @@ -84,13 +85,18 @@ void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { addEpIntoEpSet(pEpSet, host, port); } -void syncUtilnodeInfo2raftId(const SNodeInfo* pNodeInfo, SyncGroupId vgId, SRaftId* raftId) { +bool syncUtilnodeInfo2raftId(const SNodeInfo* pNodeInfo, SyncGroupId vgId, SRaftId* raftId) { uint32_t ipv4 = taosGetIpv4FromFqdn(pNodeInfo->nodeFqdn); - ASSERT(ipv4 != 0xFFFFFFFF); + if (ipv4 == 0xFFFFFFFF || ipv4 == 1) { + sError("failed to resolve ipv4 addr. fqdn: %s", pNodeInfo->nodeFqdn); + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return false; + } char ipbuf[128] = {0}; tinet_ntoa(ipbuf, ipv4); raftId->addr = syncUtilAddr2U64(ipbuf, pNodeInfo->nodePort); raftId->vgId = vgId; + return true; } bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2) { @@ -310,4 +316,4 @@ void syncUtilJson2Line(char* jsonStr) { q++; } } -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index 1d46d71a05..641bb32d2d 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -24,7 +24,10 @@ static void voteGrantedClearVotes(SVotesGranted *pVotesGranted) { SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { SVotesGranted *pVotesGranted = taosMemoryMalloc(sizeof(SVotesGranted)); - ASSERT(pVotesGranted != NULL); + if (pVotesGranted == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } memset(pVotesGranted, 0, sizeof(SVotesGranted)); pVotesGranted->replicas = &(pSyncNode->replicasId); From 5738ba08dc0a5c2fe552a0c8a8b8ac0399249fad Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 28 Sep 2022 14:07:12 +0800 Subject: [PATCH 40/71] feat(query): interp support return timestamp with pseudo column _irowts TD-18842 --- source/libs/executor/src/timewindowoperator.c | 54 +++++++++++++------ 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 4671570802..163d13f7f0 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2095,12 +2095,17 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp bool hasInterp = true; for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; - // SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); + int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); + if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { + colDataAppend(pDst, rows, (char*)&pSliceInfo->current, false); + continue; + } + + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + // SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); switch (pSliceInfo->fillType) { case TSDB_FILL_NULL: { colDataAppendNULL(pDst, rows); @@ -2346,19 +2351,24 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { if (ts == pSliceInfo->current) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); + int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); - if (colDataIsNull_s(pSrc, i)) { - colDataAppendNULL(pDst, pResBlock->info.rows); - continue; - } + if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { + colDataAppend(pDst, pResBlock->info.rows, (char *)&pSliceInfo->current, false); + } else { + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); - char* v = colDataGetData(pSrc, i); - colDataAppend(pDst, pResBlock->info.rows, v, false); + if (colDataIsNull_s(pSrc, i)) { + colDataAppendNULL(pDst, pResBlock->info.rows); + continue; + } + + char* v = colDataGetData(pSrc, i); + colDataAppend(pDst, pResBlock->info.rows, v, false); + } } pResBlock->info.rows += 1; @@ -2478,14 +2488,24 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); + int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); - char* v = colDataGetData(pSrc, i); - colDataAppend(pDst, pResBlock->info.rows, v, false); + if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { + colDataAppend(pDst, pResBlock->info.rows, (char *)&pSliceInfo->current, false); + } else { + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); + + if (colDataIsNull_s(pSrc, i)) { + colDataAppendNULL(pDst, pResBlock->info.rows); + continue; + } + + char* v = colDataGetData(pSrc, i); + colDataAppend(pDst, pResBlock->info.rows, v, false); + } } pResBlock->info.rows += 1; From a2c1c8eaf3102165f0d54ce12da30a10f40e4bc7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 28 Sep 2022 14:08:08 +0800 Subject: [PATCH 41/71] add test cases --- tests/system-test/2-query/interp.py | 225 +++++++++++++++++++++++++++- 1 file changed, 222 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 7bf0191ec1..059513f737 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -356,8 +356,227 @@ class TDTestCase: tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(linear)") tdSql.checkRows(0) + tdLog.printNoPrefix("==========step8:test _irowts with interp") - tdLog.printNoPrefix("==========step8:test intra block interpolation") + # fill null + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(null)") + tdSql.checkRows(9) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:09.500') + tdSql.checkData(4, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:10.500') + tdSql.checkData(6, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(null)") + tdSql.checkRows(13) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(6, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + tdSql.checkData(9, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(10, 0, '2020-02-01 00:00:14.000') + tdSql.checkData(11, 0, '2020-02-01 00:00:15.000') + tdSql.checkData(12, 0, '2020-02-01 00:00:16.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(null)") + tdSql.checkRows(6) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + + # fill value + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(value, 1)") + tdSql.checkRows(9) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:09.500') + tdSql.checkData(4, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:10.500') + tdSql.checkData(6, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") + tdSql.checkRows(13) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(6, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + tdSql.checkData(9, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(10, 0, '2020-02-01 00:00:14.000') + tdSql.checkData(11, 0, '2020-02-01 00:00:15.000') + tdSql.checkData(12, 0, '2020-02-01 00:00:16.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(value, 1)") + tdSql.checkRows(6) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + + # fill prev + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(prev)") + tdSql.checkRows(9) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:09.500') + tdSql.checkData(4, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:10.500') + tdSql.checkData(6, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(prev)") + tdSql.checkRows(12) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(6, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:12.000') + tdSql.checkData(8, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(9, 0, '2020-02-01 00:00:14.000') + tdSql.checkData(10, 0, '2020-02-01 00:00:15.000') + tdSql.checkData(11, 0, '2020-02-01 00:00:16.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(prev)") + tdSql.checkRows(6) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + + # fill next + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(next)") + tdSql.checkRows(9) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:09.500') + tdSql.checkData(4, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:10.500') + tdSql.checkData(6, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") + tdSql.checkRows(12) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(6, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + tdSql.checkData(9, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(10, 0, '2020-02-01 00:00:14.000') + tdSql.checkData(11, 0, '2020-02-01 00:00:15.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(next)") + tdSql.checkRows(6) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + + # fill linear + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(linear)") + tdSql.checkRows(9) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:09.500') + tdSql.checkData(4, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:10.500') + tdSql.checkData(6, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(6, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:12.000') + tdSql.checkData(8, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(9, 0, '2020-02-01 00:00:14.000') + tdSql.checkData(10, 0, '2020-02-01 00:00:15.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(linear)") + tdSql.checkRows(6) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + + return + + tdLog.printNoPrefix("==========step9:test intra block interpolation") tdSql.execute(f"drop database {dbname}"); tdSql.prepare() @@ -551,7 +770,7 @@ class TDTestCase: tdSql.checkData(0, 0, 15) tdSql.checkData(1, 0, 15) - tdLog.printNoPrefix("==========step9:test multi-interp cases") + tdLog.printNoPrefix("==========step10:test multi-interp cases") tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(null)") tdSql.checkRows(5) tdSql.checkCols(4) @@ -601,7 +820,7 @@ class TDTestCase: for i in range (tdSql.queryCols): tdSql.checkData(0, i, 13) - tdLog.printNoPrefix("==========step10:test error cases") + tdLog.printNoPrefix("==========step11:test error cases") tdSql.error(f"select interp(c0) from {dbname}.{tbname}") tdSql.error(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05')") From 528a3273c0295ee5afb403b37e79178997ab50ed Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 28 Sep 2022 14:11:34 +0800 Subject: [PATCH 42/71] fix: memory leak --- source/client/src/clientEnv.c | 2 +- source/client/src/clientImpl.c | 6 ++++-- source/client/src/clientMain.c | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 6a4ef3d821..90e31877a4 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -19,6 +19,7 @@ #include "functionMgt.h" #include "os.h" #include "query.h" +#include "qworker.h" #include "scheduler.h" #include "tcache.h" #include "tglobal.h" @@ -27,7 +28,6 @@ #include "trpc.h" #include "tsched.h" #include "ttime.h" -#include "qworker.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index be907ea1e2..e1cbef7000 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -483,8 +483,8 @@ void setResPrecision(SReqResultInfo* pResInfo, int32_t precision) { int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SArray* pDbVgList) { SArray* nodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); - char *policy = (tsQueryPolicy == QUERY_POLICY_VNODE) ? "vnode" : "client"; - + char* policy = (tsQueryPolicy == QUERY_POLICY_VNODE) ? "vnode" : "client"; + int32_t dbNum = taosArrayGetSize(pDbVgList); for (int32_t i = 0; i < dbNum; ++i) { SArray* pVg = taosArrayGetP(pDbVgList, i); @@ -906,6 +906,8 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); pRequest->prevCode = code; schedulerFreeJob(&pRequest->body.queryJob, 0); + qDestroyQuery(pRequest->pQuery); + pRequest->pQuery = NULL; doAsyncQuery(pRequest, true); return; } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 650b16f855..8812634607 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -20,13 +20,13 @@ #include "functionMgt.h" #include "os.h" #include "query.h" +#include "qworker.h" #include "scheduler.h" #include "tglobal.h" #include "tmsg.h" #include "tref.h" #include "trpc.h" #include "version.h" -#include "qworker.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 From 49f4aa8f5958174c687a00fad753756a2fc51e72 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 28 Sep 2022 14:08:08 +0800 Subject: [PATCH 43/71] add test cases --- tests/system-test/2-query/interp.py | 36 ++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 059513f737..bee20710b5 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -574,7 +574,41 @@ class TDTestCase: tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') - return + # multiple _irowts + tdSql.query(f"select interp(c0),_irowts from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkCols(2) + + tdSql.checkData(0, 1, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 1, '2020-02-01 00:00:06.000') + tdSql.checkData(2, 1, '2020-02-01 00:00:07.000') + tdSql.checkData(3, 1, '2020-02-01 00:00:08.000') + tdSql.checkData(4, 1, '2020-02-01 00:00:09.000') + tdSql.checkData(5, 1, '2020-02-01 00:00:10.000') + tdSql.checkData(6, 1, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 1, '2020-02-01 00:00:12.000') + tdSql.checkData(8, 1, '2020-02-01 00:00:13.000') + tdSql.checkData(9, 1, '2020-02-01 00:00:14.000') + tdSql.checkData(10, 1, '2020-02-01 00:00:15.000') + + tdSql.query(f"select _irowts, interp(c0), interp(c0), _irowts from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkCols(4) + + cols = (0, 3) + for i in cols: + tdSql.checkData(0, i, '2020-02-01 00:00:05.000') + tdSql.checkData(1, i, '2020-02-01 00:00:06.000') + tdSql.checkData(2, i, '2020-02-01 00:00:07.000') + tdSql.checkData(3, i, '2020-02-01 00:00:08.000') + tdSql.checkData(4, i, '2020-02-01 00:00:09.000') + tdSql.checkData(5, i, '2020-02-01 00:00:10.000') + tdSql.checkData(6, i, '2020-02-01 00:00:11.000') + tdSql.checkData(7, i, '2020-02-01 00:00:12.000') + tdSql.checkData(8, i, '2020-02-01 00:00:13.000') + tdSql.checkData(9, i, '2020-02-01 00:00:14.000') + tdSql.checkData(10, i, '2020-02-01 00:00:15.000') + tdLog.printNoPrefix("==========step9:test intra block interpolation") tdSql.execute(f"drop database {dbname}"); From 027a0c11bb0560a8ff0891d79f2b25faf5e7408d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 28 Sep 2022 14:18:32 +0800 Subject: [PATCH 44/71] refactor(query): do some internal refactor. --- source/client/src/clientEnv.c | 5 +++-- source/client/src/clientImpl.c | 27 +++++++++++++-------------- source/client/src/clientMain.c | 8 ++++++-- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 6a4ef3d821..9ba529783b 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -85,11 +85,12 @@ static void deregisterRequest(SRequestObj *pRequest) { atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 - "us, planner:%" PRId64 "us, exec:%" PRId64 "us", + "us, planner:%" PRId64 "us, exec:%" PRId64 "us, reqId:0x%"PRIx64, duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd, pRequest->metric.planEnd - pRequest->metric.semanticEnd, - pRequest->metric.resultReady - pRequest->metric.planEnd); + pRequest->metric.resultReady - pRequest->metric.planEnd, pRequest->requestId); + atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index be907ea1e2..dd541b3ac1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -870,8 +870,9 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { SRequestObj* pRequest = (SRequestObj*)param; - pRequest->code = code; + STscObj* pTscObj = pRequest->pTscObj; + pRequest->code = code; pRequest->metric.resultReady = taosGetTimestampUs(); if (pResult) { @@ -879,31 +880,28 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { memcpy(&pRequest->body.resInfo.execRes, pResult, sizeof(*pResult)); } - if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_DELETE == pRequest->type || - TDMT_VND_CREATE_TABLE == pRequest->type) { + int32_t type = pRequest->type; + if (TDMT_VND_SUBMIT == type || TDMT_VND_DELETE == type || TDMT_VND_CREATE_TABLE == type) { if (pResult) { pRequest->body.resInfo.numOfRows = pResult->numOfRows; - if (TDMT_VND_SUBMIT == pRequest->type) { - STscObj* pTscObj = pRequest->pTscObj; + + // record the insert rows + if (TDMT_VND_SUBMIT == type) { SAppClusterSummary* pActivity = &pTscObj->pAppInfo->summary; atomic_add_fetch_64((int64_t*)&pActivity->numOfInsertRows, pResult->numOfRows); } } schedulerFreeJob(&pRequest->body.queryJob, 0); - - pRequest->metric.execEnd = taosGetTimestampUs(); } taosMemoryFree(pResult); + tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); - tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, - tstrerror(code), pRequest->requestId); - - STscObj* pTscObj = pRequest->pTscObj; if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) { - tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, - pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); + tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64, + pRequest->self, tstrerror(code), pRequest->retry, pRequest->requestId); pRequest->prevCode = code; schedulerFreeJob(&pRequest->body.queryJob, 0); doAsyncQuery(pRequest, true); @@ -915,7 +913,8 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { removeMeta(pTscObj, pRequest->targetTableList); } - handleQueryExecRsp(pRequest); + pRequest->metric.execEnd = taosGetTimestampUs(); + code = handleQueryExecRsp(pRequest); // return to client pRequest->body.queryFp(pRequest->body.param, pRequest, code); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 650b16f855..dd2046121e 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -700,6 +700,7 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { SQuery *pQuery = pRequest->pQuery; pRequest->metric.ctgEnd = taosGetTimestampUs(); + qDebug("0x%" PRIx64 " start to semantic analysis, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId); if (code == TSDB_CODE_SUCCESS) { code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery); @@ -723,13 +724,16 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { destorySqlParseWrapper(pWrapper); - tscDebug("0x%" PRIx64 " analysis semantics completed, start async query, reqId:0x%" PRIx64, pRequest->self, - pRequest->requestId); + double el = (pRequest->metric.semanticEnd - pRequest->metric.ctgEnd)/1000.0; + tscDebug("0x%" PRIx64 " analysis semantics completed, start async query, elapsed time:%.2f ms, reqId:0x%" PRIx64, + pRequest->self, el, pRequest->requestId); + launchAsyncQuery(pRequest, pQuery, pResultMeta); } else { destorySqlParseWrapper(pWrapper); qDestroyQuery(pRequest->pQuery); pRequest->pQuery = NULL; + if (NEED_CLIENT_HANDLE_ERROR(code)) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); From e36eed852f9b8d2ac0fc265f000447c52c2b4c92 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 28 Sep 2022 14:39:24 +0800 Subject: [PATCH 45/71] refactor:do some internal refactor. --- source/client/src/clientEnv.c | 9 ++++----- source/libs/scheduler/src/schJob.c | 5 ++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 9ba529783b..2962aaa3b6 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -70,11 +70,10 @@ static void deregisterRequest(SRequestObj *pRequest) { int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); - int64_t nowUs = taosGetTimestampUs(); - int64_t duration = nowUs - pRequest->metric.start; - tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 - " ms, current:%d, app current:%d", - pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst); + int64_t duration = taosGetTimestampUs() - pRequest->metric.start; + tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%.2f ms, " + "current:%d, app current:%d", + pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst); if (QUERY_NODE_VNODE_MODIF_STMT == pRequest->stmtType) { tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 5e47c0a0ed..69495c8b7a 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -47,8 +47,7 @@ void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode) { return; _return: - - SCH_JOB_DLOG("job errCode updated to %x - %s", errCode, tstrerror(errCode)); + SCH_JOB_DLOG("job errCode updated to %s", tstrerror(errCode)); } bool schJobDone(SSchJob *pJob) { @@ -491,7 +490,7 @@ int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { int32_t code = atomic_load_32(&pJob->errCode); if (code) { - SCH_JOB_DLOG("job failed with error: %s", tstrerror(code)); + SCH_JOB_DLOG("job failed with error %s", tstrerror(code)); } schPostJobRes(pJob, 0); From 89fc9ae5adeae663f3ab02ab5c2b510a686a3574 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 28 Sep 2022 14:45:24 +0800 Subject: [PATCH 46/71] docs changes --- docs/en/12-taos-sql/10-function.md | 1 + docs/zh/12-taos-sql/10-function.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 14dedfb216..61648ddaeb 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -867,6 +867,7 @@ INTERP(expr) - The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. - Interpolation is performed based on `FILL` parameter. - `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable. +- Pseudo column `_irowts` can be used along with `INTERP`to return the timestamps associated with interpolation points(support after version 3.0.1.4). ### LAST diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 2a4d82045f..a2ef4d02e5 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -868,6 +868,7 @@ INTERP(expr) - INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（EVERY 值）进行插值。如果没有指定 EVERY，则默认窗口大小为无穷大，即从 timestamp1 开始只有一个窗口。 - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。 - INTERP 只能在一个时间序列内进行插值，因此当作用于超级表时必须跟 partition by tbname 一起使用。 +- INTERP 可以与伪列_irowts一起使用，返回插值点所对应的时间戳(3.0.1.4版本以后支持)。 ### LAST From 8c76f882fc4b7f9c094b3870db629d8698764e74 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 28 Sep 2022 14:50:04 +0800 Subject: [PATCH 47/71] fix docs --- docs/zh/12-taos-sql/10-function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index a2ef4d02e5..4c33b5232a 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -868,7 +868,7 @@ INTERP(expr) - INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（EVERY 值）进行插值。如果没有指定 EVERY，则默认窗口大小为无穷大，即从 timestamp1 开始只有一个窗口。 - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。 - INTERP 只能在一个时间序列内进行插值，因此当作用于超级表时必须跟 partition by tbname 一起使用。 -- INTERP 可以与伪列_irowts一起使用，返回插值点所对应的时间戳(3.0.1.4版本以后支持)。 +- INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(3.0.1.4版本以后支持)。 ### LAST From 81d12d2e0cd75ea080a0acdb1735ac6cbaeef664 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 28 Sep 2022 14:50:04 +0800 Subject: [PATCH 48/71] fix docs --- docs/en/12-taos-sql/10-function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 61648ddaeb..243ede5fcb 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -867,7 +867,7 @@ INTERP(expr) - The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. - Interpolation is performed based on `FILL` parameter. - `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable. -- Pseudo column `_irowts` can be used along with `INTERP`to return the timestamps associated with interpolation points(support after version 3.0.1.4). +- Pseudo column `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.1.4). ### LAST From 41c1f0e43b560bbffe390519b86fa25512383d6e Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 28 Sep 2022 14:54:08 +0800 Subject: [PATCH 49/71] fix code format --- source/libs/executor/src/timewindowoperator.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 163d13f7f0..66f739ec32 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2096,7 +2096,7 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; + int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { @@ -2104,7 +2104,7 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp continue; } - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; // SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); switch (pSliceInfo->fillType) { case TSDB_FILL_NULL: { @@ -2352,13 +2352,13 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; + int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { colDataAppend(pDst, pResBlock->info.rows, (char *)&pSliceInfo->current, false); } else { - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); if (colDataIsNull_s(pSrc, i)) { @@ -2489,13 +2489,13 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; + int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { colDataAppend(pDst, pResBlock->info.rows, (char *)&pSliceInfo->current, false); } else { - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); if (colDataIsNull_s(pSrc, i)) { From 7ebbce45428828fbedf228ad873c09534223641f Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 28 Sep 2022 15:12:28 +0800 Subject: [PATCH 50/71] docs: improve the system table chapter --- docs/en/12-taos-sql/22-meta.md | 16 ++++++++++++++-- docs/zh/12-taos-sql/22-meta.md | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/docs/en/12-taos-sql/22-meta.md b/docs/en/12-taos-sql/22-meta.md index 9bda5a0a10..8635fbc3dc 100644 --- a/docs/en/12-taos-sql/22-meta.md +++ b/docs/en/12-taos-sql/22-meta.md @@ -11,7 +11,15 @@ TDengine includes a built-in database named `INFORMATION_SCHEMA` to provide acce 4. Future versions of TDengine can add new columns to INFORMATION_SCHEMA tables without affecting existing business systems. 5. It is easier for users coming from other database management systems. For example, Oracle users can query data dictionary tables. -Note: SHOW statements are still supported for the convenience of existing users. +:::info + +- SHOW statements are still supported for the convenience of existing users. +- Some columns in the system table may be keywords, and you need to use the escape character '\`' when querying, for example, the query database test has several VGROUPs: +```sql + select `vgroups` from ins_databases where name = 'test'; +``` + +::: This document introduces the tables of INFORMATION_SCHEMA and their structure. @@ -102,7 +110,11 @@ Provides information about user-created databases. Similar to SHOW DATABASES. | 24 | wal_retention_period | INT | WAL retention period | | 25 | wal_retention_size | INT | Maximum WAL size | | 26 | wal_roll_period | INT | WAL rotation period | -| 27 | wal_segment_size | WAL file size | +| 27 | wal_segment_size | BIGINT | WAL file size | +| 28 | stt_trigger | SMALLINT | The number of files placed on the disk that trigger file merging | +| 29 | table_prefix | SMALLINT | The table name is used for the prefix length that needs to be ignored when calculating the consistent HASH | +| 30 | table_suffix | SMALLINT | The table name is used for the suffix length that needs to be ignored when calculating the consistent HASH | +| 31 | tsdb_pagesize | INT | Page size for vnode data storage engine, in KB | ## INS_FUNCTIONS diff --git a/docs/zh/12-taos-sql/22-meta.md b/docs/zh/12-taos-sql/22-meta.md index c1ffc4a757..c3d8e6868d 100644 --- a/docs/zh/12-taos-sql/22-meta.md +++ b/docs/zh/12-taos-sql/22-meta.md @@ -12,7 +12,15 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 4. TDengine 在后续演进中可以灵活的添加已有 INFORMATION_SCHEMA 中表的列，而不用担心对既有业务系统造成影响 5. 与其他数据库系统更具互操作性。例如，Oracle 数据库用户熟悉查询 Oracle 数据字典中的表 -Note: 由于 SHOW 语句已经被开发者熟悉和广泛使用，所以它们仍然被保留。 +:::info + +- 由于 SHOW 语句已经被开发者熟悉和广泛使用，所以它们仍然被保留。 +- 系统表中的一些列可能是关键字，在查询时需要使用转义符'\`'，例如查询数据库 test 有几个 VGROUP： +```sql + select `vgroups` from ins_databases where name = 'test'; +``` + +::: 本章将详细介绍 `INFORMATION_SCHEMA` 这个内置元数据库中的表和表结构。 @@ -103,7 +111,11 @@ Note: 由于 SHOW 语句已经被开发者熟悉和广泛使用，所以它们 | 24 | wal_retention_period | INT | WAL 的保存时长 | | 25 | wal_retention_size | INT | WAL 的保存上限 | | 26 | wal_roll_period | INT | wal 文件切换时长 | -| 27 | wal_segment_size | wal 单个文件大小 | +| 27 | wal_segment_size | BIGINT | wal 单个文件大小 | +| 28 | stt_trigger | SMALLINT | 表示落盘文件触发文件合并的个数 | +| 29 | table_prefix | SMALLINT | 表示表名用于计算一致性 HASH 时需要忽略的前缀长度 | +| 30 | table_suffix | SMALLINT | 表示表名用于计算一致性 HASH 时需要忽略的后缀长度 | +| 31 | tsdb_pagesize | INT | 一个 VNODE 中时序数据存储引擎的页大小 | ## INS_FUNCTIONS From c240e330677ca00dc393b12f1b78a7b7d75f6dd8 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 28 Sep 2022 15:28:39 +0800 Subject: [PATCH 51/71] docs: improve the system table chapter --- docs/en/12-taos-sql/22-meta.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/12-taos-sql/22-meta.md b/docs/en/12-taos-sql/22-meta.md index 8635fbc3dc..f9853b1c67 100644 --- a/docs/en/12-taos-sql/22-meta.md +++ b/docs/en/12-taos-sql/22-meta.md @@ -14,7 +14,7 @@ TDengine includes a built-in database named `INFORMATION_SCHEMA` to provide acce :::info - SHOW statements are still supported for the convenience of existing users. -- Some columns in the system table may be keywords, and you need to use the escape character '\`' when querying, for example, the query database test has several VGROUPs: +- Some columns in the system table may be keywords, and you need to use the escape character '\`' when querying, for example, to query the VGROUPS in the database `test`: ```sql select `vgroups` from ins_databases where name = 'test'; ``` @@ -111,9 +111,9 @@ Provides information about user-created databases. Similar to SHOW DATABASES. | 25 | wal_retention_size | INT | Maximum WAL size | | 26 | wal_roll_period | INT | WAL rotation period | | 27 | wal_segment_size | BIGINT | WAL file size | -| 28 | stt_trigger | SMALLINT | The number of files placed on the disk that trigger file merging | -| 29 | table_prefix | SMALLINT | The table name is used for the prefix length that needs to be ignored when calculating the consistent HASH | -| 30 | table_suffix | SMALLINT | The table name is used for the suffix length that needs to be ignored when calculating the consistent HASH | +| 28 | stt_trigger | SMALLINT | The threshold for number of files to trigger file merging | +| 29 | table_prefix | SMALLINT | The prefix length in the table name that is ignored when distributing table to vnode based on table name | +| 30 | table_suffix | SMALLINT | The suffix length in the table name that is ignored when distributing table to vnode based on table name | | 31 | tsdb_pagesize | INT | Page size for vnode data storage engine, in KB | ## INS_FUNCTIONS From faf07b81c111cd624ea334da09ff5c96bf92fc5e Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 28 Sep 2022 15:35:45 +0800 Subject: [PATCH 52/71] docs: improve the system table chapter --- docs/en/12-taos-sql/22-meta.md | 2 +- docs/zh/12-taos-sql/22-meta.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/12-taos-sql/22-meta.md b/docs/en/12-taos-sql/22-meta.md index f9853b1c67..0a59777946 100644 --- a/docs/en/12-taos-sql/22-meta.md +++ b/docs/en/12-taos-sql/22-meta.md @@ -114,7 +114,7 @@ Provides information about user-created databases. Similar to SHOW DATABASES. | 28 | stt_trigger | SMALLINT | The threshold for number of files to trigger file merging | | 29 | table_prefix | SMALLINT | The prefix length in the table name that is ignored when distributing table to vnode based on table name | | 30 | table_suffix | SMALLINT | The suffix length in the table name that is ignored when distributing table to vnode based on table name | -| 31 | tsdb_pagesize | INT | Page size for vnode data storage engine, in KB | +| 31 | tsdb_pagesize | INT | The page size for internal storage engine, its unit is KB | ## INS_FUNCTIONS diff --git a/docs/zh/12-taos-sql/22-meta.md b/docs/zh/12-taos-sql/22-meta.md index c3d8e6868d..c192d0e5e8 100644 --- a/docs/zh/12-taos-sql/22-meta.md +++ b/docs/zh/12-taos-sql/22-meta.md @@ -112,10 +112,10 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 | 25 | wal_retention_size | INT | WAL 的保存上限 | | 26 | wal_roll_period | INT | wal 文件切换时长 | | 27 | wal_segment_size | BIGINT | wal 单个文件大小 | -| 28 | stt_trigger | SMALLINT | 表示落盘文件触发文件合并的个数 | -| 29 | table_prefix | SMALLINT | 表示表名用于计算一致性 HASH 时需要忽略的前缀长度 | -| 30 | table_suffix | SMALLINT | 表示表名用于计算一致性 HASH 时需要忽略的后缀长度 | -| 31 | tsdb_pagesize | INT | 一个 VNODE 中时序数据存储引擎的页大小 | +| 28 | stt_trigger | SMALLINT | 触发文件合并的落盘文件的个数 | +| 29 | table_prefix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度 | +| 30 | table_suffix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度 | +| 31 | tsdb_pagesize | INT | 时序数据存储引擎中的页大小 | ## INS_FUNCTIONS From 1fe66cb8468603e23013dcfdcd1d68868c148f0c Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 28 Sep 2022 15:50:29 +0800 Subject: [PATCH 53/71] Revert "enh(tsdb): data migrate should not block data r/w and commit" --- include/common/tmsg.h | 3 +- source/common/src/tmsg.c | 6 +- source/dnode/mnode/impl/src/mndDb.c | 17 +- source/dnode/vnode/src/inc/tsdb.h | 13 +- source/dnode/vnode/src/inc/vnodeInt.h | 9 +- source/dnode/vnode/src/sma/smaRollup.c | 5 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 67 +--- source/dnode/vnode/src/tsdb/tsdbFS.c | 117 +------ source/dnode/vnode/src/tsdb/tsdbOpen.c | 4 - .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 77 +---- source/dnode/vnode/src/tsdb/tsdbRetention.c | 320 ++++-------------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 14 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 12 - source/dnode/vnode/src/vnd/vnodeSvr.c | 91 +---- 14 files changed, 101 insertions(+), 654 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 949446cc33..7aec00c7c1 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -873,8 +873,7 @@ int32_t tSerializeSTrimDbReq(void* buf, int32_t bufLen, STrimDbReq* pReq); int32_t tDeserializeSTrimDbReq(void* buf, int32_t bufLen, STrimDbReq* pReq); typedef struct { - int64_t timestamp; // unit: millisecond - int64_t maxSpeed; // 0 no limit, unit: Byte/s + int32_t timestamp; } SVTrimDbReq; int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 04aed7ca36..f4ffc4c996 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2708,8 +2708,7 @@ int32_t tSerializeSVTrimDbReq(void *buf, int32_t bufLen, SVTrimDbReq *pReq) { tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI64(&encoder, pReq->timestamp) < 0) return -1; - if (tEncodeI64(&encoder, pReq->maxSpeed) < 0) return -1; + if (tEncodeI32(&encoder, pReq->timestamp) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2722,8 +2721,7 @@ int32_t tDeserializeSVTrimDbReq(void *buf, int32_t bufLen, SVTrimDbReq *pReq) { tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI64(&decoder, &pReq->timestamp) < 0) return -1; - if (tDecodeI64(&decoder, &pReq->maxSpeed) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->timestamp) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index cb47e9dac1..a05d8dd739 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1385,19 +1385,11 @@ _OVER: return code; } -/** - * @brief trim database - * - * @param pMnode - * @param pDb - * @param maxSpeed MB/s - * @return int32_t - */ -static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb, int32_t maxSpeed) { +static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; - SVTrimDbReq trimReq = {.timestamp = taosGetTimestampMs(), .maxSpeed = maxSpeed << 20}; + SVTrimDbReq trimReq = {.timestamp = taosGetTimestampSec()}; int32_t reqLen = tSerializeSVTrimDbReq(NULL, 0, &trimReq); int32_t contLen = reqLen + sizeof(SMsgHead); @@ -1421,8 +1413,7 @@ static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb, int32_t maxSpeed) { if (code != 0) { mError("vgId:%d, failed to send vnode-trim request to vnode since 0x%x", pVgroup->vgId, code); } else { - mInfo("vgId:%d, send vnode-trim request to vnode, time:%" PRIi64 ", max speed:%" PRIi64, pVgroup->vgId, - trimReq.timestamp, trimReq.maxSpeed); + mInfo("vgId:%d, send vnode-trim request to vnode, time:%d", pVgroup->vgId, trimReq.timestamp); } sdbRelease(pSdb, pVgroup); } @@ -1452,7 +1443,7 @@ static int32_t mndProcessTrimDbReq(SRpcMsg *pReq) { goto _OVER; } - code = mndTrimDb(pMnode, pDb, trimReq.maxSpeed); + code = mndTrimDb(pMnode, pDb); _OVER: if (code != 0) { diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 180be0ff6e..916311bbee 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -64,7 +64,6 @@ typedef struct SDelFWriter SDelFWriter; typedef struct SDelFReader SDelFReader; typedef struct SRowIter SRowIter; typedef struct STsdbFS STsdbFS; -typedef struct STsdbTrimHdl STsdbTrimHdl; typedef struct SRowMerger SRowMerger; typedef struct STsdbReadSnap STsdbReadSnap; typedef struct SBlockInfo SBlockInfo; @@ -249,7 +248,6 @@ int32_t tsdbFSClose(STsdb *pTsdb); int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS); void tsdbFSDestroy(STsdbFS *pFS); int32_t tDFileSetCmprFn(const void *p1, const void *p2); -int32_t tsdbFSUpdDel(STsdb *pTsdb, STsdbFS *pFS, STsdbFS *pFSNew, int32_t maxFid); int32_t tsdbFSCommit1(STsdb *pTsdb, STsdbFS *pFS); int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFS); int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS); @@ -270,7 +268,7 @@ int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast); -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, int64_t maxSpeed); +int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); @@ -322,18 +320,10 @@ int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSc // structs ======================= struct STsdbFS { - int64_t version; SDelFile *pDelFile; SArray *aDFileSet; // SArray }; -struct STsdbTrimHdl { - volatile int8_t state; // 0 idle 1 in use - volatile int8_t commitInWait; // 0 not in wait, 1 in wait - volatile int32_t maxRetentFid; - volatile int32_t minCommitFid; -}; - struct STsdb { char *path; SVnode *pVnode; @@ -342,7 +332,6 @@ struct STsdb { SMemTable *mem; SMemTable *imem; STsdbFS fs; - STsdbTrimHdl trimHdl; SLRUCache *lruCache; TdThreadMutex lruMutex; }; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 8757dc88e6..4c8045d651 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -58,7 +58,6 @@ typedef struct STQ STQ; typedef struct SVState SVState; typedef struct SVBufPool SVBufPool; typedef struct SQWorker SQHandle; -typedef struct SVTrimDbHdl SVTrimDbHdl; typedef struct STsdbKeepCfg STsdbKeepCfg; typedef struct SMetaSnapReader SMetaSnapReader; typedef struct SMetaSnapWriter SMetaSnapWriter; @@ -145,7 +144,7 @@ int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepC int tsdbClose(STsdb** pTsdb); int32_t tsdbBegin(STsdb* pTsdb); int32_t tsdbCommit(STsdb* pTsdb); -int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now, int64_t maxSpeed); +int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, @@ -200,7 +199,7 @@ int32_t smaSyncPostCommit(SSma* pSma); int32_t smaAsyncPreCommit(SSma* pSma); int32_t smaAsyncCommit(SSma* pSma); int32_t smaAsyncPostCommit(SSma* pSma); -int32_t smaDoRetention(SSma* pSma, int64_t now, int64_t maxSpeed); +int32_t smaDoRetention(SSma* pSma, int64_t now); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); @@ -302,9 +301,6 @@ struct STsdbKeepCfg { int32_t keep1; int32_t keep2; }; -struct SVTrimDbHdl { - volatile int8_t state; // 0 not in trim, 1 in trim -}; struct SVnode { char* path; @@ -329,7 +325,6 @@ struct SVnode { bool restored; tsem_t syncSem; SQHandle* pQuery; - SVTrimDbHdl trimDbH; }; #define TD_VID(PVNODE) ((PVNODE)->config.vgId) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index e149abbace..8d1525e081 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -661,10 +661,9 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { * * @param pSma * @param now - * @param maxSpeed * @return int32_t */ -int32_t smaDoRetention(SSma *pSma, int64_t now, int64_t maxSpeed) { +int32_t smaDoRetention(SSma *pSma, int64_t now) { int32_t code = TSDB_CODE_SUCCESS; if (!VND_IS_RSMA(pSma->pVnode)) { return code; @@ -672,7 +671,7 @@ int32_t smaDoRetention(SSma *pSma, int64_t now, int64_t maxSpeed) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { - code = tsdbDoRetention(pSma->pRSmaTsdb[i], now, maxSpeed); + code = tsdbDoRetention(pSma->pRSmaTsdb[i], now); if (code) goto _end; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 0bfb90a374..5403395623 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -756,32 +756,6 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - - if (pTsdb->imem->nRow > 0) { - int32_t minCommitFid = tsdbKeyFid(pTsdb->imem->minKey, pCommitter->minutes, pCommitter->precision); - int32_t nLoops = 0; - - _wait_retention_end: - while (atomic_load_32(&pTsdb->trimHdl.maxRetentFid) >= minCommitFid) { - atomic_val_compare_exchange_8(&pTsdb->trimHdl.commitInWait, 0, 1); - if (++nLoops > 1000) { - nLoops = 0; - sched_yield(); - } - } - if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { - if (atomic_load_32(&pTsdb->trimHdl.maxRetentFid) >= minCommitFid) { - atomic_store_8(&pTsdb->trimHdl.state, 0); - goto _wait_retention_end; - } - atomic_store_32(&pTsdb->trimHdl.minCommitFid, minCommitFid); - atomic_store_8(&pTsdb->trimHdl.state, 0); - } else { - goto _wait_retention_end; - } - atomic_val_compare_exchange_8(&pTsdb->trimHdl.commitInWait, 1, 0); - } - code = tsdbFSCopy(pTsdb, &pCommitter->fs); if (code) goto _err; @@ -988,38 +962,20 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { int32_t code = 0; STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; - STsdbFS fsLatest = {0}; ASSERT(eno == 0); + code = tsdbFSCommit1(pTsdb, &pCommitter->fs); + if (code) goto _err; + // lock taosThreadRwlockWrlock(&pTsdb->rwLock); - - ASSERT(pCommitter->fs.version <= pTsdb->fs.version); - - if (pCommitter->fs.version < pTsdb->fs.version) { - if ((code = tsdbFSCopy(pTsdb, &fsLatest))) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; - } - - if ((code = tsdbFSUpdDel(pTsdb, &pCommitter->fs, &fsLatest, pTsdb->trimHdl.minCommitFid - 1))) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; - } - } - - code = tsdbFSCommit1(pTsdb, &pCommitter->fs); - if (code) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; - } // commit or rollback code = tsdbFSCommit2(pTsdb, &pCommitter->fs); if (code) { taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; + goto _err; } pTsdb->imem = NULL; @@ -1027,23 +983,20 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { // unlock taosThreadRwlockUnlock(&pTsdb->rwLock); -_exit: tsdbUnrefMemTable(pMemTable); tsdbFSDestroy(&pCommitter->fs); - tsdbFSDestroy(&fsLatest); taosArrayDestroy(pCommitter->aTbDataP); - atomic_store_32(&pTsdb->trimHdl.minCommitFid, INT32_MAX); // if (pCommitter->toMerge) { // code = tsdbMerge(pTsdb); // if (code) goto _err; // } - if(code == 0) { - tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); - } else { - tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - } - + + tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); + return code; + +_err: + tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index ef4f6930be..10926ae6ad 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -250,7 +250,7 @@ _err: void tsdbFSDestroy(STsdbFS *pFS) { if (pFS->pDelFile) { - taosMemoryFreeClear(pFS->pDelFile); + taosMemoryFree(pFS->pDelFile); } for (int32_t iSet = 0; iSet < taosArrayGetSize(pFS->aDFileSet); iSet++) { @@ -263,7 +263,7 @@ void tsdbFSDestroy(STsdbFS *pFS) { } } - pFS->aDFileSet = taosArrayDestroy(pFS->aDFileSet); + taosArrayDestroy(pFS->aDFileSet); } static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { @@ -419,7 +419,6 @@ int32_t tsdbFSOpen(STsdb *pTsdb) { int32_t code = 0; // open handle - pTsdb->fs.version = 0; pTsdb->fs.pDelFile = NULL; pTsdb->fs.aDFileSet = taosArrayInit(0, sizeof(SDFileSet)); if (pTsdb->fs.aDFileSet == NULL) { @@ -535,7 +534,6 @@ int32_t tsdbFSClose(STsdb *pTsdb) { int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { int32_t code = 0; - pFS->version = pTsdb->fs.version; pFS->pDelFile = NULL; pFS->aDFileSet = taosArrayInit(taosArrayGetSize(pTsdb->fs.aDFileSet), sizeof(SDFileSet)); if (pFS->aDFileSet == NULL) { @@ -666,9 +664,6 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { } } - // update the diskId - pDFileSet->diskId = pSet->diskId; - goto _exit; } } @@ -717,108 +712,6 @@ _exit: return code; } -/** - * @brief Update or delete DFileSet in pFS according to DFileSet (fid <= maxFid) in pFSNew. - * - * @param pTsdb - * @param pFS - * @param pFSNew - * @param maxFid - * @return int32_t - */ -int32_t tsdbFSUpdDel(STsdb *pTsdb, STsdbFS *pFS, STsdbFS *pFSNew, int32_t maxFid) { - int32_t code = 0; - int32_t nRef = 0; - char fname[TSDB_FILENAME_LEN]; - - int32_t iOld = 0; - int32_t iNew = 0; - while (true) { - int32_t nOld = taosArrayGetSize(pFS->aDFileSet); - int32_t nNew = taosArrayGetSize(pFSNew->aDFileSet); - SDFileSet fSet; - int8_t sameDisk; - - if (iOld >= nOld && iNew >= nNew) break; - - SDFileSet *pSetOld = (iOld < nOld) ? taosArrayGet(pFS->aDFileSet, iOld) : NULL; - SDFileSet *pSetNew = (iNew < nNew) ? taosArrayGet(pFSNew->aDFileSet, iNew) : NULL; - - if (pSetNew && (pSetNew->fid > maxFid)) break; - - if (pSetOld && pSetNew) { - if (pSetOld->fid == pSetNew->fid) { - goto _merge_migrate; - } else if (pSetOld->fid > pSetNew->fid) { - goto _remove_old; - } else { - ++iOld; - ASSERT(0); - } - continue; - } else { - break; - } - - _merge_migrate: - sameDisk = ((pSetOld->diskId.level == pSetNew->diskId.level) && (pSetOld->diskId.id == pSetNew->diskId.id)); - - ASSERT(pSetOld->pHeadF->commitID == pSetNew->pHeadF->commitID); - ASSERT(pSetOld->pHeadF->size == pSetNew->pHeadF->size); - ASSERT(pSetOld->pHeadF->offset == pSetNew->pHeadF->offset); - - if (!sameDisk) { - // head - *pSetOld->pHeadF = *pSetNew->pHeadF; - pSetOld->pHeadF->nRef = 1; - - // data - ASSERT(pSetOld->pDataF->size == pSetNew->pDataF->size); - *pSetOld->pDataF = *pSetNew->pDataF; - pSetOld->pDataF->nRef = 1; - - // sma - ASSERT(pSetOld->pSmaF->size == pSetNew->pSmaF->size); - *pSetOld->pSmaF = *pSetNew->pSmaF; - pSetOld->pSmaF->nRef = 1; - - // stt - ASSERT(pSetOld->nSttF == pSetNew->nSttF); - for (int32_t iStt = 0; iStt < pSetOld->nSttF; ++iStt) { - ASSERT(pSetOld->aSttF[iStt]->size == pSetNew->aSttF[iStt]->size); - ASSERT(pSetOld->aSttF[iStt]->offset == pSetNew->aSttF[iStt]->offset); - - *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; - pSetOld->aSttF[iStt]->nRef = 1; - } - - // set diskId - pSetOld->diskId = pSetNew->diskId; - } - - ++iOld; - ++iNew; - continue; - - _remove_old: - taosMemoryFree(pSetOld->pHeadF); - taosMemoryFree(pSetOld->pDataF); - for (int32_t iStt = 0; iStt < pSetOld->nSttF; ++iStt) { - taosMemoryFree(pSetOld->aSttF[iStt]); - } - taosMemoryFree(pSetOld->pSmaF); - taosArrayRemove(pFS->aDFileSet, iOld); - ++iNew; - continue; - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb fs upd/del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - int32_t tsdbFSCommit1(STsdb *pTsdb, STsdbFS *pFSNew) { int32_t code = 0; char tfname[TSDB_FILENAME_LEN]; @@ -852,8 +745,6 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { int32_t nRef; char fname[TSDB_FILENAME_LEN]; - ++pTsdb->fs.version; - // del if (pFSNew->pDelFile) { SDelFile *pDelFile = pTsdb->fs.pDelFile; @@ -1030,8 +921,8 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; pSetOld->aSttF[iStt]->nRef = 1; } else { - ASSERT(pSetOld->aSttF[iStt]->size == pSetNew->aSttF[iStt]->size); - ASSERT(pSetOld->aSttF[iStt]->offset == pSetNew->aSttF[iStt]->offset); + ASSERT(pSetOld->aSttF[iStt]->size == pSetOld->aSttF[iStt]->size); + ASSERT(pSetOld->aSttF[iStt]->offset == pSetOld->aSttF[iStt]->offset); } } } diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 8ce61d49b9..ec760e3c57 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -71,10 +71,6 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee goto _err; } - pTsdb->trimHdl.maxRetentFid = INT32_MIN; - pTsdb->trimHdl.minCommitFid = INT32_MAX; - pTsdb->trimHdl.commitInWait = 0; - tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d", TD_VID(pVnode), pTsdb->path, pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 282640b52c..fc577e3962 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -607,66 +607,7 @@ _err: return code; } -/** - * @brief send file with limited speed(rough control) - * - * @param pTsdb - * @param pFileOut - * @param pFileIn - * @param size - * @param speed 0 no limit, unit: B/s - * @return int64_t - */ -static int64_t tsdbFSendFile(STsdb *pTsdb, TdFilePtr pOutFD, TdFilePtr pInFD, int64_t size, int64_t speed) { - if (speed <= 0) { - return taosFSendFile(pOutFD, pInFD, 0, size); - } - - int64_t offset = 0; - int64_t tBytes = 0; - int64_t nBytes = 0; - int64_t startMs = 0; - int64_t cost = 0; - - while ((offset + nBytes) < size) { - if (atomic_load_8(&pTsdb->trimHdl.commitInWait) == 1) { - tsdbDebug("vgId:%d sendFile without limit since conflicts, fSize:%" PRIi64 ", maxSpeed:%" PRIi64, - TD_VID(pTsdb->pVnode), size, speed); - goto _send_remain; - } - startMs = taosGetTimestampMs(); - if ((nBytes = taosFSendFile(pOutFD, pInFD, &offset, speed)) < 0) { - return nBytes; - } - cost = taosGetTimestampMs() - startMs; - tBytes += nBytes; - - int64_t nSleep = 0; - if (cost < 0) { - nSleep = 1000; - } else if (cost < 1000) { - nSleep = 1000 - cost; - } - if (nSleep > 0) { - taosMsleep(nSleep); - tsdbDebug("vgId:%d sendFile and msleep:%" PRIi64 ", fSize:%" PRIi64 ", tBytes:%" PRIi64 " maxSpeed:%" PRIi64, - TD_VID(pTsdb->pVnode), nSleep, size, tBytes, speed); - } - } - -_send_remain: - if (offset < size) { - if ((nBytes = taosFSendFile(pOutFD, pInFD, &offset, size - offset)) < 0) { - return nBytes; - } - tBytes += nBytes; - tsdbDebug("vgId:%d sendFile remain, fSize:%" PRIi64 ", tBytes:%" PRIi64 " maxSpeed:%" PRIi64, TD_VID(pTsdb->pVnode), - size, tBytes, speed); - } - return tBytes; -} - -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, int64_t maxSpeed) { +int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { int32_t code = 0; int64_t n; int64_t size; @@ -679,7 +620,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i // head tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -689,7 +630,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i code = TAOS_SYSTEM_ERROR(errno); goto _err; } - n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage), maxSpeed); + n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage)); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -700,7 +641,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i // data tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom); tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -710,7 +651,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i code = TAOS_SYSTEM_ERROR(errno); goto _err; } - n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pDataF->size, szPage), maxSpeed); + n = taosFSendFile(pOutFD, PInFD, 0, LOGIC_TO_FILE_OFFSET(pSetFrom->pDataF->size, szPage)); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -721,7 +662,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i // sma tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom); tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -731,7 +672,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i code = TAOS_SYSTEM_ERROR(errno); goto _err; } - n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage), maxSpeed); + n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage)); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -743,7 +684,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i for (int8_t iStt = 0; iStt < pSetFrom->nSttF; iStt++) { tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[iStt], fNameFrom); tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[iStt], fNameTo); - pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -753,7 +694,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo, i code = TAOS_SYSTEM_ERROR(errno); goto _err; } - n = tsdbFSendFile(pTsdb, pOutFD, PInFD, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage), maxSpeed); + n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage)); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index f3890b7b53..2c68c57176 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -15,296 +15,98 @@ #include "tsdb.h" -enum { RETENTION_NO = 0, RETENTION_EXPIRED = 1, RETENTION_MIGRATE = 2 }; - -#define MIGRATE_MAX_SPEED (1048576 << 4) // 16 MB, vnode level -#define MIGRATE_MIN_COST (5) // second - -static bool tsdbShouldDoMigrate(STsdb *pTsdb); -static int32_t tsdbShouldDoRetention(STsdb *pTsdb, int64_t now); -static int32_t tsdbProcessRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed, int32_t retention, int8_t type); - -static bool tsdbShouldDoMigrate(STsdb *pTsdb) { - if (tfsGetLevel(pTsdb->pVnode->pTfs) < 2) { - return false; - } - - STsdbKeepCfg *keepCfg = &pTsdb->keepCfg; - if (keepCfg->keep0 == keepCfg->keep1 && keepCfg->keep1 == keepCfg->keep2) { - return false; - } - return true; -} - -static int32_t tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { - int32_t retention = RETENTION_NO; - if (taosArrayGetSize(pTsdb->fs.aDFileSet) == 0) { - return retention; - } - - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, 0); - if (tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now) < 0) { - retention |= RETENTION_EXPIRED; - } - - if (!tsdbShouldDoMigrate(pTsdb)) { - return retention; - } - - for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); ++iSet) { - pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); +static bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { + for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); + int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); + SDiskID did; if (expLevel == pSet->diskId.level) continue; - if (expLevel > 0) { - retention |= RETENTION_MIGRATE; - break; + if (expLevel < 0) { + return true; + } else { + if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { + return false; + } + + if (did.level == pSet->diskId.level) continue; + + return true; } } - return retention; + return false; } -/** - * @brief process retention - * - * @param pTsdb - * @param now - * @param maxSpeed - * @param retention - * @param type 0 RETENTION_EXPIRED, 1 RETENTION_MIGRATE - * @return int32_t - */ -static int32_t tsdbProcessRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed, int32_t retention, int8_t type) { +int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { int32_t code = 0; - int32_t nBatch = 0; - int32_t nLoops = 0; - int32_t maxFid = 0; - int64_t fSize = 0; - int64_t speed = maxSpeed > 0 ? maxSpeed : MIGRATE_MAX_SPEED; - STsdbFS fs = {0}; - STsdbFS fsLatest = {0}; - if (!(retention & type)) { - goto _exit; + if (!tsdbShouldDoRetention(pTsdb, now)) { + return code; } -_retention_loop: - // reset - maxFid = INT32_MIN; - fSize = 0; - tsdbFSDestroy(&fs); - tsdbFSDestroy(&fsLatest); - - if (atomic_load_8(&pTsdb->trimHdl.commitInWait) == 1) { - atomic_store_32(&pTsdb->trimHdl.maxRetentFid, INT32_MIN); - taosMsleep(50); - } + // do retention + STsdbFS fs; code = tsdbFSCopy(pTsdb, &fs); - if (code) goto _exit; + if (code) goto _err; - if (type == RETENTION_MIGRATE) { - int32_t fsSize = taosArrayGetSize(fs.aDFileSet); - for (int32_t iSet = 0; iSet < fsSize; ++iSet) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - SDiskID did; + for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); iSet++) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); + int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); + SDiskID did; - if (pSet->diskId.level == expLevel) continue; - - if (expLevel > 0) { - ASSERT(pSet->fid > maxFid); - maxFid = pSet->fid; - fSize += (pSet->pDataF->size + pSet->pHeadF->size + pSet->pSmaF->size); - if (fSize / speed > MIGRATE_MIN_COST) { - tsdbDebug("vgId:%d migrate loop %d with maxFid:%d", TD_VID(pTsdb->pVnode), nBatch, maxFid); - break; - } - for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) { - fSize += pSet->aSttF[iStt]->size; - } - if (fSize / speed > MIGRATE_MIN_COST) { - tsdbDebug("vgId:%d migrate loop %d with maxFid:%d", TD_VID(pTsdb->pVnode), nBatch, maxFid); - break; - } + if (expLevel < 0) { + taosMemoryFree(pSet->pHeadF); + taosMemoryFree(pSet->pDataF); + taosMemoryFree(pSet->aSttF[0]); + taosMemoryFree(pSet->pSmaF); + taosArrayRemove(fs.aDFileSet, iSet); + iSet--; + } else { + if (expLevel == 0) continue; + if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { + code = terrno; + goto _exit; } - } - } else if (type == RETENTION_EXPIRED) { - for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); ++iSet) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - SDiskID did; - if (expLevel < 0) { - ASSERT(pSet->fid > maxFid); - if (pSet->fid > maxFid) maxFid = pSet->fid; - taosMemoryFree(pSet->pHeadF); - taosMemoryFree(pSet->pDataF); - taosMemoryFree(pSet->pSmaF); - for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) { - taosMemoryFree(pSet->aSttF[iStt]); - } - taosArrayRemove(fs.aDFileSet, iSet); - --iSet; - } else { - break; - } + if (did.level == pSet->diskId.level) continue; + + // copy file to new disk (todo) + SDFileSet fSet = *pSet; + fSet.diskId = did; + + code = tsdbDFileSetCopy(pTsdb, pSet, &fSet); + if (code) goto _err; + + code = tsdbFSUpsertFSet(&fs, &fSet); + if (code) goto _err; } } - if (maxFid == INT32_MIN) goto _exit; + // do change fs + code = tsdbFSCommit1(pTsdb, &fs); + if (code) goto _err; -_commit_conflict_check: - while (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) { - if (++nLoops > 1000) { - nLoops = 0; - sched_yield(); - } - } - if (atomic_val_compare_exchange_8(&pTsdb->trimHdl.state, 0, 1) == 0) { - if (atomic_load_32(&pTsdb->trimHdl.minCommitFid) <= maxFid) { - atomic_store_8(&pTsdb->trimHdl.state, 0); - goto _commit_conflict_check; - } - atomic_store_32(&pTsdb->trimHdl.maxRetentFid, maxFid); - atomic_store_8(&pTsdb->trimHdl.state, 0); - } else { - goto _commit_conflict_check; - } - - // migrate - if (type == RETENTION_MIGRATE) { - for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); ++iSet) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - SDiskID did; - - if (pSet->fid > maxFid) break; - - tsdbDebug("vgId:%d migrate loop %d with maxFid:%d, fid:%d, did:%d, level:%d, expLevel:%d", TD_VID(pTsdb->pVnode), - nBatch, maxFid, pSet->fid, pSet->diskId.id, pSet->diskId.level, expLevel); - - if (expLevel < 0) { - taosMemoryFree(pSet->pHeadF); - taosMemoryFree(pSet->pDataF); - taosMemoryFree(pSet->pSmaF); - for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) { - taosMemoryFree(pSet->aSttF[iStt]); - } - taosArrayRemove(fs.aDFileSet, iSet); - --iSet; - } else { - if (expLevel == pSet->diskId.level) continue; - - if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { - code = terrno; - goto _exit; - } - - if (did.level == pSet->diskId.level) continue; - - // copy file to new disk - SDFileSet fSet = *pSet; - fSet.diskId = did; - - code = tsdbDFileSetCopy(pTsdb, pSet, &fSet, maxSpeed); - if (code) goto _exit; - - code = tsdbFSUpsertFSet(&fs, &fSet); - if (code) goto _exit; - } - } - } - -_merge_fs: taosThreadRwlockWrlock(&pTsdb->rwLock); - // 1) prepare fs, merge tsdbFSNew and pTsdb->fs if needed - STsdbFS *pTsdbFS = &fs; - ASSERT(fs.version <= pTsdb->fs.version); - - if (fs.version < pTsdb->fs.version) { - if ((code = tsdbFSCopy(pTsdb, &fsLatest))) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; - } - - if ((code = tsdbFSUpdDel(pTsdb, &fsLatest, &fs, maxFid))) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; - } - pTsdbFS = &fsLatest; - } - - // 2) save CURRENT - if ((code = tsdbFSCommit1(pTsdb, pTsdbFS))) { + code = tsdbFSCommit2(pTsdb, &fs); + if (code) { taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; + goto _err; } - // 3) apply the tsdbFS to pTsdb->fs - if ((code = tsdbFSCommit2(pTsdb, pTsdbFS))) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _exit; - } taosThreadRwlockUnlock(&pTsdb->rwLock); - if (type == RETENTION_MIGRATE) { - ++nBatch; - goto _retention_loop; - } - -_exit: tsdbFSDestroy(&fs); - tsdbFSDestroy(&fsLatest); - if (code != 0) { - tsdbError("vgId:%d, tsdb do retention %" PRIi8 " failed since %s", TD_VID(pTsdb->pVnode), type, tstrerror(code)); - ASSERT(0); - } - return code; -} - -/** - * @brief Data migration between multi-tier storage, including remove expired data. - * 1) firstly, remove expired DFileSet; - * 2) partition the tsdbFS by the expLevel and the estimated cost(e.g. 5s) to copy, and migrate - * DFileSet groups between multi-tier storage; - * 3) update the tsdbFS and CURRENT in the same transaction; - * 4) finish the migration - * @param pTsdb - * @param now - * @param maxSpeed - * @return int32_t - */ -int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now, int64_t maxSpeed) { - int32_t code = 0; - int32_t retention = RETENTION_NO; - - retention = tsdbShouldDoRetention(pTsdb, now); - if (retention == RETENTION_NO) { - goto _exit; - } - - // step 1: process expire - code = tsdbProcessRetention(pTsdb, now, maxSpeed, retention, RETENTION_EXPIRED); - if (code < 0) goto _exit; - - // step 2: process multi-tier migration - code = tsdbProcessRetention(pTsdb, now, maxSpeed, retention, RETENTION_MIGRATE); - if (code < 0) goto _exit; _exit: - pTsdb->trimHdl.maxRetentFid = INT32_MIN; - if (code != 0) { - tsdbError("vgId:%d, tsdb do retention %d failed since %s, time:%" PRIi64 ", max speed:%" PRIi64, - TD_VID(pTsdb->pVnode), retention, tstrerror(code), now, maxSpeed); - ASSERT(0); - // tsdbFSRollback(pTsdb->pFS); - } else { - tsdbInfo("vgId:%d, tsdb do retention %d succeed, time:%" PRIi64 ", max speed:%" PRIi64, TD_VID(pTsdb->pVnode), - retention, now, maxSpeed); - } + return code; + +_err: + tsdbError("vgId:%d, tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + ASSERT(0); + // tsdbFSRollback(pTsdb->pFS); return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 5e76e2b4d2..4999e7a49a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -509,24 +509,16 @@ void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minK *maxKey = *minKey + minutes * tsTickPerMin[precision] - 1; } -/** - * @brief get fid level by keep and days. - * - * @param fid - * @param pKeepCfg - * @param now millisecond - * @return int32_t - */ int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now) { int32_t aFid[3]; TSKEY key; if (pKeepCfg->precision == TSDB_TIME_PRECISION_MILLI) { - // now = now * 1000; + now = now * 1000; } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_MICRO) { - now = now * 1000l; - } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) { now = now * 1000000l; + } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) { + now = now * 1000000000l; } else { ASSERT(0); } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 481c8b66be..4ccfea4051 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -186,17 +186,6 @@ void vnodePreClose(SVnode *pVnode) { } } -static void vnodeTrimDbClose(SVnode *pVnode) { - int32_t nLoops = 0; - while (atomic_load_8(&pVnode->trimDbH.state) != 0) { - if (++nLoops > 1000) { - vTrace("vgId:%d, wait for trimDb task to finish", TD_VID(pVnode)); - sched_yield(); - nLoops = 0; - } - } -} - void vnodeClose(SVnode *pVnode) { if (pVnode) { vnodeCommit(pVnode); @@ -208,7 +197,6 @@ void vnodeClose(SVnode *pVnode) { smaClose(pVnode->pSma); metaClose(pVnode->pMeta); vnodeCloseBufPool(pVnode); - vnodeTrimDbClose(pVnode); // destroy handle tsem_destroy(&(pVnode->canCommit)); tsem_destroy(&pVnode->syncSem); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 464fd50ab0..6e9eba306a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -378,7 +378,6 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { pMetaRsp->precision = pVnode->config.tsdbCfg.precision; } -#if 0 static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; SVTrimDbReq trimReq = {0}; @@ -389,7 +388,7 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, goto _exit; } - vInfo("vgId:%d, trim vnode request will be processed, time:%" PRIi64, pVnode->config.vgId, trimReq.timestamp); + vInfo("vgId:%d, trim vnode request will be processed, time:%d", pVnode->config.vgId, trimReq.timestamp); // process code = tsdbDoRetention(pVnode->pTsdb, trimReq.timestamp); @@ -401,91 +400,6 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, _exit: return code; } -#endif - -typedef struct { - SVnode *pVnode; - SVTrimDbReq trimReq; -} SVndTrimDbReq; - -void *vnodeProcessTrimReqFunc(void *param) { - int32_t code = 0; - int8_t oldVal = 0; - SVndTrimDbReq *pReq = (SVndTrimDbReq *)param; - SVnode *pVnode = pReq->pVnode; - - setThreadName("vnode-trim"); - - // process - code = tsdbDoRetention(pVnode->pTsdb, pReq->trimReq.timestamp, pReq->trimReq.maxSpeed); - if (code) goto _exit; - - code = smaDoRetention(pVnode->pSma, pReq->trimReq.timestamp, pReq->trimReq.maxSpeed); - if (code) goto _exit; -_exit: - oldVal = atomic_val_compare_exchange_8(&pVnode->trimDbH.state, 1, 0); - ASSERT(oldVal == 1); - if (code) { - vError("vgId:%d, trim vnode thread failed since %s, time:%" PRIi64 ", max speed:%" PRIi64, TD_VID(pVnode), - tstrerror(code), pReq->trimReq.timestamp, pReq->trimReq.maxSpeed); - } else { - vInfo("vgId:%d, trim vnode thread finish, time:%" PRIi64 ", max speed:%" PRIi64, TD_VID(pVnode), - pReq->trimReq.timestamp, pReq->trimReq.maxSpeed); - } - taosMemoryFree(pReq); - return NULL; -} - -static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - int32_t code = 0; - SVndTrimDbReq *pVndTrimReq = taosMemoryMalloc(sizeof(SVndTrimDbReq)); - SVTrimDbHdl *pHandle = &pVnode->trimDbH; - - if (!pVndTrimReq) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - pVndTrimReq->pVnode = pVnode; - - if (tDeserializeSVTrimDbReq(pReq, len, &pVndTrimReq->trimReq) != 0) { - taosMemoryFree(pVndTrimReq); - code = TSDB_CODE_INVALID_MSG; - goto _exit; - } - - if (atomic_val_compare_exchange_8(&pHandle->state, 0, 1) != 0) { - vInfo("vgId:%d, trim vnode request ignored since duplicated req, time:%" PRIi64 ", max speed:%" PRIi64, - TD_VID(pVnode), pVndTrimReq->trimReq.timestamp, pVndTrimReq->trimReq.maxSpeed); - taosMemoryFree(pVndTrimReq); - goto _exit; - } - - vInfo("vgId:%d, trim vnode request will be processed, time:%" PRIi64 ", max speed:%" PRIi64, TD_VID(pVnode), - pVndTrimReq->trimReq.timestamp, pVndTrimReq->trimReq.maxSpeed); - - TdThreadAttr thAttr = {0}; - taosThreadAttrInit(&thAttr); - taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_DETACHED); - - TdThread tid; - if (taosThreadCreate(&tid, &thAttr, vnodeProcessTrimReqFunc, (void *)pVndTrimReq) != 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosMemoryFree(pVndTrimReq); - taosThreadAttrDestroy(&thAttr); - int8_t oldVal = atomic_val_compare_exchange_8(&pHandle->state, 1, 0); - ASSERT(oldVal == 1); - vError("vgId:%d, failed to create pthread to trim vnode since %s", TD_VID(pVnode), tstrerror(code)); - goto _exit; - } - vDebug("vgId:%d, success to create pthread to trim vnode", TD_VID(pVnode)); - - taosThreadAttrDestroy(&thAttr); - -_exit: - terrno = code; - return code; -} static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); @@ -1007,8 +921,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq } if (taosArrayGetSize(newTbUids) > 0) { - vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), - (int32_t)taosArrayGetSize(newTbUids)); + vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), (int32_t)taosArrayGetSize(newTbUids)); } tqUpdateTbUidList(pVnode->pTq, newTbUids, true); From 97061a0aeb91b5b2af75f296c3c193c61ea23fa4 Mon Sep 17 00:00:00 2001 From: WANG MINGMING Date: Wed, 28 Sep 2022 16:32:38 +0800 Subject: [PATCH 54/71] Update 04-opentsdb-json.mdx --- docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx index b0257b9cb7..aa3e5980cf 100644 --- a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -48,7 +48,8 @@ OpenTSDB JSON 格式协议采用一个 JSON 字符串表示一行或多行数据 - 对于 JSON 格式协议，TDengine 并不会自动把所有标签转成 nchar 类型， 字符串将将转为 nchar 类型， 数值将同样转换为 double 类型。 - TDengine 只接收 JSON **数组格式**的字符串，即使一行数据也需要转换成数组形式。 - +- 默认生产的子表名是根据规则生成的唯一ID值。为了让用户可以指定生成的表名，可以通过在taos.cfg里配置 smlChildTableName 参数来指定。 +举例如下：配置 smlChildTableName=tname 插入数据为 "tags": { "host": "web02","dc": "lga","tname":"cpu1"} 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 ::: ## 示例代码 @@ -94,3 +95,7 @@ taos> select * from `meters.current`; 2022-03-28 09:56:51.250 | 12.600000000 | 2.000000000 | California.SanFrancisco | Query OK, 2 row(s) in set (0.004076s) ``` + +## 查询示例 +想要查询"tags": {"location": "California.LosAngeles", "groupid": 1} 的数据，可以通过如下sql： +select * from `meters.voltage` where location="California.LosAngeles" and groupid=1 From 1ee075dce066fa9131b8a9451a5d5e99f34d0a03 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 28 Sep 2022 16:32:50 +0800 Subject: [PATCH 55/71] fix(query):set correct code. --- source/client/src/clientImpl.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index dd541b3ac1..7852a01bfb 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -815,7 +815,7 @@ int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog) { int32_t handleQueryExecRsp(SRequestObj* pRequest) { if (NULL == pRequest->body.resInfo.execRes.res) { - return TSDB_CODE_SUCCESS; + return pRequest->code; } SCatalog* pCatalog = NULL; @@ -868,6 +868,7 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { return code; } +//todo refacto the error code mgmt void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { SRequestObj* pRequest = (SRequestObj*)param; STscObj* pTscObj = pRequest->pTscObj; @@ -914,7 +915,10 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { } pRequest->metric.execEnd = taosGetTimestampUs(); - code = handleQueryExecRsp(pRequest); + int32_t code1 = handleQueryExecRsp(pRequest); + if (pRequest->code == TSDB_CODE_SUCCESS && pRequest->code != code1) { + pRequest->code = code1; + } // return to client pRequest->body.queryFp(pRequest->body.param, pRequest, code); From 43a76d6bf03f12fa346c61716814a612b5c9adc1 Mon Sep 17 00:00:00 2001 From: WANG MINGMING Date: Wed, 28 Sep 2022 16:34:00 +0800 Subject: [PATCH 56/71] Update 03-opentsdb-telnet.mdx --- docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx index 4f63e17635..58bca7f843 100644 --- a/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx +++ b/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx @@ -32,6 +32,8 @@ OpenTSDB 行协议同样采用一行字符串来表示一行数据。OpenTSDB meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3 ``` +- 默认生产的子表名是根据规则生成的唯一ID值。为了让用户可以指定生成的表名，可以通过在taos.cfg里配置 smlChildTableName 参数来指定。 +举例如下：配置 smlChildTableName=tname 插入数据为 meters.current 1648432611250 11.3 tname=cpu1 location=California.LosAngeles groupid=3 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 参考[OpenTSDB Telnet API 文档](http://opentsdb.net/docs/build/html/api_telnet/put.html)。 ## 示例代码 @@ -79,3 +81,6 @@ taos> select tbname, * from `meters.current`; t_7e7b26dd860280242c6492a16... | 2022-03-28 09:56:51.250 | 12.600000000 | 2 | California.SanFrancisco | Query OK, 4 row(s) in set (0.005399s) ``` +## 查询示例： +想要查询 location=California.LosAngeles groupid=3 的数据，可以通过如下sql: +select * from `meters.voltage` where location="California.LosAngeles" and groupid=3 From be5595547258e7f463374a359592ab347c887c47 Mon Sep 17 00:00:00 2001 From: WANG MINGMING Date: Wed, 28 Sep 2022 16:57:16 +0800 Subject: [PATCH 57/71] Update 03-opentsdb-telnet.mdx --- docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx index db9bfd7364..d88a6335cb 100644 --- a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx +++ b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx @@ -31,7 +31,7 @@ For example: ```txt meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3 ``` - +- The defult child table name is generated by rules.You can configure smlChildTableName in taos.cfg to specify chile table names, for example, `smlChildTableName=tname`. You can insert `meters.current 1648432611250 11.3 tname=cpu1 location=California.LosAngeles groupid=3` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. Please refer to [OpenTSDB Telnet API](http://opentsdb.net/docs/build/html/api_telnet/put.html) for more details. ## Examples @@ -79,3 +79,6 @@ taos> select tbname, * from `meters.current`; t_7e7b26dd860280242c6492a16... | 2022-03-28 09:56:51.250 | 12.600000000 | 2 | California.SanFrancisco | Query OK, 4 row(s) in set (0.005399s) ``` +## Query Examples +If you want query the data of `location=California.LosAngeles groupid=3`，here is the query sql： +select * from `meters.voltage` where location="California.LosAngeles" and groupid=3 From 66e63505373ff70cd0dc25d8dad48e1b4c758b17 Mon Sep 17 00:00:00 2001 From: WANG MINGMING Date: Wed, 28 Sep 2022 16:57:34 +0800 Subject: [PATCH 58/71] Update 04-opentsdb-json.mdx --- docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx index 23703f4087..e2e6d6fc9f 100644 --- a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -48,7 +48,7 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http - In JSON protocol, strings will be converted to nchar type and numeric values will be converted to double type. - Only data in array format is accepted and so an array must be used even if there is only one row. - +- The defult child table name is generated by rules.You can configure smlChildTableName in taos.cfg to specify chile table names, for example, `smlChildTableName=tname`. You can insert `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. ::: ## Examples @@ -94,3 +94,6 @@ taos> select * from `meters.current`; 2022-03-28 09:56:51.250 | 12.600000000 | 2.000000000 | California.SanFrancisco | Query OK, 2 row(s) in set (0.004076s) ``` +## Query Examples +If you want query the data of "tags": {"location": "California.LosAngeles", "groupid": 1}，here is the query sql： +select * from `meters.voltage` where location="California.LosAngeles" and groupid=1 From 0529617b7ee19a495a2cceaec901c2fee5483335 Mon Sep 17 00:00:00 2001 From: WANG MINGMING Date: Wed, 28 Sep 2022 16:58:56 +0800 Subject: [PATCH 59/71] Update 02-influxdb-line.mdx --- docs/en/07-develop/03-insert-data/02-influxdb-line.mdx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx index 11db3daeb0..c91dbba5d2 100644 --- a/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx +++ b/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx @@ -37,7 +37,8 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - All the data in `tag_set` will be converted to nchar type automatically . - Each data in `field_set` must be self-descriptive for its data type. For example 1.2f32 means a value 1.2 of float type. Without the "f" type suffix, it will be treated as type double. - Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h). - +- You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3) ::: For more details please refer to [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) and [TDengine Schemaless](/reference/schemaless/#Schemaless-Line-Protocol) @@ -64,3 +65,7 @@ For more details please refer to [InfluxDB Line Protocol](https://docs.influxdat + +## Query Examples +If you want query the data of `location=California.LosAngeles,groupid=2`，here is the query sql： +select * from `meters.voltage` where location="California.LosAngeles" and groupid=2 From 86c62bf5cbfe673fc1370badc44caeba65c94bac Mon Sep 17 00:00:00 2001 From: WANG MINGMING Date: Wed, 28 Sep 2022 16:59:10 +0800 Subject: [PATCH 60/71] Update 13-schemaless.md --- docs/en/14-reference/13-schemaless/13-schemaless.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/en/14-reference/13-schemaless/13-schemaless.md b/docs/en/14-reference/13-schemaless/13-schemaless.md index 4f50c38cbb..5b7924ce56 100644 --- a/docs/en/14-reference/13-schemaless/13-schemaless.md +++ b/docs/en/14-reference/13-schemaless/13-schemaless.md @@ -47,9 +47,8 @@ In the schemaless writing data line protocol, each data item in the field_set ne - `t`, `T`, `true`, `True`, `TRUE`, `f`, `F`, `false`, and `False` will be handled directly as BOOL types. -For example, the following data rows indicate that the t1 label is "3" (NCHAR), the t2 label is "4" (NCHAR), and the t3 label -is "t3" to the super table named `st` labeled "t3" (NCHAR), write c1 column as 3 (BIGINT), c2 column as false (BOOL), c3 column -is "passit" (BINARY), c4 column is 4 (DOUBLE), and the primary key timestamp is 1626006833639000000 in one row. +For example, the following data rows write c1 column as 3 (BIGINT), c2 column as false (BOOL), c3 column +as "passit" (BINARY), c4 column as 4 (DOUBLE), and the primary key timestamp as 1626006833639000000 to child table with the t1 label as "3" (NCHAR), the t2 label as "4" (NCHAR), and the t3 label as "t3" (NCHAR) and the super table named `st`. ```json st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 @@ -69,7 +68,7 @@ Schemaless writes process row data according to the following principles. Note that tag_key1, tag_key2 are not the original order of the tags entered by the user but the result of using the tag names in ascending order of the strings. Therefore, tag_key1 is not the first tag entered in the line protocol. The string's MD5 hash value "md5_val" is calculated after the ranking is completed. The calculation result is then combined with the string to generate the table name: "t_md5_val". "t_" is a fixed prefix that every table generated by this mapping relationship has. -You can configure smlChildTableName to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. 2. If the super table obtained by parsing the line protocol does not exist, this super table is created. 3. If the subtable obtained by the parse line protocol does not exist, Schemaless creates the sub-table according to the subtable name determined in steps 1 or 2. @@ -78,7 +77,7 @@ You can configure smlChildTableName to specify table names, for example, `smlChi NULL. 6. For BINARY or NCHAR columns, if the length of the value provided in a data row exceeds the column type limit, the maximum length of characters allowed to be stored in the column is automatically increased (only incremented and not decremented) to ensure complete preservation of the data. 7. Errors encountered throughout the processing will interrupt the writing process and return an error code. -8. It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat to false. Otherwise, data will be written out of order and a database error will occur. +8. It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3) :::tip All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed From 48c9373680121025ece9e2e3319239cb4195c10e Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 28 Sep 2022 17:00:02 +0800 Subject: [PATCH 61/71] enh: query information_schema.ins_tags optimize --- source/libs/parser/src/parTranslater.c | 111 +++++++++++++++++++-- source/libs/parser/test/mockCatalog.cpp | 4 + source/libs/planner/test/planOtherTest.cpp | 2 + 3 files changed, 111 insertions(+), 6 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 42476690c1..35e17e5e8c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2141,10 +2141,111 @@ static bool sysTableFromVnode(const char* pTable) { static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); } +static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTargetName, SName* pName, + SArray** pVgroupList) { + if (0 == pTargetName->acctId) { + return getDBVgInfoImpl(pCxt, pName, pVgroupList); + } + + if (TSDB_DB_NAME_T == pTargetName->type) { + return getDBVgInfoImpl(pCxt, pTargetName, pVgroupList); + } + + SVgroupInfo vgInfo = {0}; + int32_t code = getTableHashVgroupImpl(pCxt, pTargetName, &vgInfo); + if (TSDB_CODE_SUCCESS == code) { + *pVgroupList = taosArrayInit(1, sizeof(SVgroupInfo)); + if (NULL == *pVgroupList) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (TSDB_CODE_SUCCESS == code) { + taosArrayPush(*pVgroupList, &vgInfo); + } + return code; +} + +static int32_t getTagsTableTargetNameFromOp(STranslateContext* pCxt, SOperatorNode* pOper, SName* pName) { + if (OP_TYPE_EQUAL != pOper->opType) { + return TSDB_CODE_SUCCESS; + } + + SColumnNode* pCol = NULL; + SValueNode* pVal = NULL; + if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) { + pCol = (SColumnNode*)pOper->pLeft; + } else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) { + pVal = (SValueNode*)pOper->pLeft; + } + if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) { + pCol = (SColumnNode*)pOper->pRight; + } else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) { + pVal = (SValueNode*)pOper->pRight; + } + if (NULL == pCol || NULL == pVal) { + return TSDB_CODE_SUCCESS; + } + + if (0 == strcmp(pCol->colName, "db_name")) { + return tNameSetDbName(pName, pCxt->pParseCxt->acctId, pVal->literal, strlen(pVal->literal)); + } else if (0 == strcmp(pCol->colName, "table_name")) { + return tNameAddTbName(pName, pVal->literal, strlen(pVal->literal)); + } + + return TSDB_CODE_SUCCESS; +} + +static void getTagsTableTargetObjName(STranslateContext* pCxt, SNode* pNode, SName* pName) { + if (QUERY_NODE_OPERATOR == nodeType(pNode)) { + getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pNode, pName); + } +} + +static int32_t getTagsTableTargetNameFromCond(STranslateContext* pCxt, SLogicConditionNode* pCond, SName* pName) { + if (LOGIC_COND_TYPE_AND != pCond->condType) { + return TSDB_CODE_SUCCESS; + } + + SNode* pNode = NULL; + FOREACH(pNode, pCond->pParameterList) { getTagsTableTargetObjName(pCxt, pNode, pName); } + return TSDB_CODE_SUCCESS; +} + +static int32_t getTagsTableTargetName(STranslateContext* pCxt, SNode* pWhere, SName* pName) { + if (NULL == pWhere) { + return TSDB_CODE_SUCCESS; + } + + if (QUERY_NODE_OPERATOR == nodeType(pWhere)) { + return getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pWhere, pName); + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere)) { + return getTagsTableTargetNameFromCond(pCxt, (SLogicConditionNode*)pWhere, pName); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgroupList) { + if (!isSelectStmt(pCxt->pCurrStmt)) { + return TSDB_CODE_SUCCESS; + } + SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; + SName targetName = {0}; + int32_t code = getTagsTableTargetName(pCxt, pSelect->pWhere, &targetName); + if (TSDB_CODE_SUCCESS == code) { + code = getTagsTableVgroupListImpl(pCxt, &targetName, pName, pVgroupList); + } + return code; +} + static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { int32_t code = TSDB_CODE_SUCCESS; SArray* vgroupList = NULL; - if ('\0' != pRealTable->qualDbName[0]) { + if (0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS)) { + code = getTagsTableVgroupList(pCxt, pName, &vgroupList); + } else if ('\0' != pRealTable->qualDbName[0]) { if (0 != strcmp(pRealTable->qualDbName, TSDB_INFORMATION_SCHEMA_DB)) { code = getDBVgInfo(pCxt, pRealTable->qualDbName, &vgroupList); } @@ -2152,14 +2253,12 @@ static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, code = getDBVgInfoImpl(pCxt, pName, &vgroupList); } - if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.dbName, TSDB_INFORMATION_SCHEMA_DB) && - 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS) && isSelectStmt(pCxt->pCurrStmt) && - 0 == taosArrayGetSize(vgroupList)) { + if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS) && + isSelectStmt(pCxt->pCurrStmt) && 0 == taosArrayGetSize(vgroupList)) { ((SSelectStmt*)pCxt->pCurrStmt)->isEmptyResult = true; } - if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.dbName, TSDB_INFORMATION_SCHEMA_DB) && - 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES)) { + if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES)) { code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &vgroupList); } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 7725674200..fcaa5af05c 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -85,6 +85,10 @@ void generateInformationSchema(MockCatalogService* mcs) { .addColumn("dnode_id", TSDB_DATA_TYPE_INT) .addColumn("dnode_ep", TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN) .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TAGS, TSDB_SYSTEM_TABLE, 2) + .addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) + .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) + .done(); } void generatePerformanceSchema(MockCatalogService* mcs) { diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp index f6416b3cb1..2ad997bb46 100644 --- a/source/libs/planner/test/planOtherTest.cpp +++ b/source/libs/planner/test/planOtherTest.cpp @@ -84,6 +84,8 @@ TEST_F(PlanOtherTest, show) { run("SHOW TABLE DISTRIBUTED st1"); run("SHOW DNODE 1 VARIABLES"); + + run("SHOW TAGS FROM st1s1"); } TEST_F(PlanOtherTest, delete) { From 577f427065fe71a4b955235a3c6ff8753c5a1ffb Mon Sep 17 00:00:00 2001 From: WANG MINGMING Date: Wed, 28 Sep 2022 17:14:37 +0800 Subject: [PATCH 62/71] Update 02-influxdb-line.mdx --- docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx index f88284ad67..a107ffb1b6 100644 --- a/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx +++ b/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx @@ -37,7 +37,9 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - tag_set 中的所有的数据自动转化为 nchar 数据类型; - field_set 中的每个数据项都需要对自身的数据类型进行描述, 比如 1.2f32 代表 float 类型的数值 1.2, 如果不带类型后缀会被当作 double 处理; - timestamp 支持多种时间精度。写入数据的时候需要用参数指定时间精度，支持从小时到纳秒的 6 种时间精度。 - +- 为了提高写入的效率，默认假设同一个超级表中 field_set 的顺序是一样的（第一条数据包含所有的 field，后面的数据按照这个顺序），如果顺序不一样，需要配置参数 smlDataFormat 为 false，否则，数据写入按照相同顺序写入，库中数据会异常。（3.0.1.3之后的版本 smlDataFormat 默认为 false） [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) +- 默认生产的子表名是根据规则生成的唯一ID值。为了让用户可以指定生成的表名，可以通过在taos.cfg里配置 smlChildTableName 参数来指定。 +举例如下：配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) ::: 要了解更多可参考：[InfluxDB Line 协议官方文档](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) 和 [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) @@ -64,3 +66,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 + +## 查询示例 +比如查询 location=California.LosAngeles,groupid=2 子表的数据可以通过如下sql： +select * from meters where location=California.LosAngeles and groupid=2 From 8d510fcf1873124df8ed2464789bc16e615d2533 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 28 Sep 2022 17:28:23 +0800 Subject: [PATCH 63/71] add _irowts in EN doc --- docs/en/12-taos-sql/06-select.md | 8 ++++++++ docs/en/12-taos-sql/20-keywords.md | 1 + docs/en/12-taos-sql/29-changes.md | 13 +++++++------ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 786444ad77..1271e262ab 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -181,6 +181,14 @@ In TDengine, the first column of all tables must be a timestamp. This column is select _rowts, max(current) from meters; ``` +**\_IROWTS** + +The \_IROWTS pseudocolumns can only be used with INTERP FUNCTION. This pseudocolumns can be used to retrieve the corresponding timestamp column associated with the INTERPOLATION results. + +```sql +select _irowts, interp(current) from meters range('2020-01-01 10:00:00', '2020-01-01 10:30:00') every(1s) fill(linear); +``` + ## Query Objects `FROM` can be followed by a number of tables or super tables, or can be followed by a sub-query. diff --git a/docs/en/12-taos-sql/20-keywords.md b/docs/en/12-taos-sql/20-keywords.md index f7eb067c96..4b479b866b 100644 --- a/docs/en/12-taos-sql/20-keywords.md +++ b/docs/en/12-taos-sql/20-keywords.md @@ -343,6 +343,7 @@ The following list shows all reserved keywords: ### \_ - \_C0 +- \_IROWTS - \_QDURATION - \_QEND - \_QSTART diff --git a/docs/en/12-taos-sql/29-changes.md b/docs/en/12-taos-sql/29-changes.md index 8532eeac5d..78b6d5fc05 100644 --- a/docs/en/12-taos-sql/29-changes.md +++ b/docs/en/12-taos-sql/29-changes.md @@ -11,12 +11,13 @@ description: "This document explains how TDengine SQL has changed in version 3.0 | 1 | VARCHAR | Added | Alias of BINARY. | 2 | TIMESTAMP literal | Added | TIMESTAMP 'timestamp format' syntax now supported. | 3 | _ROWTS pseudocolumn | Added | Indicates the primary key. Alias of _C0. -| 4 | INFORMATION_SCHEMA | Added | Database for system metadata containing all schema definitions -| 5 | PERFORMANCE_SCHEMA | Added | Database for system performance information. -| 6 | Connection queries | Deprecated | Connection queries are no longer supported. The syntax and interfaces are deprecated. -| 7 | Mixed operations | Enhanced | Mixing scalar and vector operations in queries has been enhanced and is supported in all SELECT clauses. -| 8 | Tag operations | Added | Tag columns can be used in queries and clauses like data columns. -| 9 | Timeline clauses and time functions in supertables | Enhanced | When PARTITION BY is not used, data in supertables is merged into a single timeline. +| 4 | _IROWTS pseudocolumn | Added | Used to retrieve timestamps with INTERP function. +| 5 | INFORMATION_SCHEMA | Added | Database for system metadata containing all schema definitions +| 6 | PERFORMANCE_SCHEMA | Added | Database for system performance information. +| 7 | Connection queries | Deprecated | Connection queries are no longer supported. The syntax and interfaces are deprecated. +| 8 | Mixed operations | Enhanced | Mixing scalar and vector operations in queries has been enhanced and is supported in all SELECT clauses. +| 9 | Tag operations | Added | Tag columns can be used in queries and clauses like data columns. +| 10 | Timeline clauses and time functions in supertables | Enhanced | When PARTITION BY is not used, data in supertables is merged into a single timeline. ## SQL Syntax From dd59a04963ce18296cb8ec8a044d8b65d3ef059d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 28 Sep 2022 17:28:55 +0800 Subject: [PATCH 64/71] add _irowts in Docs zh --- docs/zh/12-taos-sql/06-select.md | 8 ++++++++ docs/zh/12-taos-sql/20-keywords.md | 1 + docs/zh/12-taos-sql/29-changes.md | 13 +++++++------ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 394e4a99d5..b3b8ef3887 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -182,6 +182,14 @@ TDengine 中，所有表的第一列都必须是时间戳类型，且为其主 select _rowts, max(current) from meters; ``` +**\_IROWTS** + +\_irowts 伪列只能与 interp 函数一起使用，用于返回 interp 函数插值结果对应的时间戳列。 + +```sql +select _irowts, interp(current) from meters range('2020-01-01 10:00:00', '2020-01-01 10:30:00') every(1s) fill(linear); +``` + ## 查询对象 FROM 关键字后面可以是若干个表（超级表）列表，也可以是子查询的结果。 diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md index 7530e803db..8013698fce 100644 --- a/docs/zh/12-taos-sql/20-keywords.md +++ b/docs/zh/12-taos-sql/20-keywords.md @@ -344,6 +344,7 @@ description: TDengine 保留关键字的详细列表 ### \_ - \_C0 +- \_IROWTS - \_QDURATION - \_QEND - \_QSTART diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index e63825045d..7da8e9f331 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -11,12 +11,13 @@ description: "TDengine 3.0 版本的语法变更说明" | 1 | VARCHAR | 新增 | BINARY类型的别名。 | 2 | TIMESTAMP字面量 | 新增 | 新增支持 TIMESTAMP 'timestamp format' 语法。 | 3 | _ROWTS伪列 | 新增 | 表示时间戳主键。是_C0伪列的别名。 -| 4 | INFORMATION_SCHEMA | 新增 | 包含各种SCHEMA定义的系统数据库。 -| 5 | PERFORMANCE_SCHEMA | 新增 | 包含运行信息的系统数据库。 -| 6 | 连续查询 | 废除 | 不再支持连续查询。相关的各种语法和接口废除。 -| 7 | 混合运算 | 增强 | 查询中的混合运算（标量运算和矢量运算混合）全面增强，SELECT的各个子句均全面支持符合语法语义的混合运算。 -| 8 | 标签运算 | 新增 |在查询中，标签列可以像普通列一样参与各种运算，用于各种子句。 -| 9 | 时间线子句和时间函数用于超级表查询 | 增强 |没有PARTITION BY时，超级表的数据会被合并成一条时间线。 +| 4 | _IROWTS伪列 | 新增 | 用于返回 interp 函数插值结果对应的时间戳列。 +| 5 | INFORMATION_SCHEMA | 新增 | 包含各种SCHEMA定义的系统数据库。 +| 6 | PERFORMANCE_SCHEMA | 新增 | 包含运行信息的系统数据库。 +| 7 | 连续查询 | 废除 | 不再支持连续查询。相关的各种语法和接口废除。 +| 8 | 混合运算 | 增强 | 查询中的混合运算（标量运算和矢量运算混合）全面增强，SELECT的各个子句均全面支持符合语法语义的混合运算。 +| 9 | 标签运算 | 新增 |在查询中，标签列可以像普通列一样参与各种运算，用于各种子句。 +| 10 | 时间线子句和时间函数用于超级表查询 | 增强 |没有PARTITION BY时，超级表的数据会被合并成一条时间线。 ## SQL 语句变更 From 65779d041cf21357daf378f3488741eedce5b623 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 28 Sep 2022 17:35:57 +0800 Subject: [PATCH 65/71] fix grammer --- docs/en/12-taos-sql/06-select.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 1271e262ab..c065245827 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -183,7 +183,7 @@ select _rowts, max(current) from meters; **\_IROWTS** -The \_IROWTS pseudocolumns can only be used with INTERP FUNCTION. This pseudocolumns can be used to retrieve the corresponding timestamp column associated with the INTERPOLATION results. +The \_IROWTS pseudocolumn can only be used with INTERP function. This pseudocolumn can be used to retrieve the corresponding timestamp column associated with the interpolation results. ```sql select _irowts, interp(current) from meters range('2020-01-01 10:00:00', '2020-01-01 10:30:00') every(1s) fill(linear); From 4b4f7a44f400465c46e41c81f641bbe6374a299a Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 28 Sep 2022 17:39:22 +0800 Subject: [PATCH 66/71] fix: resolve memory leak in vnodeOpen when vnodeSyncOpen failure --- source/dnode/vnode/src/tsdb/tsdbOpen.c | 2 ++ source/dnode/vnode/src/vnd/vnodeBufPool.c | 6 +++++- source/dnode/vnode/src/vnd/vnodeOpen.c | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index ec760e3c57..fcbcff9248 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -85,6 +85,8 @@ _err: int tsdbClose(STsdb **pTsdb) { if (*pTsdb) { taosThreadRwlockDestroy(&(*pTsdb)->rwLock); + tsdbMemTableDestroy((*pTsdb)->mem); + (*pTsdb)->mem = NULL; tsdbFSClose(*pTsdb); tsdbCloseCache(*pTsdb); taosMemoryFreeClear(*pTsdb); diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 5a22114ab4..6e02425b55 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -53,6 +53,10 @@ int vnodeCloseBufPool(SVnode *pVnode) { vnodeBufPoolDestroy(pPool); } + if (pVnode->inUse) { + vnodeBufPoolDestroy(pVnode->inUse); + pVnode->inUse = NULL; + } vDebug("vgId:%d, vnode buffer pool is closed", TD_VID(pVnode)); return 0; @@ -177,4 +181,4 @@ void vnodeBufPoolUnRef(SVBufPool *pPool) { taosThreadMutexUnlock(&pVnode->mutex); } -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 69785570f7..b5307cecf2 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -173,6 +173,7 @@ _err: if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); if (pVnode->pSma) smaClose(pVnode->pSma); if (pVnode->pMeta) metaClose(pVnode->pMeta); + if (pVnode->pPool) vnodeCloseBufPool(pVnode); tsem_destroy(&(pVnode->canCommit)); taosMemoryFree(pVnode); From 9e49eb03e472bdd3bb209063a798a35f736f329e Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 28 Sep 2022 18:54:39 +0800 Subject: [PATCH 67/71] fix: coverity scan for sma --- source/dnode/vnode/src/meta/metaQuery.c | 4 ++-- source/dnode/vnode/src/sma/smaCommit.c | 2 +- source/dnode/vnode/src/sma/smaEnv.c | 2 +- source/dnode/vnode/src/sma/smaFS.c | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 18 ++++++++++-------- source/dnode/vnode/src/sma/smaSnapshot.c | 10 +++++++--- source/dnode/vnode/src/sma/smaUtil.c | 8 ++++---- source/libs/stream/src/streamState.c | 2 +- 8 files changed, 27 insertions(+), 21 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index f549ef84f8..2ef4e7510e 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -872,7 +872,7 @@ SArray *metaGetSmaIdsByTable(SMeta *pMeta, tb_uid_t uid) { pSmaIdxKey = (SSmaIdxKey *)pCur->pKey; - if (taosArrayPush(pUids, &pSmaIdxKey->smaUid) < 0) { + if (!taosArrayPush(pUids, &pSmaIdxKey->smaUid)) { terrno = TSDB_CODE_OUT_OF_MEMORY; metaCloseSmaCursor(pCur); taosArrayDestroy(pUids); @@ -915,7 +915,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) { } } - if (taosArrayPush(pUids, &uid) < 0) { + if (!taosArrayPush(pUids, &uid)) { terrno = TSDB_CODE_OUT_OF_MEMORY; metaCloseSmaCursor(pCur); taosArrayDestroy(pUids); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 2b44cdcef1..fb5caad269 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -213,7 +213,7 @@ static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) { tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), committed, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); if (taosCheckExistFile(qTaskInfoFullName)) { SQTaskFile qFile = {.nRef = 1, .padding = 0, .version = committed, .size = 0}; - if (taosArrayPush(pFS->aQTaskInf, &qFile) < 0) { + if (!taosArrayPush(pFS->aQTaskInf, &qFile)) { taosWUnLockLatch(RSMA_FS_LOCK(pStat)); terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_FAILED; diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index ccf4ebb39f..64e5e49d0b 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -386,7 +386,7 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { } break; default: - smaError("vgId:%d, undefined smaType:%", SMA_VID(pSma), smaType); + smaError("vgId:%d, undefined smaType:%" PRIi8, SMA_VID(pSma), smaType); return TSDB_CODE_FAILED; } diff --git a/source/dnode/vnode/src/sma/smaFS.c b/source/dnode/vnode/src/sma/smaFS.c index a5f4e8d2e8..55378751eb 100644 --- a/source/dnode/vnode/src/sma/smaFS.c +++ b/source/dnode/vnode/src/sma/smaFS.c @@ -54,7 +54,7 @@ int32_t tdRSmaFSOpen(SSma *pSma, int64_t version) { if ((terrno = tdRSmaFSUpsertQTaskFile(RSMA_FS(pStat), &qTaskFile)) < 0) { goto _end; } - smaInfo("vgId:%d, open fs, version:%" PRIi64 ", ref:%" PRIi64, TD_VID(pVnode), qTaskFile.version, qTaskFile.nRef); + smaInfo("vgId:%d, open fs, version:%" PRIi64 ", ref:%d", TD_VID(pVnode), qTaskFile.version, qTaskFile.nRef); } _end: diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 8d1525e081..aa7f19374a 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -196,7 +196,8 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, if (!suid || !tbUids) { terrno = TSDB_CODE_INVALID_PTR; - smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr()); + smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), suid ? *suid : -1, + terrstr()); return TSDB_CODE_FAILED; } @@ -566,6 +567,7 @@ static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) } if (!taosArrayPush(pUidArray, uid)) { terrno = TSDB_CODE_OUT_OF_MEMORY; + taosArrayDestroy(pUidArray); return TSDB_CODE_FAILED; } if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) < 0) { @@ -1677,13 +1679,13 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaInfoItem *pItem = NULL; if (!(pRSmaRef = taosHashGet(smaMgmt.refHash, ¶m, POINTER_BYTES))) { - smaDebug("rsma fetch task not start since rsma info item:%p not exist in refHash:%p, rsetId:%" PRIi64, param, - *(int64_t *)¶m, smaMgmt.refHash, smaMgmt.rsetId); + smaDebug("rsma fetch task not start since rsma info item:%p not exist in refHash:%p, rsetId:%d", param, + smaMgmt.refHash, smaMgmt.rsetId); return; } if (!(pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaRef->refId))) { - smaDebug("rsma fetch task not start since rsma stat already destroyed, rsetId:%" PRIi64 " refId:%d)", + smaDebug("rsma fetch task not start since rsma stat already destroyed, rsetId:%d refId:%" PRIi64 ")", smaMgmt.rsetId, pRSmaRef->refId); // pRSmaRef freed in taosHashRemove taosHashRemove(smaMgmt.refHash, ¶m, POINTER_BYTES); return; @@ -1692,7 +1694,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { pSma = pStat->pSma; if (!(pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, pRSmaRef->suid))) { - smaDebug("rsma fetch task not start since rsma info not exist, rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, + smaDebug("rsma fetch task not start since rsma info not exist, rsetId:%d refId:%" PRIi64 ")", smaMgmt.rsetId, pRSmaRef->refId); // pRSmaRef freed in taosHashRemove tdReleaseSmaRef(smaMgmt.rsetId, pRSmaRef->refId); taosHashRemove(smaMgmt.refHash, ¶m, POINTER_BYTES); @@ -1700,7 +1702,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { } if (RSMA_INFO_IS_DEL(pRSmaInfo)) { - smaDebug("rsma fetch task not start since rsma info already deleted, rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, + smaDebug("rsma fetch task not start since rsma info already deleted, rsetId:%d refId:%" PRIi64 ")", smaMgmt.rsetId, pRSmaRef->refId); // pRSmaRef freed in taosHashRemove tdReleaseRSmaInfo(pSma, pRSmaInfo); tdReleaseSmaRef(smaMgmt.rsetId, pRSmaRef->refId); @@ -1716,7 +1718,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { case TASK_TRIGGER_STAT_PAUSED: case TASK_TRIGGER_STAT_CANCELLED: { smaDebug("vgId:%d, rsma fetch task not start for level %" PRIi8 " since stat is %" PRIi8 - ", rsetId rsetId:%" PRIi64 " refId:%d", + ", rsetId:%d refId:%" PRIi64, SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pRSmaRef->refId); if (rsmaTriggerStat == TASK_TRIGGER_STAT_PAUSED) { taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId); @@ -1844,7 +1846,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA void *msg = NULL; taosGetQitem(qall, (void **)&msg); if (msg) { - if (taosArrayPush(pSubmitArr, &msg) < 0) { + if (!taosArrayPush(pSubmitArr, &msg)) { tdFreeRSmaSubmitItems(pSubmitArr); goto _err; } diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c index 5a0167a75f..57b031847e 100644 --- a/source/dnode/vnode/src/sma/smaSnapshot.c +++ b/source/dnode/vnode/src/sma/smaSnapshot.c @@ -70,6 +70,8 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapRead return TSDB_CODE_SUCCESS; _err: + if (pReader) rsmaSnapReaderClose(&pReader); + *ppReader = NULL; smaError("vgId:%d, vnode snapshot rsma reader open failed since %s", TD_VID(pVnode), tstrerror(code)); return TSDB_CODE_FAILED; } @@ -101,8 +103,8 @@ static int32_t rsmaQTaskInfSnapReaderOpen(SRSmaSnapReader* pReader, int64_t vers if (!taosCheckExistFile(qTaskInfoFullName)) { tdRSmaFSUnRef(pSma, pStat, version); - smaInfo("vgId:%d, vnode snapshot rsma reader for qtaskinfo version %" PRIi64 " not need as %s not exists", - TD_VID(pVnode), qTaskInfoFullName); + smaInfo("vgId:%d, vnode snapshot rsma reader for qtaskinfo version %" PRIi64 " not need as %s not exist", + TD_VID(pVnode), version, qTaskInfoFullName); return TSDB_CODE_SUCCESS; } @@ -336,6 +338,7 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWrit tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), 0, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); TdFilePtr qTaskF = taosCreateFile(qTaskInfoFullName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (!qTaskF) { + taosMemoryFree(qWriter); code = TAOS_SYSTEM_ERROR(errno); smaError("vgId:%d, rsma snapshot writer open %s failed since %s", TD_VID(pSma->pVnode), qTaskInfoFullName, tstrerror(code)); @@ -356,6 +359,7 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWrit _err: smaError("vgId:%d, rsma snapshot writer open failed since %s", TD_VID(pSma->pVnode), tstrerror(code)); + if (pWriter) rsmaSnapWriterClose(pWriter, 0); *ppWriter = NULL; return code; } @@ -449,11 +453,11 @@ static int32_t rsmaSnapWriteQTaskInfo(SRSmaSnapWriter* pWriter, uint8_t* pData, code = TAOS_SYSTEM_ERROR(errno); goto _err; } + smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo %s succeed", SMA_VID(pWriter->pSma), qWriter->fname); } else { smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo is not needed", SMA_VID(pWriter->pSma)); } - smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo %s succeed", SMA_VID(pWriter->pSma), qWriter->fname); _exit: return code; diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index a4ba0a61a5..6d7b7df1ee 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -290,19 +290,19 @@ int32_t tdRemoveTFile(STFile *pTFile) { void *tdAcquireSmaRef(int32_t rsetId, int64_t refId) { void *pResult = taosAcquireRef(rsetId, refId); if (!pResult) { - smaWarn("rsma acquire ref for rsetId:%" PRIi64 " refId:%d failed since %s", rsetId, refId, terrstr()); + smaWarn("rsma acquire ref for rsetId:%d refId:%" PRIi64 " failed since %s", rsetId, refId, terrstr()); } else { - smaDebug("rsma acquire ref for rsetId:%" PRIi64 " refId:%d success", rsetId, refId); + smaDebug("rsma acquire ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId); } return pResult; } int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId) { if (taosReleaseRef(rsetId, refId) < 0) { - smaWarn("rsma release ref for rsetId:%" PRIi64 " refId:%d failed since %s", rsetId, refId, terrstr()); + smaWarn("rsma release ref for rsetId:%d refId:%" PRIi64 " failed since %s", rsetId, refId, terrstr()); return TSDB_CODE_FAILED; } - smaDebug("rsma release ref for rsetId:%" PRIi64 " refId:%d success", rsetId, refId); + smaDebug("rsma release ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId); return TSDB_CODE_SUCCESS; } \ No newline at end of file diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 7f3e155a70..3428a85823 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -29,7 +29,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath) { if (!specPath) { sprintf(statePath, "%s/%d", path, pTask->taskId); } else { - memcpy(statePath, path, 300); + strncpy(statePath, path, 300); } if (tdbOpen(statePath, 4096, 256, &pState->db) < 0) { goto _err; From 06bb551b70ac6a4c986d23cc0aa1a2f42ac9dad3 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 28 Sep 2022 18:55:33 +0800 Subject: [PATCH 68/71] enh: query information_schema.ins_tags optimize --- source/libs/parser/src/parAstParser.c | 4 ++++ source/libs/parser/src/parTranslater.c | 5 ++++- source/libs/parser/test/parShowToUse.cpp | 10 ++++++++-- source/libs/parser/test/parTestMain.cpp | 4 ++++ 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index ec5f6c4e57..968e03f97e 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -407,6 +407,10 @@ static int32_t collectMetaKeyFromShowTags(SCollectMetaKeyCxt* pCxt, SShowStmt* p if (TSDB_CODE_SUCCESS == code) { code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pTbName) { + code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, + ((SValueNode*)pStmt->pTbName)->literal, pCxt->pMetaCache); + } return code; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 35e17e5e8c..eb63630dbf 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2143,7 +2143,7 @@ static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, T static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTargetName, SName* pName, SArray** pVgroupList) { - if (0 == pTargetName->acctId) { + if (0 == pTargetName->type) { return getDBVgInfoImpl(pCxt, pName, pVgroupList); } @@ -2208,6 +2208,9 @@ static int32_t getTagsTableTargetNameFromCond(STranslateContext* pCxt, SLogicCon SNode* pNode = NULL; FOREACH(pNode, pCond->pParameterList) { getTagsTableTargetObjName(pCxt, pNode, pName); } + if ('\0' == pName->dbname[0]) { + pName->type = 0; + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/test/parShowToUse.cpp b/source/libs/parser/test/parShowToUse.cpp index f38859c832..36e70dd015 100644 --- a/source/libs/parser/test/parShowToUse.cpp +++ b/source/libs/parser/test/parShowToUse.cpp @@ -196,6 +196,12 @@ TEST_F(ParserShowToUseTest, showTableDistributed) { run("SHOW TABLE DISTRIBUTED st1"); } +TEST_F(ParserShowToUseTest, showTags) { + useDb("root", "test"); + + run("SHOW TAGS FROM st1s1"); +} + // todo SHOW topics TEST_F(ParserShowToUseTest, showUsers) { @@ -213,9 +219,9 @@ TEST_F(ParserShowToUseTest, showVariables) { TEST_F(ParserShowToUseTest, showVgroups) { useDb("root", "test"); - run("SHOW vgroups"); + run("SHOW VGROUPS"); - run("SHOW test.vgroups"); + run("SHOW test.VGROUPS"); } TEST_F(ParserShowToUseTest, showVnodes) { diff --git a/source/libs/parser/test/parTestMain.cpp b/source/libs/parser/test/parTestMain.cpp index bcaccbab07..9e67249238 100644 --- a/source/libs/parser/test/parTestMain.cpp +++ b/source/libs/parser/test/parTestMain.cpp @@ -86,6 +86,7 @@ static void parseArg(int argc, char* argv[]) { {"dump", no_argument, NULL, 'd'}, {"async", required_argument, NULL, 'a'}, {"skipSql", required_argument, NULL, 's'}, + {"log", required_argument, NULL, 'l'}, {0, 0, 0, 0} }; // clang-format on @@ -100,6 +101,9 @@ static void parseArg(int argc, char* argv[]) { case 's': setSkipSqlNum(optarg); break; + case 'l': + setLogLevel(optarg); + break; default: break; } From bf377943e09c62ff81bad59309a16b50e40602a1 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 28 Sep 2022 18:56:57 +0800 Subject: [PATCH 69/71] fix: api parameters --- source/dnode/vnode/src/sma/smaSnapshot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c index 57b031847e..4939fce20c 100644 --- a/source/dnode/vnode/src/sma/smaSnapshot.c +++ b/source/dnode/vnode/src/sma/smaSnapshot.c @@ -359,7 +359,7 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWrit _err: smaError("vgId:%d, rsma snapshot writer open failed since %s", TD_VID(pSma->pVnode), tstrerror(code)); - if (pWriter) rsmaSnapWriterClose(pWriter, 0); + if (pWriter) rsmaSnapWriterClose(&pWriter, 0); *ppWriter = NULL; return code; } From 8a74613a1342a03292560c4b5bf2961bcf4d41ba Mon Sep 17 00:00:00 2001 From: robotspace Date: Thu, 29 Sep 2022 10:02:53 +0800 Subject: [PATCH 70/71] Drop stream when test case is over. Add commands to disable built-in lualib. (#17134) --- examples/lua/README.md | 6 ++++++ examples/lua/lua51/lua_connector51.c | 24 +++++++++++++++--------- examples/lua/lua_connector.c | 16 ++++++++-------- examples/lua/test.lua | 10 ++++++++-- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/examples/lua/README.md b/examples/lua/README.md index 32d6a4cace..5abf0c1aab 100644 --- a/examples/lua/README.md +++ b/examples/lua/README.md @@ -1,6 +1,12 @@ # TDengine driver connector for Lua It's a Lua implementation for [TDengine](https://github.com/taosdata/TDengine), an open-sourced big data platform designed and optimized for the Internet of Things (IoT), Connected Cars, Industrial IoT, and IT Infrastructure and Application Monitoring. You may need to install Lua5.3 . +As TDengine is built with lua-enable with default configure, the built-in lua lib conflicts with external lua lib. The following commands require TDengine built with lua-disable. +To disable built-in lua: +``` +mkdir debug && cd debug +cmake .. -DBUILD_LUA=false && cmake --build . +``` ## Lua Dependencies - Lua: diff --git a/examples/lua/lua51/lua_connector51.c b/examples/lua/lua51/lua_connector51.c index 4c702b2aae..8a9051dd0c 100644 --- a/examples/lua/lua51/lua_connector51.c +++ b/examples/lua/lua51/lua_connector51.c @@ -29,7 +29,7 @@ static int l_connect(lua_State *L){ luaL_checktype(L, 1, LUA_TTABLE); lua_getfield(L, 1,"host"); - if (lua_isstring(L,-1)){ + if (lua_isstring(L, -1)){ host = lua_tostring(L, -1); // printf("host = %s\n", host); } @@ -58,8 +58,9 @@ static int l_connect(lua_State *L){ //printf("password = %s\n", password); } - lua_settop(L,0); + lua_settop(L, 0); + taos_init(); lua_newtable(L); int table_index = lua_gettop(L); @@ -125,7 +126,7 @@ static int l_query(lua_State *L){ //printf("row index:%d\n",rows); rows++; - lua_pushnumber(L,rows); + lua_pushnumber(L, rows); lua_newtable(L); for (int i = 0; i < num_fields; ++i) { @@ -136,15 +137,19 @@ static int l_query(lua_State *L){ lua_pushstring(L,fields[i].name); int32_t* length = taos_fetch_lengths(result); switch (fields[i].type) { + case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_TINYINT: lua_pushinteger(L,*((char *)row[i])); break; + case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_SMALLINT: lua_pushinteger(L,*((short *)row[i])); break; + case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_INT: lua_pushinteger(L,*((int *)row[i])); break; + case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_BIGINT: lua_pushinteger(L,*((int64_t *)row[i])); break; @@ -154,6 +159,7 @@ static int l_query(lua_State *L){ case TSDB_DATA_TYPE_DOUBLE: lua_pushnumber(L,*((double *)row[i])); break; + case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: //printf("type:%d, max len:%d, current len:%d\n",fields[i].type, fields[i].bytes, length[i]); @@ -197,7 +203,7 @@ void async_query_callback(void *param, TAOS_RES *result, int code){ printf("failed, reason:%s\n", taos_errstr(result)); lua_pushinteger(L, -1); lua_setfield(L, table_index, "code"); - lua_pushstring(L,"something is wrong");// taos_errstr(taos)); + lua_pushstring(L, taos_errstr(result)); lua_setfield(L, table_index, "error"); }else{ //printf("success to async query.\n"); @@ -214,9 +220,9 @@ void async_query_callback(void *param, TAOS_RES *result, int code){ static int l_async_query(lua_State *L){ int r = luaL_ref(L, LUA_REGISTRYINDEX); - TAOS * taos = (TAOS*)lua_topointer(L,1); - const char * sqlstr = lua_tostring(L,2); - // int stime = luaL_checknumber(L,3); + TAOS * taos = (TAOS*)lua_topointer(L, 1); + const char * sqlstr = lua_tostring(L, 2); + // int stime = luaL_checknumber(L, 3); lua_newtable(L); int table_index = lua_gettop(L); @@ -224,7 +230,7 @@ static int l_async_query(lua_State *L){ struct async_query_callback_param *p = malloc(sizeof(struct async_query_callback_param)); p->state = L; p->callback=r; - // printf("r:%d, L:%d\n",r,L); + // printf("r:%d, L:%d\n", r, L); taos_query_a(taos,sqlstr,async_query_callback,p); lua_pushnumber(L, 0); @@ -267,7 +273,7 @@ static const struct luaL_Reg lib[] = { extern int luaopen_luaconnector51(lua_State* L) { // luaL_register(L, "luaconnector51", lib); - lua_newtable (L); + lua_newtable(L); luaL_setfuncs(L,lib,0); return 1; } diff --git a/examples/lua/lua_connector.c b/examples/lua/lua_connector.c index ce13ab3829..c3d8bcb995 100644 --- a/examples/lua/lua_connector.c +++ b/examples/lua/lua_connector.c @@ -29,7 +29,7 @@ static int l_connect(lua_State *L){ luaL_checktype(L, 1, LUA_TTABLE); lua_getfield(L, 1,"host"); - if (lua_isstring(L,-1)){ + if (lua_isstring(L, -1)){ host = lua_tostring(L, -1); // printf("host = %s\n", host); } @@ -58,7 +58,7 @@ static int l_connect(lua_State *L){ //printf("password = %s\n", password); } - lua_settop(L,0); + lua_settop(L, 0); taos_init(); @@ -126,7 +126,7 @@ static int l_query(lua_State *L){ //printf("row index:%d\n",rows); rows++; - lua_pushnumber(L,rows); + lua_pushnumber(L, rows); lua_newtable(L); for (int i = 0; i < num_fields; ++i) { @@ -203,7 +203,7 @@ void async_query_callback(void *param, TAOS_RES *result, int code){ printf("failed, reason:%s\n", taos_errstr(result)); lua_pushinteger(L, -1); lua_setfield(L, table_index, "code"); - lua_pushstring(L,"something is wrong");// taos_errstr(taos)); + lua_pushstring(L, taos_errstr(result)); lua_setfield(L, table_index, "error"); }else{ //printf("success to async query.\n"); @@ -220,9 +220,9 @@ void async_query_callback(void *param, TAOS_RES *result, int code){ static int l_async_query(lua_State *L){ int r = luaL_ref(L, LUA_REGISTRYINDEX); - TAOS * taos = (TAOS*)lua_topointer(L,1); - const char * sqlstr = lua_tostring(L,2); - // int stime = luaL_checknumber(L,3); + TAOS * taos = (TAOS*)lua_topointer(L, 1); + const char * sqlstr = lua_tostring(L, 2); + // int stime = luaL_checknumber(L, 3); lua_newtable(L); int table_index = lua_gettop(L); @@ -230,7 +230,7 @@ static int l_async_query(lua_State *L){ struct async_query_callback_param *p = malloc(sizeof(struct async_query_callback_param)); p->state = L; p->callback=r; - // printf("r:%d, L:%d\n",r,L); + // printf("r:%d, L:%d\n", r, L); taos_query_a(taos,sqlstr,async_query_callback,p); lua_pushnumber(L, 0); diff --git a/examples/lua/test.lua b/examples/lua/test.lua index 3d725cc6a3..94415982e7 100644 --- a/examples/lua/test.lua +++ b/examples/lua/test.lua @@ -176,8 +176,14 @@ end driver.query_a(conn,"INSERT INTO therm1 VALUES ('2019-09-01 00:00:00.005', 100),('2019-09-01 00:00:00.006', 101),('2019-09-01 00:00:00.007', 102)", async_query_callback) res = driver.query(conn, "create stream stream_avg_degree into avg_degree as select avg(degree) from thermometer interval(5s) sliding(1s)") +if res.code ~=0 then + print("create stream--- failed:"..res.error) + return +else + print("create stream--- pass") +end -print("From now on we start continous insert in an definite loop, pls wait for about 10 seconds and check stream table for result.") +print("From now on we start continous insertion in an definite loop, please wait for about 10 seconds and check stream table avg_degree for result.") local loop_index = 0 while loop_index < 10 do local t = os.time()*1000 @@ -193,5 +199,5 @@ while loop_index < 10 do os.execute("sleep " .. 1) loop_index = loop_index + 1 end -driver.query(conn,"DROP STREAM IF EXISTS avg_therm_s") +driver.query(conn,"DROP STREAM IF EXISTS stream_avg_degree") driver.close(conn) From 0e734f7aee14dcef638d1a4d0c3464a0c3ff3bdf Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 29 Sep 2022 10:39:22 +0800 Subject: [PATCH 71/71] fix:timestamp out of range --- utils/test/c/sml_test.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index ca3d464da7..56ba622a9c 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -879,7 +879,7 @@ int sml_16960_Test() { "{" "\"timestamp\":" "" - "{ \"value\": 1349020800000, \"type\": \"ms\" }" + "{ \"value\": 1664418955000, \"type\": \"ms\" }" "," "\"value\":" "" @@ -916,7 +916,7 @@ int sml_16960_Test() { "{" "\"timestamp\":" "" - "{ \"value\": 1349020800001, \"type\": \"ms\" }" + "{ \"value\": 1664418955001, \"type\": \"ms\" }" "," "\"value\":" "" @@ -953,7 +953,7 @@ int sml_16960_Test() { "{" "\"timestamp\":" "" - "{ \"value\": 1349020800002, \"type\": \"ms\" }" + "{ \"value\": 1664418955002, \"type\": \"ms\" }" "," "\"value\":" "" @@ -990,7 +990,7 @@ int sml_16960_Test() { "{" "\"timestamp\":" "" - "{ \"value\": 1349020800003, \"type\": \"ms\" }" + "{ \"value\": 1664418955003, \"type\": \"ms\" }" "," "\"value\":" "" @@ -1027,7 +1027,7 @@ int sml_16960_Test() { "{" "\"timestamp\":" "" - "{ \"value\": 1349020800004, \"type\": \"ms\" }" + "{ \"value\": 1664418955004, \"type\": \"ms\" }" "," "\"value\":" ""