From f9d162bae2474137ed1bdf02a9159b75712efed2 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 13 Apr 2022 16:02:55 +0800 Subject: [PATCH 01/13] tsma use TDB --- source/dnode/vnode/CMakeLists.txt | 3 +- source/dnode/vnode/inc/vnode.h | 6 +- source/dnode/vnode/src/inc/tsdbSma.h | 68 ++++ source/dnode/vnode/src/inc/vnodeInt.h | 2 + source/dnode/vnode/src/meta/metaBDBImpl.c | 19 +- source/dnode/vnode/src/meta/metaTDBImpl.c | 356 +++++++++++++++++- source/dnode/vnode/src/tq/tq.c | 6 +- source/dnode/vnode/src/tsdb/tsdbSma.c | 218 +++++++++-- source/dnode/vnode/src/tsdb/tsdbTDBImpl.c | 137 +++++++ source/dnode/vnode/src/vnd/vnodeWrite.c | 8 +- source/dnode/vnode/test/tsdbSmaTest.cpp | 11 +- source/libs/tdb/src/db/tdbBtree.c | 25 +- .../script/tsim/sma/tsmaCreateInsertData.sim | 49 +++ 13 files changed, 829 insertions(+), 79 deletions(-) create mode 100644 source/dnode/vnode/src/inc/tsdbSma.h create mode 100644 source/dnode/vnode/src/tsdb/tsdbTDBImpl.c create mode 100644 tests/script/tsim/sma/tsmaCreateInsertData.sim diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index bd2e4213d5..4b8aa5dfaf 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -30,6 +30,7 @@ target_sources( # tsdb # "src/tsdb/tsdbBDBImpl.c" + "src/tsdb/tsdbTDBImpl.c" "src/tsdb/tsdbCommit.c" "src/tsdb/tsdbCompact.c" "src/tsdb/tsdbFile.c" @@ -40,7 +41,7 @@ target_sources( "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbScan.c" - # "src/tsdb/tsdbSma.c" + "src/tsdb/tsdbSma.c" "src/tsdb/tsdbWrite.c" # tq diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 113501a26b..aab835b958 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -357,7 +357,7 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); -STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid); +void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode); STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid); SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup); int metaGetTbNum(SMeta *pMeta); @@ -369,8 +369,8 @@ void metaCloseCtbCurosr(SMCtbCursor *pCtbCur); tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur); SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid); -void metaCloseSmaCurosr(SMSmaCursor *pSmaCur); -const char *metaSmaCursorNext(SMSmaCursor *pSmaCur); +void metaCloseSmaCursor(SMSmaCursor *pSmaCur); +int64_t metaSmaCursorNext(SMSmaCursor *pSmaCur); // Options void metaOptionsInit(SMetaCfg *pMetaCfg); diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h new file mode 100644 index 0000000000..ce03fa7c67 --- /dev/null +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_TSDB_SMA_H_ +#define _TD_VNODE_TSDB_SMA_H_ + +#include "tdbInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SSmaKey SSmaKey; + +struct SSmaKey { + TSKEY skey; + int64_t groupId; +}; + + +typedef struct SDBFile SDBFile; + +struct SDBFile { + int32_t fid; + TDB *pDB; + char *path; +}; + +int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path); +int32_t tsdbCloseDBEnv(TENV *pEnv); +int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF); +int32_t tsdbCloseDBF(SDBFile *pDBF); +int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn); +void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen); + +void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv); +void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv); +#if 0 +int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result); +int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin); +#endif + +// internal func +static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) { + int32_t len = 0; + len += taosEncodeFixedI64(pData, tsKey); + len += taosEncodeFixedI64(pData, groupId); + return len; +} + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_TSDB_SMA_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 6e1f00f931..29a45723c4 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -220,6 +220,8 @@ void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); #include "tq.h" +#include "tsdbSma.h" + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index caa101b5d0..a8270f746d 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -667,7 +667,7 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { return pTbCfg; } -STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { +void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) { STSma * pCfg = NULL; SMetaDB *pDB = pMeta->pDB; DBT key = {0}; @@ -920,7 +920,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { return pCur; } -void metaCloseSmaCurosr(SMSmaCursor *pCur) { +void metaCloseSmaCursor(SMSmaCursor *pCur) { if (pCur) { if (pCur->pCur) { pCur->pCur->close(pCur->pCur); @@ -930,7 +930,8 @@ void metaCloseSmaCurosr(SMSmaCursor *pCur) { } } -const char *metaSmaCursorNext(SMSmaCursor *pCur) { +int64_t metaSmaCursorNext(SMSmaCursor *pCur) { +#if 0 DBT skey = {0}; DBT pkey = {0}; DBT pval = {0}; @@ -946,6 +947,8 @@ const char *metaSmaCursorNext(SMSmaCursor *pCur) { } else { return NULL; } +#endif + return 0; } STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { @@ -972,7 +975,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { ++pSW->number; STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); if (tptr == NULL) { - metaCloseSmaCurosr(pCur); + metaCloseSmaCursor(pCur); tdDestroyTSmaWrapper(pSW); taosMemoryFreeClear(pSW); return NULL; @@ -980,7 +983,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { pSW->tSma = tptr; pBuf = pval.data; if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { - metaCloseSmaCurosr(pCur); + metaCloseSmaCursor(pCur); tdDestroyTSmaWrapper(pSW); taosMemoryFreeClear(pSW); return NULL; @@ -990,8 +993,8 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { break; } - metaCloseSmaCurosr(pCur); - + metaCloseSmaCursor(pCur); + return pSW; } @@ -1004,7 +1007,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { int ret; // TODO: lock? - ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &pCur, 0); + ret = pDB->pSmaIdx->cursor(pDB->pSmaIdx, NULL, &pCur, 0); if (ret != 0) { return NULL; } diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index e3acba7eb6..6f218ad72b 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -22,6 +22,8 @@ typedef struct SPoolMem { struct SPoolMem *next; } SPoolMem; +#define META_TDB_SMA_TEST + static SPoolMem *openPool(); static void clearPool(SPoolMem *pPool); static void closePool(SPoolMem *pPool); @@ -38,6 +40,10 @@ struct SMetaDB { TDB * pNtbIdx; TDB * pCtbIdx; SPoolMem *pPool; +#ifdef META_TDB_SMA_TEST + TDB *pSmaDB; + TDB *pSmaIdx; +#endif }; typedef struct __attribute__((__packed__)) { @@ -55,6 +61,11 @@ typedef struct { tb_uid_t uid; } SCtbIdxKey; +typedef struct { + tb_uid_t uid; + int64_t smaUid; +} SSmaIdxKey; + static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW); @@ -115,6 +126,17 @@ static inline int metaCtbIdxCmpr(const void *arg1, int len1, const void *arg2, i return metaUidCmpr(&pKey1->uid, sizeof(tb_uid_t), &pKey2->uid, sizeof(tb_uid_t)); } +static inline int metaSmaIdxCmpr(const void *arg1, int len1, const void *arg2, int len2) { + int c; + SSmaIdxKey *pKey1 = (SSmaIdxKey *)arg1; + SSmaIdxKey *pKey2 = (SSmaIdxKey *)arg2; + + c = metaUidCmpr(arg1, sizeof(tb_uid_t), arg2, sizeof(tb_uid_t)); + if (c) return c; + + return metaUidCmpr(&pKey1->smaUid, sizeof(int64_t), &pKey2->smaUid, sizeof(int64_t)); +} + int metaOpenDB(SMeta *pMeta) { SMetaDB *pMetaDb; int ret; @@ -143,6 +165,15 @@ int metaOpenDB(SMeta *pMeta) { return -1; } +#ifdef META_TDB_SMA_TEST + ret = tdbDbOpen("sma.db", sizeof(int64_t), TDB_VARIANT_LEN, metaUidCmpr, pMetaDb->pEnv, &(pMetaDb->pSmaDB)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } +#endif + // open schema DB ret = tdbDbOpen("schema.db", sizeof(SSchemaDbKey), TDB_VARIANT_LEN, metaSchemaKeyCmpr, pMetaDb->pEnv, &(pMetaDb->pSchemaDB)); @@ -180,6 +211,15 @@ int metaOpenDB(SMeta *pMeta) { return -1; } +#ifdef META_TDB_SMA_TEST + ret = tdbDbOpen("sma.idx", sizeof(SSmaIdxKey), 0, metaSmaIdxCmpr, pMetaDb->pEnv, &(pMetaDb->pSmaIdx)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } +#endif + pMetaDb->pPool = openPool(); tdbTxnOpen(&pMetaDb->txn, 0, poolMalloc, poolFree, pMetaDb->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); tdbBegin(pMetaDb->pEnv, NULL); @@ -193,10 +233,16 @@ void metaCloseDB(SMeta *pMeta) { tdbCommit(pMeta->pDB->pEnv, &pMeta->pDB->txn); tdbTxnClose(&pMeta->pDB->txn); clearPool(pMeta->pDB->pPool); +#ifdef META_TDB_SMA_TEST + tdbDbClose(pMeta->pDB->pSmaIdx); +#endif tdbDbClose(pMeta->pDB->pCtbIdx); tdbDbClose(pMeta->pDB->pNtbIdx); tdbDbClose(pMeta->pDB->pStbIdx); tdbDbClose(pMeta->pDB->pNameIdx); +#ifdef META_TDB_SMA_TEST + tdbDbClose(pMeta->pDB->pSmaDB); +#endif tdbDbClose(pMeta->pDB->pSchemaDB); tdbDbClose(pMeta->pDB->pTbDB); taosMemoryFree(pMeta->pDB); @@ -491,7 +537,6 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { taosMemoryFree(tbCfg.name); taosMemoryFree(tbCfg.stbCfg.pTagSchema); continue; - ; } else if (tbCfg.type == META_CHILD_TABLE) { kvRowFree(tbCfg.ctbCfg.pTag); } @@ -566,51 +611,326 @@ int metaGetTbNum(SMeta *pMeta) { return 0; } +struct SMSmaCursor { + TDBC *pCur; + tb_uid_t uid; + void *pKey; + void *pVal; + int kLen; + int vLen; +}; + STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { // TODO - ASSERT(0); - return NULL; + // ASSERT(0); + // return NULL; +#ifdef META_TDB_SMA_TEST + STSmaWrapper *pSW = NULL; + + pSW = taosMemoryCalloc(1, sizeof(*pSW)); + if (pSW == NULL) { + return NULL; + } + + SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid); + if (pCur == NULL) { + taosMemoryFree(pSW); + return NULL; + } + + void *pBuf = NULL; + SSmaIdxKey *pSmaIdxKey = NULL; + + while (true) { + // TODO: lock during iterate? + if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { + pSmaIdxKey = pCur->pKey; + ASSERT(pSmaIdxKey != NULL); + + void *pSmaVal = metaGetSmaInfoByIndex(pMeta, pSmaIdxKey->smaUid, false); + + if (pSmaVal == NULL) { + tsdbWarn("no tsma exists for indexUid: %" PRIi64, pSmaIdxKey->smaUid); + continue; + } + + + ++pSW->number; + STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); + if (tptr == NULL) { + TDB_FREE(pSmaVal); + metaCloseSmaCursor(pCur); + tdDestroyTSmaWrapper(pSW); + taosMemoryFreeClear(pSW); + return NULL; + } + pSW->tSma = tptr; + pBuf = pSmaVal; + if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { + TDB_FREE(pSmaVal); + metaCloseSmaCursor(pCur); + tdDestroyTSmaWrapper(pSW); + taosMemoryFreeClear(pSW); + return NULL; + } + TDB_FREE(pSmaVal); + continue; + } + break; + } + + metaCloseSmaCursor(pCur); + + return pSW; + +#endif } int metaRemoveSmaFromDb(SMeta *pMeta, int64_t indexUid) { // TODO ASSERT(0); +#ifndef META_TDB_SMA_TEST + DBT key = {0}; + + key.data = (void *)indexName; + key.size = strlen(indexName); + + metaDBWLock(pMeta->pDB); + // TODO: No guarantee of consistence. + // Use transaction or DB->sync() for some guarantee. + pMeta->pDB->pSmaDB->del(pMeta->pDB->pSmaDB, NULL, &key, 0); + metaDBULock(pMeta->pDB); +#endif return 0; } int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { // TODO - ASSERT(0); + // ASSERT(0); + +#ifdef META_TDB_SMA_TEST + int32_t ret = 0; + SMetaDB *pMetaDb = pMeta->pDB; + void *pBuf = NULL, *qBuf = NULL; + void *key = {0}, *val = {0}; + + // save sma info + int32_t len = tEncodeTSma(NULL, pSmaCfg); + pBuf = taosMemoryCalloc(1, len); + if (pBuf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + key = (void *)&pSmaCfg->indexUid; + qBuf = pBuf; + tEncodeTSma(&qBuf, pSmaCfg); + val = pBuf; + + int32_t kLen = sizeof(pSmaCfg->indexUid); + int32_t vLen = POINTER_DISTANCE(qBuf, pBuf); + + ret = tdbDbInsert(pMeta->pDB->pSmaDB, key, kLen, val, vLen, &pMetaDb->txn); + if (ret < 0) { + taosMemoryFreeClear(pBuf); + return -1; + } + + // add sma idx + SSmaIdxKey smaIdxKey; + smaIdxKey.uid = pSmaCfg->tableUid; + smaIdxKey.smaUid = pSmaCfg->indexUid; + key = &smaIdxKey; + kLen = sizeof(smaIdxKey); + val = NULL; + vLen = 0; + + ret = tdbDbInsert(pMeta->pDB->pSmaIdx, key, kLen, val, vLen, &pMetaDb->txn); + if (ret < 0) { + taosMemoryFreeClear(pBuf); + return -1; + } + + // release + taosMemoryFreeClear(pBuf); + + if (pMeta->pDB->pPool->size > 0) { + metaCommit(pMeta); + } + +#endif return 0; } -STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { +void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) { // TODO - ASSERT(0); - return NULL; + // ASSERT(0); + // return NULL; +#ifdef META_TDB_SMA_TEST + SMetaDB *pDB = pMeta->pDB; + void *pKey = NULL; + void *pVal = NULL; + int kLen = 0; + int vLen = 0; + int ret = -1; + + // Set key + pKey = (void *)&indexUid; + kLen = sizeof(indexUid); + + // Query + ret = tdbDbGet(pDB->pSmaDB, pKey, kLen, &pVal, &vLen); + if (ret != 0 || !pVal) { + return NULL; + } + + if (!isDecode) { + // return raw value + return pVal; + } + + // Decode + STSma *pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma)); + if (pCfg == NULL) { + taosMemoryFree(pVal); + return NULL; + } + + void *pBuf = pVal; + if (tDecodeTSma(pBuf, pCfg) == NULL) { + tdDestroyTSma(pCfg); + taosMemoryFree(pCfg); + TDB_FREE(pVal); + return NULL; + } + + TDB_FREE(pVal); + return pCfg; +#endif } -const char *metaSmaCursorNext(SMSmaCursor *pCur) { +/** + * @brief + * + * @param pMeta + * @param uid 0 means iterate all uids. + * @return SMSmaCursor* + */ +SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { // TODO - ASSERT(0); - return NULL; + // ASSERT(0); + // return NULL; +#ifdef META_TDB_SMA_TEST + SMSmaCursor *pCur = NULL; + SMetaDB *pDB = pMeta->pDB; + int ret; + + pCur = (SMSmaCursor *)taosMemoryCalloc(1, sizeof(*pCur)); + if (pCur == NULL) { + return NULL; + } + + pCur->uid = uid; + ret = tdbDbcOpen(pDB->pSmaIdx, &(pCur->pCur)); + if ((ret != 0) || (pCur->pCur == NULL)) { + taosMemoryFree(pCur); + return NULL; + } + + if (uid != 0) { + // TODO: move to the specific uid + } + + return pCur; +#endif } -void metaCloseSmaCurosr(SMSmaCursor *pCur) { +/** + * @brief + * + * @param pCur + * @return int64_t smaIndexUid + */ +int64_t metaSmaCursorNext(SMSmaCursor *pCur) { // TODO - ASSERT(0); + // ASSERT(0); + // return NULL; +#ifdef META_TDB_SMA_TEST + int ret; + void *pBuf; + SSmaIdxKey *smaIdxKey; + + ret = tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, &pCur->pVal, &pCur->vLen); + if (ret < 0) { + return 0; + } + smaIdxKey = pCur->pKey; + return smaIdxKey->smaUid; +#endif +} + +void metaCloseSmaCursor(SMSmaCursor *pCur) { + // TODO + // ASSERT(0); +#ifdef META_TDB_SMA_TEST + if (pCur) { + if (pCur->pCur) { + tdbDbcClose(pCur->pCur); + } + + taosMemoryFree(pCur); + } +#endif } SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { // TODO // ASSERT(0); // comment this line to pass CI - return NULL; -} + // return NULL: +#ifdef META_TDB_SMA_TEST + SArray *pUids = NULL; + SMetaDB *pDB = pMeta->pDB; + void *pKey; -SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { - // TODO - ASSERT(0); - return NULL; + // TODO: lock? + SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, 0); + if (pCur == NULL) { + return NULL; + } + // TODO: lock? + + SSmaIdxKey *pSmaIdxKey = NULL; + tb_uid_t uid = 0; + while (true) { + // TODO: lock during iterate? + if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) { + ASSERT(pSmaIdxKey != NULL); + pSmaIdxKey = pCur->pKey; + + if (pSmaIdxKey->uid == 0 || pSmaIdxKey->uid == uid) { + continue; + } + uid = pSmaIdxKey->uid; + + if (!pUids) { + pUids = taosArrayInit(16, sizeof(tb_uid_t)); + if (!pUids) { + metaCloseSmaCursor(pCur); + return NULL; + } + } + + taosArrayPush(pUids, &uid); + + continue; + } + break; + } + + metaCloseSmaCursor(pCur); + + return pUids; +#endif } static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0d4acd4594..794427475e 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -82,9 +82,9 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi memcpy(data, msg, msgLen); if (msgType == TDMT_VND_SUBMIT) { - // if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg) != 0) { - // return -1; - // } + if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg) != 0) { + return -1; + } } SRpcMsg req = { diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 07b7d62165..bfdad836f1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -38,6 +38,29 @@ typedef enum { SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma } ESmaStorageLevel; +typedef struct SPoolMem { + int64_t size; + struct SPoolMem *prev; + struct SPoolMem *next; +} SPoolMem; + +struct SSmaEnv { + TdThreadRwlock lock; + TXN txn; + SPoolMem *pPool; + SDiskID did; + TENV *dbEnv; // TODO: If it's better to put it in smaIndex level? + char *path; // relative path + SSmaStat *pStat; +}; + +#define SMA_ENV_LOCK(env) ((env)->lock) +#define SMA_ENV_DID(env) ((env)->did) +#define SMA_ENV_ENV(env) ((env)->dbEnv) +#define SMA_ENV_PATH(env) ((env)->path) +#define SMA_ENV_STAT(env) ((env)->pStat) +#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems) + typedef struct { STsdb *pTsdb; SDBFile dFile; @@ -104,7 +127,8 @@ static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH); static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit); static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, int32_t fid); -static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen); +static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, + TXN *txn); static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel); static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid); @@ -117,9 +141,121 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg); // mgmt interface static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); +// Pool Memory +static SPoolMem *openPool(); +static void clearPool(SPoolMem *pPool); +static void closePool(SPoolMem *pPool); +static void *poolMalloc(void *arg, size_t size); +static void poolFree(void *arg, void *ptr); + +static int tsdbSmaBeginCommit(SSmaEnv *pEnv); +static int tsdbSmaEndCommit(SSmaEnv *pEnv); + // implementation -static FORCE_INLINE int16_t tsdbTSmaAdd(STsdb *pTsdb, int16_t n) { return atomic_add_fetch_16(&REPO_TSMA_NUM(pTsdb), n); } -static FORCE_INLINE int16_t tsdbTSmaSub(STsdb *pTsdb, int16_t n) { return atomic_sub_fetch_16(&REPO_TSMA_NUM(pTsdb), n); } +static FORCE_INLINE int16_t tsdbTSmaAdd(STsdb *pTsdb, int16_t n) { + return atomic_add_fetch_16(&REPO_TSMA_NUM(pTsdb), n); +} +static FORCE_INLINE int16_t tsdbTSmaSub(STsdb *pTsdb, int16_t n) { + return atomic_sub_fetch_16(&REPO_TSMA_NUM(pTsdb), n); +} + +static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) { + int code = taosThreadRwlockRdlock(&(pEnv->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) { + int code = taosThreadRwlockWrlock(&(pEnv->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) { + int code = taosThreadRwlockUnlock(&(pEnv->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static SPoolMem *openPool() { + SPoolMem *pPool = (SPoolMem *)tdbOsMalloc(sizeof(*pPool)); + + pPool->prev = pPool->next = pPool; + pPool->size = 0; + + return pPool; +} + +static void clearPool(SPoolMem *pPool) { + if (!pPool) return; + + SPoolMem *pMem; + + do { + pMem = pPool->next; + + if (pMem == pPool) break; + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + tdbOsFree(pMem); + } while (1); + + assert(pPool->size == 0); +} + +static void closePool(SPoolMem *pPool) { + if (pPool) { + clearPool(pPool); + tdbOsFree(pPool); + } +} + +static void *poolMalloc(void *arg, size_t size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = (SPoolMem *)tdbOsMalloc(sizeof(*pMem) + size); + if (pMem == NULL) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void poolFree(void *arg, void *ptr) { + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = &(((SPoolMem *)ptr)[-1]); + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + tdbOsFree(pMem); +} int32_t tsdbInitSma(STsdb *pTsdb) { // tSma @@ -213,7 +349,12 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) char aname[TSDB_FILENAME_LEN] = {0}; tfsAbsoluteName(pTsdb->pTfs, did, path, aname); - if (tsdbOpenBDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) { + if (tsdbOpenDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) { + tsdbFreeSmaEnv(pEnv); + return NULL; + } + + if ((pEnv->pPool = openPool()) == NULL) { tsdbFreeSmaEnv(pEnv); return NULL; } @@ -248,7 +389,8 @@ void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) { taosMemoryFreeClear(pSmaEnv->pStat); taosMemoryFreeClear(pSmaEnv->path); taosThreadRwlockDestroy(&(pSmaEnv->lock)); - tsdbCloseBDBEnv(pSmaEnv->dbEnv); + tsdbCloseDBEnv(pSmaEnv->dbEnv); + closePool(pSmaEnv->pPool); } } @@ -414,7 +556,7 @@ static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t } // cache smaMeta - STSma *pSma = metaGetSmaInfoByIndex(pTsdb->pMeta, indexUid); + STSma *pSma = metaGetSmaInfoByIndex(pTsdb->pMeta, indexUid, true); if (pSma == NULL) { terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; taosHashCleanup(pItem->expiredWindows); @@ -498,10 +640,6 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg) { return TSDB_CODE_FAILED; } -#ifndef TSDB_SMA_TEST - TSKEY expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE]; -#endif - // Firstly, assume that tSma can only be created on super table/normal table. // getActiveTimeWindow @@ -563,6 +701,10 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg) { TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, winSKey); + + // TODO: release only when suid changes. + tdDestroyTSmaWrapper(pSW); + taosMemoryFreeClear(pSW); } } @@ -676,10 +818,12 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) { * @param dataLen * @return int32_t */ -static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) { +static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, + TXN *txn) { SDBFile *pDBFile = &pSmaH->dFile; + // TODO: insert sma data blocks into B+Tree(TDB) - if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) { + if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { tsdbWarn("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); return TSDB_CODE_FAILED; @@ -826,6 +970,30 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe return daysPerFile; } +static int tsdbSmaBeginCommit(SSmaEnv *pEnv) { + TXN *pTxn = &pEnv->txn; + // start a new txn + tdbTxnOpen(pTxn, 0, poolMalloc, poolFree, pEnv->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + if (tdbBegin(pEnv->dbEnv, pTxn) != 0) { + tsdbWarn("tsdbSma tdb restart txn fail"); + return -1; + } + return 0; +} + +static int tsdbSmaEndCommit(SSmaEnv *pEnv) { + TXN *pTxn = &pEnv->txn; + + // Commit current txn + if (tdbCommit(pEnv->dbEnv, pTxn) != 0) { + tsdbWarn("tsdbSma tdb commit fail"); + return -1; + } + tdbTxnClose(pTxn); + clearPool(pEnv->pPool); + return 0; +} + /** * @brief Insert/Update Time-range-wise SMA data. * - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g. @@ -911,14 +1079,10 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char int64_t groupId = pDataBlock->info.groupId; for (int32_t j = 0; j < rows; ++j) { printf("|"); - TSKEY skey = 1649295200000; // TSKEY_INITIAL_VAL; // the start key of TS window by interval + TSKEY skey = TSKEY_INITIAL_VAL; // the start key of TS window by interval void *pSmaKey = &smaKey; bool isStartKey = false; - { - // just for debugging - isStartKey = true; - tsdbEncodeTSmaKey(groupId, skey, &pSmaKey); - } + int32_t tlen = 0; // reset the len pDataBuf = &dataBuf; // reset the buf for (int32_t k = 0; k < colNum; ++k) { @@ -929,7 +1093,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char if (!isStartKey) { isStartKey = true; skey = *(TSKEY *)var; - printf("==> skey = %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId); + printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId); tsdbEncodeTSmaKey(groupId, skey, &pSmaKey); } else { printf(" %" PRIi64 " |", *(int64_t *)var); @@ -1010,6 +1174,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char // TODO: tsdbStartTSmaCommit(); if (fid != tSmaH.dFile.fid) { if (tSmaH.dFile.fid != TSDB_IVLD_FID) { + tsdbSmaEndCommit(pEnv); tsdbCloseDBF(&tSmaH.dFile); } tsdbSetTSmaDataFile(&tSmaH, indexUid, fid); @@ -1020,12 +1185,14 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_FAILED; } + tsdbSmaBeginCommit(pEnv); } - if (tsdbInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen) != 0) { + if (tsdbInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) { tsdbWarn("vgId:%d insert tSma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 " since %s", REPO_ID(pTsdb), indexUid, skey, groupId, tstrerror(terrno)); + tsdbSmaEndCommit(pEnv); tsdbDestroyTSmaWriteH(&tSmaH); tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_FAILED; @@ -1044,9 +1211,10 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char printf("\n"); } } - + tsdbSmaEndCommit(pEnv); // TODO: not commit for every insert tsdbDestroyTSmaWriteH(&tSmaH); tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_SUCCESS; } @@ -1370,8 +1538,8 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, tsdbDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", REPO_ID(pTsdb), tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); - void *result = NULL; - uint32_t valueSize = 0; + void *result = NULL; + int32_t valueSize = 0; if ((result = tsdbGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize)) == NULL) { tsdbWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", REPO_ID(pTsdb), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); @@ -1422,7 +1590,7 @@ int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) { return -1; } tsdbDebug("vgId:%d TDMT_VND_CREATE_SMA msg received for %s:%" PRIi64, REPO_ID(pTsdb), vCreateSmaReq.tSma.indexName, - vCreateSmaReq.tSma.indexUid); + vCreateSmaReq.tSma.indexUid); // record current timezone of server side vCreateSmaReq.tSma.timezoneInt = tsTimezone; @@ -1464,7 +1632,7 @@ int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) { return -1; } - tsdbTSmaSub(pTsdb, 1); + tsdbTSmaSub(pTsdb, 1); // TODO: return directly or go on follow steps? return TSDB_CODE_SUCCESS; diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c new file mode 100644 index 0000000000..519ecd3fa0 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC + +#include "vnodeInt.h" + +int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) { + int ret = 0; + + if (path == NULL) return -1; + + ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param + + if (ret != 0) { + tsdbError("Failed to create tsdb db env, ret = %d", ret); + return -1; + } + + return 0; +} + +int32_t tsdbCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); } + +static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { + const SSmaKey *pKey1 = (const SSmaKey *)arg1; + const SSmaKey *pKey2 = (const SSmaKey *)arg2; + + ASSERT(len1 == len2 && len1 == sizeof(SSmaKey)); + + if (pKey1->skey < pKey2->skey) { + return -1; + } else if (pKey1->skey > pKey2->skey) { + return 1; + } + if (pKey1->groupId < pKey2->groupId) { + return -1; + } else if (pKey1->groupId > pKey2->groupId) { + return 1; + } + + return 0; +} + +static int32_t tsdbOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { + int ret; + FKeyComparator compFunc; + + // Create a database + compFunc = tsdbSmaKeyCmpr; + ret = tdbDbOpen(pFName, TDB_VARIANT_LEN, TDB_VARIANT_LEN, compFunc, pEnv, ppDB); + + return 0; +} + +static int32_t tsdbCloseDBDb(TDB *pDB) { return tdbDbClose(pDB); } + +int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF) { + // TEnv is shared by a group of SDBFile + if (!pEnv || !pDBF) { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } + + // Open DBF + if (tsdbOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + tsdbCloseDBDb(pDBF->pDB); + return -1; + } + + return 0; +} + +int32_t tsdbCloseDBF(SDBFile *pDBF) { + int32_t ret = 0; + if (pDBF->pDB) { + ret = tsdbCloseDBDb(pDBF->pDB); + pDBF->pDB = NULL; + } + taosMemoryFreeClear(pDBF->path); + return ret; +} + +int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { + int32_t ret; + + ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + if (ret < 0) { + tsdbError("Failed to create insert sma data into db, ret = %d", ret); + return -1; + } + + return 0; +} + +void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen) { + void *result; + void *pVal; + int ret; + + ret = tdbDbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); + + if (ret < 0) { + tsdbError("Failed to get sma data from db, ret = %d", ret); + return NULL; + } + + ASSERT(*valLen >= 0); + + result = taosMemoryMalloc(*valLen); + + if (result == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + // TODO: lock? + // TODO: Would the key/value be destoryed during return the data? + // TODO: How about the key is updated while value length is changed? The original value buffer would be freed + // automatically? + memcpy(result, pVal, *valLen); + + return result; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index bf2260c51c..0449319dc2 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -19,7 +19,7 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { // TODO // blockDebugShowData(data); - // tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data); + tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data); } void vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { @@ -232,9 +232,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { tdDestroyTSma(&vCreateSmaReq.tSma); // TODO: return directly or go on follow steps? #endif - // if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { - // // TODO - // } + if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { + // TODO + } // } break; // case TDMT_VND_CANCEL_SMA: { // timeRangeSMA // } break; diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 6a4adfe4f8..da874716f2 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -210,7 +210,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { // get value by indexName STSma *qSmaCfg = NULL; - qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid1); + qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid1, true); assert(qSmaCfg != NULL); printf("name1 = %s\n", qSmaCfg->indexName); printf("timezone1 = %" PRIi8 "\n", qSmaCfg->timezoneInt); @@ -221,7 +221,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { tdDestroyTSma(qSmaCfg); taosMemoryFreeClear(qSmaCfg); - qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid2); + qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid2, true); assert(qSmaCfg != NULL); printf("name2 = %s\n", qSmaCfg->indexName); printf("timezone2 = %" PRIi8 "\n", qSmaCfg->timezoneInt); @@ -233,11 +233,12 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { taosMemoryFreeClear(qSmaCfg); // get index name by table uid +#if 0 SMSmaCursor *pSmaCur = metaOpenSmaCursor(pMeta, tbUid); assert(pSmaCur != NULL); uint32_t indexCnt = 0; while (1) { - const char *indexName = metaSmaCursorNext(pSmaCur); + const char *indexName = (const char *)metaSmaCursorNext(pSmaCur); if (indexName == NULL) { break; } @@ -245,8 +246,8 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { ++indexCnt; } EXPECT_EQ(indexCnt, nCntTSma); - metaCloseSmaCurosr(pSmaCur); - + metaCloseSmaCursor(pSmaCur); +#endif // get wrapper by table uid STSmaWrapper *pSW = metaGetSmaInfoByTable(pMeta, tbUid); assert(pSW != NULL); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 9df749bec7..8a95635ddc 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1196,21 +1196,22 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { return -1; } - // TODO: vLen may be zero - pVal = TDB_REALLOC(*ppVal, cd.vLen); - if (pVal == NULL) { - TDB_FREE(pKey); - return -1; - } - *ppKey = pKey; - *ppVal = pVal; - *kLen = cd.kLen; - *vLen = cd.vLen; - memcpy(pKey, cd.pKey, cd.kLen); - memcpy(pVal, cd.pVal, cd.vLen); + + if (ppVal) { + // TODO: vLen may be zero + pVal = TDB_REALLOC(*ppVal, cd.vLen); + if (pVal == NULL) { + TDB_FREE(pKey); + return -1; + } + + *ppVal = pVal; + *vLen = cd.vLen; + memcpy(pVal, cd.pVal, cd.vLen); + } ret = tdbBtcMoveToNext(pBtc); if (ret < 0) { diff --git a/tests/script/tsim/sma/tsmaCreateInsertData.sim b/tests/script/tsim/sma/tsmaCreateInsertData.sim new file mode 100644 index 0000000000..87140e5d84 --- /dev/null +++ b/tests/script/tsim/sma/tsmaCreateInsertData.sim @@ -0,0 +1,49 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database d1 +sql show databases +if $rows != 2 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use d1 + +print =============== create super table, include column type for count/sum/min/max/first +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +sql create table ct1 using stb tags(1000) + +sql show tables +if $rows != 1 then + return -1 +endi + +print =============== insert data, mode1: one row one table in sql +sql insert into ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) + + +print =============== create sma index from super table +sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(5m,10s) sliding(2m) +print $data00 $data01 $data02 $data03 +sleep 300 + +print =============== trigger stream to execute sma aggr task and insert sma data into sma store +sql insert into ct1 values(now+5s, 20, 20.0, 30.0) +#=================================================================== + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From d9e8a40f26e7702be9bf72345528a52a89fcf2aa Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 13 Apr 2022 16:05:23 +0800 Subject: [PATCH 02/13] add tsma create/insert data case --- tests/script/jenkins/basic.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index c5c041be91..c14d4fe27b 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -80,4 +80,7 @@ ./test.sh -f tsim/qnode/basic1.sim -m ./test.sh -f tsim/mnode/basic1.sim -m +# --- sma +./test.sh -f tsim/sma/tsmaCreateInsertData.sim + #======================b1-end=============== From 7cf63f0990ef8e4d87ca7bc281133e1891cf6de0 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 13 Apr 2022 16:09:55 +0800 Subject: [PATCH 03/13] remove sleep --- tests/script/tsim/sma/tsmaCreateInsertData.sim | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/script/tsim/sma/tsmaCreateInsertData.sim b/tests/script/tsim/sma/tsmaCreateInsertData.sim index 87140e5d84..9fc3700da1 100644 --- a/tests/script/tsim/sma/tsmaCreateInsertData.sim +++ b/tests/script/tsim/sma/tsmaCreateInsertData.sim @@ -39,7 +39,6 @@ sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s print =============== create sma index from super table sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(5m,10s) sliding(2m) print $data00 $data01 $data02 $data03 -sleep 300 print =============== trigger stream to execute sma aggr task and insert sma data into sma store sql insert into ct1 values(now+5s, 20, 20.0, 30.0) From 11e772d4d4074c9160db4f5089770a1664ffb2a0 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 13 Apr 2022 16:21:54 +0800 Subject: [PATCH 04/13] fix(grant): taosd -k. --- include/os/osFile.h | 2 +- include/os/osSystem.h | 7 ++++ source/dnode/mgmt/CMakeLists.txt | 2 +- source/dnode/mgmt/exe/dndMain.c | 2 +- source/dnode/mnode/impl/CMakeLists.txt | 7 ++++ .../dnode/mnode/impl/inc/mndGrant.h | 4 +-- source/dnode/mnode/impl/src/mndGrant.c | 6 ++-- source/dnode/mnode/impl/src/mnode.c | 2 ++ source/os/src/osFile.c | 2 +- source/os/src/osSystem.c | 32 +++++++++++++++++++ 10 files changed, 57 insertions(+), 9 deletions(-) rename include/common/tgrant.h => source/dnode/mnode/impl/inc/mndGrant.h (92%) diff --git a/include/os/osFile.h b/include/os/osFile.h index 89b58cdd65..36ca6fb8bb 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -79,7 +79,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); void taosFprintfFile(TdFilePtr pFile, const char *format, ...); -int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf); +int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict ptrBuf); int32_t taosEOFFile(TdFilePtr pFile); diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 15959a2d8c..33b0a46ee9 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -29,6 +29,13 @@ extern "C" { #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID #endif +typedef struct TdCmd *TdCmdPtr; + +TdCmdPtr taosOpenCmd(const char *cmd); +int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf); +int32_t taosEOFCmd(TdCmdPtr pCmd); +int64_t taosCloseCmd(TdCmdPtr *ppCmd); + void* taosLoadDll(const char* filename); void* taosLoadSym(void* handle, char* name); void taosCloseDll(void* handle); diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 297e3a08d7..8fb4b02479 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -20,6 +20,7 @@ add_executable(taosd ${EXEC_SRC}) target_include_directories( taosd PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/impl/inc" ) target_link_libraries(taosd dnode) @@ -28,7 +29,6 @@ IF (TD_GRANT) ENDIF () IF (TD_USB_DONGLE) TARGET_LINK_LIBRARIES(taosd usb_dongle) -else() ENDIF () if(${BUILD_TEST}) diff --git a/source/dnode/mgmt/exe/dndMain.c b/source/dnode/mgmt/exe/dndMain.c index 66ab2a6dcd..80c431fe36 100644 --- a/source/dnode/mgmt/exe/dndMain.c +++ b/source/dnode/mgmt/exe/dndMain.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "dndInt.h" #include "tconfig.h" -#include "tgrant.h" +#include "mndGrant.h" static struct { bool dumpConfig; diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 341dbf6135..8cb5e2a528 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -9,6 +9,13 @@ target_link_libraries( mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser ) +IF (TD_GRANT) + TARGET_LINK_LIBRARIES(mnode grant) +ENDIF () +IF (TD_USB_DONGLE) + TARGET_LINK_LIBRARIES(mnode usb_dongle) +ENDIF () + if(${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/include/common/tgrant.h b/source/dnode/mnode/impl/inc/mndGrant.h similarity index 92% rename from include/common/tgrant.h rename to source/dnode/mnode/impl/inc/mndGrant.h index 962af0ddc4..ad3dc7f79d 100644 --- a/include/common/tgrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -36,8 +36,8 @@ typedef enum { TSDB_GRANT_CPU_CORES, } EGrantType; -int32_t grantInit(); -void grantCleanUp(); +int32_t mndInitGrant(); +void mndCleanupGrant(); void grantParseParameter(); int32_t grantCheck(EGrantType grant); void grantReset(EGrantType grant, uint64_t value); diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index cdf3271789..37f87142e6 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -17,11 +17,11 @@ #ifndef _GRANT #include "os.h" #include "taoserror.h" -#include "tgrant.h" +#include "mndGrant.h" #include "mndInt.h" -int32_t grantInit() { return TSDB_CODE_SUCCESS; } -void grantCleanUp() {} +int32_t mndInitGrant(SMnode *pMnode) { return TSDB_CODE_SUCCESS; } +void mndCleanupGrant() {} void grantParseParameter() { mError("can't parsed parameter k"); } int32_t grantCheck(EGrantType grant) { return TSDB_CODE_SUCCESS; } void grantReset(EGrantType grant, uint64_t value) {} diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 5a1780530c..cad6396338 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -40,6 +40,7 @@ #include "mndUser.h" #include "mndVgroup.h" #include "mndQuery.h" +#include "mndGrant.h" #define MQ_TIMER_MS 3000 #define TRNAS_TIMER_MS 6000 @@ -197,6 +198,7 @@ static int32_t mndInitSteps(SMnode *pMnode, bool deploy) { if (mndAllocStep(pMnode, "mnode-qnode", mndInitBnode, mndCleanupBnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1; if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1; if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 4bd6b9e5cd..4b55d4912c 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -768,7 +768,7 @@ int32_t taosUmaskFile(int32_t maskVal) { } int32_t taosGetErrorFile(TdFilePtr pFile) { return errno; } -int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict__ ptrBuf) { +int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { if (pFile == NULL) { return -1; } diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index a36b3d41d0..665f6370e1 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -29,6 +29,8 @@ struct termios oldtio; #endif +typedef struct FILE TdCmd; + void* taosLoadDll(const char* filename) { #if defined(WINDOWS) return NULL; @@ -178,3 +180,33 @@ void resetTerminalMode() { } #endif } + +TdCmdPtr taosOpenCmd(const char *cmd) { + if (cmd == NULL) return NULL; + return (TdCmdPtr)popen(cmd, "r"); +} + +int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf) { + if (pCmd == NULL) { + return -1; + } + + size_t len = 0; + return getline(ptrBuf, &len, (FILE*)pCmd); +} + +int32_t taosEOFCmd(TdCmdPtr pCmd) { + if (pCmd == NULL) { + return 0; + } + return feof((FILE*)pCmd); +} + +int64_t taosCloseCmd(TdCmdPtr *ppCmd) { + if (ppCmd == NULL || *ppCmd == NULL) { + return 0; + } + pclose((FILE*)(*ppCmd)); + *ppCmd = NULL; + return 0; +} From 222987c002125434357038c16636366ab115e779 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 13 Apr 2022 16:39:36 +0800 Subject: [PATCH 05/13] Revert "fix(grant): taosd -k." This reverts commit 11e772d4d4074c9160db4f5089770a1664ffb2a0. --- .../inc/mndGrant.h => include/common/tgrant.h | 4 +-- include/os/osFile.h | 2 +- include/os/osSystem.h | 7 ---- source/dnode/mgmt/CMakeLists.txt | 2 +- source/dnode/mgmt/exe/dndMain.c | 2 +- source/dnode/mnode/impl/CMakeLists.txt | 7 ---- source/dnode/mnode/impl/src/mndGrant.c | 6 ++-- source/dnode/mnode/impl/src/mnode.c | 2 -- source/os/src/osFile.c | 2 +- source/os/src/osSystem.c | 32 ------------------- 10 files changed, 9 insertions(+), 57 deletions(-) rename source/dnode/mnode/impl/inc/mndGrant.h => include/common/tgrant.h (92%) diff --git a/source/dnode/mnode/impl/inc/mndGrant.h b/include/common/tgrant.h similarity index 92% rename from source/dnode/mnode/impl/inc/mndGrant.h rename to include/common/tgrant.h index ad3dc7f79d..962af0ddc4 100644 --- a/source/dnode/mnode/impl/inc/mndGrant.h +++ b/include/common/tgrant.h @@ -36,8 +36,8 @@ typedef enum { TSDB_GRANT_CPU_CORES, } EGrantType; -int32_t mndInitGrant(); -void mndCleanupGrant(); +int32_t grantInit(); +void grantCleanUp(); void grantParseParameter(); int32_t grantCheck(EGrantType grant); void grantReset(EGrantType grant, uint64_t value); diff --git a/include/os/osFile.h b/include/os/osFile.h index 36ca6fb8bb..89b58cdd65 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -79,7 +79,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); void taosFprintfFile(TdFilePtr pFile, const char *format, ...); -int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict ptrBuf); +int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf); int32_t taosEOFFile(TdFilePtr pFile); diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 33b0a46ee9..15959a2d8c 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -29,13 +29,6 @@ extern "C" { #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID #endif -typedef struct TdCmd *TdCmdPtr; - -TdCmdPtr taosOpenCmd(const char *cmd); -int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf); -int32_t taosEOFCmd(TdCmdPtr pCmd); -int64_t taosCloseCmd(TdCmdPtr *ppCmd); - void* taosLoadDll(const char* filename); void* taosLoadSym(void* handle, char* name); void taosCloseDll(void* handle); diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 8fb4b02479..297e3a08d7 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -20,7 +20,6 @@ add_executable(taosd ${EXEC_SRC}) target_include_directories( taosd PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" - PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/impl/inc" ) target_link_libraries(taosd dnode) @@ -29,6 +28,7 @@ IF (TD_GRANT) ENDIF () IF (TD_USB_DONGLE) TARGET_LINK_LIBRARIES(taosd usb_dongle) +else() ENDIF () if(${BUILD_TEST}) diff --git a/source/dnode/mgmt/exe/dndMain.c b/source/dnode/mgmt/exe/dndMain.c index 80c431fe36..66ab2a6dcd 100644 --- a/source/dnode/mgmt/exe/dndMain.c +++ b/source/dnode/mgmt/exe/dndMain.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "dndInt.h" #include "tconfig.h" -#include "mndGrant.h" +#include "tgrant.h" static struct { bool dumpConfig; diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 8cb5e2a528..341dbf6135 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -9,13 +9,6 @@ target_link_libraries( mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser ) -IF (TD_GRANT) - TARGET_LINK_LIBRARIES(mnode grant) -ENDIF () -IF (TD_USB_DONGLE) - TARGET_LINK_LIBRARIES(mnode usb_dongle) -ENDIF () - if(${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index 37f87142e6..cdf3271789 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -17,11 +17,11 @@ #ifndef _GRANT #include "os.h" #include "taoserror.h" -#include "mndGrant.h" +#include "tgrant.h" #include "mndInt.h" -int32_t mndInitGrant(SMnode *pMnode) { return TSDB_CODE_SUCCESS; } -void mndCleanupGrant() {} +int32_t grantInit() { return TSDB_CODE_SUCCESS; } +void grantCleanUp() {} void grantParseParameter() { mError("can't parsed parameter k"); } int32_t grantCheck(EGrantType grant) { return TSDB_CODE_SUCCESS; } void grantReset(EGrantType grant, uint64_t value) {} diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index cad6396338..5a1780530c 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -40,7 +40,6 @@ #include "mndUser.h" #include "mndVgroup.h" #include "mndQuery.h" -#include "mndGrant.h" #define MQ_TIMER_MS 3000 #define TRNAS_TIMER_MS 6000 @@ -198,7 +197,6 @@ static int32_t mndInitSteps(SMnode *pMnode, bool deploy) { if (mndAllocStep(pMnode, "mnode-qnode", mndInitBnode, mndCleanupBnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; - if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1; if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1; if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 4b55d4912c..4bd6b9e5cd 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -768,7 +768,7 @@ int32_t taosUmaskFile(int32_t maskVal) { } int32_t taosGetErrorFile(TdFilePtr pFile) { return errno; } -int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { +int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict__ ptrBuf) { if (pFile == NULL) { return -1; } diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index 665f6370e1..a36b3d41d0 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -29,8 +29,6 @@ struct termios oldtio; #endif -typedef struct FILE TdCmd; - void* taosLoadDll(const char* filename) { #if defined(WINDOWS) return NULL; @@ -180,33 +178,3 @@ void resetTerminalMode() { } #endif } - -TdCmdPtr taosOpenCmd(const char *cmd) { - if (cmd == NULL) return NULL; - return (TdCmdPtr)popen(cmd, "r"); -} - -int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf) { - if (pCmd == NULL) { - return -1; - } - - size_t len = 0; - return getline(ptrBuf, &len, (FILE*)pCmd); -} - -int32_t taosEOFCmd(TdCmdPtr pCmd) { - if (pCmd == NULL) { - return 0; - } - return feof((FILE*)pCmd); -} - -int64_t taosCloseCmd(TdCmdPtr *ppCmd) { - if (ppCmd == NULL || *ppCmd == NULL) { - return 0; - } - pclose((FILE*)(*ppCmd)); - *ppCmd = NULL; - return 0; -} From 5034cf4e3f67cccafb2d230524294d303ae012a6 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 13 Apr 2022 16:48:29 +0800 Subject: [PATCH 06/13] fix(grant): taosd -k. --- include/os/osFile.h | 2 +- include/os/osSystem.h | 7 ++++ source/dnode/mgmt/CMakeLists.txt | 2 +- source/dnode/mgmt/exe/dndMain.c | 2 +- source/dnode/mnode/impl/CMakeLists.txt | 7 ++++ .../dnode/mnode/impl/inc/mndGrant.h | 4 +-- source/dnode/mnode/impl/src/mndGrant.c | 6 ++-- source/dnode/mnode/impl/src/mnode.c | 2 ++ source/os/src/osFile.c | 2 +- source/os/src/osSystem.c | 32 +++++++++++++++++++ 10 files changed, 57 insertions(+), 9 deletions(-) rename include/common/tgrant.h => source/dnode/mnode/impl/inc/mndGrant.h (92%) diff --git a/include/os/osFile.h b/include/os/osFile.h index 89b58cdd65..36ca6fb8bb 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -79,7 +79,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); void taosFprintfFile(TdFilePtr pFile, const char *format, ...); -int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf); +int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict ptrBuf); int32_t taosEOFFile(TdFilePtr pFile); diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 15959a2d8c..33b0a46ee9 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -29,6 +29,13 @@ extern "C" { #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID #endif +typedef struct TdCmd *TdCmdPtr; + +TdCmdPtr taosOpenCmd(const char *cmd); +int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf); +int32_t taosEOFCmd(TdCmdPtr pCmd); +int64_t taosCloseCmd(TdCmdPtr *ppCmd); + void* taosLoadDll(const char* filename); void* taosLoadSym(void* handle, char* name); void taosCloseDll(void* handle); diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 297e3a08d7..8fb4b02479 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -20,6 +20,7 @@ add_executable(taosd ${EXEC_SRC}) target_include_directories( taosd PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/impl/inc" ) target_link_libraries(taosd dnode) @@ -28,7 +29,6 @@ IF (TD_GRANT) ENDIF () IF (TD_USB_DONGLE) TARGET_LINK_LIBRARIES(taosd usb_dongle) -else() ENDIF () if(${BUILD_TEST}) diff --git a/source/dnode/mgmt/exe/dndMain.c b/source/dnode/mgmt/exe/dndMain.c index 66ab2a6dcd..80c431fe36 100644 --- a/source/dnode/mgmt/exe/dndMain.c +++ b/source/dnode/mgmt/exe/dndMain.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "dndInt.h" #include "tconfig.h" -#include "tgrant.h" +#include "mndGrant.h" static struct { bool dumpConfig; diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 341dbf6135..8cb5e2a528 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -9,6 +9,13 @@ target_link_libraries( mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser ) +IF (TD_GRANT) + TARGET_LINK_LIBRARIES(mnode grant) +ENDIF () +IF (TD_USB_DONGLE) + TARGET_LINK_LIBRARIES(mnode usb_dongle) +ENDIF () + if(${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/include/common/tgrant.h b/source/dnode/mnode/impl/inc/mndGrant.h similarity index 92% rename from include/common/tgrant.h rename to source/dnode/mnode/impl/inc/mndGrant.h index 962af0ddc4..ad3dc7f79d 100644 --- a/include/common/tgrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -36,8 +36,8 @@ typedef enum { TSDB_GRANT_CPU_CORES, } EGrantType; -int32_t grantInit(); -void grantCleanUp(); +int32_t mndInitGrant(); +void mndCleanupGrant(); void grantParseParameter(); int32_t grantCheck(EGrantType grant); void grantReset(EGrantType grant, uint64_t value); diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index cdf3271789..37f87142e6 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -17,11 +17,11 @@ #ifndef _GRANT #include "os.h" #include "taoserror.h" -#include "tgrant.h" +#include "mndGrant.h" #include "mndInt.h" -int32_t grantInit() { return TSDB_CODE_SUCCESS; } -void grantCleanUp() {} +int32_t mndInitGrant(SMnode *pMnode) { return TSDB_CODE_SUCCESS; } +void mndCleanupGrant() {} void grantParseParameter() { mError("can't parsed parameter k"); } int32_t grantCheck(EGrantType grant) { return TSDB_CODE_SUCCESS; } void grantReset(EGrantType grant, uint64_t value) {} diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 5a1780530c..cad6396338 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -40,6 +40,7 @@ #include "mndUser.h" #include "mndVgroup.h" #include "mndQuery.h" +#include "mndGrant.h" #define MQ_TIMER_MS 3000 #define TRNAS_TIMER_MS 6000 @@ -197,6 +198,7 @@ static int32_t mndInitSteps(SMnode *pMnode, bool deploy) { if (mndAllocStep(pMnode, "mnode-qnode", mndInitBnode, mndCleanupBnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1; if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1; if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 4bd6b9e5cd..4b55d4912c 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -768,7 +768,7 @@ int32_t taosUmaskFile(int32_t maskVal) { } int32_t taosGetErrorFile(TdFilePtr pFile) { return errno; } -int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict__ ptrBuf) { +int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { if (pFile == NULL) { return -1; } diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index a36b3d41d0..665f6370e1 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -29,6 +29,8 @@ struct termios oldtio; #endif +typedef struct FILE TdCmd; + void* taosLoadDll(const char* filename) { #if defined(WINDOWS) return NULL; @@ -178,3 +180,33 @@ void resetTerminalMode() { } #endif } + +TdCmdPtr taosOpenCmd(const char *cmd) { + if (cmd == NULL) return NULL; + return (TdCmdPtr)popen(cmd, "r"); +} + +int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf) { + if (pCmd == NULL) { + return -1; + } + + size_t len = 0; + return getline(ptrBuf, &len, (FILE*)pCmd); +} + +int32_t taosEOFCmd(TdCmdPtr pCmd) { + if (pCmd == NULL) { + return 0; + } + return feof((FILE*)pCmd); +} + +int64_t taosCloseCmd(TdCmdPtr *ppCmd) { + if (ppCmd == NULL || *ppCmd == NULL) { + return 0; + } + pclose((FILE*)(*ppCmd)); + *ppCmd = NULL; + return 0; +} From 9a09f15de8bf754dcbf0c6ad291985bcbfc5a660 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao <36554565+glzhao89@users.noreply.github.com> Date: Wed, 13 Apr 2022 12:38:57 +0800 Subject: [PATCH 07/13] feat(query): add timediff function TD-14243 --- include/libs/scalar/scalar.h | 5 +- source/libs/function/src/builtins.c | 14 +++ source/libs/scalar/src/sclfunc.c | 136 ++++++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 2 deletions(-) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index caea52a5ef..10b4866a96 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -31,8 +31,8 @@ pNode will be freed in API; */ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes); -/* -pDst need to freed in caller +/* +pDst need to freed in caller */ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst); @@ -77,6 +77,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); bool getTimePseudoFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 36de0d1149..27067fc966 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -423,6 +423,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = timeTruncateFunction, .finalizeFunc = NULL }, + { + .name = "timediff", + .type = FUNCTION_TYPE_TIMEDIFF, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = checkAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = timeDiffFunction, + .finalizeFunc = NULL + }, { .name = "_rowts", .type = FUNCTION_TYPE_ROWTS, @@ -651,6 +661,10 @@ int32_t checkAndGetResultType(SFunctionNode* pFunc) { pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; break; } + case FUNCTION_TYPE_TIMEDIFF: { + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; + break; + } case FUNCTION_TYPE_TBNAME: { // todo diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index b53dc955de..76059cf2e1 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1104,6 +1104,142 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara return TSDB_CODE_SUCCESS; } +int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 2 && inputNum != 3) { + return TSDB_CODE_FAILED; + } + + int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]); + int64_t timeUnit = -1, timeVal[2] = {0}; + if (inputNum == 3) { + if (GET_PARAM_TYPE(&pInput[2]) != TSDB_DATA_TYPE_BIGINT) { + return TSDB_CODE_FAILED; + } + GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); + } + + char *input[2]; + for (int32_t k = 0; k < 2; ++k) { + int32_t type = GET_PARAM_TYPE(&pInput[k]); + if (type != TSDB_DATA_TYPE_BIGINT && type != TSDB_DATA_TYPE_TIMESTAMP && + type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) { + return TSDB_CODE_FAILED; + } + + if (IS_VAR_DATA_TYPE(type)) { + input[k] = pInput[k].columnData->pData + pInput[k].columnData->varmeta.offset[0]; + } else { + input[k] = pInput[k].columnData->pData; + } + } + + for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { + for (int32_t k = 0; k < 2; ++k) { + if (colDataIsNull_s(pInput[0].columnData, i)) { + colDataAppendNULL(pOutput->columnData, i); + continue; + } + + int32_t type = GET_PARAM_TYPE(&pInput[k]); + if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ + convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k]); + } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) { /* unix timestamp or ts column*/ + GET_TYPED_DATA(timeVal[k], int64_t, type, input[k]); + if (type == TSDB_DATA_TYPE_TIMESTAMP) { + int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : + (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); + int64_t timeValSec = timeVal[k] / factor; + if (timeValSec < 1000000000) { + timeVal[k] = timeValSec; + } + } + + char buf[20] = {0}; + NUM_TO_STRING(TSDB_DATA_TYPE_BIGINT, &timeVal[k], sizeof(buf), buf); + int32_t tsDigits = (int32_t)strlen(buf); + if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { + timeVal[k] = timeVal[k] * 1000000000; + } else if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { + timeVal[k] = timeVal[k] * 1000000; + } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { + timeVal[k] = timeVal[k] * 1000; + } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + timeVal[k] = timeVal[k]; + } + } + + if (IS_VAR_DATA_TYPE(type)) { + input[k] += varDataTLen(input[k]); + } else { + input[k] += tDataTypes[type].bytes; + } + } + + int64_t result = (timeVal[0] >= timeVal[1]) ? (timeVal[0] - timeVal[1]) : + (timeVal[1] - timeVal[0]); + + if (timeUnit < 0) { // if no time unit given use db precision + switch(timePrec) { + case TSDB_TIME_PRECISION_MILLI: { + result = result / 1000000; + break; + } + case TSDB_TIME_PRECISION_MICRO: { + result = result / 1000; + break; + } + case TSDB_TIME_PRECISION_NANO: { + result = result / 1; + break; + } + } + } else { + int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : + (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); + timeUnit = timeUnit * 1000 / factor; + switch(timeUnit) { + case 0: { /* 1u */ + result = result / 1000; + break; + } + case 1: { /* 1a */ + result = result / 1000000; + break; + } + case 1000: { /* 1s */ + result = result / 1000000000; + break; + } + case 60000: { /* 1m */ + result = result / 1000000000 / 60; + break; + } + case 3600000: { /* 1h */ + result = result / 1000000000 / 3600; + break; + } + case 86400000: { /* 1d */ + result = result / 1000000000 / 86400; + break; + } + case 604800000: { /* 1w */ + result = result / 1000000000 / 604800; + break; + } + default: { + break; + } + } + } + + colDataAppend(pOutput->columnData, i, (char *)&result, false); + } + + pOutput->numOfRows = pInput->numOfRows; + + return TSDB_CODE_SUCCESS; +} + int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return doScalarFunctionUnique(pInput, inputNum, pOutput, atan); } From e5b0b962d7e4f7a72a19375f5a79e23f6c3e94e4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 13 Apr 2022 17:00:41 +0800 Subject: [PATCH 08/13] test: adjust test case for show databases result on 3.0 (#11454) * [TD-13558]: taos shell refactor add taosTools as submodule * add tools/taos-tools * add more client interface for taosTools compile * update taos-tools * update taos-tools * refactor shell * [TD-13558]: taos shell test speed * [TD-13558]: taos -n startup works * taos -n rpc works * taos -n server works * cleanup code since no endPort in 3.0 * update taos-tools * [TD-13558]: taos -C works * improve taos shell -c WIP * update taos-tools * add demoapi.c * adjust show databases result for 3.0 --- tests/pytest/insert/binary.py | 41 +++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/tests/pytest/insert/binary.py b/tests/pytest/insert/binary.py index 0cbb7876c6..567a2eaad5 100644 --- a/tests/pytest/insert/binary.py +++ b/tests/pytest/insert/binary.py @@ -13,6 +13,23 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) + def getPath(self, tool="taos"): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + return paths[0] + def run(self): tdSql.prepare() @@ -53,15 +70,31 @@ class TDTestCase: tdLog.info("tdSql.checkData(0, 0, '34567')") tdSql.checkData(0, 0, '34567') tdLog.info("insert into tb values (now+4a, \"'';\")") - config_dir = subprocess.check_output(str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), stderr=subprocess.STDOUT, shell=True).decode('utf-8').replace('\n', '') - result = ''.join(os.popen(r"""taos -s "insert into db.tb values (now+4a, \"'';\")" -c %s"""%(config_dir)).readlines()) - if "Query OK" not in result: tdLog.exit("err:insert '';") + config_dir = subprocess.check_output( + str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), + stderr=subprocess.STDOUT, + shell=True).decode('utf-8').replace( + '\n', + '') + + binPath = self.getPath("taos") + if (binPath == ""): + tdLog.exit("taos not found!") + else: + tdLog.info("taos found: %s" % binPath) + + result = ''.join( + os.popen( + r"""%s -s "insert into db.tb values (now+4a, \"'';\")" -c %s""" % + (binPath, (config_dir))).readlines()) + if "Query OK" not in result: + tdLog.exit("err:insert '';") tdLog.info('drop database db') tdSql.execute('drop database db') tdLog.info('show databases') tdSql.query('show databases') tdLog.info('tdSql.checkRow(0)') - tdSql.checkRows(0) + tdSql.checkRows(1) # convert end def stop(self): From eabfc7d31cf38134d405bcc6c33adaffc50d91a4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao <36554565+glzhao89@users.noreply.github.com> Date: Wed, 13 Apr 2022 12:38:57 +0800 Subject: [PATCH 09/13] feat(query): add timediff function TD-14243 --- source/libs/scalar/src/sclfunc.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 76059cf2e1..db62a6b33d 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1168,10 +1168,12 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p } } - if (IS_VAR_DATA_TYPE(type)) { - input[k] += varDataTLen(input[k]); - } else { - input[k] += tDataTypes[type].bytes; + if (pInput[k].numOfRows != 1) { + if (IS_VAR_DATA_TYPE(type)) { + input[k] += varDataTLen(input[k]); + } else { + input[k] += tDataTypes[type].bytes; + } } } From 6eba8ad1ab461b80f8348c923f1966a3b8b7d4e2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 13 Apr 2022 17:50:33 +0800 Subject: [PATCH 10/13] test: add platform logic (#11459) * [TD-13558]: taos shell refactor add taosTools as submodule * add tools/taos-tools * add more client interface for taosTools compile * update taos-tools * update taos-tools * refactor shell * [TD-13558]: taos shell test speed * [TD-13558]: taos -n startup works * taos -n rpc works * taos -n server works * cleanup code since no endPort in 3.0 * update taos-tools * [TD-13558]: taos -C works * improve taos shell -c WIP * update taos-tools * add demoapi.c * adjust show databases result for 3.0 * test: add platform logic --- tests/pytest/insert/binary.py | 44 ++++++++++++++++------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/tests/pytest/insert/binary.py b/tests/pytest/insert/binary.py index 567a2eaad5..ffd1d6cb8c 100644 --- a/tests/pytest/insert/binary.py +++ b/tests/pytest/insert/binary.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import platform import sys from util.log import * from util.cases import * @@ -70,32 +71,27 @@ class TDTestCase: tdLog.info("tdSql.checkData(0, 0, '34567')") tdSql.checkData(0, 0, '34567') tdLog.info("insert into tb values (now+4a, \"'';\")") - config_dir = subprocess.check_output( - str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), - stderr=subprocess.STDOUT, - shell=True).decode('utf-8').replace( - '\n', - '') - binPath = self.getPath("taos") - if (binPath == ""): - tdLog.exit("taos not found!") - else: - tdLog.info("taos found: %s" % binPath) + if platform.system() == "Linux": + config_dir = subprocess.check_output( + str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), + stderr=subprocess.STDOUT, + shell=True).decode('utf-8').replace( + '\n', + '') - result = ''.join( - os.popen( - r"""%s -s "insert into db.tb values (now+4a, \"'';\")" -c %s""" % - (binPath, (config_dir))).readlines()) - if "Query OK" not in result: - tdLog.exit("err:insert '';") - tdLog.info('drop database db') - tdSql.execute('drop database db') - tdLog.info('show databases') - tdSql.query('show databases') - tdLog.info('tdSql.checkRow(0)') - tdSql.checkRows(1) -# convert end + binPath = self.getPath("taos") + if (binPath == ""): + tdLog.exit("taos not found!") + else: + tdLog.info("taos found: %s" % binPath) + + result = ''.join( + os.popen( + r"""%s -s "insert into db.tb values (now+4a, \"'';\")" -c %s""" % + (binPath, (config_dir))).readlines()) + if "Query OK" not in result: + tdLog.exit("err:insert '';") def stop(self): tdSql.close() From 7b3a8834981eeb64d6abf56ff74f071fdfc72466 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 13 Apr 2022 18:23:05 +0800 Subject: [PATCH 11/13] test: add tmq test cases --- tests/script/tsim/tmq/insertDataV1.sim | 16 +- tests/script/tsim/tmq/insertDataV4.sim | 14 +- tests/script/tsim/tmq/insertFixedDataV2.sim | 51 ++++ tests/script/tsim/tmq/insertFixedDataV4.sim | 51 ++++ .../tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim | 265 +++++++++++++++++ .../tsim/tmq/main2Con1Cgrp1TopicFrStb.sim | 270 ++++++++++++++++++ .../tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim | 265 +++++++++++++++++ .../tsim/tmq/main2Con1Cgrp2TopicFrStb.sim | 270 ++++++++++++++++++ .../tsim/tmq/mainConsumerInMultiTopic.sim | 52 ++-- .../tsim/tmq/mainConsumerInOneTopic.sim | 76 ++--- tests/test/c/tmqSim.c | 175 +++++++++++- 11 files changed, 1424 insertions(+), 81 deletions(-) create mode 100644 tests/script/tsim/tmq/insertFixedDataV2.sim create mode 100644 tests/script/tsim/tmq/insertFixedDataV4.sim create mode 100644 tests/script/tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim create mode 100644 tests/script/tsim/tmq/main2Con1Cgrp1TopicFrStb.sim create mode 100644 tests/script/tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim create mode 100644 tests/script/tsim/tmq/main2Con1Cgrp2TopicFrStb.sim diff --git a/tests/script/tsim/tmq/insertDataV1.sim b/tests/script/tsim/tmq/insertDataV1.sim index a349c55dbd..0df74d53f8 100644 --- a/tests/script/tsim/tmq/insertDataV1.sim +++ b/tests/script/tsim/tmq/insertDataV1.sim @@ -22,23 +22,19 @@ while $i < $tbNum $x = 0 while $x < $rowNum - $c = $x / 10 - $c = $c * 10 - $c = $x - $c - $binary = ' . binary - $binary = $binary . $c + $binary = $binary . $i $binary = $binary . ' - #print ====> insert into $tb values ($tstart , $c , $x , $binary ) - #print ====> insert into ntb values ($tstart , $c , $x , $binary ) - sql insert into $tb values ($tstart , $c , $x , $binary ) - sql insert into ntb values ($tstart , $c , $x , $binary ) + #print ====> insert into $tb values ($tstart , $i , $x , $binary ) + #print ====> insert into ntb values ($tstart , $i , $x , $binary ) + sql insert into $tb values ($tstart , $i , $x , $binary ) + sql insert into ntb values ($tstart , 999 , 999 , 'binary-ntb' ) $tstart = $tstart + 1 $x = $x + 1 endw - #print ====> insert rows: $rowNum into $tb and ntb + print ====> insert rows: $rowNum into $tb and ntb $i = $i + 1 # $tstart = 1640966400000 diff --git a/tests/script/tsim/tmq/insertDataV4.sim b/tests/script/tsim/tmq/insertDataV4.sim index 72c1358e7f..dbd52f56b8 100644 --- a/tests/script/tsim/tmq/insertDataV4.sim +++ b/tests/script/tsim/tmq/insertDataV4.sim @@ -22,18 +22,14 @@ while $i < $tbNum $x = 0 while $x < $rowNum - $c = $x / 10 - $c = $c * 10 - $c = $x - $c - $binary = ' . binary - $binary = $binary . $c + $binary = $binary . $i $binary = $binary . ' - #print ====> insert into $tb values ($tstart , $c , $x , $binary ) - #print ====> insert into ntb values ($tstart , $c , $x , $binary ) - sql insert into $tb values ($tstart , $c , $x , $binary ) - sql insert into ntb values ($tstart , $c , $x , $binary ) + #print ====> insert into $tb values ($tstart , $i , $x , $binary ) + #print ====> insert into ntb values ($tstart , $i , $x , $binary ) + sql insert into $tb values ($tstart , $i , $x , $binary ) + sql insert into ntb values ($tstart , 999 , 999 , 'binary-ntb' ) $tstart = $tstart + 1 $x = $x + 1 endw diff --git a/tests/script/tsim/tmq/insertFixedDataV2.sim b/tests/script/tsim/tmq/insertFixedDataV2.sim new file mode 100644 index 0000000000..a93be3e5a6 --- /dev/null +++ b/tests/script/tsim/tmq/insertFixedDataV2.sim @@ -0,0 +1,51 @@ + +sql connect + +print ================ insert data +$dbNamme = d0 +$tbPrefix = ct +$tbNum = 10 +$rowNum = 100 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + +$loopInsertNum = 10 + +sql use $dbNamme + +$loopIndex = 0 + +loop_insert: +print ====> loop $loopIndex insert +$loopIndex = $loopIndex + 1 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + + $x = 0 + while $x < $rowNum + $binary = ' . binary + $binary = $binary . $i + $binary = $binary . ' + + #print ====> insert into $tb values ($tstart , $i , $x , $binary ) + #print ====> insert into ntb values ($tstart , $i , $x , $binary ) + sql insert into $tb values ($tstart , $i , $x , $binary ) + sql insert into ntb values ($tstart , 999 , 999 , 'binary-ntb' ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + #print ====> insert rows: $rowNum into $tb and ntb + + $i = $i + 1 +# $tstart = 1640966400000 +endw + + +if $loopIndex < $loopInsertNum then + goto loop_insert +endi + +print ====> insert data end =========== + diff --git a/tests/script/tsim/tmq/insertFixedDataV4.sim b/tests/script/tsim/tmq/insertFixedDataV4.sim new file mode 100644 index 0000000000..9f7f86747f --- /dev/null +++ b/tests/script/tsim/tmq/insertFixedDataV4.sim @@ -0,0 +1,51 @@ + +sql connect + +print ================ insert data +$dbNamme = d1 +$tbPrefix = ct +$tbNum = 10 +$rowNum = 100 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + +$loopInsertNum = 10 + +sql use $dbNamme + +$loopIndex = 0 + +loop_insert: +print ====> loop $loopIndex insert +$loopIndex = $loopIndex + 1 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + + $x = 0 + while $x < $rowNum + $binary = ' . binary + $binary = $binary . $i + $binary = $binary . ' + + #print ====> insert into $tb values ($tstart , $i , $x , $binary ) + #print ====> insert into ntb values ($tstart , $i , $x , $binary ) + sql insert into $tb values ($tstart , $i , $x , $binary ) + sql insert into ntb values ($tstart , 999 , 999 , 'binary-ntb' ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + #print ====> insert rows: $rowNum into $tb and ntb + + $i = $i + 1 +# $tstart = 1640966400000 +endw + + +if $loopIndex < $loopInsertNum then + goto loop_insert +endi + +print ====> insert data end =========== + diff --git a/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim b/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim new file mode 100644 index 0000000000..c301219ad6 --- /dev/null +++ b/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim @@ -0,0 +1,265 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +# scene1: vgroups=2, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene2: vgroups=2, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene3: vgroups=4, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene4: vgroups=4, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## +######## This test case include scene1 and scene3 +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +$loop_cnt = 0 +$vgroups = 2 +$dbNamme = d0 +loop_vgroups: +print =============== create database $dbNamme vgroups $vgroups +sql create database $dbNamme vgroups $vgroups +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 + +if $loop_cnt == 0 then + if $rows != 2 then + return -1 + endi + if $data02 != 2 then # vgroups + print vgroups: $data02 + return -1 + endi +else + if $rows != 3 then + return -1 + endi + if $data00 == d1 then + if $data02 != 4 then # vgroups + print vgroups: $data02 + return -1 + endi + else + if $data12 != 4 then # vgroups + print vgroups: $data12 + return -1 + endi + endi +endi + +sql use $dbNamme + +print =============== create super table +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +$tbPrefix = ct +$tbNum = 100 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $i = $i + 1 +endw + +print =============== create normal table +sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + +print =============== create topics from child table + +sql create topic topic_ctb_column as select ts, c1, c3 from ct0 +sql create topic topic_ctb_all as select * from ct0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 + +#sql create topic topic_ntb_column as select ts, c1, c3 from ntb +#sql create topic topic_ntb_all as select * from ntb +#sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb + +sql show tables +if $rows != 101 then + return -1 +endi + +print =============== run_back insert data + +if $loop_cnt == 0 then + run_back tsim/tmq/insertFixedDataV2.sim +else + run_back tsim/tmq/insertFixedDataV4.sim +endi + +#sleep 1000 + +#$rowNum = 1000 +#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +# +#$i = 0 +#while $i < $tbNum +# $tb = $tbPrefix . $i +# +# $x = 0 +# while $x < $rowNum +# $c = $x / 10 +# $c = $c * 10 +# $c = $x - $c +# +# $binary = ' . binary +# $binary = $binary . $c +# $binary = $binary . ' +# +# sql insert into $tb values ($tstart , $c , $x , $binary ) +# sql insert into ntb values ($tstart , $c , $x , $binary ) +# $tstart = $tstart + 1 +# $x = $x + 1 +# endw +# +# $i = $i + 1 +# $tstart = 1640966400000 +#endw + +#root@trd02 /home $ tmq_sim --help +# -c Configuration directory, default is +# -d The name of the database for cosumer, no default +# -t The topic string for cosumer, no default +# -k The key-value string for cosumer, no default +# -g showMsgFlag, default is 0 +# + +$tbNum = 10 +$consumeDelay = 10 +$expectMsgCntFromCtb = 300 +$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum +print consumeDelay: $consumeDelay +print insert data child num: $tbNum +print expectMsgCntFromCtb: $expectMsgCntFromCtb +print expectMsgCntFromStb: $expectMsgCntFromStb + +# supported key: +# group.id: +# enable.auto.commit: +# auto.offset.reset: +# td.connect.ip: +# td.connect.user:root +# td.connect.pass:taosdata +# td.connect.port:6030 +# td.connect.db:db + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $rowNum +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0 +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0 +print cmd result----> $system_content +if $system_content != success then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0 +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0 +print cmd result----> $system_content +if $system_content != success then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0 +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 0 +print cmd result----> $system_content +if $system_content != success then + return -1 +endi + +#$expect_result = @{consume success: @ +#$expect_result = $expect_result . $totalMsgCnt +#$expect_result = $expect_result . @, @ +#$expect_result = $expect_result . 0} +#print expect_result----> $expect_result +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#$expect_result = @{consume success: @ +#$expect_result = $expect_result . $expectConsumeMsgCnt +#$expect_result = $expect_result . @, @ +#$expect_result = $expect_result . 0} +#print expect_result----> $expect_result +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +##print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +##system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +##print cmd result----> $system_content +###if $system_content != @{consume success: 10000, 0}@ then +##if $system_content != success then +## return -1 +##endi +# +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +#print cmd result----> $system_content +##if $system_content != @{consume success: 10000, 0}@ then +#if $system_content != success then +# return -1 +#endi + +if $loop_cnt == 0 then + $loop_cnt = 1 + $vgroups = 4 + $dbNamme = d1 + goto loop_vgroups +endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrStb.sim b/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrStb.sim new file mode 100644 index 0000000000..0f38a8b58b --- /dev/null +++ b/tests/script/tsim/tmq/main2Con1Cgrp1TopicFrStb.sim @@ -0,0 +1,270 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +# scene1: vgroups=2, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene2: vgroups=2, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene3: vgroups=4, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene4: vgroups=4, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## +######## This test case include scene1 and scene3 +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +$loop_cnt = 0 +$vgroups = 2 +$dbNamme = d0 +loop_vgroups: +print =============== create database $dbNamme vgroups $vgroups +sql create database $dbNamme vgroups $vgroups +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 + +if $loop_cnt == 0 then + if $rows != 2 then + return -1 + endi + if $data02 != 2 then # vgroups + print vgroups: $data02 + return -1 + endi +else + if $rows != 3 then + return -1 + endi + if $data00 == d1 then + if $data02 != 4 then # vgroups + print vgroups: $data02 + return -1 + endi + else + if $data12 != 4 then # vgroups + print vgroups: $data12 + return -1 + endi + endi +endi + +sql use $dbNamme + +print =============== create super table +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +$tbPrefix = ct +$tbNum = 100 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $i = $i + 1 +endw + +print =============== create normal table +sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + +print =============== create multi topics. notes: now only support: +print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb +print =============== will support: * from stb + +sql create topic topic_stb_column as select ts, c1, c3 from stb +#sql create topic topic_stb_all as select * from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +sql create topic topic_ctb_column as select ts, c1, c3 from ct0 +sql create topic topic_ctb_all as select * from ct0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 + +sql create topic topic_ntb_column as select ts, c1, c3 from ntb +sql create topic topic_ntb_all as select * from ntb +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb + +sql show tables +if $rows != 101 then + return -1 +endi + +print =============== run_back insert data + +if $loop_cnt == 0 then + run_back tsim/tmq/insertFixedDataV2.sim +else + run_back tsim/tmq/insertFixedDataV4.sim +endi + +#sleep 1000 + +#$rowNum = 1000 +#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +# +#$i = 0 +#while $i < $tbNum +# $tb = $tbPrefix . $i +# +# $x = 0 +# while $x < $rowNum +# $c = $x / 10 +# $c = $c * 10 +# $c = $x - $c +# +# $binary = ' . binary +# $binary = $binary . $c +# $binary = $binary . ' +# +# sql insert into $tb values ($tstart , $c , $x , $binary ) +# sql insert into ntb values ($tstart , $c , $x , $binary ) +# $tstart = $tstart + 1 +# $x = $x + 1 +# endw +# +# $i = $i + 1 +# $tstart = 1640966400000 +#endw + +#root@trd02 /home $ tmq_sim --help +# -c Configuration directory, default is +# -d The name of the database for cosumer, no default +# -t The topic string for cosumer, no default +# -k The key-value string for cosumer, no default +# -g showMsgFlag, default is 0 +# + +$tbNum = 10 +$consumeDelay = 10 +$expectMsgCntFromCtb = 300 +$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum +print consumeDelay: $consumeDelay +print insert data child num: $tbNum +print expectMsgCntFromCtb: $expectMsgCntFromCtb +print expectMsgCntFromStb: $expectMsgCntFromStb + +# supported key: +# group.id: +# enable.auto.commit: +# auto.offset.reset: +# td.connect.ip: +# td.connect.user:root +# td.connect.pass:taosdata +# td.connect.port:6030 +# td.connect.db:db + +#$expect_result = @{consume success: @ +#$expect_result = $expect_result . $rowNum +#$expect_result = $expect_result . @, @ +#$expect_result = $expect_result . 0} +#print expect_result----> $expect_result +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#$expect_result = @{consume success: @ +#$expect_result = $expect_result . $totalMsgCnt +#$expect_result = $expect_result . @, @ +#$expect_result = $expect_result . 0} +#print expect_result----> $expect_result +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $expectConsumeMsgCnt +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +print cmd result----> $system_content +if $system_content != success then + return -1 +endi + +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +#print cmd result----> $system_content +##if $system_content != @{consume success: 10000, 0}@ then +#if $system_content != success then +# return -1 +#endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +print cmd result----> $system_content +#if $system_content != @{consume success: 10000, 0}@ then +if $system_content != success then + return -1 +endi +if $loop_cnt == 0 then + $loop_cnt = 1 + $vgroups = 4 + $dbNamme = d1 + goto loop_vgroups +endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim b/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim new file mode 100644 index 0000000000..99d1c6e442 --- /dev/null +++ b/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim @@ -0,0 +1,265 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +# scene1: vgroups=2, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene2: vgroups=2, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene3: vgroups=4, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene4: vgroups=4, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## +######## This test case include scene1 and scene3 +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +$loop_cnt = 0 +$vgroups = 2 +$dbNamme = d0 +loop_vgroups: +print =============== create database $dbNamme vgroups $vgroups +sql create database $dbNamme vgroups $vgroups +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 + +if $loop_cnt == 0 then + if $rows != 2 then + return -1 + endi + if $data02 != 2 then # vgroups + print vgroups: $data02 + return -1 + endi +else + if $rows != 3 then + return -1 + endi + if $data00 == d1 then + if $data02 != 4 then # vgroups + print vgroups: $data02 + return -1 + endi + else + if $data12 != 4 then # vgroups + print vgroups: $data12 + return -1 + endi + endi +endi + +sql use $dbNamme + +print =============== create super table +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +$tbPrefix = ct +$tbNum = 100 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $i = $i + 1 +endw + +print =============== create normal table +sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + +print =============== create topics from child table + +sql create topic topic_ctb_column as select ts, c1, c3 from ct0 +sql create topic topic_ctb_all as select * from ct0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 + +#sql create topic topic_ntb_column as select ts, c1, c3 from ntb +#sql create topic topic_ntb_all as select * from ntb +#sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb + +sql show tables +if $rows != 101 then + return -1 +endi + +print =============== run_back insert data + +if $loop_cnt == 0 then + run_back tsim/tmq/insertFixedDataV2.sim +else + run_back tsim/tmq/insertFixedDataV4.sim +endi + +#sleep 1000 + +#$rowNum = 1000 +#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +# +#$i = 0 +#while $i < $tbNum +# $tb = $tbPrefix . $i +# +# $x = 0 +# while $x < $rowNum +# $c = $x / 10 +# $c = $c * 10 +# $c = $x - $c +# +# $binary = ' . binary +# $binary = $binary . $c +# $binary = $binary . ' +# +# sql insert into $tb values ($tstart , $c , $x , $binary ) +# sql insert into ntb values ($tstart , $c , $x , $binary ) +# $tstart = $tstart + 1 +# $x = $x + 1 +# endw +# +# $i = $i + 1 +# $tstart = 1640966400000 +#endw + +#root@trd02 /home $ tmq_sim --help +# -c Configuration directory, default is +# -d The name of the database for cosumer, no default +# -t The topic string for cosumer, no default +# -k The key-value string for cosumer, no default +# -g showMsgFlag, default is 0 +# + +$tbNum = 10 +$consumeDelay = 10 +$expectMsgCntFromCtb = 300 +$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum +print consumeDelay: $consumeDelay +print insert data child num: $tbNum +print expectMsgCntFromCtb: $expectMsgCntFromCtb +print expectMsgCntFromStb: $expectMsgCntFromStb + +# supported key: +# group.id: +# enable.auto.commit: +# auto.offset.reset: +# td.connect.ip: +# td.connect.user:root +# td.connect.pass:taosdata +# td.connect.port:6030 +# td.connect.db:db + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $rowNum +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 +print cmd result----> $system_content +if $system_content != success then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 +print cmd result----> $system_content +if $system_content != success then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 +print cmd result----> $system_content +if $system_content != success then + return -1 +endi + +#$expect_result = @{consume success: @ +#$expect_result = $expect_result . $totalMsgCnt +#$expect_result = $expect_result . @, @ +#$expect_result = $expect_result . 0} +#print expect_result----> $expect_result +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb -j 1 +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#$expect_result = @{consume success: @ +#$expect_result = $expect_result . $expectConsumeMsgCnt +#$expect_result = $expect_result . @, @ +#$expect_result = $expect_result . 0} +#print expect_result----> $expect_result +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +##print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +##system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +##print cmd result----> $system_content +###if $system_content != @{consume success: 10000, 0}@ then +##if $system_content != success then +## return -1 +##endi +# +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +#print cmd result----> $system_content +##if $system_content != @{consume success: 10000, 0}@ then +#if $system_content != success then +# return -1 +#endi + +if $loop_cnt == 0 then + $loop_cnt = 1 + $vgroups = 4 + $dbNamme = d1 + goto loop_vgroups +endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrStb.sim b/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrStb.sim new file mode 100644 index 0000000000..76ad75aead --- /dev/null +++ b/tests/script/tsim/tmq/main2Con1Cgrp2TopicFrStb.sim @@ -0,0 +1,270 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +# scene1: vgroups=2, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene2: vgroups=2, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene3: vgroups=4, one topic for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene4: vgroups=4, multi topics for two consumers, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## +######## This test case include scene1 and scene3 +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +$loop_cnt = 0 +$vgroups = 2 +$dbNamme = d0 +loop_vgroups: +print =============== create database $dbNamme vgroups $vgroups +sql create database $dbNamme vgroups $vgroups +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 + +if $loop_cnt == 0 then + if $rows != 2 then + return -1 + endi + if $data02 != 2 then # vgroups + print vgroups: $data02 + return -1 + endi +else + if $rows != 3 then + return -1 + endi + if $data00 == d1 then + if $data02 != 4 then # vgroups + print vgroups: $data02 + return -1 + endi + else + if $data12 != 4 then # vgroups + print vgroups: $data12 + return -1 + endi + endi +endi + +sql use $dbNamme + +print =============== create super table +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +$tbPrefix = ct +$tbNum = 100 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $i = $i + 1 +endw + +print =============== create normal table +sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + +print =============== create multi topics. notes: now only support: +print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb +print =============== will support: * from stb + +sql create topic topic_stb_column as select ts, c1, c3 from stb +#sql create topic topic_stb_all as select * from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +sql create topic topic_ctb_column as select ts, c1, c3 from ct0 +sql create topic topic_ctb_all as select * from ct0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 + +sql create topic topic_ntb_column as select ts, c1, c3 from ntb +sql create topic topic_ntb_all as select * from ntb +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb + +sql show tables +if $rows != 101 then + return -1 +endi + +print =============== run_back insert data + +if $loop_cnt == 0 then + run_back tsim/tmq/insertFixedDataV2.sim +else + run_back tsim/tmq/insertFixedDataV4.sim +endi + +#sleep 1000 + +#$rowNum = 1000 +#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +# +#$i = 0 +#while $i < $tbNum +# $tb = $tbPrefix . $i +# +# $x = 0 +# while $x < $rowNum +# $c = $x / 10 +# $c = $c * 10 +# $c = $x - $c +# +# $binary = ' . binary +# $binary = $binary . $c +# $binary = $binary . ' +# +# sql insert into $tb values ($tstart , $c , $x , $binary ) +# sql insert into ntb values ($tstart , $c , $x , $binary ) +# $tstart = $tstart + 1 +# $x = $x + 1 +# endw +# +# $i = $i + 1 +# $tstart = 1640966400000 +#endw + +#root@trd02 /home $ tmq_sim --help +# -c Configuration directory, default is +# -d The name of the database for cosumer, no default +# -t The topic string for cosumer, no default +# -k The key-value string for cosumer, no default +# -g showMsgFlag, default is 0 +# + +$tbNum = 10 +$consumeDelay = 10 +$expectMsgCntFromCtb = 300 +$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum +print consumeDelay: $consumeDelay +print insert data child num: $tbNum +print expectMsgCntFromCtb: $expectMsgCntFromCtb +print expectMsgCntFromStb: $expectMsgCntFromStb + +# supported key: +# group.id: +# enable.auto.commit: +# auto.offset.reset: +# td.connect.ip: +# td.connect.user:root +# td.connect.pass:taosdata +# td.connect.port:6030 +# td.connect.db:db + +#$expect_result = @{consume success: @ +#$expect_result = $expect_result . $rowNum +#$expect_result = $expect_result . @, @ +#$expect_result = $expect_result . 0} +#print expect_result----> $expect_result +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_column" -k1 "group.id:tg2" -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_all" -k1 "group.id:tg2" -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ctb_function" -k1 "group.id:tg2" -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#$expect_result = @{consume success: @ +#$expect_result = $expect_result . $totalMsgCnt +#$expect_result = $expect_result . @, @ +#$expect_result = $expect_result . 0} +#print expect_result----> $expect_result +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_column" -k1 "group.id:tg2" -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_all" -k1 "group.id:tg2" -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi +# +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_ntb_function" -k1 "group.id:tg2" -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +#print cmd result----> $system_content +#if $system_content != success then +# return -1 +#endi + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $expectMsgCntFromStb +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1 +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_function" -k1 "group.id:tg2" -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1 +print cmd result----> $system_content +if $system_content != success then + return -1 +endi + +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1 +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_all" -k1 "group.id:tg2" -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1 +#print cmd result----> $system_content +##if $system_content != @{consume success: 10000, 0}@ then +#if $system_content != success then +# return -1 +#endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1 +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t1 "topic_stb_column" -k1 "group.id:tg2" -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb -j 1 +print cmd result----> $system_content +#if $system_content != @{consume success: 10000, 0}@ then +if $system_content != success then + return -1 +endi +if $loop_cnt == 0 then + $loop_cnt = 1 + $vgroups = 4 + $dbNamme = d1 + goto loop_vgroups +endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/mainConsumerInMultiTopic.sim b/tests/script/tsim/tmq/mainConsumerInMultiTopic.sim index 0df7a8ba57..095809abe9 100644 --- a/tests/script/tsim/tmq/mainConsumerInMultiTopic.sim +++ b/tests/script/tsim/tmq/mainConsumerInMultiTopic.sim @@ -120,9 +120,9 @@ endi print =============== run_back insert data if $loop_cnt == 0 then - run_back tsim/tmq/insertDataV1.sim + run_back tsim/tmq/insertFixedDataV2.sim else - run_back tsim/tmq/insertDataV4.sim + run_back tsim/tmq/insertFixedDataV4.sim endi #sleep 1000 @@ -162,9 +162,16 @@ endi # -g showMsgFlag, default is 0 # -$consumeDelay = 50 -$consumeMsgCntFromTopic = 1000 -print consumeMsgCntFromTopic: $consumeMsgCntFromTopic , consumeDelay: $consumeDelay +$tbNum = 10 +$consumeDelay = 5 +$expectMsgCntFromCtb = 1000 +$expectMsgCntFromNtb = 1000 +$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum +print consumeDelay: $consumeDelay +print insert data child num: $tbNum +print expectMsgCntFromCtb: $expectMsgCntFromCtb +print expectMsgCntFromStb: $expectMsgCntFromStb + # supported key: # group.id: @@ -177,49 +184,50 @@ print consumeMsgCntFromTopic: $consumeMsgCntFromTopic , consumeDelay: $consumeDe # td.connect.db:db $numOfTopics = 2 -$expectConsumeMsgCnt = $consumeMsgCntFromTopic * $numOfTopics +$expectMsgCntFromStb = $expectMsgCntFromStb * $numOfTopics $expect_result = @{consume success: @ -$expect_result = $expect_result . $expectConsumeMsgCnt +$expect_result = $expect_result . $expectMsgCntFromStb $expect_result = $expect_result . @, @ $expect_result = $expect_result . 0} print expect_result----> $expect_result -#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb print cmd result----> $system_content #if $system_content != @{consume success: 20000, 0}@ then -if $system_content < $expect_result then +if $system_content != $expect_result then return -1 endi $numOfTopics = 3 -$expectConsumeMsgCnt = $consumeMsgCntFromTopic * $numOfTopics +$expectMsgCntFromCtb = $expectMsgCntFromCtb * $numOfTopics $expect_result = @{consume success: @ -$expect_result = $expect_result . $expectConsumeMsgCnt +$expect_result = $expect_result . $expectMsgCntFromCtb $expect_result = $expect_result . @, @ $expect_result = $expect_result . 0} print expect_result----> $expect_result -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb print cmd result----> $system_content #if $system_content != @{consume success: 300, 0}@ then -if $system_content < $expectConsumeMsgCnt then +if $system_content != $expect_result then return -1 endi $numOfTopics = 3 -$expectConsumeMsgCnt = $consumeMsgCntFromTopic * $numOfTopics +$expectMsgCntFromNtb = $expectMsgCntFromNtb * $tbNum +$expectMsgCntFromNtb = $expectMsgCntFromNtb * $numOfTopics $expect_result = @{consume success: @ -$expect_result = $expect_result . $expectConsumeMsgCnt +$expect_result = $expect_result . $expectMsgCntFromNtb $expect_result = $expect_result . @, @ $expect_result = $expect_result . 0} print expect_result----> $expect_result -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromNtb +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromNtb print cmd result----> $system_content #if $system_content != @{consume success: 30000, 0}@ then -if $system_content < $expectConsumeMsgCnt then +if $system_content != $expect_result then return -1 endi diff --git a/tests/script/tsim/tmq/mainConsumerInOneTopic.sim b/tests/script/tsim/tmq/mainConsumerInOneTopic.sim index b9a867921e..fc20e2db23 100644 --- a/tests/script/tsim/tmq/mainConsumerInOneTopic.sim +++ b/tests/script/tsim/tmq/mainConsumerInOneTopic.sim @@ -120,9 +120,9 @@ endi print =============== run_back insert data if $loop_cnt == 0 then - run_back tsim/tmq/insertDataV1.sim + run_back tsim/tmq/insertFixedDataV2.sim else - run_back tsim/tmq/insertDataV4.sim + run_back tsim/tmq/insertFixedDataV4.sim endi #sleep 1000 @@ -162,9 +162,15 @@ endi # -g showMsgFlag, default is 0 # -$consumeDelay = 50 -$expectConsumeMsgCnt = 1000 -print expectConsumeMsgCnt: $expectConsumeMsgCnt , consumeDelay: $consumeDelay +$tbNum = 10 +$consumeDelay = 5 +$expectMsgCntFromCtb = 1000 +$expectMsgCntFromStb = $expectMsgCntFromCtb * $tbNum +print consumeDelay: $consumeDelay +print insert data child num: $tbNum +print expectMsgCntFromCtb: $expectMsgCntFromCtb +print expectMsgCntFromStb: $expectMsgCntFromStb + # supported key: # group.id: @@ -177,82 +183,82 @@ print expectConsumeMsgCnt: $expectConsumeMsgCnt , consumeDelay: $consumeDelay # td.connect.db:db $expect_result = @{consume success: @ -$expect_result = $expect_result . $expectConsumeMsgCnt +$expect_result = $expect_result . $expectMsgCntFromStb $expect_result = $expect_result . @, @ $expect_result = $expect_result . 0} print expect_result----> $expect_result -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb print cmd result----> $system_content -if $system_content < $expectConsumeMsgCnt then +if $system_content != $expect_result then return -1 endi -#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb #print cmd result----> $system_content ##if $system_content != @{consume success: 10000, 0}@ then -#if $system_content < $expectConsumeMsgCnt then +#if $system_content != $expect_result then # return -1 #endi -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromStb print cmd result----> $system_content #if $system_content != @{consume success: 10000, 0}@ then -if $system_content < $expectConsumeMsgCnt then +if $system_content != $expect_result then return -1 endi $expect_result = @{consume success: @ -$expect_result = $expect_result . $rowNum +$expect_result = $expect_result . $expectMsgCntFromCtb $expect_result = $expect_result . @, @ $expect_result = $expect_result . 0} print expect_result----> $expect_result -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb print cmd result----> $system_content -if $system_content < $expectConsumeMsgCnt then +if $system_content != $expect_result then return -1 endi -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb print cmd result----> $system_content -if $system_content < $expectConsumeMsgCnt then +if $system_content != $expect_result then return -1 endi -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb print cmd result----> $system_content -if $system_content < $expectConsumeMsgCnt then +if $system_content != $expect_result then return -1 endi $expect_result = @{consume success: @ -$expect_result = $expect_result . $totalMsgCnt +$expect_result = $expect_result . $expectMsgCntFromStb $expect_result = $expect_result . @, @ $expect_result = $expect_result . 0} print expect_result----> $expect_result -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb print cmd result----> $system_content -if $system_content < $expectConsumeMsgCnt then +if $system_content != $expect_result then return -1 endi -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb print cmd result----> $system_content -if $system_content < $expectConsumeMsgCnt then +if $system_content != $expect_result then return -1 endi -print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt -system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectMsgCntFromCtb print cmd result----> $system_content -if $system_content < $expectConsumeMsgCnt then +if $system_content != $expect_result then return -1 endi diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 702d88f258..86d0bef1a9 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -34,14 +34,23 @@ #define MAX_SQL_STR_LEN (1024 * 1024) #define MAX_ROW_STR_LEN (16 * 1024) +typedef struct { + int32_t expectMsgCnt; + int32_t consumeMsgCnt; + TdThread thread; +} SThreadInfo; + typedef struct { // input from argvs char dbName[32]; char topicString[256]; char keyString[1024]; + char topicString1[256]; + char keyString1[1024]; int32_t showMsgFlag; int32_t consumeDelay; // unit s int32_t consumeMsgCnt; + int32_t checkMode; // save result after parse agrvs int32_t numOfTopic; @@ -50,6 +59,13 @@ typedef struct { int32_t numOfKey; char key[32][64]; char value[32][64]; + + int32_t numOfTopic1; + char topics1[32][64]; + + int32_t numOfKey1; + char key1[32][64]; + char value1[32][64]; } SConfInfo; static SConfInfo g_stConfInfo; @@ -69,12 +85,18 @@ static void printHelp() { printf("%s%s%s\n", indent, indent, "The topic string for cosumer, no default "); printf("%s%s\n", indent, "-k"); printf("%s%s%s\n", indent, indent, "The key-value string for cosumer, no default "); + printf("%s%s\n", indent, "-t1"); + printf("%s%s%s\n", indent, indent, "The topic1 string for cosumer, no default "); + printf("%s%s\n", indent, "-k1"); + printf("%s%s%s\n", indent, indent, "The key1-value1 string for cosumer, no default "); printf("%s%s\n", indent, "-g"); printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); printf("%s%s\n", indent, "-y"); printf("%s%s%s%d\n", indent, indent, "consume delay, default is s", g_stConfInfo.consumeDelay); printf("%s%s\n", indent, "-m"); printf("%s%s%s%d\n", indent, indent, "consume msg count, default is s", g_stConfInfo.consumeMsgCnt); + printf("%s%s\n", indent, "-j"); + printf("%s%s%s%d\n", indent, indent, "check mode, default is s", g_stConfInfo.checkMode); exit(EXIT_SUCCESS); } @@ -96,12 +118,18 @@ void parseArgument(int32_t argc, char* argv[]) { strcpy(g_stConfInfo.topicString, argv[++i]); } else if (strcmp(argv[i], "-k") == 0) { strcpy(g_stConfInfo.keyString, argv[++i]); + } else if (strcmp(argv[i], "-t1") == 0) { + strcpy(g_stConfInfo.topicString1, argv[++i]); + } else if (strcmp(argv[i], "-k1") == 0) { + strcpy(g_stConfInfo.keyString1, argv[++i]); } else if (strcmp(argv[i], "-g") == 0) { g_stConfInfo.showMsgFlag = atol(argv[++i]); } else if (strcmp(argv[i], "-y") == 0) { g_stConfInfo.consumeDelay = atol(argv[++i]); } else if (strcmp(argv[i], "-m") == 0) { g_stConfInfo.consumeMsgCnt = atol(argv[++i]); + } else if (strcmp(argv[i], "-j") == 0) { + g_stConfInfo.checkMode = atol(argv[++i]); } else { printf("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); @@ -154,7 +182,18 @@ void parseInputString() { ltrim(g_stConfInfo.topics[g_stConfInfo.numOfTopic]); // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); g_stConfInfo.numOfTopic++; + + token = strtok(NULL, delim); + } + token = strtok(g_stConfInfo.topicString1, delim); + while(token != NULL) { + //printf("%s\n", token ); + strcpy(g_stConfInfo.topics1[g_stConfInfo.numOfTopic1], token); + ltrim(g_stConfInfo.topics1[g_stConfInfo.numOfTopic1]); + //printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); + g_stConfInfo.numOfTopic1++; + token = strtok(NULL, delim); } @@ -171,11 +210,28 @@ void parseInputString() { // g_stConfInfo.value[g_stConfInfo.numOfKey]); g_stConfInfo.numOfKey++; } + + token = strtok(NULL, delim); + } + token = strtok(g_stConfInfo.keyString1, delim); + while(token != NULL) { + //printf("%s\n", token ); + { + char* pstr = token; + ltrim(pstr); + char *ret = strchr(pstr, ch); + memcpy(g_stConfInfo.key1[g_stConfInfo.numOfKey1], pstr, ret-pstr); + strcpy(g_stConfInfo.value1[g_stConfInfo.numOfKey1], ret+1); + //printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], g_stConfInfo.value[g_stConfInfo.numOfKey]); + g_stConfInfo.numOfKey1++; + } + token = strtok(NULL, delim); } } + static int running = 1; /*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/ @@ -225,6 +281,40 @@ tmq_list_t* build_topic_list() { return topic_list; } + +tmq_t* build_consumer_x() { + char sqlStr[1024] = {0}; + + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + sprintf(sqlStr, "use %s", g_stConfInfo.dbName); + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + exit(-1); + } + taos_free_result(pRes); + + tmq_conf_t* conf = tmq_conf_new(); + //tmq_conf_set(conf, "group.id", "tg2"); + for (int32_t i = 0; i < g_stConfInfo.numOfKey1; i++) { + tmq_conf_set(conf, g_stConfInfo.key1[i], g_stConfInfo.value1[i]); + } + tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); + return tmq; +} + +tmq_list_t* build_topic_list_x() { + tmq_list_t* topic_list = tmq_list_new(); + //tmq_list_append(topic_list, "test_stb_topic_1"); + for (int32_t i = 0; i < g_stConfInfo.numOfTopic1; i++) { + tmq_list_append(topic_list, g_stConfInfo.topics1[i]); + } + return topic_list; +} + void loop_consume(tmq_t* tmq) { tmq_resp_err_t err; @@ -262,7 +352,7 @@ void loop_consume(tmq_t* tmq) { printf("{consume success: %d, %d}", totalMsgs, totalRows); } -void parallel_consume(tmq_t* tmq) { +int32_t parallel_consume(tmq_t* tmq, int threadLable) { tmq_resp_err_t err; int32_t totalMsgs = 0; @@ -273,7 +363,9 @@ void parallel_consume(tmq_t* tmq) { if (tmqMsg) { totalMsgs++; -#if 0 + //printf("threadFlag: %d, totalMsgs: %d\n", threadLable, totalMsgs); + + #if 0 TAOS_ROW row; while (NULL != (row = tmq_get_row(tmqMsg))) { totalRows++; @@ -300,13 +392,65 @@ void parallel_consume(tmq_t* tmq) { exit(-1); } - printf("%d", totalMsgs); // output to sim for check result + //printf("%d", totalMsgs); // output to sim for check result + return totalMsgs; } + +void *threadFunc(void *param) { + int32_t totalMsgs = 0; + + SThreadInfo *pInfo = (SThreadInfo *)param; + + tmq_t* tmq = build_consumer_x(); + tmq_list_t* topic_list = build_topic_list_x(); + if ((NULL == tmq) || (NULL == topic_list)){ + return NULL; + } + + tmq_resp_err_t err = tmq_subscribe(tmq, topic_list); + if (err) { + printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); + exit(-1); + } + + //if (0 == g_stConfInfo.consumeMsgCnt) { + // loop_consume(tmq); + //} else { + pInfo->consumeMsgCnt = parallel_consume(tmq, 1); + //} + + err = tmq_unsubscribe(tmq); + if (err) { + printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); + pInfo->consumeMsgCnt = -1; + return NULL; + } + + return NULL; +} + + int main(int32_t argc, char* argv[]) { parseArgument(argc, argv); parseInputString(); + int32_t numOfThreads = 1; + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + SThreadInfo *pInfo = (SThreadInfo *)taosMemoryCalloc(numOfThreads, sizeof(SThreadInfo)); + + if (g_stConfInfo.numOfTopic1) { + // pthread_create one thread to consume + for (int32_t i = 0; i < numOfThreads; ++i) { + pInfo[i].expectMsgCnt = 0; + pInfo[i].consumeMsgCnt = 0; + taosThreadCreate(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); + } + } + + int32_t totalMsgs = 0; tmq_t* tmq = build_consumer(); tmq_list_t* topic_list = build_topic_list(); if ((NULL == tmq) || (NULL == topic_list)) { @@ -319,10 +463,10 @@ int main(int32_t argc, char* argv[]) { exit(-1); } - if (0 == g_stConfInfo.consumeMsgCnt) { + if (0 == g_stConfInfo.numOfTopic1) { loop_consume(tmq); } else { - parallel_consume(tmq); + totalMsgs = parallel_consume(tmq, 0); } err = tmq_unsubscribe(tmq); @@ -331,6 +475,27 @@ int main(int32_t argc, char* argv[]) { exit(-1); } + if (g_stConfInfo.numOfTopic1) { + for (int32_t i = 0; i < numOfThreads; i++) { + taosThreadJoin(pInfo[i].thread, NULL); + } + + //printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); + if (0 == g_stConfInfo.checkMode) { + if ((totalMsgs + pInfo->consumeMsgCnt) == g_stConfInfo.consumeMsgCnt) { + printf("success"); + } else { + printf("fail, consumer msg cnt: %d, %d", totalMsgs, pInfo->consumeMsgCnt); + } + } else if (1 == g_stConfInfo.checkMode) { + if ((totalMsgs == g_stConfInfo.consumeMsgCnt) && (pInfo->consumeMsgCnt == g_stConfInfo.consumeMsgCnt)) { + printf("success"); + } else { + printf("fail, consumer msg cnt: %d, %d", totalMsgs, pInfo->consumeMsgCnt); + } + } + } + return 0; } From 6c1774524a1b46e072527e7bfd5ac4f073c95363 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 13 Apr 2022 18:53:53 +0800 Subject: [PATCH 12/13] test:add test cases --- tests/script/jenkins/basic.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index c14d4fe27b..730937333b 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -59,6 +59,15 @@ ./test.sh -f tsim/tmq/oneTopic.sim ./test.sh -f tsim/tmq/multiTopic.sim +./test.sh -f tsim/tmq/mainConsumerInMultiTopic.sim +./test.sh -f tsim/tmq/mainConsumerInOneTopic.sim + +#fail ./test.sh -f tsim/tmq/main2Con1Cgrp1TopicFrCtb.sim +#fail ./test.sh -f tsim/tmq/main2Con1Cgrp1TopicFrStb.sim +./test.sh -f tsim/tmq/main2Con1Cgrp2TopicFrCtb.sim +./test.sh -f tsim/tmq/main2Con1Cgrp2TopicFrStb.sim + + # --- stable ./test.sh -f tsim/stable/disk.sim ./test.sh -f tsim/stable/dnode3.sim From 13735d725715922aba2a8881ace4c52a992f635c Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 13 Apr 2022 19:04:25 +0800 Subject: [PATCH 13/13] feat(query): distributed splitting of child/normal table JOIN --- include/libs/nodes/nodes.h | 1 + source/libs/executor/src/scanoperator.c | 2 +- source/libs/nodes/src/nodesTraverseFuncs.c | 50 ++-- source/libs/parser/src/parAstCreater.c | 22 +- .../libs/parser/test/mockCatalogService.cpp | 11 +- source/libs/planner/src/planLogicCreater.c | 1 - source/libs/planner/src/planOptimizer.c | 244 ++++++++++++++++-- source/libs/planner/src/planSpliter.c | 147 ++++++++--- source/libs/planner/test/plannerTest.cpp | 16 +- 9 files changed, 396 insertions(+), 98 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 3a1d7954a7..4ffacf3787 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -231,6 +231,7 @@ typedef enum EDealRes { DEAL_RES_CONTINUE = 1, DEAL_RES_IGNORE_CHILD, DEAL_RES_ERROR, + DEAL_RES_END } EDealRes; typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4179999c4b..5079a49a62 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -548,7 +548,7 @@ EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { char* dbName = nodesGetValueFromNode(node); strncpy(pContext, varDataVal(dbName), varDataLen(dbName)); *((char*)pContext + varDataLen(dbName)) = 0; - return DEAL_RES_ERROR; // stop walk + return DEAL_RES_END; // stop walk } default: break; diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 4a782cce08..99a08923bb 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -46,7 +46,7 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker case QUERY_NODE_OPERATOR: { SOperatorNode* pOpNode = (SOperatorNode*)pNode; res = walkNode(pOpNode->pLeft, order, walker, pContext); - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = walkNode(pOpNode->pRight, order, walker, pContext); } break; @@ -63,10 +63,10 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker case QUERY_NODE_JOIN_TABLE: { SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode; res = walkNode(pJoinTableNode->pLeft, order, walker, pContext); - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = walkNode(pJoinTableNode->pRight, order, walker, pContext); } - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = walkNode(pJoinTableNode->pOnCond, order, walker, pContext); } break; @@ -80,7 +80,7 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker case QUERY_NODE_STATE_WINDOW: { SStateWindowNode* pState = (SStateWindowNode*)pNode; res = walkNode(pState->pExpr, order, walker, pContext); - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = walkNode(pState->pCol, order, walker, pContext); } break; @@ -88,7 +88,7 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker case QUERY_NODE_SESSION_WINDOW: { SSessionWindowNode* pSession = (SSessionWindowNode*)pNode; res = walkNode(pSession->pCol, order, walker, pContext); - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = walkNode(pSession->pGap, order, walker, pContext); } break; @@ -96,16 +96,16 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker case QUERY_NODE_INTERVAL_WINDOW: { SIntervalWindowNode* pInterval = (SIntervalWindowNode*)pNode; res = walkNode(pInterval->pInterval, order, walker, pContext); - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = walkNode(pInterval->pOffset, order, walker, pContext); } - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = walkNode(pInterval->pSliding, order, walker, pContext); } - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = walkNode(pInterval->pFill, order, walker, pContext); } - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = walkNode(pInterval->pCol, order, walker, pContext); } break; @@ -126,7 +126,7 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker break; } - if (DEAL_RES_ERROR != res && TRAVERSAL_POSTORDER == order) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res && TRAVERSAL_POSTORDER == order) { res = walker(pNode, pContext); } @@ -136,8 +136,9 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalker walker, void* pContext) { SNode* node; FOREACH(node, pNodeList) { - if (DEAL_RES_ERROR == walkNode(node, order, walker, pContext)) { - return DEAL_RES_ERROR; + EDealRes res = walkNode(node, order, walker, pContext); + if (DEAL_RES_ERROR == res || DEAL_RES_END == res) { + return res; } } return DEAL_RES_CONTINUE; @@ -185,7 +186,7 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit case QUERY_NODE_OPERATOR: { SOperatorNode* pOpNode = (SOperatorNode*)pNode; res = rewriteNode(&(pOpNode->pLeft), order, rewriter, pContext); - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = rewriteNode(&(pOpNode->pRight), order, rewriter, pContext); } break; @@ -202,10 +203,10 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit case QUERY_NODE_JOIN_TABLE: { SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode; res = rewriteNode(&(pJoinTableNode->pLeft), order, rewriter, pContext); - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = rewriteNode(&(pJoinTableNode->pRight), order, rewriter, pContext); } - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = rewriteNode(&(pJoinTableNode->pOnCond), order, rewriter, pContext); } break; @@ -219,7 +220,7 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit case QUERY_NODE_STATE_WINDOW: { SStateWindowNode* pState = (SStateWindowNode*)pNode; res = rewriteNode(&pState->pExpr, order, rewriter, pContext); - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = rewriteNode(&pState->pCol, order, rewriter, pContext); } break; @@ -227,7 +228,7 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit case QUERY_NODE_SESSION_WINDOW: { SSessionWindowNode* pSession = (SSessionWindowNode*)pNode; res = rewriteNode(&pSession->pCol, order, rewriter, pContext); - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = rewriteNode(&pSession->pGap, order, rewriter, pContext); } break; @@ -235,16 +236,16 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit case QUERY_NODE_INTERVAL_WINDOW: { SIntervalWindowNode* pInterval = (SIntervalWindowNode*)pNode; res = rewriteNode(&(pInterval->pInterval), order, rewriter, pContext); - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = rewriteNode(&(pInterval->pOffset), order, rewriter, pContext); } - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = rewriteNode(&(pInterval->pSliding), order, rewriter, pContext); } - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = rewriteNode(&(pInterval->pFill), order, rewriter, pContext); } - if (DEAL_RES_ERROR != res) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = rewriteNode(&(pInterval->pCol), order, rewriter, pContext); } break; @@ -265,7 +266,7 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit break; } - if (DEAL_RES_ERROR != res && TRAVERSAL_POSTORDER == order) { + if (DEAL_RES_ERROR != res && DEAL_RES_END != res && TRAVERSAL_POSTORDER == order) { res = rewriter(pRawNode, pContext); } @@ -275,8 +276,9 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit static EDealRes rewriteList(SNodeList* pNodeList, ETraversalOrder order, FNodeRewriter rewriter, void* pContext) { SNode** pNode; FOREACH_FOR_REWRITE(pNode, pNodeList) { - if (DEAL_RES_ERROR == rewriteNode(pNode, order, rewriter, pContext)) { - return DEAL_RES_ERROR; + EDealRes res = rewriteNode(pNode, order, rewriter, pContext); + if (DEAL_RES_ERROR == res || DEAL_RES_END == res) { + return res; } } return DEAL_RES_CONTINUE; diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 318f7ca5f7..9a26e38c98 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -328,8 +328,26 @@ SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType typ CHECK_OUT_OF_MEM(cond); cond->condType = type; cond->pParameterList = nodesMakeList(); - nodesListAppend(cond->pParameterList, pParam1); - nodesListAppend(cond->pParameterList, pParam2); + if ((QUERY_NODE_LOGIC_CONDITION == nodeType(pParam1) && type != ((SLogicConditionNode*)pParam1)->condType) || + (QUERY_NODE_LOGIC_CONDITION == nodeType(pParam2) && type != ((SLogicConditionNode*)pParam2)->condType)) { + nodesListAppend(cond->pParameterList, pParam1); + nodesListAppend(cond->pParameterList, pParam2); + } else { + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pParam1)) { + nodesListAppendList(cond->pParameterList, ((SLogicConditionNode*)pParam1)->pParameterList); + ((SLogicConditionNode*)pParam1)->pParameterList = NULL; + nodesDestroyNode(pParam1); + } else { + nodesListAppend(cond->pParameterList, pParam1); + } + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pParam2)) { + nodesListAppendList(cond->pParameterList, ((SLogicConditionNode*)pParam2)->pParameterList); + ((SLogicConditionNode*)pParam2)->pParameterList = NULL; + nodesDestroyNode(pParam2); + } else { + nodesListAppend(cond->pParameterList, pParam2); + } + } return (SNode*)cond; } diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 402caeb252..3da3678563 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -146,6 +146,7 @@ public: meta_[db][tbname].reset(new MockTableMeta()); meta_[db][tbname]->schema = table.release(); meta_[db][tbname]->schema->uid = id_++; + meta_[db][tbname]->schema->tableType = TSDB_CHILD_TABLE; SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0,}; addEpIntoEpSet(&vgroup.epSet, "dnode_1", 6030); @@ -197,11 +198,11 @@ public: std::cout << "Table:" << table.first << std::endl; std::cout << SH("Field") << SH("Type") << SH("DataType") << IH("Bytes") << std::endl; std::cout << SL(3, 1) << std::endl; - int16_t numOfTags = schema->tableInfo.numOfTags; - int16_t numOfFields = numOfTags + schema->tableInfo.numOfColumns; + int16_t numOfColumns = schema->tableInfo.numOfColumns; + int16_t numOfFields = numOfColumns + schema->tableInfo.numOfTags; for (int16_t i = 0; i < numOfFields; ++i) { const SSchema* col = schema->schema + i; - std::cout << SF(std::string(col->name)) << SH(ftToString(i, numOfTags)) << SH(dtToString(col->type)) << IF(col->bytes) << std::endl; + std::cout << SF(std::string(col->name)) << SH(ftToString(i, numOfColumns)) << SH(dtToString(col->type)) << IF(col->bytes) << std::endl; } std::cout << std::endl; } @@ -262,8 +263,8 @@ private: return tDataTypes[type].name; } - std::string ftToString(int16_t colid, int16_t numOfTags) const { - return (0 == colid ? "column" : (colid <= numOfTags ? "tag" : "column")); + std::string ftToString(int16_t colid, int16_t numOfColumns) const { + return (0 == colid ? "column" : (colid <= numOfColumns ? "tag" : "column")); } STableMeta* getTableSchemaMeta(const std::string& db, const std::string& tbname) const { diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 1d8400e1eb..3c60b62434 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -694,7 +694,6 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS } return code; - return TSDB_CODE_SUCCESS; } static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index c56e113d40..19e6718fe8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -41,6 +41,21 @@ typedef struct SOsdInfo { SNodeList* pDsoFuncs; } SOsdInfo; +typedef struct SCpdIsMultiTableCondCxt { + SNodeList* pLeftCols; + SNodeList* pRightCols; + bool havaLeftCol; + bool haveRightCol; +} SCpdIsMultiTableCondCxt; + +typedef enum ECondAction { + COND_ACTION_STAY = 1, + COND_ACTION_PUSH_JOIN, + COND_ACTION_PUSH_LEFT_CHILD, + COND_ACTION_PUSH_RIGHT_CHILD + // after supporting outer join, there are other possibilities +} ECondAction; + static bool osdMayBeOptimized(SLogicNode* pNode) { if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_OSD)) { return false; @@ -152,34 +167,227 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* return TSDB_CODE_SUCCESS; } -static int32_t cpdPartitionCondition(SJoinLogicNode* pJoin, SNodeList** pMultiTableCond, SNodeList** pSingleTableCond) { - // todo +static bool belongThisTable(SNode* pCondCol, SNodeList* pTableCols) { + SNode* pTableCol = NULL; + FOREACH(pTableCol, pTableCols) { + if (nodesEqualNode(pCondCol, pTableCol)) { + return true; + } + } + return false; +} + +static EDealRes cpdIsMultiTableCondImpl(SNode* pNode, void* pContext) { + SCpdIsMultiTableCondCxt* pCxt = pContext; + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + if (belongThisTable(pNode, pCxt->pLeftCols)) { + pCxt->havaLeftCol = true; + } else if (belongThisTable(pNode, pCxt->pRightCols)) { + pCxt->haveRightCol = true; + } + return pCxt->havaLeftCol && pCxt->haveRightCol ? DEAL_RES_END : DEAL_RES_CONTINUE; + } + return DEAL_RES_CONTINUE; +} + +static ECondAction cpdCondAction(EJoinType joinType, SNodeList* pLeftCols, SNodeList* pRightCols, SNode* pNode) { + SCpdIsMultiTableCondCxt cxt = { .pLeftCols = pLeftCols, .pRightCols = pRightCols, .havaLeftCol = false, .haveRightCol = false }; + nodesWalkExpr(pNode, cpdIsMultiTableCondImpl, &cxt); + return (JOIN_TYPE_INNER != joinType ? COND_ACTION_STAY : + (cxt.havaLeftCol && cxt.haveRightCol ? COND_ACTION_PUSH_JOIN : (cxt.havaLeftCol ? COND_ACTION_PUSH_LEFT_CHILD : COND_ACTION_PUSH_RIGHT_CHILD))); +} + +static int32_t cpdMakeCond(SNodeList** pConds, SNode** pCond) { + if (NULL == *pConds) { + return TSDB_CODE_SUCCESS; + } + + if (1 == LIST_LENGTH(*pConds)) { + *pCond = nodesListGetNode(*pConds, 0); + nodesClearList(*pConds); + } else { + SLogicConditionNode* pLogicCond = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + if (NULL == pLogicCond) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pLogicCond->condType = LOGIC_COND_TYPE_AND; + pLogicCond->pParameterList = *pConds; + *pCond = (SNode*)pLogicCond; + } + *pConds = NULL; + return TSDB_CODE_SUCCESS; } -static int32_t cpdPushJoinCondToOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNodeList* pMultiTableCond) { - // todo +static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, SNode** pRightChildCond) { + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pJoin->node.pConditions; + if (LOGIC_COND_TYPE_AND != pLogicCond->condType) { + return TSDB_CODE_SUCCESS; + } + + SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; + SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; + int32_t code = TSDB_CODE_SUCCESS; + + SNodeList* pOnConds = NULL; + SNodeList* pLeftChildConds = NULL; + SNodeList* pRightChildConds = NULL; + SNodeList* pRemainConds = NULL; + SNode* pCond = NULL; + FOREACH(pCond, pLogicCond->pParameterList) { + ECondAction condAction = cpdCondAction(pJoin->joinType, pLeftCols, pRightCols, pCond); + if (COND_ACTION_PUSH_JOIN == condAction) { + code = nodesListMakeAppend(&pOnConds, nodesCloneNode(pCond)); + } else if (COND_ACTION_PUSH_LEFT_CHILD == condAction) { + code = nodesListMakeAppend(&pLeftChildConds, nodesCloneNode(pCond)); + } else if (COND_ACTION_PUSH_RIGHT_CHILD == condAction) { + code = nodesListMakeAppend(&pRightChildConds, nodesCloneNode(pCond)); + } else { + code = nodesListMakeAppend(&pRemainConds, nodesCloneNode(pCond)); + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + + SNode* pTempOnCond = NULL; + SNode* pTempLeftChildCond = NULL; + SNode* pTempRightChildCond = NULL; + SNode* pTempRemainCond = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = cpdMakeCond(&pOnConds, &pTempOnCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = cpdMakeCond(&pLeftChildConds, &pTempLeftChildCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = cpdMakeCond(&pRightChildConds, &pTempRightChildCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = cpdMakeCond(&pRemainConds, &pTempRemainCond); + } + + if (TSDB_CODE_SUCCESS == code) { + *pOnCond = pTempOnCond; + *pLeftChildCond = pTempLeftChildCond; + *pRightChildCond = pTempRightChildCond; + nodesDestroyNode(pJoin->node.pConditions); + pJoin->node.pConditions = pTempRemainCond; + } else { + nodesDestroyList(pOnConds); + nodesDestroyList(pLeftChildConds); + nodesDestroyList(pRightChildConds); + nodesDestroyList(pRemainConds); + nodesDestroyNode(pTempOnCond); + nodesDestroyNode(pTempLeftChildCond); + nodesDestroyNode(pTempRightChildCond); + nodesDestroyNode(pTempRemainCond); + } + + return code; +} + +static int32_t cpdPartitionOpCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, SNode** pRightChildCond) { + SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; + SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; + ECondAction condAction = cpdCondAction(pJoin->joinType, pLeftCols, pRightCols, pJoin->node.pConditions); + if (COND_ACTION_STAY == condAction) { + return TSDB_CODE_SUCCESS; + } else if (COND_ACTION_PUSH_JOIN == condAction) { + *pOnCond = pJoin->node.pConditions; + } else if (COND_ACTION_PUSH_LEFT_CHILD == condAction) { + *pLeftChildCond = pJoin->node.pConditions; + } else if (COND_ACTION_PUSH_RIGHT_CHILD == condAction) { + *pRightChildCond = pJoin->node.pConditions; + } + pJoin->node.pConditions = NULL; return TSDB_CODE_SUCCESS; } -static int32_t cpdPushJoinCondToChildren(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNodeList* pSingleTableCond) { - // todo - return TSDB_CODE_SUCCESS; +static int32_t cpdPartitionCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, SNode** pRightChildCond) { + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->node.pConditions)) { + return cpdPartitionLogicCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond); + } else { + return cpdPartitionOpCond(pJoin, pOnCond, pLeftChildCond, pRightChildCond); + } +} + +static int32_t cpdCondAppend(SOptimizeContext* pCxt, SNode** pCond, SNode** pAdditionalCond) { + if (NULL == *pCond) { + TSWAP(*pCond, *pAdditionalCond, SNode*); + return TSDB_CODE_SUCCESS; + } + + int32_t code = TSDB_CODE_SUCCESS; + if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pCond)) { + code = nodesListAppend(((SLogicConditionNode*)*pCond)->pParameterList, *pAdditionalCond); + if (TSDB_CODE_SUCCESS == code) { + *pAdditionalCond = NULL; + } + } else { + SLogicConditionNode* pLogicCond = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + if (NULL == pLogicCond) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pLogicCond->condType = LOGIC_COND_TYPE_AND; + code = nodesListMakeAppend(&pLogicCond->pParameterList, *pAdditionalCond); + if (TSDB_CODE_SUCCESS == code) { + *pAdditionalCond = NULL; + code = nodesListMakeAppend(&pLogicCond->pParameterList, *pCond); + } + if (TSDB_CODE_SUCCESS == code) { + *pCond = (SNode*)pLogicCond; + } else { + nodesDestroyNode(pLogicCond); + } + } + return code; +} + +static int32_t cpdPushCondToOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode** pCond) { + return cpdCondAppend(pCxt, &pJoin->pOnConditions, pCond); +} + +static int32_t cpdPushCondToScan(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pCond) { + return cpdCondAppend(pCxt, &pScan->node.pConditions, pCond); +} + +static int32_t cpdPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) { + switch (nodeType(pChild)) { + case QUERY_NODE_LOGIC_PLAN_SCAN: + return cpdPushCondToScan(pCxt, (SScanLogicNode*)pChild, pCond); + default: + break; + } + return TSDB_CODE_PLAN_INTERNAL_ERROR; } static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { - if (NULL != pJoin->node.pConditions) { - SNodeList* pMultiTableCond = NULL; - SNodeList* pSingleTableCond = NULL; - int32_t code = cpdPartitionCondition(pJoin, &pMultiTableCond, &pSingleTableCond); - if (TSDB_CODE_SUCCESS == code && NULL != pMultiTableCond) { - code = cpdPushJoinCondToOnCond(pCxt, pJoin, pMultiTableCond); - } - if (TSDB_CODE_SUCCESS == code && NULL != pSingleTableCond) { - code = cpdPushJoinCondToChildren(pCxt, pJoin, pSingleTableCond); - } + if (NULL == pJoin->node.pConditions) { + return TSDB_CODE_SUCCESS; } - return TSDB_CODE_SUCCESS; + + SNode* pOnCond = NULL; + SNode* pLeftChildCond = NULL; + SNode* pRightChildCond = NULL; + int32_t code = cpdPartitionCond(pJoin, &pOnCond, &pLeftChildCond, &pRightChildCond); + if (TSDB_CODE_SUCCESS == code && NULL != pOnCond) { + code = cpdPushCondToOnCond(pCxt, pJoin, &pOnCond); + } + if (TSDB_CODE_SUCCESS == code && NULL != pLeftChildCond) { + code = cpdPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0), &pLeftChildCond); + } + if (TSDB_CODE_SUCCESS == code && NULL != pRightChildCond) { + code = cpdPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond); + } + + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pOnCond); + nodesDestroyNode(pLeftChildCond); + nodesDestroyNode(pRightChildCond); + } + + return code; } static int32_t cpdPushAggCondition(SOptimizeContext* pCxt, SAggLogicNode* pAgg) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 0b21052955..e54cf33934 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -18,6 +18,7 @@ #define SPLIT_FLAG_MASK(n) (1 << n) #define SPLIT_FLAG_STS SPLIT_FLAG_MASK(0) +#define SPLIT_FLAG_CTJ SPLIT_FLAG_MASK(1) #define SPLIT_FLAG_SET_MASK(val, mask) (val) |= (mask) #define SPLIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0) @@ -39,43 +40,14 @@ typedef struct SStsInfo { SLogicSubplan* pSubplan; } SStsInfo; -static SLogicNode* stsMatchByNode(SLogicNode* pNode) { - if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && - NULL != ((SScanLogicNode*)pNode)->pVgroupList && ((SScanLogicNode*)pNode)->pVgroupList->numOfVgroups > 1) { - return pNode; - } - SNode* pChild; - FOREACH(pChild, pNode->pChildren) { - SLogicNode* pSplitNode = stsMatchByNode((SLogicNode*)pChild); - if (NULL != pSplitNode) { - return pSplitNode; - } - } - return NULL; -} +typedef struct SCtjInfo { + SScanLogicNode* pScan; + SLogicSubplan* pSubplan; +} SCtjInfo; -static void stsFindSplitNode(SLogicSubplan* pSubplan, SStsInfo* pInfo) { - SLogicNode* pSplitNode = stsMatchByNode(pSubplan->pNode); - if (NULL != pSplitNode) { - pInfo->pScan = (SScanLogicNode*)pSplitNode; - pInfo->pSubplan = pSubplan; - } -} -static void stsMatch(SSplitContext* pCxt, SLogicSubplan* pSubplan, SStsInfo* pInfo) { - if (!SPLIT_FLAG_TEST_MASK(pSubplan->splitFlag, SPLIT_FLAG_STS)) { - stsFindSplitNode(pSubplan, pInfo); - } - SNode* pChild; - FOREACH(pChild, pSubplan->pChildren) { - stsMatch(pCxt, (SLogicSubplan*)pChild, pInfo); - if (NULL != pInfo->pScan) { - break; - } - } - return; -} +typedef bool (*FSplFindSplitNode)(SLogicSubplan* pSubplan, SStsInfo* pInfo); -static SLogicSubplan* stsCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pScan) { +static SLogicSubplan* splCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pScan, int32_t flag) { SLogicSubplan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); if (NULL == pSubplan) { return NULL; @@ -84,11 +56,11 @@ static SLogicSubplan* stsCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pSubplan->subplanType = SUBPLAN_TYPE_SCAN; pSubplan->pNode = (SLogicNode*)nodesCloneNode(pScan); TSWAP(pSubplan->pVgroupList, ((SScanLogicNode*)pSubplan->pNode)->pVgroupList, SVgroupsInfo*); - SPLIT_FLAG_SET_MASK(pSubplan->splitFlag, SPLIT_FLAG_STS); + SPLIT_FLAG_SET_MASK(pSubplan->splitFlag, flag); return pSubplan; } -static int32_t stsCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SScanLogicNode* pScan) { +static int32_t splCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SScanLogicNode* pScan, ESubplanType subplanType) { SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); if (NULL == pExchange) { return TSDB_CODE_OUT_OF_MEMORY; @@ -119,10 +91,48 @@ static int32_t stsCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla return TSDB_CODE_FAILED; } +static bool splMatch(SSplitContext* pCxt, SLogicSubplan* pSubplan, int32_t flag, FSplFindSplitNode func, void* pInfo) { + if (!SPLIT_FLAG_TEST_MASK(pSubplan->splitFlag, flag)) { + if (func(pSubplan, pInfo)) { + return true; + } + } + SNode* pChild; + FOREACH(pChild, pSubplan->pChildren) { + if (splMatch(pCxt, (SLogicSubplan*)pChild, flag, func, pInfo)) { + return true; + } + } + return false; +} + +static SLogicNode* stsMatchByNode(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && + NULL != ((SScanLogicNode*)pNode)->pVgroupList && ((SScanLogicNode*)pNode)->pVgroupList->numOfVgroups > 1) { + return pNode; + } + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + SLogicNode* pSplitNode = stsMatchByNode((SLogicNode*)pChild); + if (NULL != pSplitNode) { + return pSplitNode; + } + } + return NULL; +} + +static bool stsFindSplitNode(SLogicSubplan* pSubplan, SStsInfo* pInfo) { + SLogicNode* pSplitNode = stsMatchByNode(pSubplan->pNode); + if (NULL != pSplitNode) { + pInfo->pScan = (SScanLogicNode*)pSplitNode; + pInfo->pSubplan = pSubplan; + } + return NULL != pSplitNode; +} + static int32_t stsSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { SStsInfo info = {0}; - stsMatch(pCxt, pSubplan, &info); - if (NULL == info.pScan) { + if (!splMatch(pCxt, pSubplan, SPLIT_FLAG_STS, stsFindSplitNode, &info)) { return TSDB_CODE_SUCCESS; } if (NULL == info.pSubplan->pChildren) { @@ -131,9 +141,61 @@ static int32_t stsSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { return TSDB_CODE_OUT_OF_MEMORY; } } - int32_t code = nodesListStrictAppend(info.pSubplan->pChildren, stsCreateScanSubplan(pCxt, info.pScan)); + int32_t code = nodesListStrictAppend(info.pSubplan->pChildren, splCreateScanSubplan(pCxt, info.pScan, SPLIT_FLAG_STS)); if (TSDB_CODE_SUCCESS == code) { - code = stsCreateExchangeNode(pCxt, info.pSubplan, info.pScan); + code = splCreateExchangeNode(pCxt, info.pSubplan, info.pScan, SUBPLAN_TYPE_MERGE); + } + ++(pCxt->groupId); + pCxt->split = true; + return code; +} + +static bool ctjIsSingleTable(int8_t tableType) { + return (TSDB_CHILD_TABLE == tableType || TSDB_NORMAL_TABLE == tableType); +} + +static SLogicNode* ctjMatchByNode(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pNode)) { + SLogicNode* pLeft = (SLogicNode*)nodesListGetNode(pNode->pChildren, 0); + SLogicNode* pRight = (SLogicNode*)nodesListGetNode(pNode->pChildren, 1); + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pLeft) && ctjIsSingleTable(((SScanLogicNode*)pLeft)->pMeta->tableType) && + QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pRight) && ctjIsSingleTable(((SScanLogicNode*)pRight)->pMeta->tableType)) { + return pRight; + } + } + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + SLogicNode* pSplitNode = ctjMatchByNode((SLogicNode*)pChild); + if (NULL != pSplitNode) { + return pSplitNode; + } + } + return NULL; +} + +static bool ctjFindSplitNode(SLogicSubplan* pSubplan, SStsInfo* pInfo) { + SLogicNode* pSplitNode = ctjMatchByNode(pSubplan->pNode); + if (NULL != pSplitNode) { + pInfo->pScan = (SScanLogicNode*)pSplitNode; + pInfo->pSubplan = pSubplan; + } + return NULL != pSplitNode; +} + +static int32_t ctjSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { + SCtjInfo info = {0}; + if (!splMatch(pCxt, pSubplan, SPLIT_FLAG_CTJ, ctjFindSplitNode, &info)) { + return TSDB_CODE_SUCCESS; + } + if (NULL == info.pSubplan->pChildren) { + info.pSubplan->pChildren = nodesMakeList(); + if (NULL == info.pSubplan->pChildren) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + int32_t code = nodesListStrictAppend(info.pSubplan->pChildren, splCreateScanSubplan(pCxt, info.pScan, SPLIT_FLAG_CTJ)); + if (TSDB_CODE_SUCCESS == code) { + code = splCreateExchangeNode(pCxt, info.pSubplan, info.pScan, info.pSubplan->subplanType); } ++(pCxt->groupId); pCxt->split = true; @@ -141,7 +203,8 @@ static int32_t stsSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { } static const SSplitRule splitRuleSet[] = { - { .pName = "SuperTableScan", .splitFunc = stsSplit } + { .pName = "SuperTableScan", .splitFunc = stsSplit }, + { .pName = "ChildTableJoin", .splitFunc = ctjSplit }, }; static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule)); diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index fd0084c01e..69c76fc3ae 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -70,6 +70,12 @@ protected: cout << "unformatted logic plan : " << endl; cout << toString((const SNode*)pLogicNode, false) << endl; + code = optimizeLogicPlan(&cxt, pLogicNode); + if (code != TSDB_CODE_SUCCESS) { + cout << "sql:[" << cxt_.pSql << "] optimizeLogicPlan code:" << code << ", strerror:" << tstrerror(code) << endl; + return false; + } + SLogicSubplan* pLogicSubplan = nullptr; code = splitLogicPlan(&cxt, pLogicNode, &pLogicSubplan); if (code != TSDB_CODE_SUCCESS) { @@ -174,13 +180,13 @@ TEST_F(PlannerTest, selectStableBasic) { TEST_F(PlannerTest, selectJoin) { setDatabase("root", "test"); - bind("SELECT * FROM st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); - ASSERT_TRUE(run()); + // bind("SELECT t1.c1, t2.c2 FROM st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); + // ASSERT_TRUE(run()); - bind("SELECT * FROM st1s1 t1 join st1s2 t2 on t1.ts = t2.ts where t1.c1 > t2.c1"); - ASSERT_TRUE(run()); + // bind("SELECT t1.*, t2.* FROM st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); + // ASSERT_TRUE(run()); - bind("SELECT t1.* FROM st1s1 t1 join st1s2 t2 on t1.ts = t2.ts where t1.c1 > t2.c1"); + bind("SELECT t1.c1, t2.c1 FROM st1s1 t1 join st1s2 t2 on t1.ts = t2.ts where t1.c1 > t2.c1 and t1.c2 = 'abc' and t2.c2 = 'qwe'"); ASSERT_TRUE(run()); }