From 762086c22be6ef2996977c8d05e6e954b2a438c6 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 26 Mar 2022 18:10:26 +0800 Subject: [PATCH 01/38] feature/qnode --- source/libs/catalog/inc/catalogInt.h | 17 +- source/libs/catalog/src/catalog.c | 187 +----------------- source/libs/catalog/src/catalogDbg.c | 222 ++++++++++++++++++++++ source/libs/catalog/test/catalogTests.cpp | 82 ++++---- 4 files changed, 284 insertions(+), 224 deletions(-) create mode 100644 source/libs/catalog/src/catalogDbg.c diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 9bd5c1c016..a5887e8807 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -112,7 +112,13 @@ typedef struct SCtgRuntimeStat { } SCtgRuntimeStat; typedef struct SCtgCacheStat { - + uint64_t clusterNum; + uint64_t dbNum; + uint64_t tblNum; + uint64_t vgHitNum; + uint64_t vgMissNum; + uint64_t tblHitNum; + uint64_t tblMissNum; } SCtgCacheStat; typedef struct SCatalogStat { @@ -186,7 +192,7 @@ typedef struct SCatalogMgmt { bool exit; SRWLatch lock; SCtgQueue queue; - TdThread updateThread; + TdThread updateThread; SHashObj *pCluster; //key: clusterId, value: SCatalog* SCatalogStat stat; SCatalogCfg cfg; @@ -206,6 +212,10 @@ typedef struct SCtgAction { #define CTG_STAT_ADD(n) atomic_add_fetch_64(&(n), 1) #define CTG_STAT_SUB(n) atomic_sub_fetch_64(&(n), 1) +#define CTG_STAT_GET(n) atomic_load_64(&(n)) + +#define CTG_RUNTIME_STAT_ADD(_item) (CTG_STAT_ADD(gCtgMgmt.stat.runtime._item)) +#define CTG_CACHE_STAT_ADD(_item) (CTG_STAT_ADD(gCtgMgmt.stat.cache._item)) #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE) #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE) @@ -291,6 +301,9 @@ typedef struct SCtgAction { #define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) +extern void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p); +extern void ctgdShowClusterCache(SCatalog* pCtg); +extern int32_t ctgdShowCacheInfo(void); #ifdef __cplusplus } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6eab0d280b..73b363ec5c 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -24,8 +24,8 @@ int32_t ctgActRemoveDB(SCtgMetaAction *action); int32_t ctgActRemoveStb(SCtgMetaAction *action); int32_t ctgActRemoveTbl(SCtgMetaAction *action); +extern SCtgDebug gCTGDebug; SCatalogMgmt gCtgMgmt = {0}; -SCtgDebug gCTGDebug = {0}; SCtgAction gCtgAction[CTG_ACT_MAX] = {{ CTG_ACT_UPDATE_VG, "update vgInfo", @@ -53,181 +53,6 @@ SCtgAction gCtgAction[CTG_ACT_MAX] = {{ } }; -int32_t ctgDbgEnableDebug(char *option) { - if (0 == strcasecmp(option, "lock")) { - gCTGDebug.lockEnable = true; - qDebug("lock debug enabled"); - return TSDB_CODE_SUCCESS; - } - - if (0 == strcasecmp(option, "cache")) { - gCTGDebug.cacheEnable = true; - qDebug("cache debug enabled"); - return TSDB_CODE_SUCCESS; - } - - if (0 == strcasecmp(option, "api")) { - gCTGDebug.apiEnable = true; - qDebug("api debug enabled"); - return TSDB_CODE_SUCCESS; - } - - if (0 == strcasecmp(option, "meta")) { - gCTGDebug.metaEnable = true; - qDebug("api debug enabled"); - return TSDB_CODE_SUCCESS; - } - - qError("invalid debug option:%s", option); - - return TSDB_CODE_CTG_INTERNAL_ERROR; -} - -int32_t ctgDbgGetStatNum(char *option, void *res) { - if (0 == strcasecmp(option, "runtime.qDoneNum")) { - *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum); - return TSDB_CODE_SUCCESS; - } - - qError("invalid stat option:%s", option); - - return TSDB_CODE_CTG_INTERNAL_ERROR; -} - -int32_t ctgDbgGetTbMetaNum(SCtgDBCache *dbCache) { - return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0; -} - -int32_t ctgDbgGetStbNum(SCtgDBCache *dbCache) { - return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0; -} - -int32_t ctgDbgGetRentNum(SCtgRentMgmt *rent) { - int32_t num = 0; - for (uint16_t i = 0; i < rent->slotNum; ++i) { - SCtgRentSlot *slot = &rent->slots[i]; - if (NULL == slot->meta) { - continue; - } - - num += taosArrayGetSize(slot->meta); - } - - return num; -} - -int32_t ctgDbgGetClusterCacheNum(SCatalog* pCtg, int32_t type) { - if (NULL == pCtg || NULL == pCtg->dbCache) { - return 0; - } - - switch (type) { - case CTG_DBG_DB_NUM: - return (int32_t)taosHashGetSize(pCtg->dbCache); - case CTG_DBG_DB_RENT_NUM: - return ctgDbgGetRentNum(&pCtg->dbRent); - case CTG_DBG_STB_RENT_NUM: - return ctgDbgGetRentNum(&pCtg->stbRent); - default: - break; - } - - SCtgDBCache *dbCache = NULL; - int32_t num = 0; - void *pIter = taosHashIterate(pCtg->dbCache, NULL); - while (pIter) { - dbCache = (SCtgDBCache *)pIter; - switch (type) { - case CTG_DBG_META_NUM: - num += ctgDbgGetTbMetaNum(dbCache); - break; - case CTG_DBG_STB_NUM: - num += ctgDbgGetStbNum(dbCache); - break; - default: - ctgError("invalid type:%d", type); - break; - } - pIter = taosHashIterate(pCtg->dbCache, pIter); - } - - return num; -} - -void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) { - if (!gCTGDebug.metaEnable) { - return; - } - - STableComInfo *c = &p->tableInfo; - - if (TSDB_CHILD_TABLE == p->tableType) { - ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid); - return; - } else { - ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d", - tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize); - } - - int32_t colNum = c->numOfColumns + c->numOfTags; - for (int32_t i = 0; i < colNum; ++i) { - SSchema *s = &p->schema[i]; - ctgDebug("[%d] name:%s, type:%d, colId:%d, bytes:%d", i, s->name, s->type, s->colId, s->bytes); - } -} - -void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { - if (NULL == dbHash || !gCTGDebug.cacheEnable) { - return; - } - - int32_t i = 0; - SCtgDBCache *dbCache = NULL; - void *pIter = taosHashIterate(dbHash, NULL); - while (pIter) { - char *dbFName = NULL; - size_t len = 0; - - dbCache = (SCtgDBCache *)pIter; - - dbFName = taosHashGetKey(pIter, &len); - - int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0; - int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0; - int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; - int32_t hashMethod = -1; - int32_t vgNum = 0; - - if (dbCache->vgInfo) { - vgVersion = dbCache->vgInfo->vgVersion; - hashMethod = dbCache->vgInfo->hashMethod; - if (dbCache->vgInfo->vgHash) { - vgNum = taosHashGetSize(dbCache->vgInfo->vgHash); - } - } - - ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d", - i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum); - - pIter = taosHashIterate(dbHash, pIter); - } -} - - - - -void ctgDbgShowClusterCache(SCatalog* pCtg) { - if (!gCTGDebug.cacheEnable || NULL == pCtg) { - return; - } - - ctgDebug("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg); - ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), - ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)); - - ctgDbgShowDBCache(pCtg, pCtg->dbCache); -} - void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { if (NULL == mgmt->slots) { @@ -361,7 +186,7 @@ int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); CTG_QUEUE_ADD(); - CTG_STAT_ADD(gCtgMgmt.stat.runtime.qNum); + CTG_RUNTIME_STAT_ADD(qNum); tsem_post(&gCtgMgmt.queue.reqSem); @@ -1595,7 +1420,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui } ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); - ctgDbgShowTableMeta(pCtg, tbName, meta); + ctgdShowTableMeta(pCtg, tbName, meta); if (!isStb) { CTG_UNLOCK(CTG_READ, &tbCache->metaLock); @@ -1962,7 +1787,7 @@ _return: if (*pTableMeta) { ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType); - ctgDbgShowTableMeta(pCtg, pTableName->tname, *pTableMeta); + ctgdShowTableMeta(pCtg, pTableName->tname, *pTableMeta); } CTG_RET(code); @@ -2161,9 +1986,9 @@ void* ctgUpdateThreadFunc(void* param) { tsem_post(&gCtgMgmt.queue.rspSem); } - CTG_STAT_ADD(gCtgMgmt.stat.runtime.qDoneNum); + CTG_RUNTIME_STAT_ADD(qDoneNum); - ctgDbgShowClusterCache(pCtg); + ctgdShowClusterCache(pCtg); } CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); diff --git a/source/libs/catalog/src/catalogDbg.c b/source/libs/catalog/src/catalogDbg.c new file mode 100644 index 0000000000..1d4ad0082c --- /dev/null +++ b/source/libs/catalog/src/catalogDbg.c @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "trpc.h" +#include "query.h" +#include "tname.h" +#include "catalogInt.h" + +extern SCatalogMgmt gCtgMgmt; +SCtgDebug gCTGDebug = {0}; + +int32_t ctgdEnableDebug(char *option) { + if (0 == strcasecmp(option, "lock")) { + gCTGDebug.lockEnable = true; + qDebug("lock debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "cache")) { + gCTGDebug.cacheEnable = true; + qDebug("cache debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "api")) { + gCTGDebug.apiEnable = true; + qDebug("api debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "meta")) { + gCTGDebug.metaEnable = true; + qDebug("api debug enabled"); + return TSDB_CODE_SUCCESS; + } + + qError("invalid debug option:%s", option); + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} + +int32_t ctgdGetStatNum(char *option, void *res) { + if (0 == strcasecmp(option, "runtime.qDoneNum")) { + *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum); + return TSDB_CODE_SUCCESS; + } + + qError("invalid stat option:%s", option); + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} + +int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) { + return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0; +} + +int32_t ctgdGetStbNum(SCtgDBCache *dbCache) { + return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0; +} + +int32_t ctgdGetRentNum(SCtgRentMgmt *rent) { + int32_t num = 0; + for (uint16_t i = 0; i < rent->slotNum; ++i) { + SCtgRentSlot *slot = &rent->slots[i]; + if (NULL == slot->meta) { + continue; + } + + num += taosArrayGetSize(slot->meta); + } + + return num; +} + +int32_t ctgdGetClusterCacheNum(SCatalog* pCtg, int32_t type) { + if (NULL == pCtg || NULL == pCtg->dbCache) { + return 0; + } + + switch (type) { + case CTG_DBG_DB_NUM: + return (int32_t)taosHashGetSize(pCtg->dbCache); + case CTG_DBG_DB_RENT_NUM: + return ctgdGetRentNum(&pCtg->dbRent); + case CTG_DBG_STB_RENT_NUM: + return ctgdGetRentNum(&pCtg->stbRent); + default: + break; + } + + SCtgDBCache *dbCache = NULL; + int32_t num = 0; + void *pIter = taosHashIterate(pCtg->dbCache, NULL); + while (pIter) { + dbCache = (SCtgDBCache *)pIter; + switch (type) { + case CTG_DBG_META_NUM: + num += ctgdGetTbMetaNum(dbCache); + break; + case CTG_DBG_STB_NUM: + num += ctgdGetStbNum(dbCache); + break; + default: + ctgError("invalid type:%d", type); + break; + } + pIter = taosHashIterate(pCtg->dbCache, pIter); + } + + return num; +} + +void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) { + if (!gCTGDebug.metaEnable) { + return; + } + + STableComInfo *c = &p->tableInfo; + + if (TSDB_CHILD_TABLE == p->tableType) { + ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid); + return; + } else { + ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d", + tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize); + } + + int32_t colNum = c->numOfColumns + c->numOfTags; + for (int32_t i = 0; i < colNum; ++i) { + SSchema *s = &p->schema[i]; + ctgDebug("[%d] name:%s, type:%d, colId:%d, bytes:%d", i, s->name, s->type, s->colId, s->bytes); + } +} + +void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { + if (NULL == dbHash || !gCTGDebug.cacheEnable) { + return; + } + + int32_t i = 0; + SCtgDBCache *dbCache = NULL; + void *pIter = taosHashIterate(dbHash, NULL); + while (pIter) { + char *dbFName = NULL; + size_t len = 0; + + dbCache = (SCtgDBCache *)pIter; + + dbFName = taosHashGetKey(pIter, &len); + + int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0; + int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0; + int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; + int32_t hashMethod = -1; + int32_t vgNum = 0; + + if (dbCache->vgInfo) { + vgVersion = dbCache->vgInfo->vgVersion; + hashMethod = dbCache->vgInfo->hashMethod; + if (dbCache->vgInfo->vgHash) { + vgNum = taosHashGetSize(dbCache->vgInfo->vgHash); + } + } + + ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d", + i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum); + + pIter = taosHashIterate(dbHash, pIter); + } +} + + + + +void ctgdShowClusterCache(SCatalog* pCtg) { + if (!gCTGDebug.cacheEnable || NULL == pCtg) { + return; + } + + ctgDebug("## cluster %"PRIx64" %p cache Info BEGIN ##", pCtg->clusterId, pCtg); + ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), + ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)); + + ctgdShowDBCache(pCtg, pCtg->dbCache); + + ctgDebug("## cluster %"PRIx64" %p cache Info END ##", pCtg->clusterId, pCtg); +} + +int32_t ctgdShowCacheInfo(void) { + if (!gCTGDebug.cacheEnable) { + return TSDB_CODE_CTG_OUT_OF_SERVICE; + } + + CTG_API_ENTER(); + + SCatalog *pCtg = NULL; + void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + while (pIter) { + pCtg = *(SCatalog **)pIter; + + if (pCtg) { + ctgdShowClusterCache(pCtg); + } + + pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); + } + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); +} + diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index e62819b078..27d08f6428 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -39,10 +39,10 @@ namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, int32_t *exist, int32_t flag, uint64_t *dbId); -extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); +extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); -extern "C" int32_t ctgDbgEnableDebug(char *option); -extern "C" int32_t ctgDbgGetStatNum(char *option, void *res); +extern "C" int32_t ctgdEnableDebug(char *option); +extern "C" int32_t ctgdGetStatNum(char *option, void *res); void ctgTestSetRspTableMeta(); void ctgTestSetRspCTableMeta(); @@ -140,9 +140,9 @@ void ctgTestInitLogFile() { qDebugFlag = 159; strcpy(tsLogDir, "/var/log/taos"); - ctgDbgEnableDebug("api"); - ctgDbgEnableDebug("meta"); - ctgDbgEnableDebug("cache"); + ctgdEnableDebug("api"); + ctgdEnableDebug("meta"); + ctgdEnableDebug("cache"); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); @@ -879,7 +879,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(vgInfo.vgId, 8); ASSERT_EQ(vgInfo.epSet.numOfEps, 3); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { taosMsleep(50); } @@ -899,7 +899,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { taosMsleep(50); } else { @@ -994,7 +994,7 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { taosMsleep(50); } else { @@ -1103,7 +1103,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { taosMsleep(50); } else { @@ -1130,7 +1130,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (2 != n) { taosMsleep(50); } else { @@ -1228,7 +1228,7 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { taosMsleep(50); } else { @@ -1241,8 +1241,8 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(code, 0); while (true) { - int32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); - int32_t m = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM); + int32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + int32_t m = ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM); if (n || m) { taosMsleep(50); } else { @@ -1251,11 +1251,11 @@ TEST(tableMeta, rmStbMeta) { } - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1298,7 +1298,7 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (true) { - uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { taosMsleep(50); } else { @@ -1318,7 +1318,7 @@ TEST(tableMeta, updateStbMeta) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n != 3) { taosMsleep(50); } else { @@ -1326,11 +1326,11 @@ TEST(tableMeta, updateStbMeta) { } } - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); - ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); + ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1); code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); ASSERT_EQ(code, 0); @@ -1388,7 +1388,7 @@ TEST(refreshGetMeta, normal2normal) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1409,7 +1409,7 @@ TEST(refreshGetMeta, normal2normal) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -1467,7 +1467,7 @@ TEST(refreshGetMeta, normal2notexist) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1488,7 +1488,7 @@ TEST(refreshGetMeta, normal2notexist) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -1541,7 +1541,7 @@ TEST(refreshGetMeta, normal2child) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1562,7 +1562,7 @@ TEST(refreshGetMeta, normal2child) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -1625,7 +1625,7 @@ TEST(refreshGetMeta, stable2child) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1647,7 +1647,7 @@ TEST(refreshGetMeta, stable2child) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -1710,7 +1710,7 @@ TEST(refreshGetMeta, stable2stable) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1732,7 +1732,7 @@ TEST(refreshGetMeta, stable2stable) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -1798,7 +1798,7 @@ TEST(refreshGetMeta, child2stable) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -1818,7 +1818,7 @@ TEST(refreshGetMeta, child2stable) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); taosMemoryFreeClear(tableMeta); - while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + while (2 != ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); } @@ -2015,7 +2015,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n > 0) { break; } @@ -2041,7 +2041,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { while (true) { uint64_t n = 0; - ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + ctgdGetStatNum("runtime.qDoneNum", (void *)&n); if (n != 3) { taosMsleep(50); } else { @@ -2266,7 +2266,7 @@ TEST(rentTest, allRent) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) { + while (ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) { taosMsleep(50); } From 9e3f03ad745602fef4f0f07e6636a05d2225e5cd Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 28 Mar 2022 13:49:21 +0800 Subject: [PATCH 02/38] feature/qnode --- source/libs/catalog/inc/catalogInt.h | 12 +- source/libs/catalog/src/catalog.c | 145 ++++++++++++++-------- source/libs/catalog/test/catalogTests.cpp | 8 +- 3 files changed, 105 insertions(+), 60 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index a5887e8807..09f51dc03e 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -115,6 +115,7 @@ typedef struct SCtgCacheStat { uint64_t clusterNum; uint64_t dbNum; uint64_t tblNum; + uint64_t stblNum; uint64_t vgHitNum; uint64_t vgMissNum; uint64_t tblHitNum; @@ -210,12 +211,13 @@ typedef struct SCtgAction { #define CTG_QUEUE_ADD() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) #define CTG_QUEUE_SUB() atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) -#define CTG_STAT_ADD(n) atomic_add_fetch_64(&(n), 1) -#define CTG_STAT_SUB(n) atomic_sub_fetch_64(&(n), 1) -#define CTG_STAT_GET(n) atomic_load_64(&(n)) +#define CTG_STAT_ADD(_item, _n) atomic_add_fetch_64(&(_item), _n) +#define CTG_STAT_SUB(_item, _n) atomic_sub_fetch_64(&(_item), _n) +#define CTG_STAT_GET(_item) atomic_load_64(&(_item)) -#define CTG_RUNTIME_STAT_ADD(_item) (CTG_STAT_ADD(gCtgMgmt.stat.runtime._item)) -#define CTG_CACHE_STAT_ADD(_item) (CTG_STAT_ADD(gCtgMgmt.stat.cache._item)) +#define CTG_RUNTIME_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.runtime.item, n)) +#define CTG_CACHE_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.cache.item, n)) +#define CTG_CACHE_STAT_SUB(item, n) (CTG_STAT_SUB(gCtgMgmt.stat.cache.item, n)) #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE) #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 73b363ec5c..04f93c31f3 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -74,15 +74,19 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { CTG_LOCK(CTG_WRITE, &cache->stbLock); if (cache->stbCache) { + int32_t stblNum = taosHashGetSize(cache->stbCache); taosHashCleanup(cache->stbCache); cache->stbCache = NULL; + CTG_CACHE_STAT_SUB(stblNum, stblNum); } CTG_UNLOCK(CTG_WRITE, &cache->stbLock); CTG_LOCK(CTG_WRITE, &cache->metaLock); if (cache->metaCache) { + int32_t tblNum = taosHashGetSize(cache->metaCache); taosHashCleanup(cache->metaCache); cache->metaCache = NULL; + CTG_CACHE_STAT_SUB(tblNum, tblNum); } CTG_UNLOCK(CTG_WRITE, &cache->metaLock); } @@ -118,6 +122,8 @@ void ctgFreeHandle(SCatalog* pCtg) { ctgFreeMetaRent(&pCtg->stbRent); if (pCtg->dbCache) { + int32_t dbNum = taosHashGetSize(pCtg->dbCache); + void *pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { SCtgDBCache *dbCache = pIter; @@ -130,6 +136,8 @@ void ctgFreeHandle(SCatalog* pCtg) { } taosHashCleanup(pCtg->dbCache); + + CTG_CACHE_STAT_SUB(dbNum, dbNum); } taosMemoryFree(pCtg); @@ -186,7 +194,7 @@ int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); CTG_QUEUE_ADD(); - CTG_RUNTIME_STAT_ADD(qNum); + CTG_RUNTIME_STAT_ADD(qNum, 1); tsem_post(&gCtgMgmt.queue.reqSem); @@ -445,34 +453,45 @@ int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) { + SCtgDBCache *dbCache = NULL; + if (NULL == pCtg->dbCache) { - *pCache = NULL; - *inCache = false; - ctgWarn("empty db cache, dbFName:%s", dbFName); - return TSDB_CODE_SUCCESS; + ctgDebug("empty db cache, dbFName:%s", dbFName); + goto _return; } - SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { - *pCache = NULL; - *inCache = false; - return TSDB_CODE_SUCCESS; + ctgDebug("db %s not in cache", dbFName); + goto _return; } ctgAcquireVgInfo(pCtg, dbCache, inCache); if (!(*inCache)) { - ctgReleaseDBCache(pCtg, dbCache); - - *pCache = NULL; - return TSDB_CODE_SUCCESS; + ctgDebug("vgInfo of db %s not in cache", dbFName); + goto _return; } *pCache = dbCache; *inCache = true; + CTG_CACHE_STAT_ADD(vgHitNum, 1); + ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + +_return: + + if (dbCache) { + ctgReleaseDBCache(pCtg, dbCache); + } + + *pCache = NULL; + *inCache = false; + + CTG_CACHE_STAT_ADD(vgMissNum, 1); + return TSDB_CODE_SUCCESS; } @@ -588,11 +607,10 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, } -int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist, int32_t flag, uint64_t *dbId) { +int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, bool *inCache, int32_t flag, uint64_t *dbId) { if (NULL == pCtg->dbCache) { - *exist = 0; - ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname); - return TSDB_CODE_SUCCESS; + ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); + goto _return; } char dbFName[TSDB_DB_FNAME_LEN] = {0}; @@ -607,8 +625,8 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { - *exist = 0; - return TSDB_CODE_SUCCESS; + ctgDebug("db %s not in cache", pTableName->tname); + goto _return; } int32_t sz = 0; @@ -617,13 +635,11 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == *pTableMeta) { - *exist = 0; ctgReleaseDBCache(pCtg, dbCache); ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); - return TSDB_CODE_SUCCESS; + goto _return; } - *exist = 1; if (dbId) { *dbId = dbCache->dbId; } @@ -633,6 +649,10 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable if (tbMeta->tableType != TSDB_CHILD_TABLE) { ctgReleaseDBCache(pCtg, dbCache); ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname); + + *inCache = true; + CTG_CACHE_STAT_ADD(tblHitNum, 1); + return TSDB_CODE_SUCCESS; } @@ -644,8 +664,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable ctgReleaseDBCache(pCtg, dbCache); ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); taosMemoryFreeClear(*pTableMeta); - *exist = 0; - return TSDB_CODE_SUCCESS; + goto _return; } if ((*stbMeta)->suid != tbMeta->suid) { @@ -671,8 +690,18 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable ctgReleaseDBCache(pCtg, dbCache); + *inCache = true; + CTG_CACHE_STAT_ADD(tblHitNum, 1); + ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); + return TSDB_CODE_SUCCESS; + +_return: + + *inCache = false; + CTG_CACHE_STAT_ADD(tblMissNum, 1); + return TSDB_CODE_SUCCESS; } @@ -1202,6 +1231,8 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } + + CTG_CACHE_STAT_ADD(dbNum, 1); SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); @@ -1261,6 +1292,8 @@ int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } + CTG_CACHE_STAT_SUB(dbNum, 1); + ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); return TSDB_CODE_SUCCESS; @@ -1393,6 +1426,8 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui CTG_LOCK(CTG_WRITE, &tbCache->stbLock); if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + } else { + CTG_CACHE_STAT_SUB(stblNum, 1); } CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); @@ -1419,6 +1454,10 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + if (NULL == orig) { + CTG_CACHE_STAT_ADD(tblNum, 1); + } + ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); ctgdShowTableMeta(pCtg, tbName, meta); @@ -1440,6 +1479,8 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + + CTG_CACHE_STAT_ADD(stblNum, 1); CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); @@ -1699,7 +1740,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - int32_t exist = 0; + bool inCache = false; int32_t code = 0; uint64_t dbId = 0; uint64_t suid = 0; @@ -1709,11 +1750,11 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons CTG_FLAG_SET_INF_DB(flag); } - CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist, flag, &dbId)); + CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &inCache, flag, &dbId)); int32_t tbType = 0; - if (exist) { + if (inCache) { if (CTG_FLAG_MATCH_STB(flag, (*pTableMeta)->tableType) && ((!CTG_FLAG_IS_FORCE_UPDATE(flag)) || (CTG_FLAG_IS_INF_DB(flag)))) { goto _return; } @@ -1755,8 +1796,8 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons SName stbName = *pTableName; strcpy(stbName.tname, output->tbName); - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist, flag, NULL)); - if (0 == exist) { + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &inCache, flag, NULL)); + if (!inCache) { ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); continue; } @@ -1768,7 +1809,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons _return: - if (CTG_TABLE_NOT_EXIST(code) && exist) { + if (CTG_TABLE_NOT_EXIST(code) && inCache) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; if (CTG_FLAG_IS_INF_DB(flag)) { strcpy(dbFName, pTableName->dbname); @@ -1900,12 +1941,16 @@ int32_t ctgActRemoveStb(SCtgMetaAction *action) { CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + } else { + CTG_CACHE_STAT_SUB(stblNum, 1); } CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); - } + } else { + CTG_CACHE_STAT_SUB(tblNum, 1); + } CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); @@ -1944,6 +1989,8 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } else { + CTG_CACHE_STAT_SUB(tblNum, 1); } CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); @@ -1986,7 +2033,7 @@ void* ctgUpdateThreadFunc(void* param) { tsem_post(&gCtgMgmt.queue.rspSem); } - CTG_RUNTIME_STAT_ADD(qDoneNum); + CTG_RUNTIME_STAT_ADD(qDoneNum, 1); ctgdShowClusterCache(pCtg); } @@ -2208,6 +2255,8 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { } *catalogHandle = clusterCtg; + + CTG_CACHE_STAT_ADD(clusterNum, 1); return TSDB_CODE_SUCCESS; @@ -2228,10 +2277,12 @@ void catalogFreeHandle(SCatalog* pCtg) { return; } + CTG_CACHE_STAT_SUB(clusterNum, 1); + uint64_t clusterId = pCtg->clusterId; ctgFreeHandle(pCtg); - + ctgInfo("handle freed, culsterId:%"PRIx64, clusterId); } @@ -2242,24 +2293,12 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == pCtg->dbCache) { - *version = CTG_DEFAULT_INVALID_VERSION; - ctgInfo("empty db cache, dbFName:%s", dbFName); - CTG_API_LEAVE(TSDB_CODE_SUCCESS); - } - SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - *version = CTG_DEFAULT_INVALID_VERSION; - CTG_API_LEAVE(TSDB_CODE_SUCCESS); - } - bool inCache = false; - ctgAcquireVgInfo(pCtg, dbCache, &inCache); - if (!inCache) { - ctgReleaseDBCache(pCtg, dbCache); + int32_t code = 0; + CTG_ERR_JRET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache)); + if (!inCache) { *version = CTG_DEFAULT_INVALID_VERSION; CTG_API_LEAVE(TSDB_CODE_SUCCESS); } @@ -2274,6 +2313,10 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers ctgDebug("Got db vgVersion from cache, dbFName:%s, vgVersion:%d", dbFName, *version); CTG_API_LEAVE(TSDB_CODE_SUCCESS); + +_return: + + CTG_API_LEAVE(code); } int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SArray** vgroupList) { @@ -2374,11 +2417,11 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) { } STableMeta *tblMeta = NULL; - int32_t exist = 0; + bool inCache = false; uint64_t dbId = 0; - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &exist, 0, &dbId)); + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &inCache, 0, &dbId)); - if (0 == exist) { + if (!inCache) { ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname); goto _return; } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 27d08f6428..73d0dc2011 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -38,7 +38,7 @@ namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, - int32_t *exist, int32_t flag, uint64_t *dbId); + bool *inCache, int32_t flag, uint64_t *dbId); extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); extern "C" int32_t ctgdEnableDebug(char *option); @@ -786,15 +786,15 @@ void *ctgTestGetCtableMetaThread(void *param) { int32_t code = 0; int32_t n = 0; STableMeta *tbMeta = NULL; - int32_t exist = 0; + bool inCache = false; SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; strcpy(cn.dbname, "db1"); strcpy(cn.tname, ctgTestCTablename); while (!ctgTestStop) { - code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist, 0, NULL); - if (code || 0 == exist) { + code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &inCache, 0, NULL); + if (code || !inCache) { assert(0); } From f867538446cd02cf60417efe06cea70d8e21eccf Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 28 Mar 2022 19:48:47 +0800 Subject: [PATCH 03/38] feature/qnode --- include/libs/qcom/query.h | 6 + source/libs/qcom/inc/queryInt.h | 19 +++ source/libs/qcom/src/queryExplain.c | 199 ++++++++++++++++++++++++++++ 3 files changed, 224 insertions(+) create mode 100644 source/libs/qcom/src/queryExplain.c diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index f487e5ae22..c3870665a6 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -53,6 +53,12 @@ typedef struct SIndexMeta { } SIndexMeta; +typedef struct SPhysiNodeExplainResNode { + SNodeList* pChildren; + SPhysiNode* pNode; + void* pExecInfo; +} SPhysiNodeExplainResNode; + /* * ASSERT(sizeof(SCTableMeta) == 24) * ASSERT(tableType == TSDB_CHILD_TABLE) diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h index 75c1e134cd..86a1e04ae5 100644 --- a/source/libs/qcom/inc/queryInt.h +++ b/source/libs/qcom/inc/queryInt.h @@ -20,6 +20,25 @@ extern "C" { #endif +#define QUERY_EXPLAIN_MAX_RES_LEN 1024 + +#define EXPLAIN_TAG_SCAN_FORMAT "Tag scan on %s" +#define EXPLAIN_ORDER_FORMAT "Order: " + +typedef struct SQueryExplainRowInfo { + int32_t level; + int32_t len; + char *buf; +} SQueryExplainRowInfo; + + +#define QUERY_EXPLAIN_NEWLINE(_format, ...) tlen = snprintf(tbuf + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, _format, __VA_ARGS__) +#define QUERY_EXPLAIN_APPEND(_format, ...) tlen += snprintf(tbuf + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, _format, __VA_ARGS__) + + +#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) #ifdef __cplusplus diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/qcom/src/queryExplain.c new file mode 100644 index 0000000000..d547e67170 --- /dev/null +++ b/source/libs/qcom/src/queryExplain.c @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "queryInt.h" +#include "query.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-truncation" + +void qFreeExplainRes(SPhysiNodeExplainRes *res) { + +} + +int32_t qMakeExplainResChildrenInfo(SPhysiNode *pNode, void *pExecInfo, SNodeList **pChildren) { + +} + +int32_t qMakeExplainResNode(SPhysiNode *pNode, void *pExecInfo, SPhysiNodeExplainResNode **pRes) { + if (NULL == pNode) { + *pRes = NULL; + qError("physical node is NULL"); + return TSDB_CODE_QRY_APP_ERROR; + } + + SPhysiNodeExplainResNode *res = calloc(1, sizeof(SPhysiNodeExplainResNode)); + if (NULL == res) { + qError("calloc SPhysiNodeExplainRes failed"); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + int32_t code = 0; + res->pNode = pNode; + res->pExecInfo = pExecInfo; + QRY_ERR_JRET(qMakeExplainResChildrenInfo(pNode, pExecInfo, &res->pChildren)); + + *pRes = res; + + return TSDB_CODE_SUCCESS; + +_return: + + qFreeExplainRes(res); + + QRY_RET(code); +} + +int32_t qMakeTaskExplainResTree(struct SSubplan *plan, void *pExecTree, SPhysiNodeExplainResNode **pRes) { + char *tbuf = taosMemoryMalloc(QUERY_EXPLAIN_MAX_RES_LEN); + if (NULL == tbuf) { + qError("malloc size %d failed", QUERY_EXPLAIN_MAX_RES_LEN); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + void *pExecInfo = NULL; // TODO + int32_t code = qMakeExplainResNode(plan->pNode, pExecInfo, pRes); + + taosMemoryFree(tbuf); + + QRY_RET(code); +} + +int32_t qExplainResNodeAppendExecInfo(void *pExecInfo, char *tbuf) { + +} + +int32_t qExplainResAppendRow(SArray *pRows, char *tbuf, int32_t len, int32_t level) { + SQueryExplainRowInfo row = {0}; + row.buf = strdup(tbuf); + if (NULL == row.buf) { + qError("strdup %s failed", tbuf); + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + row.level = level; + row.len = len; + + if (taosArrayPush(pRows, &row)) { + qError("taosArrayPush row to explain res rows failed"); + taosMemoryFree(row.buf); + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t qExplainResNodeToRowsImpl(SPhysiNodeExplainResNode *pResNode, SArray *pRows, char *tbuf, int32_t level) { + int32_t tlen = 0; + SPhysiNode* pNode = pResNode->pNode; + if (NULL == pNode) { + qError("pyhsical node in explain res node is NULL"); + return TSDB_CODE_QRY_APP_ERROR; + } + + switch (pNode->type) { + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { + STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainResNodeAppendExecInfo(pResNode->pExecInfo, tbuf)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + } + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + case QUERY_NODE_PHYSICAL_PLAN_JOIN: + case QUERY_NODE_PHYSICAL_PLAN_AGG: + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: + case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: + case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: + case QUERY_NODE_PHYSICAL_PLAN_INSERT: + default: + qError("not supported physical node type %d", pNode->type); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t qExplainResNodeToRows(SPhysiNodeExplainResNode *pResNode, SArray *pRsp, char *tbuf, int32_t level) { + if (NULL == pResNode) { + qError("explain res node is NULL"); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + int32_t code = 0; + QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, pRsp, tbuf, level)); + + SNode* pNode = NULL; + FOREACH(pNode, pResNode->pChildren) { + QRY_ERR_RET(qExplainResNodeToRows((SPhysiNodeExplainResNode *)pNode, pRsp, tbuf, level + 1)); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t qMakeTaskExplainResRows(SPhysiNodeExplainResNode *pResNode, SRetrieveTableRsp **pRsp) { + if (NULL == pResNode) { + qError("explain res node is NULL"); + QRY_RET(TSDB_CODE_QRY_APP_ERROR); + } + + int32_t code = 0; + char *tbuf = taosMemoryMalloc(QUERY_EXPLAIN_MAX_RES_LEN); + if (NULL == tbuf) { + qError("malloc size %d failed", QUERY_EXPLAIN_MAX_RES_LEN); + QRY_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); + if (NULL == rows) { + qError("taosArrayInit SQueryExplainRowInfo failed"); + QRY_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + QRY_ERR_JRET(qExplainResNodeToRows(pResNode, rows, tbuf, 0)); + + int32_t rspSize = sizeof(SRetrieveTableRsp) + ; + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); + if (NULL == rsp) { + qError("malloc SRetrieveTableRsp failed"); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + QRY_ERR_JRET(qExplainRowsToRsp(rows, rsp)); + + *pRsp = rsp; + rsp = NULL; + +_return: + + taosMemoryFree(tbuf); + taosMemoryFree(rsp); + taosArrayDestroy(rows); + + QRY_RET(code); +} + + +#pragma GCC diagnostic pop From d1102c5f96a2d69fd759627d6790c6fbbf62733f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 29 Mar 2022 08:55:22 +0800 Subject: [PATCH 04/38] feature/qnode --- source/libs/qcom/inc/queryInt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h index 86a1e04ae5..2de0c89a13 100644 --- a/source/libs/qcom/inc/queryInt.h +++ b/source/libs/qcom/inc/queryInt.h @@ -32,7 +32,7 @@ typedef struct SQueryExplainRowInfo { } SQueryExplainRowInfo; -#define QUERY_EXPLAIN_NEWLINE(_format, ...) tlen = snprintf(tbuf + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, _format, __VA_ARGS__) +#define QUERY_EXPLAIN_NEWLINE(_format, ...) tlen = snprintf(tbuf, QUERY_EXPLAIN_MAX_RES_LEN, _format, __VA_ARGS__) #define QUERY_EXPLAIN_APPEND(_format, ...) tlen += snprintf(tbuf + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, _format, __VA_ARGS__) From c8e132052a02fc787205a76f67273b469cea7d3d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 29 Mar 2022 19:51:11 +0800 Subject: [PATCH 05/38] feature/qnode --- include/libs/qcom/query.h | 4 +- source/libs/nodes/src/nodesToSQLFuncs.c | 134 +++++++++ source/libs/nodes/src/nodesTraverseFuncs.c | 3 +- source/libs/nodes/src/nodesUtilFuncs.c | 63 ++++ source/libs/qcom/inc/queryInt.h | 26 +- source/libs/qcom/src/queryExplain.c | 321 ++++++++++++++++++--- source/libs/scheduler/src/scheduler.c | 15 +- 7 files changed, 519 insertions(+), 47 deletions(-) create mode 100644 source/libs/nodes/src/nodesToSQLFuncs.c diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index c3870665a6..c0fc8c630e 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -53,11 +53,11 @@ typedef struct SIndexMeta { } SIndexMeta; -typedef struct SPhysiNodeExplainResNode { +typedef struct SExplainResNode { SNodeList* pChildren; SPhysiNode* pNode; void* pExecInfo; -} SPhysiNodeExplainResNode; +} SExplainResNode; /* * ASSERT(sizeof(SCTableMeta) == 24) diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c new file mode 100644 index 0000000000..6e4cd0f04e --- /dev/null +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "cmdnodes.h" +#include "nodesUtil.h" +#include "plannodes.h" +#include "querynodes.h" +#include "taos.h" +#include "taoserror.h" +#include "thash.h" + +char *gOperatorStr[] = {NULL, "+", "-", "*", "/", "%", "&", "|", ">", ">=", "<", "<=", "=", "<>", + "IN", "NOT IN", "LIKE", "NOT LIKE", "MATCH", "NMATCH", "IS NULL", "IS NOT NULL", + "IS TRUE", "IS FALSE", "IS UNKNOWN", "IS NOT TRUE", "IS NOT FALSE", "IS NOT UNKNOWN"}; +char *gLogicConditionStr[] = {"AND", "OR", "NOT"}; + +int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { + switch (pNode->type) { + case QUERY_NODE_COLUMN: { + SColumnNode *colNode = (SColumnNode *)pNode; + *len = 0; + if (colNode->dbName[0]) { + *len += snprintf(buf, bufSize - *len, "`%s`.", colNode->dbName); + } + + if (colNode->tableAlias[0]) { + *len += snprintf(buf, bufSize - *len, "`%s`.", colNode->tableAlias); + } else if (colNode->tableName[0]) { + *len += snprintf(buf, bufSize - *len, "`%s`.", colNode->tableName); + } + + *len += snprintf(buf, bufSize - *len, "`%s`", colNode->colName); + + return TSDB_CODE_SUCCESS; + } + case QUERY_NODE_VALUE:{ + SValueNode *colNode = (SValueNode *)pNode; + char *t = nodesGetStrValueFromNode(colNode); + if (NULL == t) { + nodesError("fail to get str value from valueNode"); + return TSDB_CODE_QRY_APP_ERROR; + } + + *len += snprintf(buf, bufSize - *len, "%s", t); + taosMemoryFree(t); + + return TSDB_CODE_SUCCESS; + } + case QUERY_NODE_OPERATOR: { + SOperatorNode* pOpNode = (SOperatorNode*)pNode; + *len += snprintf(buf, bufSize - *len, "("); + if (pOpNode->pLeft) { + QRY_ERR_RET(nodesNodeToSQL(pOpNode->pLeft, buf, bufSize, len)); + } + + if (pOpNode->opType >= (sizeof(gOperatorStr) / sizeof(gOperatorStr[0]))) { + nodesError("unknown operation type:%d", pOpNode->opType); + return TSDB_CODE_QRY_APP_ERROR; + } + + *len += snprintf(buf, bufSize - *len, " %s ", gOperatorStr[pOpNode->opType]); + + if (pOpNode->pRight) { + QRY_ERR_RET(nodesNodeToSQL(pOpNode->pRight, buf, bufSize, len)); + } + + *len += snprintf(buf, bufSize - *len, ")"); + + return TSDB_CODE_SUCCESS; + } + case QUERY_NODE_LOGIC_CONDITION:{ + SLogicConditionNode* pLogicNode = (SLogicConditionNode*)pNode; + SNode* node = NULL; + bool first = true; + + *len += snprintf(buf, bufSize - *len, "("); + + FOREACH(node, pLogicNode->pParameterList) { + if (!first) { + *len += snprintf(buf, bufSize - *len, " %s ", gLogicConditionStr[pLogicNode->condType]); + } + QRY_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); + first = false; + } + + *len += snprintf(buf, bufSize - *len, ")"); + + return TSDB_CODE_SUCCESS; + } + case QUERY_NODE_FUNCTION:{ + SFunctionNode* pFuncNode = (SFunctionNode*)pNode; + SNode* node = NULL; + bool first = true; + + *len += snprintf(buf, bufSize - *len, "%s(", pFuncNode->functionName); + + FOREACH(node, pFuncNode->pParameterList) { + if (!first) { + *len += snprintf(buf, bufSize - *len, ", "); + } + QRY_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); + first = false; + } + + *len += snprintf(buf, bufSize - *len, ")"); + + return TSDB_CODE_SUCCESS; + } + case QUERY_NODE_NODE_LIST:{ + SNodeListNode* pListNode = (SNodeListNode *)pNode; + + //TODO + + return TSDB_CODE_SUCCESS; + } + default: + break; + } + + nodesError("nodesNodeToSQL unknown node = %s", nodesNodeName(pNode->type)); + return TSDB_CODE_QRY_APP_ERROR; +} diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 7eaa049946..bbd0473edd 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -17,7 +17,8 @@ typedef enum ETraversalOrder { TRAVERSAL_PREORDER = 1, - TRAVERSAL_POSTORDER + TRAVERSAL_INORDER, + TRAVERSAL_POSTORDER, } ETraversalOrder; static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalker walker, void* pContext); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 035a2f1caa..7b2be5e355 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -467,6 +467,69 @@ void* nodesGetValueFromNode(SValueNode *pNode) { return NULL; } +char* nodesGetStrValueFromNode(SValueNode *pNode) { + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_BOOL: + void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE); + if (NULL == buf) { + return NULL; + } + + sprintf(buf, "%s", pNode->datum.b ? "true" : "false"); + return buf; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE); + if (NULL == buf) { + return NULL; + } + + sprintf(buf, "%" PRId64, pNode->datum.i); + return buf; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: { + void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE); + if (NULL == buf) { + return NULL; + } + + sprintf(buf, "%" PRIu64, pNode->datum.u); + return buf; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE); + if (NULL == buf) { + return NULL; + } + + sprintf(buf, "%e", pNode->datum.d); + return buf; + } + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: { + void *buf = taosMemoryMalloc(varDataLen(pNode->datum.p) + 1); + if (NULL == buf) { + return NULL; + } + + strncpy(buf, varDataVal(pNode->datum.p), varDataLen(pNode->datum.p) + 1); + return buf; + } + default: + break; + } + + return NULL; +} + bool nodesIsExprNode(const SNode* pNode) { ENodeType type = nodeType(pNode); return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || QUERY_NODE_FUNCTION == type); diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h index 2de0c89a13..7b6b7d44ef 100644 --- a/source/libs/qcom/inc/queryInt.h +++ b/source/libs/qcom/inc/queryInt.h @@ -22,8 +22,24 @@ extern "C" { #define QUERY_EXPLAIN_MAX_RES_LEN 1024 -#define EXPLAIN_TAG_SCAN_FORMAT "Tag scan on %s" -#define EXPLAIN_ORDER_FORMAT "Order: " +#define EXPLAIN_TAG_SCAN_FORMAT "Tag scan on %s columns=%d" +#define EXPLAIN_TBL_SCAN_FORMAT "Table scan on %s columns=%d" +#define EXPLAIN_SYSTBL_SCAN_FORMAT "System table scan on %s columns=%d" +#define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d" +#define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d" +#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d groups=%d width=%d" +#define EXPLAIN_EXCHANGE_FORMAT "Exchange %d:1 width=%d" +#define EXPLAIN_SORT_FORMAT "Sort on %d columns width=%d" +#define EXPLAIN_INTERVAL_FORMAT "Interval on column %s functions=%d interval=%d%c offset=%d%c sliding=%d%c width=%d" +#define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d" + +#define EXPLAIN_ORDER_FORMAT "Order: %s" +#define EXPLAIN_FILTER_FORMAT "Filter: " +#define EXPLAIN_FILL_FORMAT "Fill: %s" +#define EXPLAIN_ON_CONDITIONS_FORMAT "ON Conditions: " +#define EXPLAIN_TIMERANGE_FORMAT "Time range: [%" PRId64 ", %" PRId64 "]" +#define EXPLAIN_LOOPS_FORMAT "loops %d" +#define EXPLAIN_REVERSE_FORMAT "reverse %d" typedef struct SQueryExplainRowInfo { int32_t level; @@ -31,9 +47,11 @@ typedef struct SQueryExplainRowInfo { char *buf; } SQueryExplainRowInfo; +#define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending") +#define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") -#define QUERY_EXPLAIN_NEWLINE(_format, ...) tlen = snprintf(tbuf, QUERY_EXPLAIN_MAX_RES_LEN, _format, __VA_ARGS__) -#define QUERY_EXPLAIN_APPEND(_format, ...) tlen += snprintf(tbuf + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, _format, __VA_ARGS__) +#define QUERY_EXPLAIN_NEWLINE(...) tlen = snprintf(tbuf, QUERY_EXPLAIN_MAX_RES_LEN, __VA_ARGS__) +#define QUERY_EXPLAIN_APPEND(...) tlen += snprintf(tbuf + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, __VA_ARGS__) #define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/qcom/src/queryExplain.c index d547e67170..d6da9ab0e2 100644 --- a/source/libs/qcom/src/queryExplain.c +++ b/source/libs/qcom/src/queryExplain.c @@ -19,22 +19,124 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation" -void qFreeExplainRes(SPhysiNodeExplainRes *res) { +void qFreeExplainRes(SExplainResNode *res) { } +char *qFillModeString(EFillMode mode) { + switch (mode) { + case FILL_MODE_NONE: + return "none"; + case FILL_MODE_VALUE: + return "value"; + case FILL_MODE_PREV: + return "prev"; + case FILL_MODE_NULL: + return "null"; + case FILL_MODE_LINEAR: + return "linear"; + case FILL_MODE_NEXT: + return "next"; + default: + return "unknown"; + } +} + +char *qGetNameFromColumnNode(SNode *pNode) { + if (NULL == pNode || QUERY_NODE_COLUMN != pNode->type) { + return "NULL"; + } + + return ((SColumnNode *)pNode)->colName; +} + int32_t qMakeExplainResChildrenInfo(SPhysiNode *pNode, void *pExecInfo, SNodeList **pChildren) { + int32_t tlen = 0; + SNodeList *pPhysiChildren = NULL; + + switch (pNode->type) { + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { + STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; + pPhysiChildren = pTagScanNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ + STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; + pPhysiChildren = pTblScanNode->scan.node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ + SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; + pPhysiChildren = pSTblScanNode->scan.node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ + SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; + pPhysiChildren = pPrjNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ + SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; + pPhysiChildren = pJoinNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_AGG:{ + SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; + pPhysiChildren = pAggNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ + SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; + pPhysiChildren = pExchNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SORT:{ + SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; + pPhysiChildren = pSortNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ + SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; + pPhysiChildren = pIntNode->window.node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ + SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode; + pPhysiChildren = pSessNode->window.node.pChildren; + break; + } + default: + qError("not supported physical node type %d", pNode->type); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + if (pPhysiChildren) { + *pChildren = nodesMakeList(); + if (NULL == *pChildren) { + qError("nodesMakeList failed"); + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + SNode* node = NULL; + SExplainResNode *pResNode = NULL; + FOREACH(node, pPhysiChildren) { + QRY_ERR_RET(qMakeExplainResNode((SPhysiNode *)node, pExecInfo, &pResNode)); + QRY_ERR_RET(nodesListAppend(*pChildren, pResNode)); + } + + return TSDB_CODE_SUCCESS; } -int32_t qMakeExplainResNode(SPhysiNode *pNode, void *pExecInfo, SPhysiNodeExplainResNode **pRes) { +int32_t qMakeExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes) { if (NULL == pNode) { *pRes = NULL; qError("physical node is NULL"); return TSDB_CODE_QRY_APP_ERROR; } - SPhysiNodeExplainResNode *res = calloc(1, sizeof(SPhysiNodeExplainResNode)); + SExplainResNode *res = taosMemoryCalloc(1, sizeof(SExplainResNode)); if (NULL == res) { qError("calloc SPhysiNodeExplainRes failed"); return TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -56,7 +158,7 @@ _return: QRY_RET(code); } -int32_t qMakeTaskExplainResTree(struct SSubplan *plan, void *pExecTree, SPhysiNodeExplainResNode **pRes) { +int32_t qMakeTaskExplainResTree(struct SSubplan *plan, void *pExecTree, SExplainResNode **pRes) { char *tbuf = taosMemoryMalloc(QUERY_EXPLAIN_MAX_RES_LEN); if (NULL == tbuf) { qError("malloc size %d failed", QUERY_EXPLAIN_MAX_RES_LEN); @@ -71,7 +173,7 @@ int32_t qMakeTaskExplainResTree(struct SSubplan *plan, void *pExecTree, SPhysiNo QRY_RET(code); } -int32_t qExplainResNodeAppendExecInfo(void *pExecInfo, char *tbuf) { +int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t tlen) { } @@ -96,7 +198,7 @@ int32_t qExplainResAppendRow(SArray *pRows, char *tbuf, int32_t len, int32_t lev } -int32_t qExplainResNodeToRowsImpl(SPhysiNodeExplainResNode *pResNode, SArray *pRows, char *tbuf, int32_t level) { +int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SArray *pRows, char *tbuf, int32_t level) { int32_t tlen = 0; SPhysiNode* pNode = pResNode->pNode; if (NULL == pNode) { @@ -107,25 +209,177 @@ int32_t qExplainResNodeToRowsImpl(SPhysiNodeExplainResNode *pResNode, SArray *pR switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname); + QUERY_EXPLAIN_NEWLINE(EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname, pTagScanNode->pScanCols->length); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainResNodeAppendExecInfo(pResNode->pExecInfo, tbuf)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf)); + } + QUERY_EXPLAIN_APPEND(EXPLAIN_LOOPS_FORMAT, pTagScanNode->count); + if (pTagScanNode->reverse) { + QUERY_EXPLAIN_APPEND(EXPLAIN_REVERSE_FORMAT, pTagScanNode->reverse); } QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + QUERY_EXPLAIN_NEWLINE(EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + break; } - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_PROJECT: - case QUERY_NODE_PHYSICAL_PLAN_JOIN: - case QUERY_NODE_PHYSICAL_PLAN_AGG: - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: - case QUERY_NODE_PHYSICAL_PLAN_SORT: - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: - case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: - case QUERY_NODE_PHYSICAL_PLAN_INSERT: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ + STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname, pTblScanNode->scan.pScanCols->length); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QUERY_EXPLAIN_APPEND(EXPLAIN_LOOPS_FORMAT, pTblScanNode->scan.count); + if (pTblScanNode->scan.reverse) { + QUERY_EXPLAIN_APPEND(EXPLAIN_REVERSE_FORMAT, pTblScanNode->scan.reverse); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + QUERY_EXPLAIN_NEWLINE(EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + + QUERY_EXPLAIN_NEWLINE(EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + + if (pTblScanNode->pScanConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->pScanConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ + SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname, pSTblScanNode->scan.pScanCols->length); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QUERY_EXPLAIN_APPEND(EXPLAIN_LOOPS_FORMAT, pSTblScanNode->scan.count); + if (pSTblScanNode->scan.reverse) { + QUERY_EXPLAIN_APPEND(EXPLAIN_REVERSE_FORMAT, pSTblScanNode->scan.reverse); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + QUERY_EXPLAIN_NEWLINE(EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + break; + } + case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ + SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_PROJECTION_FORMAT, pPrjNode->pProjections->length, pPrjNode->node.pOutputDataBlockDesc->resultRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + if (pPrjNode->node.pConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ + SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType), pJoinNode->pTargets->length, pJoinNode->node.pOutputDataBlockDesc->resultRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + if (pJoinNode->node.pConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + + QUERY_EXPLAIN_NEWLINE(EXPLAIN_ON_CONDITIONS_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + break; + } + case QUERY_NODE_PHYSICAL_PLAN_AGG:{ + SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length, pAggNode->pGroupKeys->length, pAggNode->node.pOutputDataBlockDesc->resultRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + if (pAggNode->node.pConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ + SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_EXCHANGE_FORMAT, pExchNode->pSrcEndPoints->length, pExchNode->node.pOutputDataBlockDesc->resultRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + if (pExchNode->node.pConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SORT:{ + SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_SORT_FORMAT, pSortNode->pSortKeys->length, pSortNode->node.pOutputDataBlockDesc->resultRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + if (pSortNode->node.pConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ + SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, + pIntNode->interval, pIntNode->intervalUnit, pIntNode->offset, pIntNode->intervalUnit, pIntNode->sliding, pIntNode->slidingUnit, pIntNode->window.node.pOutputDataBlockDesc->resultRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + if (pIntNode->pFill) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILL_FORMAT, qFillModeString(pIntNode->pFill->mode)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + + if (pIntNode->window.node.pConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ + SSessionWinodwPhysiNode *pIntNode = (SSessionWinodwPhysiNode *)pNode; + QUERY_EXPLAIN_NEWLINE(EXPLAIN_SESSION_FORMAT, pIntNode->gap, pIntNode->window.pFuncs->length, pIntNode->window.node.pOutputDataBlockDesc->resultRowSize); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + } + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + + if (pIntNode->window.node.pConditions) { + QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + } + break; + } default: qError("not supported physical node type %d", pNode->type); return TSDB_CODE_QRY_APP_ERROR; @@ -135,7 +389,7 @@ int32_t qExplainResNodeToRowsImpl(SPhysiNodeExplainResNode *pResNode, SArray *pR } -int32_t qExplainResNodeToRows(SPhysiNodeExplainResNode *pResNode, SArray *pRsp, char *tbuf, int32_t level) { +int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SArray *pRsp, char *tbuf, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -146,14 +400,22 @@ int32_t qExplainResNodeToRows(SPhysiNodeExplainResNode *pResNode, SArray *pRsp, SNode* pNode = NULL; FOREACH(pNode, pResNode->pChildren) { - QRY_ERR_RET(qExplainResNodeToRows((SPhysiNodeExplainResNode *)pNode, pRsp, tbuf, level + 1)); + QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, pRsp, tbuf, level + 1)); } return TSDB_CODE_SUCCESS; } +int32_t qExplainRowsToRsp(SArray *rows, SRetrieveTableRsp **pRsp) { + int32_t rspSize = sizeof(SRetrieveTableRsp) + ; + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); + if (NULL == rsp) { + qError("malloc SRetrieveTableRsp failed"); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } +} -int32_t qMakeTaskExplainResRows(SPhysiNodeExplainResNode *pResNode, SRetrieveTableRsp **pRsp) { +int32_t qMakeTaskExplainResRows(SExplainResNode *pResNode, SRetrieveTableRsp **pRsp) { if (NULL == pResNode) { qError("explain res node is NULL"); QRY_RET(TSDB_CODE_QRY_APP_ERROR); @@ -174,22 +436,11 @@ int32_t qMakeTaskExplainResRows(SPhysiNodeExplainResNode *pResNode, SRetrieveTab QRY_ERR_JRET(qExplainResNodeToRows(pResNode, rows, tbuf, 0)); - int32_t rspSize = sizeof(SRetrieveTableRsp) + ; - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); - if (NULL == rsp) { - qError("malloc SRetrieveTableRsp failed"); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - QRY_ERR_JRET(qExplainRowsToRsp(rows, rsp)); - - *pRsp = rsp; - rsp = NULL; + QRY_ERR_JRET(qExplainRowsToRsp(rows, pRsp)); _return: taosMemoryFree(tbuf); - taosMemoryFree(rsp); taosArrayDestroy(rows); QRY_RET(code); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 5af13d97ca..fdb0b6498c 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -2126,19 +2126,24 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD tsem_init(&pJob->rspSem, 0, 0); - pJob->refId = taosAddRef(schMgmt.jobRef, pJob); - if (pJob->refId < 0) { - SCH_JOB_ELOG("taosHashPut job failed, error:%s", tstrerror(terrno)); + int64_t refId = taosAddRef(schMgmt.jobRef, pJob); + if (refId < 0) { + SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); SCH_ERR_JRET(terrno); } + if (NULL == schAcquireJob(refId)) { + SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId); + SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + pJob->refId = refId; + SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId); pJob->status = JOB_TASK_STATUS_NOT_START; SCH_ERR_JRET(schLaunchJob(pJob)); - schAcquireJob(pJob->refId); - *job = pJob->refId; if (syncSchedule) { From 6982f2b7bda6660bdfc1d04b8ebd688b211a741f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 30 Mar 2022 17:09:30 +0800 Subject: [PATCH 06/38] feature/qnode --- include/libs/nodes/nodes.h | 2 + include/libs/nodes/plannodes.h | 12 + include/libs/nodes/querynodes.h | 1 + include/libs/qcom/query.h | 9 +- include/util/tdef.h | 4 + source/libs/nodes/inc/nodesUtil.h | 16 +- source/libs/nodes/src/nodesToSQLFuncs.c | 30 +- source/libs/nodes/src/nodesUtilFuncs.c | 3 +- source/libs/qcom/CMakeLists.txt | 2 +- source/libs/qcom/inc/queryInt.h | 33 ++- source/libs/qcom/src/queryExplain.c | 335 +++++++++++++++-------- source/libs/scalar/inc/filterInt.h | 2 - source/libs/scheduler/inc/schedulerInt.h | 7 +- source/libs/scheduler/src/scheduler.c | 136 ++++++++- 14 files changed, 447 insertions(+), 145 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 9b8739a4f3..031445ac79 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -210,6 +210,8 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode); int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen); int32_t nodesStringToList(const char* pStr, SNodeList** pList); +int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len); + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 08792b6f8f..fb4def4440 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -283,11 +283,23 @@ typedef struct SSubplan { SDataSinkNode* pDataSink; // data of the subplan flow into the datasink } SSubplan; +typedef enum EExplainMode { + EXPLAIN_MODE_DISABLE = 1, + EXPLAIN_MODE_STATIC, + EXPLAIN_MODE_ANALYZE +} EExplainMode; + +typedef struct SExplainInfo { + EExplainMode mode; + bool verbose; +} SExplainInfo; + typedef struct SQueryPlan { ENodeType type; uint64_t queryId; int32_t numOfSubplans; SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. + SExplainInfo explainInfo; } SQueryPlan; #ifdef __cplusplus diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 66f60bde98..2f4f938652 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -300,6 +300,7 @@ bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery); void* nodesGetValueFromNode(SValueNode *pNode); +char* nodesGetStrValueFromNode(SValueNode *pNode); #ifdef __cplusplus } diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index c0fc8c630e..2a9dd28819 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -53,11 +53,6 @@ typedef struct SIndexMeta { } SIndexMeta; -typedef struct SExplainResNode { - SNodeList* pChildren; - SPhysiNode* pNode; - void* pExecInfo; -} SExplainResNode; /* * ASSERT(sizeof(SCTableMeta) == 24) @@ -179,6 +174,10 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); char *jobTaskStatusStr(int32_t status); +int32_t qAppendTaskExplainResRows(void **pRowCtx, struct SSubplan *plan, void *pExecTree, int32_t level); +int32_t qGetExplainRspFromRowCtx(void *ctx, SRetrieveTableRsp **pRsp); +void qFreeExplainRowCtx(void *ctx); + SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen); diff --git a/include/util/tdef.h b/include/util/tdef.h index 193be4a3e6..7db06bbafb 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -224,6 +224,7 @@ typedef enum ELogicConditionType { #define TSDB_APP_NAME_LEN TSDB_UNI_LEN #define TSDB_STB_COMMENT_LEN 1024 + /** * In some scenarios uint16_t (0~65535) is used to store the row len. * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. @@ -471,6 +472,9 @@ enum { #define QND_VGID 1 #define VND_VGID 0 +#define MAX_NUM_STR_SIZE 40 + + #ifdef __cplusplus } #endif diff --git a/source/libs/nodes/inc/nodesUtil.h b/source/libs/nodes/inc/nodesUtil.h index de00b6bca4..976044c16f 100644 --- a/source/libs/nodes/inc/nodesUtil.h +++ b/source/libs/nodes/inc/nodesUtil.h @@ -20,12 +20,16 @@ extern "C" { #endif -#define nodesFatal(param, ...) qFatal("NODES: " param, __VA_ARGS__) -#define nodesError(param, ...) qError("NODES: " param, __VA_ARGS__) -#define nodesWarn(param, ...) qWarn("NODES: " param, __VA_ARGS__) -#define nodesInfo(param, ...) qInfo("NODES: " param, __VA_ARGS__) -#define nodesDebug(param, ...) qDebug("NODES: " param, __VA_ARGS__) -#define nodesTrace(param, ...) qTrace("NODES: " param, __VA_ARGS__) +#define nodesFatal(...) qFatal("NODES: " __VA_ARGS__) +#define nodesError(...) qError("NODES: " __VA_ARGS__) +#define nodesWarn(...) qWarn("NODES: " __VA_ARGS__) +#define nodesInfo(...) qInfo("NODES: " __VA_ARGS__) +#define nodesDebug(...) qDebug("NODES: " __VA_ARGS__) +#define nodesTrace(...) qTrace("NODES: " __VA_ARGS__) + +#define NODES_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define NODES_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define NODES_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) #ifdef __cplusplus } diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index 6e4cd0f04e..fa941cfe9c 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -50,7 +50,7 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { char *t = nodesGetStrValueFromNode(colNode); if (NULL == t) { nodesError("fail to get str value from valueNode"); - return TSDB_CODE_QRY_APP_ERROR; + NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } *len += snprintf(buf, bufSize - *len, "%s", t); @@ -62,18 +62,18 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { SOperatorNode* pOpNode = (SOperatorNode*)pNode; *len += snprintf(buf, bufSize - *len, "("); if (pOpNode->pLeft) { - QRY_ERR_RET(nodesNodeToSQL(pOpNode->pLeft, buf, bufSize, len)); + NODES_ERR_RET(nodesNodeToSQL(pOpNode->pLeft, buf, bufSize, len)); } if (pOpNode->opType >= (sizeof(gOperatorStr) / sizeof(gOperatorStr[0]))) { nodesError("unknown operation type:%d", pOpNode->opType); - return TSDB_CODE_QRY_APP_ERROR; + NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } *len += snprintf(buf, bufSize - *len, " %s ", gOperatorStr[pOpNode->opType]); if (pOpNode->pRight) { - QRY_ERR_RET(nodesNodeToSQL(pOpNode->pRight, buf, bufSize, len)); + NODES_ERR_RET(nodesNodeToSQL(pOpNode->pRight, buf, bufSize, len)); } *len += snprintf(buf, bufSize - *len, ")"); @@ -91,7 +91,7 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { if (!first) { *len += snprintf(buf, bufSize - *len, " %s ", gLogicConditionStr[pLogicNode->condType]); } - QRY_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); + NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); first = false; } @@ -110,7 +110,7 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { if (!first) { *len += snprintf(buf, bufSize - *len, ", "); } - QRY_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); + NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); first = false; } @@ -120,8 +120,20 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { } case QUERY_NODE_NODE_LIST:{ SNodeListNode* pListNode = (SNodeListNode *)pNode; - - //TODO + SNode* node = NULL; + bool first = true; + + *len += snprintf(buf, bufSize - *len, "("); + + FOREACH(node, pListNode->pNodeList) { + if (!first) { + *len += snprintf(buf, bufSize - *len, ", "); + } + NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); + first = false; + } + + *len += snprintf(buf, bufSize - *len, ")"); return TSDB_CODE_SUCCESS; } @@ -130,5 +142,5 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { } nodesError("nodesNodeToSQL unknown node = %s", nodesNodeName(pNode->type)); - return TSDB_CODE_QRY_APP_ERROR; + NODES_RET(TSDB_CODE_QRY_APP_ERROR); } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 7b2be5e355..2725ad85dd 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -469,7 +469,7 @@ void* nodesGetValueFromNode(SValueNode *pNode) { char* nodesGetStrValueFromNode(SValueNode *pNode) { switch (pNode->node.resType.type) { - case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_BOOL: { void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE); if (NULL == buf) { return NULL; @@ -477,6 +477,7 @@ char* nodesGetStrValueFromNode(SValueNode *pNode) { sprintf(buf, "%s", pNode->datum.b ? "true" : "false"); return buf; + } case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: diff --git a/source/libs/qcom/CMakeLists.txt b/source/libs/qcom/CMakeLists.txt index a9bf0f5594..d50047e592 100644 --- a/source/libs/qcom/CMakeLists.txt +++ b/source/libs/qcom/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( qcom - PRIVATE os util transport + PRIVATE os util transport nodes ) if(${BUILD_TEST}) diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h index 7b6b7d44ef..6fc15aacfc 100644 --- a/source/libs/qcom/inc/queryInt.h +++ b/source/libs/qcom/inc/queryInt.h @@ -19,9 +19,12 @@ #ifdef __cplusplus extern "C" { #endif +#include "nodes.h" +#include "plannodes.h" #define QUERY_EXPLAIN_MAX_RES_LEN 1024 +//newline area #define EXPLAIN_TAG_SCAN_FORMAT "Tag scan on %s columns=%d" #define EXPLAIN_TBL_SCAN_FORMAT "Table scan on %s columns=%d" #define EXPLAIN_SYSTBL_SCAN_FORMAT "System table scan on %s columns=%d" @@ -30,29 +33,47 @@ extern "C" { #define EXPLAIN_AGG_FORMAT "Aggragate functions=%d groups=%d width=%d" #define EXPLAIN_EXCHANGE_FORMAT "Exchange %d:1 width=%d" #define EXPLAIN_SORT_FORMAT "Sort on %d columns width=%d" -#define EXPLAIN_INTERVAL_FORMAT "Interval on column %s functions=%d interval=%d%c offset=%d%c sliding=%d%c width=%d" +#define EXPLAIN_INTERVAL_FORMAT "Interval on column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d" #define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d" - #define EXPLAIN_ORDER_FORMAT "Order: %s" #define EXPLAIN_FILTER_FORMAT "Filter: " #define EXPLAIN_FILL_FORMAT "Fill: %s" -#define EXPLAIN_ON_CONDITIONS_FORMAT "ON Conditions: " +#define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: " #define EXPLAIN_TIMERANGE_FORMAT "Time range: [%" PRId64 ", %" PRId64 "]" + +//append area #define EXPLAIN_LOOPS_FORMAT "loops %d" #define EXPLAIN_REVERSE_FORMAT "reverse %d" + +typedef struct SExplainResNode { + SNodeList* pChildren; + SPhysiNode* pNode; + void* pExecInfo; +} SExplainResNode; + typedef struct SQueryExplainRowInfo { int32_t level; int32_t len; char *buf; } SQueryExplainRowInfo; +typedef struct SExplainRowCtx { + int32_t totalSize; + SArray *rows; +} SExplainRowCtx; + #define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending") #define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") -#define QUERY_EXPLAIN_NEWLINE(...) tlen = snprintf(tbuf, QUERY_EXPLAIN_MAX_RES_LEN, __VA_ARGS__) -#define QUERY_EXPLAIN_APPEND(...) tlen += snprintf(tbuf + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, __VA_ARGS__) - +#define EXPLAIN_ROW_NEW(level, ...) \ + do { \ + tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, QUERY_EXPLAIN_MAX_RES_LEN, "%*s", level, ""); \ + tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, __VA_ARGS__); \ + } while (0) + +#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, __VA_ARGS__) +#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; } while (0) #define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/qcom/src/queryExplain.c index d6da9ab0e2..d352a29e3c 100644 --- a/source/libs/qcom/src/queryExplain.c +++ b/source/libs/qcom/src/queryExplain.c @@ -15,12 +15,41 @@ #include "queryInt.h" #include "query.h" +#include "plannodes.h" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wformat-truncation" +int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes); -void qFreeExplainRes(SExplainResNode *res) { +void qFreeExplainResTree(SExplainResNode *res) { + if (NULL == res) { + return; + } + + taosMemoryFreeClear(res->pExecInfo); + + SNode* node = NULL; + FOREACH(node, res->pChildren) { + qFreeExplainResTree((SExplainResNode *)node); + } + nodesDestroyList(res->pChildren); + + taosMemoryFreeClear(res); +} + +void qFreeExplainRowCtx(void *ctx) { + if (NULL == ctx) { + return; + } + + SExplainRowCtx *pCtx = (SExplainRowCtx *)ctx; + int32_t rowSize = taosArrayGetSize(pCtx->rows); + for (int32_t i = 0; i < rowSize; ++i) { + SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); + taosMemoryFreeClear(row->buf); + } + + taosArrayDestroy(pCtx->rows); + taosMemoryFree(pCtx); } char *qFillModeString(EFillMode mode) { @@ -50,7 +79,7 @@ char *qGetNameFromColumnNode(SNode *pNode) { return ((SColumnNode *)pNode)->colName; } -int32_t qMakeExplainResChildrenInfo(SPhysiNode *pNode, void *pExecInfo, SNodeList **pChildren) { +int32_t qGenerateExplainResChildren(SPhysiNode *pNode, void *pExecInfo, SNodeList **pChildren) { int32_t tlen = 0; SNodeList *pPhysiChildren = NULL; @@ -122,14 +151,14 @@ int32_t qMakeExplainResChildrenInfo(SPhysiNode *pNode, void *pExecInfo, SNodeLis SNode* node = NULL; SExplainResNode *pResNode = NULL; FOREACH(node, pPhysiChildren) { - QRY_ERR_RET(qMakeExplainResNode((SPhysiNode *)node, pExecInfo, &pResNode)); + QRY_ERR_RET(qGenerateExplainResNode((SPhysiNode *)node, pExecInfo, &pResNode)); QRY_ERR_RET(nodesListAppend(*pChildren, pResNode)); } return TSDB_CODE_SUCCESS; } -int32_t qMakeExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes) { +int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes) { if (NULL == pNode) { *pRes = NULL; qError("physical node is NULL"); @@ -145,7 +174,7 @@ int32_t qMakeExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode int32_t code = 0; res->pNode = pNode; res->pExecInfo = pExecInfo; - QRY_ERR_JRET(qMakeExplainResChildrenInfo(pNode, pExecInfo, &res->pChildren)); + QRY_ERR_JRET(qGenerateExplainResChildren(pNode, pExecInfo, &res->pChildren)); *pRes = res; @@ -153,31 +182,27 @@ int32_t qMakeExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode _return: - qFreeExplainRes(res); + qFreeExplainResTree(res); QRY_RET(code); } -int32_t qMakeTaskExplainResTree(struct SSubplan *plan, void *pExecTree, SExplainResNode **pRes) { - char *tbuf = taosMemoryMalloc(QUERY_EXPLAIN_MAX_RES_LEN); - if (NULL == tbuf) { - qError("malloc size %d failed", QUERY_EXPLAIN_MAX_RES_LEN); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - +int32_t qGenerateExplainResNodeTree(struct SSubplan *plan, void *pExecTree, SExplainResNode **pRes) { void *pExecInfo = NULL; // TODO - int32_t code = qMakeExplainResNode(plan->pNode, pExecInfo, pRes); - - taosMemoryFree(tbuf); - - QRY_RET(code); + QRY_RET(qGenerateExplainResNode(plan->pNode, pExecInfo, pRes)); } -int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t tlen) { - +int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t *len) { + int32_t tlen = *len; + + EXPLAIN_ROW_APPEND("(exec info here)"); + + *len = tlen; + + return TSDB_CODE_SUCCESS; } -int32_t qExplainResAppendRow(SArray *pRows, char *tbuf, int32_t len, int32_t level) { +int32_t qExplainResAppendRow(SExplainRowCtx *ctx, char *tbuf, int32_t len, int32_t level) { SQueryExplainRowInfo row = {0}; row.buf = strdup(tbuf); if (NULL == row.buf) { @@ -187,8 +212,9 @@ int32_t qExplainResAppendRow(SArray *pRows, char *tbuf, int32_t len, int32_t lev row.level = level; row.len = len; + ctx->totalSize += len; - if (taosArrayPush(pRows, &row)) { + if (taosArrayPush(ctx->rows, &row)) { qError("taosArrayPush row to explain res rows failed"); taosMemoryFree(row.buf); QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -198,7 +224,7 @@ int32_t qExplainResAppendRow(SArray *pRows, char *tbuf, int32_t len, int32_t lev } -int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SArray *pRows, char *tbuf, int32_t level) { +int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx, char *tbuf, int32_t level) { int32_t tlen = 0; SPhysiNode* pNode = pResNode->pNode; if (NULL == pNode) { @@ -209,174 +235,198 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SArray *pRows, char switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname, pTagScanNode->pScanCols->length); + EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname, pTagScanNode->pScanCols->length); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QUERY_EXPLAIN_APPEND(EXPLAIN_LOOPS_FORMAT, pTagScanNode->count); + EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTagScanNode->count); if (pTagScanNode->reverse) { - QUERY_EXPLAIN_APPEND(EXPLAIN_REVERSE_FORMAT, pTagScanNode->reverse); + EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTagScanNode->reverse); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - QUERY_EXPLAIN_NEWLINE(EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); break; } case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname, pTblScanNode->scan.pScanCols->length); + EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname, pTblScanNode->scan.pScanCols->length); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QUERY_EXPLAIN_APPEND(EXPLAIN_LOOPS_FORMAT, pTblScanNode->scan.count); + EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTblScanNode->scan.count); if (pTblScanNode->scan.reverse) { - QUERY_EXPLAIN_APPEND(EXPLAIN_REVERSE_FORMAT, pTblScanNode->scan.reverse); + EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTblScanNode->scan.reverse); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - QUERY_EXPLAIN_NEWLINE(EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - QUERY_EXPLAIN_NEWLINE(EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pTblScanNode->pScanConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->pScanConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname, pSTblScanNode->scan.pScanCols->length); + EXPLAIN_ROW_NEW(level, EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname, pSTblScanNode->scan.pScanCols->length); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QUERY_EXPLAIN_APPEND(EXPLAIN_LOOPS_FORMAT, pSTblScanNode->scan.count); + EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pSTblScanNode->scan.count); if (pSTblScanNode->scan.reverse) { - QUERY_EXPLAIN_APPEND(EXPLAIN_REVERSE_FORMAT, pSTblScanNode->scan.reverse); + EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pSTblScanNode->scan.reverse); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - QUERY_EXPLAIN_NEWLINE(EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); break; } case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_PROJECTION_FORMAT, pPrjNode->pProjections->length, pPrjNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_PROJECTION_FORMAT, pPrjNode->pProjections->length, pPrjNode->node.pOutputDataBlockDesc->resultRowSize); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (pPrjNode->node.pConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType), pJoinNode->pTargets->length, pJoinNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType), pJoinNode->pTargets->length, pJoinNode->node.pOutputDataBlockDesc->resultRowSize); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (pJoinNode->node.pConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } - QUERY_EXPLAIN_NEWLINE(EXPLAIN_ON_CONDITIONS_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); break; } case QUERY_NODE_PHYSICAL_PLAN_AGG:{ SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length, pAggNode->pGroupKeys->length, pAggNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length, pAggNode->pGroupKeys->length, pAggNode->node.pOutputDataBlockDesc->resultRowSize); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (pAggNode->node.pConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_EXCHANGE_FORMAT, pExchNode->pSrcEndPoints->length, pExchNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, pExchNode->pSrcEndPoints->length, pExchNode->node.pOutputDataBlockDesc->resultRowSize); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (pExchNode->node.pConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } case QUERY_NODE_PHYSICAL_PLAN_SORT:{ SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_SORT_FORMAT, pSortNode->pSortKeys->length, pSortNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT, pSortNode->pSortKeys->length, pSortNode->node.pOutputDataBlockDesc->resultRowSize); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (pSortNode->node.pConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, + EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, pIntNode->interval, pIntNode->intervalUnit, pIntNode->offset, pIntNode->intervalUnit, pIntNode->sliding, pIntNode->slidingUnit, pIntNode->window.node.pOutputDataBlockDesc->resultRowSize); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (pIntNode->pFill) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILL_FORMAT, qFillModeString(pIntNode->pFill->mode)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_FORMAT, qFillModeString(pIntNode->pFill->mode)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } if (pIntNode->window.node.pConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ SSessionWinodwPhysiNode *pIntNode = (SSessionWinodwPhysiNode *)pNode; - QUERY_EXPLAIN_NEWLINE(EXPLAIN_SESSION_FORMAT, pIntNode->gap, pIntNode->window.pFuncs->length, pIntNode->window.node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT, pIntNode->gap, pIntNode->window.pFuncs->length, pIntNode->window.node.pOutputDataBlockDesc->resultRowSize); if (pResNode->pExecInfo) { - QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, tlen)); + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (pIntNode->window.node.pConditions) { - QUERY_EXPLAIN_NEWLINE(EXPLAIN_FILTER_FORMAT); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); - QRY_ERR_RET(qExplainResAppendRow(pRows, tbuf, tlen, level + 1)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } @@ -389,33 +439,24 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SArray *pRows, char } -int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SArray *pRsp, char *tbuf, int32_t level) { +int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainRowCtx *ctx, char *tbuf, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } int32_t code = 0; - QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, pRsp, tbuf, level)); + QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, ctx, tbuf, level)); SNode* pNode = NULL; FOREACH(pNode, pResNode->pChildren) { - QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, pRsp, tbuf, level + 1)); + QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, tbuf, level + 1)); } return TSDB_CODE_SUCCESS; } -int32_t qExplainRowsToRsp(SArray *rows, SRetrieveTableRsp **pRsp) { - int32_t rspSize = sizeof(SRetrieveTableRsp) + ; - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); - if (NULL == rsp) { - qError("malloc SRetrieveTableRsp failed"); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } -} - -int32_t qMakeTaskExplainResRows(SExplainResNode *pResNode, SRetrieveTableRsp **pRsp) { +int32_t qGenerateExplainResRowsFromNode(SExplainResNode *pResNode, SExplainRowCtx *ctx, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); QRY_RET(TSDB_CODE_QRY_APP_ERROR); @@ -427,24 +468,92 @@ int32_t qMakeTaskExplainResRows(SExplainResNode *pResNode, SRetrieveTableRsp **p qError("malloc size %d failed", QUERY_EXPLAIN_MAX_RES_LEN); QRY_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - - SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); - if (NULL == rows) { - qError("taosArrayInit SQueryExplainRowInfo failed"); - QRY_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - QRY_ERR_JRET(qExplainResNodeToRows(pResNode, rows, tbuf, 0)); - - QRY_ERR_JRET(qExplainRowsToRsp(rows, pRsp)); + QRY_ERR_JRET(qExplainResNodeToRows(pResNode, ctx, tbuf, level)); _return: taosMemoryFree(tbuf); - taosArrayDestroy(rows); QRY_RET(code); } +int32_t qAppendTaskExplainResRows(void **pRowCtx, struct SSubplan *plan, void *pExecTree, int32_t level) { + SExplainResNode *node = NULL; + int32_t code = 0; + + QRY_ERR_RET(qGenerateExplainResNodeTree(plan, pExecTree, &node)); + + if (NULL == *pRowCtx) { + *pRowCtx = taosMemoryCalloc(1, sizeof(SExplainRowCtx)); + if (NULL == *pRowCtx) { + qError("calloc SExplainRowCtx failed"); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); + if (NULL == rows) { + qError("taosArrayInit SQueryExplainRowInfo failed"); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + (*pRowCtx)->rows = rows; + } + + QRY_ERR_JRET(qGenerateExplainResRowsFromNode(node, *pRowCtx, level)); + +_return: + + qFreeExplainResTree(node); + + if (code) { + taosArrayDestroy((*pRowCtx)->rows); + taosMemoryFree(*pRowCtx); + *pRowCtx = NULL; + } + + QRY_RET(code); +} + + +int32_t qGetExplainRspFromRowCtx(void *ctx, SRetrieveTableRsp **pRsp) { + SExplainRowCtx *pCtx = (SExplainRowCtx *)ctx; + int32_t rowNum = taosArrayGetSize(pCtx->rows); + if (rowNum <= 0) { + qError("empty explain res rows"); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + int32_t colNum = 1; + int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->totalSize; + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); + if (NULL == rsp) { + qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + rsp->completed = 1; + rsp->numOfRows = rowNum; + + *(int32_t *)rsp->data = pCtx->totalSize; + + int32_t *offset = (int32_t *)((char *)rsp->data + sizeof(int32_t)); + char *data = (char *)(offset + rowNum); + int32_t tOffset = 0; + + for (int32_t i = 0; i < rowNum; ++i) { + SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); + *offset = tOffset; + tOffset += row->len; + + memcpy(data, row->buf, row->len); + + ++offset; + data += row->len; + } + + return TSDB_CODE_SUCCESS; +} + + -#pragma GCC diagnostic pop diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 834a722bd8..3e04e7b30a 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -36,8 +36,6 @@ extern "C" { #define FILTER_DUMMY_EMPTY_OPTR 127 -#define MAX_NUM_STR_SIZE 40 - #define FILTER_RM_UNIT_MIN_ROWS 100 enum { diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 518da6e2b8..579a995056 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -38,6 +38,11 @@ enum { SCH_WRITE, }; +typedef struct SSchExplainGroup { + int32_t nodeNum; + SSubplan *plan; +} SSchExplainGroup; + typedef struct SSchTrans { void *transInst; void *transHandle; @@ -142,7 +147,7 @@ typedef struct SSchTask { } SSchTask; typedef struct SSchJobAttr { - bool needFetch; + bool analyzeExplain; bool syncSchedule; bool queryJob; bool needFlowCtrl; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index fdb0b6498c..376febcdf3 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -476,6 +476,90 @@ _return: SCH_RET(code); } +int32_t schValidateAndBuildJobExplain(SQueryPlan *pDag, SSchJob *pJob) { + int32_t code = 0; + pJob->queryId = pDag->queryId; + + if (pDag->numOfSubplans <= 0) { + SCH_JOB_ELOG("invalid subplan num:%d", pDag->numOfSubplans); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans); + if (levelNum <= 0) { + SCH_JOB_ELOG("invalid level num:%d", levelNum); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SHashObj *groupHash = taosHashInit(SCHEDULE_DEFAULT_MAX_TASK_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == groupHash) { + SCH_JOB_ELOG("groupHash %d failed", SCHEDULE_DEFAULT_MAX_TASK_NUM); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pJob->subPlans = pDag->pSubplans; + + SNodeListNode *plans = NULL; + int32_t taskNum = 0; + SSchExplainGroup *pGroup = NULL; + void *rowCtx = NULL; + + for (int32_t i = 0; i < levelNum; ++i) { + plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); + if (NULL == plans) { + SCH_JOB_ELOG("empty level plan, level:%d", i); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + taskNum = (int32_t)LIST_LENGTH(plans->pNodeList); + if (taskNum <= 0) { + SCH_JOB_ELOG("invalid level plan number:%d, level:%d", taskNum, i); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + for (int32_t n = 0; n < taskNum; ++n) { + SSubplan *plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); + pGroup = taosHashGet(groupHash, &plan->id.groupId, sizeof(plan->id.groupId)); + if (pGroup) { + ++pGroup->nodeNum; + continue; + } + + SSchExplainGroup group = {.nodeNum = 1, .plan = plan}; + if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { + SCH_TASK_ELOG("taosHashPut to explainGroupHash failed, taskIdx:%d", n); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + void *pIter = taosHashIterate(groupHash, NULL); + while (pIter) { + pGroup = (SSchExplainGroup *)pIter; + + SCH_ERR_JRET(qAppendTaskExplainResRows(&rowCtx, pGroup->plan, NULL, i)); + + pIter = taosHashIterate(groupHash, pIter); + } + + taosHashClear(groupHash); + + SCH_JOB_DLOG("level initialized, taskNum:%d", taskNum); + } + + SRetrieveTableRsp *pRsp = NULL; + SCH_ERR_JRET(qGetExplainRspFromRowCtx(rowCtx, &pRsp)); + + pJob->resData = pRsp; + +_return: + + taosHashCleanup(groupHash); + qFreeExplainRowCtx(rowCtx); + + SCH_RET(code); +} + + int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { if (NULL != pTask->candidateAddrs) { return TSDB_CODE_SUCCESS; @@ -2093,6 +2177,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + pJob->attr.analyzeExplain = (EXPLAIN_MODE_ANALYZE == pDag->explainInfo.mode); pJob->attr.syncSchedule = syncSchedule; pJob->transport = transport; pJob->sql = sql; @@ -2163,6 +2248,51 @@ _return: SCH_RET(code); } +int32_t schStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, + bool syncSchedule) { + qDebug("QID:0x%" PRIx64 " job started", pDag->queryId); + + int32_t code = 0; + SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); + if (NULL == pJob) { + qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pJob->sql = sql; + pJob->attr.queryJob = true; + + SCH_ERR_JRET(schValidateAndBuildJobExplain(pDag, pJob)); + + int64_t refId = taosAddRef(schMgmt.jobRef, pJob); + if (refId < 0) { + SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); + SCH_ERR_JRET(terrno); + } + + if (NULL == schAcquireJob(refId)) { + SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId); + SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + pJob->refId = refId; + + SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId); + + pJob->status = JOB_TASK_STATUS_PARTIAL_SUCCEED; + *job = pJob->refId; + SCH_JOB_DLOG("job exec done, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); + + schReleaseJob(pJob->refId); + + return TSDB_CODE_SUCCESS; + +_return: + + schFreeJobImpl(pJob); + SCH_RET(code); +} + int32_t schedulerInit(SSchedulerCfg *cfg) { if (schMgmt.jobRef) { qError("scheduler already initialized"); @@ -2211,7 +2341,11 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true)); + if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { + SCH_ERR_RET(schStaticExplain(transport, nodeList, pDag, pJob, sql, true)); + } else { + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true)); + } SSchJob *job = schAcquireJob(*pJob); From 100d3230a01fb3587347904756361afb3dc0f7e4 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 30 Mar 2022 17:21:47 +0800 Subject: [PATCH 07/38] feature/qnode --- include/libs/qcom/query.h | 2 +- source/libs/qcom/src/queryExplain.c | 17 ++++++++++------- source/libs/scheduler/src/scheduler.c | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 2a9dd28819..dd2dccc02d 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -174,7 +174,7 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); char *jobTaskStatusStr(int32_t status); -int32_t qAppendTaskExplainResRows(void **pRowCtx, struct SSubplan *plan, void *pExecTree, int32_t level); +int32_t qAppendTaskExplainResRows(void **pRowCtx, void *plan, void *pExecTree, int32_t level); int32_t qGetExplainRspFromRowCtx(void *ctx, SRetrieveTableRsp **pRsp); void qFreeExplainRowCtx(void *ctx); diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/qcom/src/queryExplain.c index d352a29e3c..e1bdf96331 100644 --- a/source/libs/qcom/src/queryExplain.c +++ b/source/libs/qcom/src/queryExplain.c @@ -478,11 +478,13 @@ _return: QRY_RET(code); } -int32_t qAppendTaskExplainResRows(void **pRowCtx, struct SSubplan *plan, void *pExecTree, int32_t level) { +int32_t qAppendTaskExplainResRows(void **pRowCtx, void *plan, void *pExecTree, int32_t level) { SExplainResNode *node = NULL; int32_t code = 0; + struct SSubplan *pPlan = (struct SSubplan *)plan; + SExplainRowCtx *ctx = (SExplainRowCtx *)*pRowCtx; - QRY_ERR_RET(qGenerateExplainResNodeTree(plan, pExecTree, &node)); + QRY_ERR_RET(qGenerateExplainResNodeTree(pPlan, pExecTree, &node)); if (NULL == *pRowCtx) { *pRowCtx = taosMemoryCalloc(1, sizeof(SExplainRowCtx)); @@ -491,24 +493,25 @@ int32_t qAppendTaskExplainResRows(void **pRowCtx, struct SSubplan *plan, void *p QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + ctx = (SExplainRowCtx *)*pRowCtx; + SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); if (NULL == rows) { qError("taosArrayInit SQueryExplainRowInfo failed"); QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - (*pRowCtx)->rows = rows; + ctx->rows = rows; } - - QRY_ERR_JRET(qGenerateExplainResRowsFromNode(node, *pRowCtx, level)); + + QRY_ERR_JRET(qGenerateExplainResRowsFromNode(node, ctx, level)); _return: qFreeExplainResTree(node); if (code) { - taosArrayDestroy((*pRowCtx)->rows); - taosMemoryFree(*pRowCtx); + qFreeExplainRowCtx(*pRowCtx); *pRowCtx = NULL; } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 376febcdf3..d11dafb19b 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -527,7 +527,7 @@ int32_t schValidateAndBuildJobExplain(SQueryPlan *pDag, SSchJob *pJob) { SSchExplainGroup group = {.nodeNum = 1, .plan = plan}; if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { - SCH_TASK_ELOG("taosHashPut to explainGroupHash failed, taskIdx:%d", n); + SCH_JOB_ELOG("taosHashPut to explainGroupHash failed, taskIdx:%d", n); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } } From 08bd344d2ff0bfdca8fa18233076a16a9920d976 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 30 Mar 2022 17:59:05 +0800 Subject: [PATCH 08/38] feature/qnode --- source/libs/qcom/src/queryExplain.c | 22 +++++++++++----------- source/libs/scheduler/inc/schedulerInt.h | 8 ++++---- source/libs/scheduler/src/scheduler.c | 12 ++++++++---- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/qcom/src/queryExplain.c index e1bdf96331..91ca7f54b0 100644 --- a/source/libs/qcom/src/queryExplain.c +++ b/source/libs/qcom/src/queryExplain.c @@ -31,7 +31,7 @@ void qFreeExplainResTree(SExplainResNode *res) { FOREACH(node, res->pChildren) { qFreeExplainResTree((SExplainResNode *)node); } - nodesDestroyList(res->pChildren); + nodesClearList(res->pChildren); taosMemoryFreeClear(res); } @@ -214,7 +214,7 @@ int32_t qExplainResAppendRow(SExplainRowCtx *ctx, char *tbuf, int32_t len, int32 row.len = len; ctx->totalSize += len; - if (taosArrayPush(ctx->rows, &row)) { + if (NULL == taosArrayPush(ctx->rows, &row)) { qError("taosArrayPush row to explain res rows failed"); taosMemoryFree(row.buf); QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -273,9 +273,9 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - if (pTblScanNode->pScanConditions) { + if (pTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->pScanConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -301,7 +301,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_PROJECTION_FORMAT, pPrjNode->pProjections->length, pPrjNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_PROJECTION_FORMAT, pPrjNode->pProjections->length, pPrjNode->node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -318,7 +318,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType), pJoinNode->pTargets->length, pJoinNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType), pJoinNode->pTargets->length, pJoinNode->node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -340,7 +340,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } case QUERY_NODE_PHYSICAL_PLAN_AGG:{ SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length, pAggNode->pGroupKeys->length, pAggNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length, pAggNode->pGroupKeys->length, pAggNode->node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -357,7 +357,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, pExchNode->pSrcEndPoints->length, pExchNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, pExchNode->pSrcEndPoints->length, pExchNode->node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -374,7 +374,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } case QUERY_NODE_PHYSICAL_PLAN_SORT:{ SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT, pSortNode->pSortKeys->length, pSortNode->node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT, pSortNode->pSortKeys->length, pSortNode->node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -392,7 +392,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, - pIntNode->interval, pIntNode->intervalUnit, pIntNode->offset, pIntNode->intervalUnit, pIntNode->sliding, pIntNode->slidingUnit, pIntNode->window.node.pOutputDataBlockDesc->resultRowSize); + pIntNode->interval, pIntNode->intervalUnit, pIntNode->offset, pIntNode->intervalUnit, pIntNode->sliding, pIntNode->slidingUnit, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -415,7 +415,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ SSessionWinodwPhysiNode *pIntNode = (SSessionWinodwPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT, pIntNode->gap, pIntNode->window.pFuncs->length, pIntNode->window.node.pOutputDataBlockDesc->resultRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT, pIntNode->gap, pIntNode->window.pFuncs->length, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 579a995056..1b2c7d54e2 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -147,10 +147,10 @@ typedef struct SSchTask { } SSchTask; typedef struct SSchJobAttr { - bool analyzeExplain; - bool syncSchedule; - bool queryJob; - bool needFlowCtrl; + EExplainMode explainMode; + bool syncSchedule; + bool queryJob; + bool needFlowCtrl; } SSchJobAttr; typedef struct SSchJob { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index d11dafb19b..a26b5f32f3 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -2177,7 +2177,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - pJob->attr.analyzeExplain = (EXPLAIN_MODE_ANALYZE == pDag->explainInfo.mode); + pJob->attr.explainMode = pDag->explainInfo.mode; pJob->attr.syncSchedule = syncSchedule; pJob->transport = transport; pJob->sql = sql; @@ -2528,11 +2528,15 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { SCH_JOB_DLOG("job already succeed, status:%s", jobTaskStatusStr(status)); goto _return; } else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_ERR_JRET(schFetchFromRemote(pJob)); + if (!pJob->attr.explainMode == EXPLAIN_MODE_STATIC) { + SCH_ERR_JRET(schFetchFromRemote(pJob)); + tsem_wait(&pJob->rspSem); + } + } else { + SCH_JOB_ELOG("job status error for fetch, status:%s", jobTaskStatusStr(status)); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } - tsem_wait(&pJob->rspSem); - status = SCH_GET_JOB_STATUS(pJob); if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) { From 35f8a337af018d3afcfdebe9258f14349ac3fc77 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 30 Mar 2022 19:37:21 +0800 Subject: [PATCH 09/38] feature/qnode --- include/util/tdef.h | 1 + source/libs/parser/src/parTranslater.c | 1 + source/libs/qcom/inc/queryInt.h | 12 ++++------ source/libs/qcom/src/queryExplain.c | 33 ++++++++++++++------------ source/libs/scheduler/src/scheduler.c | 3 ++- 5 files changed, 27 insertions(+), 23 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 4b53cde6f5..38c0000ccc 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -389,6 +389,7 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_EXPLAIN_RATIO 0.001 #define TSDB_EXPLAIN_RESULT_ROW_SIZE 1024 +#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN" #define TSDB_MAX_JOIN_TABLE_NUM 10 #define TSDB_MAX_UNION_CLAUSE 5 diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d453193d0f..ee22529f1d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1944,6 +1944,7 @@ int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** p } (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE; + strcpy((*pSchema)[0].name, TSDB_EXPLAIN_RESULT_COLUMN_NAME); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h index 6fc15aacfc..510ba92a86 100644 --- a/source/libs/qcom/inc/queryInt.h +++ b/source/libs/qcom/inc/queryInt.h @@ -22,8 +22,6 @@ extern "C" { #include "nodes.h" #include "plannodes.h" -#define QUERY_EXPLAIN_MAX_RES_LEN 1024 - //newline area #define EXPLAIN_TAG_SCAN_FORMAT "Tag scan on %s columns=%d" #define EXPLAIN_TBL_SCAN_FORMAT "Table scan on %s columns=%d" @@ -42,8 +40,8 @@ extern "C" { #define EXPLAIN_TIMERANGE_FORMAT "Time range: [%" PRId64 ", %" PRId64 "]" //append area -#define EXPLAIN_LOOPS_FORMAT "loops %d" -#define EXPLAIN_REVERSE_FORMAT "reverse %d" +#define EXPLAIN_LOOPS_FORMAT "loops=%d" +#define EXPLAIN_REVERSE_FORMAT "reverse=%d" typedef struct SExplainResNode { @@ -68,11 +66,11 @@ typedef struct SExplainRowCtx { #define EXPLAIN_ROW_NEW(level, ...) \ do { \ - tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, QUERY_EXPLAIN_MAX_RES_LEN, "%*s", level, ""); \ - tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, __VA_ARGS__); \ + tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s", (level) * 2, ""); \ + tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \ } while (0) -#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, QUERY_EXPLAIN_MAX_RES_LEN - tlen, __VA_ARGS__) +#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__) #define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; } while (0) #define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/qcom/src/queryExplain.c index 91ca7f54b0..b2b061cee0 100644 --- a/source/libs/qcom/src/queryExplain.c +++ b/source/libs/qcom/src/queryExplain.c @@ -204,12 +204,13 @@ int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t *len) { int32_t qExplainResAppendRow(SExplainRowCtx *ctx, char *tbuf, int32_t len, int32_t level) { SQueryExplainRowInfo row = {0}; - row.buf = strdup(tbuf); + row.buf = taosMemoryMalloc(len); if (NULL == row.buf) { - qError("strdup %s failed", tbuf); + qError("taosMemoryMalloc %d failed", len); QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + memcpy(row.buf, tbuf, len); row.level = level; row.len = len; ctx->totalSize += len; @@ -275,7 +276,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -310,7 +311,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pPrjNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -327,13 +328,13 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pJoinNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); break; @@ -349,7 +350,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pAggNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -366,7 +367,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pExchNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -383,7 +384,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pSortNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -407,7 +408,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pIntNode->window.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -424,7 +425,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx if (pIntNode->window.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, QUERY_EXPLAIN_MAX_RES_LEN, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -463,9 +464,9 @@ int32_t qGenerateExplainResRowsFromNode(SExplainResNode *pResNode, SExplainRowCt } int32_t code = 0; - char *tbuf = taosMemoryMalloc(QUERY_EXPLAIN_MAX_RES_LEN); + char *tbuf = taosMemoryMalloc(TSDB_EXPLAIN_RESULT_ROW_SIZE); if (NULL == tbuf) { - qError("malloc size %d failed", QUERY_EXPLAIN_MAX_RES_LEN); + qError("malloc size %d failed", TSDB_EXPLAIN_RESULT_ROW_SIZE); QRY_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -536,9 +537,9 @@ int32_t qGetExplainRspFromRowCtx(void *ctx, SRetrieveTableRsp **pRsp) { } rsp->completed = 1; - rsp->numOfRows = rowNum; + rsp->numOfRows = htonl(rowNum); - *(int32_t *)rsp->data = pCtx->totalSize; + *(int32_t *)rsp->data = htonl(pCtx->totalSize); int32_t *offset = (int32_t *)((char *)rsp->data + sizeof(int32_t)); char *data = (char *)(offset + rowNum); @@ -555,6 +556,8 @@ int32_t qGetExplainRspFromRowCtx(void *ctx, SRetrieveTableRsp **pRsp) { data += row->len; } + *pRsp = rsp; + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index a26b5f32f3..4a84536d00 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -2261,6 +2261,7 @@ int32_t schStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDag, i pJob->sql = sql; pJob->attr.queryJob = true; + pJob->attr.explainMode = pDag->explainInfo.mode; SCH_ERR_JRET(schValidateAndBuildJobExplain(pDag, pJob)); @@ -2528,7 +2529,7 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { SCH_JOB_DLOG("job already succeed, status:%s", jobTaskStatusStr(status)); goto _return; } else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { - if (!pJob->attr.explainMode == EXPLAIN_MODE_STATIC) { + if (!(pJob->attr.explainMode == EXPLAIN_MODE_STATIC)) { SCH_ERR_JRET(schFetchFromRemote(pJob)); tsem_wait(&pJob->rspSem); } From a1d40669dced293cb289bcfd608b2cb3cd20c52c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 31 Mar 2022 11:21:53 +0800 Subject: [PATCH 10/38] add fuzzy search --- source/libs/index/src/index.c | 2 +- source/libs/index/src/indexCache.c | 4 ++-- source/libs/index/src/indexFstCommon.c | 2 +- source/libs/index/src/{indexSparse.c => indexFstSparse.c} | 0 4 files changed, 4 insertions(+), 4 deletions(-) rename source/libs/index/src/{indexSparse.c => indexFstSparse.c} (100%) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index d3ca3a1acf..7d52abcd1b 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -27,7 +27,7 @@ #endif #define INDEX_NUM_OF_THREADS 4 -#define INDEX_QUEUE_SIZE 200 +#define INDEX_QUEUE_SIZE 200 void* indexQhandle = NULL; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index ca26cf38e5..df3c0b6e7b 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -21,8 +21,8 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later -#define MEM_TERM_LIMIT 10 * 10000 -#define MEM_THRESHOLD 1024 * 1024 +#define MEM_TERM_LIMIT 10 * 10000 +#define MEM_THRESHOLD 1024 * 1024 #define MEM_ESTIMATE_RADIO 1.5 static void indexMemRef(MemTable* tbl); diff --git a/source/libs/index/src/indexFstCommon.c b/source/libs/index/src/indexFstCommon.c index e2544c7ac3..902e68ce09 100644 --- a/source/libs/index/src/indexFstCommon.c +++ b/source/libs/index/src/indexFstCommon.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 TAOS Data, Inc. +YAML:9:25: error: unknown key 'AlignConsecutiveMacros' * Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 diff --git a/source/libs/index/src/indexSparse.c b/source/libs/index/src/indexFstSparse.c similarity index 100% rename from source/libs/index/src/indexSparse.c rename to source/libs/index/src/indexFstSparse.c From 9da56a412ab99239ae60fa02c0d85c64a5d0f55d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 31 Mar 2022 13:56:09 +0800 Subject: [PATCH 11/38] add fuzzy search --- source/libs/index/inc/indexFstRegex.h | 2 +- source/libs/index/src/indexFstRegex.c | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/index/inc/indexFstRegex.h b/source/libs/index/inc/indexFstRegex.h index 50b9cae7ff..a58906c242 100644 --- a/source/libs/index/inc/indexFstRegex.h +++ b/source/libs/index/inc/indexFstRegex.h @@ -63,7 +63,7 @@ typedef struct { FstRegex *regexCreate(const char *str); -void regexSetup(FstRegex *regex, uint32_t size, const char *str); +// void regexSetup(FstRegex *regex, uint32_t size, const char *str); // uint32_t regexStart() diff --git a/source/libs/index/src/indexFstRegex.c b/source/libs/index/src/indexFstRegex.c index ec41a7f58e..2b8c8b6708 100644 --- a/source/libs/index/src/indexFstRegex.c +++ b/source/libs/index/src/indexFstRegex.c @@ -14,6 +14,7 @@ */ #include "indexFstRegex.h" +#include "indexFstDfa.h" #include "indexFstSparse.h" FstRegex *regexCreate(const char *str) { @@ -26,9 +27,11 @@ FstRegex *regexCreate(const char *str) { memcpy(orig, str, sz); regex->orig = orig; -} -void regexSetup(FstRegex *regex, uint32_t size, const char *str) { - // return - // return; + // construct insts based on str + SArray *insts = NULL; + + FstDfaBuilder *builder = dfaBuilderCreate(insts); + regex->dfa = dfaBuilderBuild(builder); + return regex; } From aae1ba31ac617b8639f9d51097f0302fe49ebbc6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 31 Mar 2022 14:45:26 +0800 Subject: [PATCH 12/38] add fuzzy search --- source/libs/index/inc/indexFstRegex.h | 4 ++++ source/libs/index/src/indexFstRegex.c | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/source/libs/index/inc/indexFstRegex.h b/source/libs/index/inc/indexFstRegex.h index a58906c242..a80e768feb 100644 --- a/source/libs/index/inc/indexFstRegex.h +++ b/source/libs/index/inc/indexFstRegex.h @@ -63,6 +63,10 @@ typedef struct { FstRegex *regexCreate(const char *str); +uint32_t regexAutomStart(FstRegex *regex); +bool regexAutomIsMatch(FstRegex *regex, uint32_t state); +bool regexAutomCanMatch(FstRegex *regex, uint32_t state, bool null); +bool regexAutomAccept(FstRegex *regex, uint32_t state, uint8_t byte, uint32_t *result); // void regexSetup(FstRegex *regex, uint32_t size, const char *str); // uint32_t regexStart() diff --git a/source/libs/index/src/indexFstRegex.c b/source/libs/index/src/indexFstRegex.c index 2b8c8b6708..33eeae802e 100644 --- a/source/libs/index/src/indexFstRegex.c +++ b/source/libs/index/src/indexFstRegex.c @@ -35,3 +35,27 @@ FstRegex *regexCreate(const char *str) { regex->dfa = dfaBuilderBuild(builder); return regex; } + +uint32_t regexAutomStart(FstRegex *regex) { + ///// no nothing + return 0; +} +bool regexAutomIsMatch(FstRegex *regex, uint32_t state) { + if (regex->dfa != NULL && dfaIsMatch(regex->dfa, state)) { + return true; + } else { + return false; + } +} + +bool regexAutomCanMatch(FstRegex *regex, uint32_t state, bool null) { + // make frame happy + return null; +} + +bool regexAutomAccept(FstRegex *regex, uint32_t state, uint8_t byte, uint32_t *result) { + if (regex->dfa == NULL) { + return false; + } + return dfaAccept(regex->dfa, state, byte, result); +} From 3e5da8bedbd39db852ae7935d04d901602d11463 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 31 Mar 2022 15:52:54 +0800 Subject: [PATCH 13/38] fix --- example/src/tmq.c | 52 +++++++++++-- example/src/tstream.c | 4 +- include/common/tcommon.h | 29 ++++--- include/common/trow.h | 78 +++++++++---------- include/libs/wal/wal.h | 34 ++++---- source/client/src/tmq.c | 15 +++- source/dnode/mnode/impl/inc/mndDef.h | 3 + source/dnode/vnode/src/inc/vnd.h | 2 +- source/dnode/vnode/src/meta/metaTbUid.c | 2 +- source/dnode/vnode/src/tq/tq.c | 91 +++++++++++++--------- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/dnode/vnode/src/vnd/vnodeQuery.c | 2 +- source/dnode/vnode/src/vnd/vnodeWrite.c | 1 + source/libs/executor/src/executorimpl.c | 5 +- source/libs/planner/src/planPhysiCreater.c | 5 ++ source/libs/stream/src/tstream.c | 2 +- source/libs/wal/src/walRead.c | 23 +++++- source/util/src/tjson.c | 13 ++-- tests/script/tsim/tmq/basic1.sim | 2 +- tests/test/c/tmqSim.c | 2 +- 20 files changed, 240 insertions(+), 127 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index 8757104ad9..efb4d1830e 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -28,7 +28,7 @@ int32_t init_env() { return -1; } - TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); if (taos_errno(pRes) != 0) { printf("error in create db, reason:%s\n", taos_errstr(pRes)); return -1; @@ -42,25 +42,33 @@ int32_t init_env() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)"); + pRes = + taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)"); if (taos_errno(pRes) != 0) { printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); return -1; } taos_free_result(pRes); - pRes = taos_query(pConn, "create table if not exists tu1 using st1 tags(1)"); + pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000)"); if (taos_errno(pRes) != 0) { printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); return -1; } taos_free_result(pRes); - pRes = taos_query(pConn, "create table if not exists tu2 using st1 tags(2)"); + pRes = taos_query(pConn, "create table if not exists ct1 using st1 tags(2000)"); if (taos_errno(pRes) != 0) { printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes)); return -1; } + + pRes = taos_query(pConn, "create table if not exists ct3 using st1 tags(3000)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); return 0; } @@ -82,12 +90,40 @@ int32_t create_topic() { /*const char* sql = "select * from tu1";*/ /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/ - pRes = taos_query(pConn, "create topic test_stb_topic_1 as select * from tu1"); + pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1 from ct1"); if (taos_errno(pRes) != 0) { - printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes)); + printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; } taos_free_result(pRes); + +#if 0 + pRes = taos_query(pConn, "insert into tu1 values(now, 1, 1.0, 'bi1')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + pRes = taos_query(pConn, "insert into tu1 values(now+1d, 1, 1.0, 'bi1')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + pRes = taos_query(pConn, "insert into tu2 values(now, 2, 2.0, 'bi2')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + pRes = taos_query(pConn, "insert into tu2 values(now+1d, 2, 2.0, 'bi2')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); +#endif + taos_close(pConn); return 0; } @@ -115,7 +151,7 @@ tmq_t* build_consumer() { tmq_list_t* build_topic_list() { tmq_list_t* topic_list = tmq_list_new(); - tmq_list_append(topic_list, "test_stb_topic_1"); + tmq_list_append(topic_list, "topic_ctb_column"); return topic_list; } @@ -215,8 +251,8 @@ int main(int argc, char* argv[]) { if (argc > 1) { printf("env init\n"); code = init_env(); + create_topic(); } - create_topic(); tmq_t* tmq = build_consumer(); tmq_list_t* topic_list = build_topic_list(); /*perf_loop(tmq, topic_list);*/ diff --git a/example/src/tstream.c b/example/src/tstream.c index 40d8ff9b0b..8ffa932bd2 100644 --- a/example/src/tstream.c +++ b/example/src/tstream.c @@ -20,7 +20,7 @@ #include "taos.h" int32_t init_env() { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010); + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { return -1; } @@ -65,7 +65,7 @@ int32_t init_env() { int32_t create_stream() { printf("create stream\n"); TAOS_RES* pRes; - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010); + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { return -1; } diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 51eabb7d61..b57de3660b 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -37,6 +37,14 @@ enum { TMQ_MSG_TYPE__EP_RSP, }; +enum { + STREAM_TRIGGER__AT_ONCE = 1, + STREAM_TRIGGER__WINDOW_CLOSE, + STREAM_TRIGGER__BY_COUNT, + STREAM_TRIGGER__BY_BATCH_COUNT, + STREAM_TRIGGER__BY_EVENT_TIME, +}; + typedef struct { uint32_t numOfTables; SArray* pGroupList; @@ -54,13 +62,16 @@ typedef struct SColumnDataAgg { } SColumnDataAgg; typedef struct SDataBlockInfo { - STimeWindow window; - int32_t rows; - int32_t rowSize; - int16_t numOfCols; - int16_t hasVarCol; - union {int64_t uid; int64_t blockId;}; - int64_t groupId; // no need to serialize + STimeWindow window; + int32_t rows; + int32_t rowSize; + int16_t numOfCols; + int16_t hasVarCol; + union { + int64_t uid; + int64_t blockId; + }; + int64_t groupId; // no need to serialize } SDataBlockInfo; typedef struct SSDataBlock { @@ -93,7 +104,7 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock); int32_t tEncodeDataBlocks(void** buf, const SArray* blocks); void* tDecodeDataBlocks(const void* buf, SArray** blocks); -void colDataDestroy(SColumnInfoData* pColData) ; +void colDataDestroy(SColumnInfoData* pColData); static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) { // WARNING: do not use info.numOfCols, @@ -239,7 +250,7 @@ typedef struct SSessionWindow { SColumn col; } SSessionWindow; -#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) diff --git a/include/common/trow.h b/include/common/trow.h index df28bc9962..fb6d3ece44 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -47,21 +47,21 @@ extern "C" { #define TD_VTYPE_NONE 0x0U // none or unknown/undefined #define TD_VTYPE_NULL 0x01U // null val #define TD_VTYPE_NORM 0x02U // normal val: not none, not null -#define TD_VTYPE_MAX 0x03U // +#define TD_VTYPE_MAX 0x03U // #define TD_VTYPE_NONE_BYTE 0x0U #define TD_VTYPE_NULL_BYTE 0x55U #define TD_VTYPE_NORM_BYTE 0xAAU -#define TD_ROWS_ALL_NORM 0x01U +#define TD_ROWS_ALL_NORM 0x01U #define TD_ROWS_NULL_NORM 0x0U -#define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM) // all rows of SDataCol/SBlockCol is NORM +#define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM) // all rows of SDataCol/SBlockCol is NORM #define TD_SET_COL_ROWS_BTIMAP(c, v) ((c)->bitmap = (v)) -#define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM) -#define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM) +#define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM) +#define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM) -#define KvConvertRatio (0.9f) +#define KvConvertRatio (0.9f) #define isSelectKVRow(klen, tlen) ((klen) < ((tlen)*KvConvertRatio)) #ifdef TD_SUPPORT_BITMAP @@ -98,7 +98,7 @@ typedef void *SRow; typedef struct { TDRowValT valType; - void * val; + void *val; } SCellVal; typedef struct { @@ -158,43 +158,43 @@ typedef struct { int16_t nBitmaps; int16_t nBoundBitmaps; int32_t offset; - void * pBitmap; - void * pOffset; + void *pBitmap; + void *pOffset; int32_t extendedRowSize; } SRowBuilder; -#define TD_ROW_HEAD_LEN (sizeof(STSRow)) +#define TD_ROW_HEAD_LEN (sizeof(STSRow)) #define TD_ROW_NCOLS_LEN (sizeof(col_id_t)) -#define TD_ROW_INFO(r) ((r)->info) -#define TD_ROW_TYPE(r) ((r)->type) -#define TD_ROW_DELETE(r) ((r)->del) -#define TD_ROW_ENDIAN(r) ((r)->endian) -#define TD_ROW_SVER(r) ((r)->sver) -#define TD_ROW_NCOLS(r) ((r)->data) // only valid for SKvRow -#define TD_ROW_DATA(r) ((r)->data) -#define TD_ROW_LEN(r) ((r)->len) -#define TD_ROW_KEY(r) ((r)->ts) +#define TD_ROW_INFO(r) ((r)->info) +#define TD_ROW_TYPE(r) ((r)->type) +#define TD_ROW_DELETE(r) ((r)->del) +#define TD_ROW_ENDIAN(r) ((r)->endian) +#define TD_ROW_SVER(r) ((r)->sver) +#define TD_ROW_NCOLS(r) ((r)->data) // only valid for SKvRow +#define TD_ROW_DATA(r) ((r)->data) +#define TD_ROW_LEN(r) ((r)->len) +#define TD_ROW_KEY(r) ((r)->ts) #define TD_ROW_KEY_ADDR(r) (r) // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and // (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined. #define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) (schemaTLen(s) + TD_ROW_HEAD_LEN) -#define TD_ROW_SET_INFO(r, i) (TD_ROW_INFO(r) = (i)) -#define TD_ROW_SET_TYPE(r, t) (TD_ROW_TYPE(r) = (t)) -#define TD_ROW_SET_DELETE(r) (TD_ROW_DELETE(r) = 1) -#define TD_ROW_SET_SVER(r, v) (TD_ROW_SVER(r) = (v)) -#define TD_ROW_SET_LEN(r, l) (TD_ROW_LEN(r) = (l)) +#define TD_ROW_SET_INFO(r, i) (TD_ROW_INFO(r) = (i)) +#define TD_ROW_SET_TYPE(r, t) (TD_ROW_TYPE(r) = (t)) +#define TD_ROW_SET_DELETE(r) (TD_ROW_DELETE(r) = 1) +#define TD_ROW_SET_SVER(r, v) (TD_ROW_SVER(r) = (v)) +#define TD_ROW_SET_LEN(r, l) (TD_ROW_LEN(r) = (l)) #define TD_ROW_SET_NCOLS(r, n) (*(col_id_t *)TD_ROW_NCOLS(r) = (n)) #define TD_ROW_IS_DELETED(r) (TD_ROW_DELETE(r) == 1) -#define TD_IS_TP_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_TP) -#define TD_IS_KV_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_KV) -#define TD_IS_TP_ROW_T(t) ((t) == TD_ROW_TP) -#define TD_IS_KV_ROW_T(t) ((t) == TD_ROW_KV) +#define TD_IS_TP_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_TP) +#define TD_IS_KV_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_KV) +#define TD_IS_TP_ROW_T(t) ((t) == TD_ROW_TP) +#define TD_IS_KV_ROW_T(t) ((t) == TD_ROW_KV) -#define TD_BOOL_STR(b) ((b) ? "true" : "false") +#define TD_BOOL_STR(b) ((b) ? "true" : "false") #define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert)) #define TD_ROW_COL_IDX(r) POINTER_SHIFT(TD_ROW_DATA(r), sizeof(col_id_t)) @@ -275,7 +275,7 @@ static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TD } int16_t nBytes = colIdx / TD_VTYPE_PARTS; int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char * pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); switch (nOffset) { case 0: *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); @@ -313,7 +313,7 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD } int16_t nBytes = colIdx / TD_VTYPE_PARTS; int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char * pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); switch (nOffset) { case 0: *pValType = (((*pDestByte) & 0xC0) >> 6); @@ -620,7 +620,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa if (tdValIsNorm(valType, val, colType)) { // ts key stored in STSRow.ts SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char * ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); pColIdx->colId = colId; pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN @@ -638,7 +638,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa // NULL/None value else { SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char * ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); pColIdx->colId = colId; pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN const void *nullVal = getNullValue(colType); @@ -775,8 +775,8 @@ static FORCE_INLINE int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, v typedef struct { STSchema *pSchema; - STSRow * pRow; - void * pBitmap; + STSRow *pRow; + void *pBitmap; uint32_t offset; col_id_t maxColId; col_id_t colIdx; // [PRIMARYKEY_TIMESTAMP_COL_ID, nSchemaCols], PRIMARYKEY_TIMESTAMP_COL_ID equals 1 @@ -881,7 +881,7 @@ static FORCE_INLINE bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colTy // internal static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal) { - STSRow * pRow = pIter->pRow; + STSRow *pRow = pIter->pRow; SKvRowIdx *pKvIdx = NULL; bool colFound = false; col_id_t kvNCols = tdRowGetNCols(pRow); @@ -939,7 +939,6 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col while (pIter->colIdx <= pSchema->numOfCols) { pCol = &pSchema->columns[pIter->colIdx]; if (colId == pCol->colId) { - ++pIter->colIdx; break; } else if (colId < pCol->colId) { ++pIter->colIdx; @@ -948,7 +947,8 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col return false; } } - return tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + ++pIter->colIdx; } else if (TD_IS_KV_ROW(pIter->pRow)) { return tdGetKvRowValOfColEx(pIter, colId, colType, &pIter->kvIdx, pVal); } else { @@ -1076,7 +1076,7 @@ typedef struct { typedef struct { STSchema *pSchema; - STSRow * pRow; + STSRow *pRow; } STSRowReader; typedef struct { diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 8b1ac3ed8d..e635227eaa 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -61,16 +61,16 @@ extern "C" { } \ } -#define WAL_HEAD_VER 0 +#define WAL_HEAD_VER 0 #define WAL_NOSUFFIX_LEN 20 -#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1) -#define WAL_LOG_SUFFIX "log" +#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1) +#define WAL_LOG_SUFFIX "log" #define WAL_INDEX_SUFFIX "idx" -#define WAL_REFRESH_MS 1000 -#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead)) -#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) -#define WAL_FILE_LEN (WAL_PATH_LEN + 32) -#define WAL_MAGIC 0xFAFBFCFDULL +#define WAL_REFRESH_MS 1000 +#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead)) +#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) +#define WAL_FILE_LEN (WAL_PATH_LEN + 32) +#define WAL_MAGIC 0xFAFBFCFDULL #define WAL_CUR_FAILED 1 @@ -150,14 +150,15 @@ typedef struct SWal { } SWal; // WAL HANDLE typedef struct SWalReadHandle { - SWal *pWal; - TdFilePtr pReadLogTFile; - TdFilePtr pReadIdxTFile; - int64_t curFileFirstVer; - int64_t curVersion; - int64_t capacity; - int64_t status; // if cursor valid - SWalHead *pHead; + SWal *pWal; + TdFilePtr pReadLogTFile; + TdFilePtr pReadIdxTFile; + int64_t curFileFirstVer; + int64_t curVersion; + int64_t capacity; + int64_t status; // if cursor valid + TdThreadMutex mutex; + SWalHead *pHead; } SWalReadHandle; #pragma pack(pop) @@ -191,6 +192,7 @@ int32_t walEndSnapshot(SWal *); SWalReadHandle *walOpenReadHandle(SWal *); void walCloseReadHandle(SWalReadHandle *); int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); +int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead); // deprecated #if 0 diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 1fa267ae7e..5d1e95be7c 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -255,6 +255,7 @@ void tmqClearUnhandleMsg(tmq_t* tmq) { break; } + msg = NULL; taosReadAllQitems(tmq->mqueue, tmq->qall); while (1) { taosGetQitem(tmq->qall, (void**)&msg); @@ -787,7 +788,7 @@ void tmqShowMsg(tmq_message_t* tmq_message) { static bool noPrintSchema; char pBuf[128]; SMqPollRsp* pRsp = &tmq_message->msg; - int32_t colNum = pRsp->schema->nCols; + int32_t colNum = 2; if (!noPrintSchema) { printf("|"); for (int32_t i = 0; i < colNum; i++) { @@ -838,6 +839,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch; int32_t tmqEpoch = atomic_load_32(&tmq->epoch); if (msgEpoch < tmqEpoch) { + /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/ tsem_post(&tmq->rspSem); tscWarn("discard rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch); return 0; @@ -886,6 +888,9 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { goto WRITE_QUEUE_FAIL; } + tscError("tmq recv poll: vg %d, req offset %ld, rsp offset %ld", pParam->pVg->vgId, pRsp->msg.reqOffset, + pRsp->msg.rspOffset); + pRsp->vg = pParam->pVg; taosWriteQitem(tmq->mqueue, pRsp); atomic_add_fetch_32(&tmq->readyRequest, 1); @@ -902,6 +907,7 @@ WRITE_QUEUE_FAIL: bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { /*printf("call update ep %d\n", epoch);*/ + /*printf("tmq update ep epoch %d to epoch %d\n", tmq->epoch, epoch);*/ bool set = false; int32_t topicNumGet = taosArrayGetSize(pRsp->topics); char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; @@ -932,6 +938,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { for (int32_t k = 0; k < vgNumCur; k++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k); sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId); + /*printf("epoch %d vg %d build %s\n", epoch, pVgCur->vgId, vgKey);*/ taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); } break; @@ -945,9 +952,12 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); int64_t offset = pVgEp->offset; + /*printf("epoch %d vg %d offset og to %ld\n", epoch, pVgEp->vgId, offset);*/ if (pOffset != NULL) { offset = *pOffset; + /*printf("epoch %d vg %d found %s\n", epoch, pVgEp->vgId, vgKey);*/ } + /*printf("epoch %d vg %d offset set to %ld\n", epoch, pVgEp->vgId, offset);*/ SMqClientVg clientVg = { .pollCnt = 0, .currentOffset = offset, @@ -1195,6 +1205,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus != TMQ_VG_STATUS__IDLE) { + /*printf("skip vg %d\n", pVg->vgId);*/ continue; } SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); @@ -1238,6 +1249,8 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { int64_t transporterId = 0; /*printf("send poll\n");*/ atomic_add_fetch_32(&tmq->waitingRequest, 1); + /*tscDebug("tmq send poll: vg %d, req offset %ld", pVg->vgId, pVg->currentOffset);*/ + /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; tmq->pollCnt++; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 6976d83abd..7891c690e3 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -735,6 +735,9 @@ typedef struct { int8_t createdBy; // STREAM_CREATED_BY__USER or SMA int32_t fixedSinkVgId; // 0 for shuffle int64_t smaId; // 0 for unused + int8_t trigger; + int32_t triggerParam; + int64_t waterMark; char* sql; char* logicalPlan; char* physicalPlan; diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index e5d1f952a8..aa4a9fc1de 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -194,7 +194,7 @@ void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version); int tqCommit(STQ*); -int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); int32_t tqProcessSetConnReq(STQ* pTq, char* msg); int32_t tqProcessRebReq(STQ* pTq, char* msg); int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); diff --git a/source/dnode/vnode/src/meta/metaTbUid.c b/source/dnode/vnode/src/meta/metaTbUid.c index cad1eba134..b488630024 100644 --- a/source/dnode/vnode/src/meta/metaTbUid.c +++ b/source/dnode/vnode/src/meta/metaTbUid.c @@ -27,5 +27,5 @@ void metaCloseUidGnrt(SMeta *pMeta) { /* TODO */ tb_uid_t metaGenerateUid(SMeta *pMeta) { // Generate a new table UID - return ++(pMeta->uidGnrt.nextUid); + return tGenIdPI64(); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 06a2350aab..f34fa9e19c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -250,7 +250,7 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu return 0; } -int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { +int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqPollReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; int64_t fetchOffset; @@ -264,6 +264,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { fetchOffset = pReq->currentOffset + 1; } + printf("tmq poll vg %d req %ld %ld\n", pTq->pVnode->vgId, pReq->currentOffset, fetchOffset); + SMqPollRsp rsp = { /*.consumerId = consumerId,*/ .numOfTopics = 0, @@ -288,62 +290,76 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { rsp.reqOffset = pReq->currentOffset; rsp.skipLogNum = 0; - SWalHead* pHead; while (1) { /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/ - if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { + SWalReadHead* pHead; + if (walReadWithHandle_s(pTopic->pReadhandle, fetchOffset, &pHead) < 0) { // TODO: no more log, set timer to wait blocking time // if data inserted during waiting, launch query and // response to user break; } - int8_t pos = fetchOffset % TQ_BUFFER_SIZE; - pHead = pTopic->pReadhandle->pHead; - if (pHead->head.msgType == TDMT_VND_SUBMIT) { - SSubmitReq* pCont = (SSubmitReq*)&pHead->head.body; - qTaskInfo_t task = pTopic->buffer.output[pos].task; + /*printf("vg %d offset %ld msgType %d from epoch %d\n", pTq->pVnode->vgId, fetchOffset, pHead->msgType, + * pReq->epoch);*/ + /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/ + /*pHead = pTopic->pReadhandle->pHead;*/ + if (pHead->msgType == TDMT_VND_SUBMIT) { + SSubmitReq* pCont = (SSubmitReq*)&pHead->body; + printf("from topic %s from consumer\n", pTopic->topicName, consumerId); + qTaskInfo_t task = pTopic->buffer.output[workerId].task; + ASSERT(task); qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK); SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); while (1) { - SSDataBlock* pDataBlock; + SSDataBlock* pDataBlock = NULL; uint64_t ts; if (qExecTask(task, &pDataBlock, &ts) < 0) { ASSERT(false); } if (pDataBlock == NULL) { - fetchOffset++; - pos = fetchOffset % TQ_BUFFER_SIZE; - rsp.skipLogNum++; + /*pos = fetchOffset % TQ_BUFFER_SIZE;*/ break; } taosArrayPush(pRes, pDataBlock); - rsp.schema = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; - rsp.rspOffset = fetchOffset; - - rsp.numOfTopics = 1; - rsp.pBlockData = pRes; - - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - pMsg->code = -1; - return -1; - } - ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; - ((SMqRspHead*)buf)->epoch = pReq->epoch; - ((SMqRspHead*)buf)->consumerId = consumerId; - - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqPollRsp(&abuf, &rsp); - /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/ - pMsg->pCont = buf; - pMsg->contLen = tlen; - pMsg->code = 0; - tmsgSendRsp(pMsg); - return 0; } + + if (taosArrayGetSize(pRes) == 0) { + fetchOffset++; + rsp.skipLogNum++; + taosArrayDestroy(pRes); + continue; + } + rsp.schema = pTopic->buffer.output[workerId].pReadHandle->pSchemaWrapper; + rsp.rspOffset = fetchOffset; + + rsp.numOfTopics = 1; + rsp.pBlockData = pRes; + + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + pMsg->code = -1; + taosMemoryFree(pHead); + return -1; + } + ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; + ((SMqRspHead*)buf)->epoch = pReq->epoch; + ((SMqRspHead*)buf)->consumerId = consumerId; + + void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + tEncodeSMqPollRsp(&abuf, &rsp); + /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/ + pMsg->pCont = buf; + pMsg->contLen = tlen; + pMsg->code = 0; + printf("vg %d offset %ld msgType %d from epoch %d actual rsp\n", pTq->pVnode->vgId, fetchOffset, pHead->msgType, + pReq->epoch); + tmsgSendRsp(pMsg); + taosMemoryFree(pHead); + return 0; } else { + taosMemoryFree(pHead); fetchOffset++; rsp.skipLogNum++; } @@ -368,6 +384,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { pMsg->contLen = tlen; pMsg->code = 0; tmsgSendRsp(pMsg); + printf("vg %d offset %ld from epoch %d not rsp\n", pTq->pVnode->vgId, fetchOffset, pReq->epoch); /*}*/ return 0; @@ -432,7 +449,9 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { }; pTopic->buffer.output[i].pReadHandle = pReadHandle; pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle); + ASSERT(pTopic->buffer.output[i].task); } + printf("set topic %s to consumer %ld\n", pTopic->topicName, req.consumerId); taosArrayPush(pConsumer->topics, pTopic); tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer); tqHandleCommit(pTq->tqMeta, req.consumerId); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 9c0b0802ab..344b4fa655 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -167,7 +167,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { break; } - if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) { + if (colDataAppend(pColData, curRow, sVal.val, false) < 0) { taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock); return NULL; } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 598647f797..053c03cc97 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -66,7 +66,7 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { case TDMT_VND_TABLE_META: return vnodeGetTableMeta(pVnode, pMsg); case TDMT_VND_CONSUME: - return tqProcessPollReq(pVnode->pTq, pMsg); + return tqProcessPollReq(pVnode->pTq, pMsg, pInfo->workerId); case TDMT_VND_TASK_PIPE_EXEC: case TDMT_VND_TASK_MERGE_EXEC: return tqProcessTaskExec(pVnode->pTq, msgstr, msgLen, pInfo->workerId); diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 8875090b43..631727fd5b 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -163,6 +163,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // } break; case TDMT_VND_SUBMIT: + printf("vnode %d write data %ld\n", pVnode->vgId, ver); if (pVnode->config.streamMode == 0) { if (tsdbInsertData(pVnode->pTsdb, (SSubmitReq *)ptr, NULL) < 0) { // TODO: handle error diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 2df18f135c..d6c07b6659 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1551,8 +1551,8 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); tsCols = (int64_t*)pColDataInfo->pData; - assert(tsCols[0] == pSDataBlock->info.window.skey && - tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); + /*assert(tsCols[0] == pSDataBlock->info.window.skey &&*/ + /*tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey);*/ } int32_t startPos = ascScan? 0 : (pSDataBlock->info.rows - 1); @@ -6994,6 +6994,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator, bool* newgroup finalizeUpdatedResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset); blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->binfo.capacity); + initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pInfo->binfo.pRes, pInfo->binfo.capacity, pInfo->binfo.rowCellInfoOffset); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 9ef1c01cd1..d8a44e7156 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -719,6 +719,11 @@ static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExc if (NULL == pScan->pScanCols) { code = TSDB_CODE_OUT_OF_MEMORY; } + + if (TSDB_CODE_SUCCESS == code) { + code = sortScanCols(pScan->pScanCols); + } + if (TSDB_CODE_SUCCESS == code) { code = addDataBlockSlots(pCxt, pScan->pScanCols, pScan->node.pOutputDataBlockDesc); } diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c index fb6c0f6c12..e2c5979a5a 100644 --- a/source/libs/stream/src/tstream.c +++ b/source/libs/stream/src/tstream.c @@ -38,7 +38,7 @@ static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg* req.taskId = pTask->fixedEpDispatcher.taskId; } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { - // TODO fix tbname issue + // TODO use general name rule of schemaless char ctbName[TSDB_TABLE_FNAME_LEN + 22]; // all groupId must be the same in an array SSDataBlock* pBlock = taosArrayGet(data, 0); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 5296a16703..2545eeab4b 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -30,6 +30,9 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { pRead->curFileFirstVer = -1; pRead->capacity = 0; pRead->status = 0; + + taosThreadMutexInit(&pRead->mutex, NULL); + pRead->pHead = taosMemoryMalloc(sizeof(SWalHead)); if (pRead->pHead == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; @@ -135,6 +138,22 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { return 0; } +int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead) { + taosThreadMutexLock(&pRead->mutex); + if (walReadWithHandle(pRead, ver) < 0) { + taosThreadMutexUnlock(&pRead->mutex); + return -1; + } + *ppHead = taosMemoryMalloc(sizeof(SWalReadHead) + pRead->pHead->head.len); + if (*ppHead == NULL) { + taosThreadMutexUnlock(&pRead->mutex); + return -1; + } + memcpy(*ppHead, &pRead->pHead->head, sizeof(SWalReadHead) + pRead->pHead->head.len); + taosThreadMutexUnlock(&pRead->mutex); + return 0; +} + int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { int code; // TODO: check wal life @@ -145,7 +164,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } } - if (!taosValidFile(pRead->pReadLogTFile)) return -1; + if (!taosValidFile(pRead->pReadLogTFile)) { + return -1; + } code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead)); if (code != sizeof(SWalHead)) { diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 0efcf517a9..93a843fee8 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -121,7 +121,8 @@ int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj) { return tjsonAddItemToArray(pJson, pJobj); } -int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void* pArray, int32_t itemSize, int32_t num) { +int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void* pArray, int32_t itemSize, + int32_t num) { if (num > 0) { SJson* pJsonArray = tjsonAddArrayToObject(pJson, pName); if (NULL == pJsonArray) { @@ -168,7 +169,7 @@ int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal } *pVal = strtol(p, NULL, 10); - return (errno == EINVAL || errno == ERANGE) ? TSDB_CODE_FAILED:TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } int32_t tjsonGetIntValue(const SJson* pJson, const char* pName, int32_t* pVal) { @@ -199,19 +200,19 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV } *pVal = strtoul(p, NULL, 10); - return (errno == ERANGE||errno == EINVAL) ? TSDB_CODE_FAILED:TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } int32_t tjsonGetUIntValue(const SJson* pJson, const char* pName, uint32_t* pVal) { uint64_t val = 0; - int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); + int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); *pVal = val; return code; } int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal) { uint64_t val = 0; - int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); + int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); *pVal = val; return code; } @@ -264,7 +265,7 @@ int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, v int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize) { const cJSON* jArray = tjsonGetObjectItem(pJson, pName); - int32_t size = (NULL == jArray ? 0 : tjsonGetArraySize(jArray)); + int32_t size = (NULL == jArray ? 0 : tjsonGetArraySize(jArray)); for (int32_t i = 0; i < size; ++i) { int32_t code = func(tjsonGetArrayItem(jArray, i), (char*)pArray + itemSize * i); if (TSDB_CODE_SUCCESS != code) { diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index fe6a7a0660..f84d70f711 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -35,7 +35,7 @@ sql connect $dbNamme = d0 print =============== create database , vgroup 1 -sql create database $dbNamme vgroups 1 +sql create database $dbNamme vgroups 2 sql show databases print $data00 $data01 $data02 if $rows != 2 then diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 4d3108500e..38264331c1 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -226,7 +226,7 @@ void loop_consume(tmq_t* tmq) { int32_t totalRows = 0; int32_t skipLogNum = 0; while (running) { - tmq_message_t* tmqMsg = tmq_consumer_poll(tmq, 1); + tmq_message_t* tmqMsg = tmq_consumer_poll(tmq, 3000); if (tmqMsg) { totalMsgs++; From 236b8adad3d7e460b1dca075b057fb082e2ce5eb Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 31 Mar 2022 16:00:47 +0800 Subject: [PATCH 14/38] feature/qnode --- include/common/ttime.h | 1 + include/libs/qcom/query.h | 7 +- include/os/osTime.h | 8 + source/common/src/ttime.c | 29 ++- source/libs/planner/src/planLogicCreater.c | 2 +- source/libs/qcom/inc/queryInt.h | 52 ++-- source/libs/qcom/src/queryExplain.c | 290 ++++++++++++--------- source/libs/scheduler/inc/schedulerInt.h | 6 +- source/libs/scheduler/src/scheduler.c | 42 +-- 9 files changed, 263 insertions(+), 174 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index 57af24e635..2209cc998f 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -62,6 +62,7 @@ int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t t void deltaToUtcInitOnce(); int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); +int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit); void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index dd2dccc02d..5908b0b875 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -174,9 +174,10 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); char *jobTaskStatusStr(int32_t status); -int32_t qAppendTaskExplainResRows(void **pRowCtx, void *plan, void *pExecTree, int32_t level); -int32_t qGetExplainRspFromRowCtx(void *ctx, SRetrieveTableRsp **pRsp); -void qFreeExplainRowCtx(void *ctx); +int32_t qInitExplainCtx(void **pCtx, SHashObj *groupHash, bool verbose); +int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level); +int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp); +void qFreeExplainCtx(void *ctx); SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); diff --git a/include/os/osTime.h b/include/os/osTime.h index 9e426455dc..766fec0fbd 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -46,6 +46,14 @@ extern "C" { #define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24) #define MILLISECOND_PER_WEEK (MILLISECOND_PER_DAY * 7) +#define NANOSECOND_PER_USEC (1000L) +#define NANOSECOND_PER_MSEC (1000000L) +#define NANOSECOND_PER_SEC (1000000000L) +#define NANOSECOND_PER_MINUTE (NANOSECOND_PER_SEC * 60) +#define NANOSECOND_PER_HOUR (NANOSECOND_PER_MINUTE * 60) +#define NANOSECOND_PER_DAY (NANOSECOND_PER_HOUR * 24) +#define NANOSECOND_PER_WEEK (NANOSECOND_PER_DAY * 7) + int32_t taosGetTimeOfDay(struct timeval *tv); //@return timestamp in second diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 510a2809e7..23b19b55e7 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -370,6 +370,33 @@ int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrec return (int64_t)((double)time * factors[fromPrecision][toPrecision]); } +int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) { + assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || + fromPrecision == TSDB_TIME_PRECISION_NANO); + static double factors[3] = {1000000., 1000., 1.}; + switch (toUnit) { + case 's': + return time * factors[fromPrecision] / NANOSECOND_PER_SEC; + case 'm': + return time * factors[fromPrecision] / NANOSECOND_PER_MINUTE; + case 'h': + return time * factors[fromPrecision] / NANOSECOND_PER_HOUR; + case 'd': + return time * factors[fromPrecision] / NANOSECOND_PER_DAY; + case 'w': + return time * factors[fromPrecision] / NANOSECOND_PER_WEEK; + case 'a': + return time * factors[fromPrecision] / NANOSECOND_PER_MSEC; + case 'u': + return time * factors[fromPrecision] / NANOSECOND_PER_USEC; + case 'b': + return time * factors[fromPrecision]; + default: { + return -1; + } + } +} + static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { switch (unit) { case 's': @@ -688,4 +715,4 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm); tstrncpy(buf, ts, bufLen); -} \ No newline at end of file +} diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 7f6d8826e6..f57f476d51 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -288,7 +288,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect // set the output if (TSDB_CODE_SUCCESS == code) { pJoin->node.pTargets = nodesCloneList(pLeft->pTargets); - if (NULL == pJoin->pOnConditions) { + if (NULL == pJoin->node.pTargets) { code = TSDB_CODE_OUT_OF_MEMORY; } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h index 510ba92a86..06d5b07dc7 100644 --- a/source/libs/qcom/inc/queryInt.h +++ b/source/libs/qcom/inc/queryInt.h @@ -23,25 +23,34 @@ extern "C" { #include "plannodes.h" //newline area -#define EXPLAIN_TAG_SCAN_FORMAT "Tag scan on %s columns=%d" -#define EXPLAIN_TBL_SCAN_FORMAT "Table scan on %s columns=%d" -#define EXPLAIN_SYSTBL_SCAN_FORMAT "System table scan on %s columns=%d" +#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d" +#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d" +#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d" #define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d" #define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d" -#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d groups=%d width=%d" -#define EXPLAIN_EXCHANGE_FORMAT "Exchange %d:1 width=%d" -#define EXPLAIN_SORT_FORMAT "Sort on %d columns width=%d" -#define EXPLAIN_INTERVAL_FORMAT "Interval on column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d" +#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d" +#define EXPLAIN_EXCHANGE_FORMAT "Data Exchange %d:1 width=%d" +#define EXPLAIN_SORT_FORMAT "Sort on %d Column(s) width=%d" +#define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d" #define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d" #define EXPLAIN_ORDER_FORMAT "Order: %s" #define EXPLAIN_FILTER_FORMAT "Filter: " #define EXPLAIN_FILL_FORMAT "Fill: %s" #define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: " -#define EXPLAIN_TIMERANGE_FORMAT "Time range: [%" PRId64 ", %" PRId64 "]" +#define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]" //append area -#define EXPLAIN_LOOPS_FORMAT "loops=%d" -#define EXPLAIN_REVERSE_FORMAT "reverse=%d" +#define EXPLAIN_GROUPS_FORMAT " groups=%d" +#define EXPLAIN_WIDTH_FORMAT " width=%d" +#define EXPLAIN_LOOPS_FORMAT " loops=%d" +#define EXPLAIN_REVERSE_FORMAT " reverse=%d" + +//TODO MOVE TO LIB +typedef struct SExplainGroup { + int32_t nodeNum; + SSubplan *plan; + void *execInfo; //TODO +} SExplainGroup; typedef struct SExplainResNode { @@ -56,22 +65,27 @@ typedef struct SQueryExplainRowInfo { char *buf; } SQueryExplainRowInfo; -typedef struct SExplainRowCtx { - int32_t totalSize; - SArray *rows; -} SExplainRowCtx; +typedef struct SExplainCtx { + int32_t totalSize; + bool verbose; + char *tbuf; + SArray *rows; + SHashObj *groupHash; +} SExplainCtx; #define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending") #define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") -#define EXPLAIN_ROW_NEW(level, ...) \ - do { \ - tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s", (level) * 2, ""); \ - tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \ +#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u))) + +#define EXPLAIN_ROW_NEW(level, ...) \ + do { \ + tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", (isVerboseLine ? "" : "-> ")); \ + tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \ } while (0) #define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__) -#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; } while (0) +#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; isVerboseLine = true; } while (0) #define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/qcom/src/queryExplain.c index b2b061cee0..af092e842e 100644 --- a/source/libs/qcom/src/queryExplain.c +++ b/source/libs/qcom/src/queryExplain.c @@ -36,22 +36,62 @@ void qFreeExplainResTree(SExplainResNode *res) { taosMemoryFreeClear(res); } -void qFreeExplainRowCtx(void *ctx) { +void qFreeExplainCtx(void *ctx) { if (NULL == ctx) { return; } - SExplainRowCtx *pCtx = (SExplainRowCtx *)ctx; + SExplainCtx *pCtx = (SExplainCtx *)ctx; int32_t rowSize = taosArrayGetSize(pCtx->rows); for (int32_t i = 0; i < rowSize; ++i) { SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); taosMemoryFreeClear(row->buf); } + taosHashCleanup(pCtx->groupHash); taosArrayDestroy(pCtx->rows); taosMemoryFree(pCtx); } +int32_t qInitExplainCtx(void **pCtx, SHashObj *groupHash, bool verbose) { + int32_t code = 0; + SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx)); + if (NULL == ctx) { + qError("calloc SExplainCtx failed"); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); + if (NULL == rows) { + qError("taosArrayInit SQueryExplainRowInfo failed"); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + char *tbuf = taosMemoryMalloc(TSDB_EXPLAIN_RESULT_ROW_SIZE); + if (NULL == tbuf) { + qError("malloc size %d failed", TSDB_EXPLAIN_RESULT_ROW_SIZE); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + ctx->verbose = verbose; + ctx->tbuf = tbuf; + ctx->rows = rows; + ctx->groupHash = groupHash; + + *pCtx = ctx; + + return TSDB_CODE_SUCCESS; + +_return: + + taosArrayDestroy(rows); + taosHashCleanup(groupHash); + taosMemoryFree(ctx); + + QRY_RET(code); +} + + char *qFillModeString(EFillMode mode) { switch (mode) { case FILL_MODE_NONE: @@ -187,11 +227,6 @@ _return: QRY_RET(code); } -int32_t qGenerateExplainResNodeTree(struct SSubplan *plan, void *pExecTree, SExplainResNode **pRes) { - void *pExecInfo = NULL; // TODO - QRY_RET(qGenerateExplainResNode(plan->pNode, pExecInfo, pRes)); -} - int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t *len) { int32_t tlen = *len; @@ -202,7 +237,7 @@ int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t *len) { return TSDB_CODE_SUCCESS; } -int32_t qExplainResAppendRow(SExplainRowCtx *ctx, char *tbuf, int32_t len, int32_t level) { +int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) { SQueryExplainRowInfo row = {0}; row.buf = taosMemoryMalloc(len); if (NULL == row.buf) { @@ -225,8 +260,11 @@ int32_t qExplainResAppendRow(SExplainRowCtx *ctx, char *tbuf, int32_t len, int32 } -int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx, char *tbuf, int32_t level) { +int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { int32_t tlen = 0; + bool isVerboseLine = false; + char *tbuf = ctx->tbuf; + bool verbose = ctx->verbose; SPhysiNode* pNode = pResNode->pNode; if (NULL == pNode) { qError("pyhsical node in explain res node is NULL"); @@ -246,10 +284,12 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } break; } case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: @@ -265,20 +305,22 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - if (pTblScanNode->scan.node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (pTblScanNode->scan.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } } break; } @@ -294,10 +336,12 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } break; } case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ @@ -309,11 +353,13 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (pPrjNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (verbose) { + if (pPrjNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } } break; } @@ -326,51 +372,70 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (pJoinNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + if (verbose) { + if (pJoinNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } - - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); break; } case QUERY_NODE_PHYSICAL_PLAN_AGG:{ SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length, pAggNode->pGroupKeys->length, pAggNode->node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length); + if (pAggNode->pGroupKeys) { + EXPLAIN_ROW_APPEND(EXPLAIN_GROUPS_FORMAT, pAggNode->pGroupKeys->length); + } + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->outputRowSize); + if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (pAggNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (verbose) { + if (pAggNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } } break; } case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, pExchNode->pSrcEndPoints->length, pExchNode->node.pOutputDataBlockDesc->outputRowSize); + SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcGroupId, sizeof(pExchNode->srcGroupId)); + if (NULL == group) { + qError("exchange src group %d not in groupHash", pExchNode->srcGroupId); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, group->nodeNum, pExchNode->node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (pExchNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (verbose) { + if (pExchNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } } + + QRY_ERR_RET(qAppendTaskExplainResRows(ctx, pExchNode->srcGroupId, level + 1)); break; } case QUERY_NODE_PHYSICAL_PLAN_SORT:{ @@ -382,35 +447,42 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (pSortNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (verbose) { + if (pSortNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } } break; } case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, - pIntNode->interval, pIntNode->intervalUnit, pIntNode->offset, pIntNode->intervalUnit, pIntNode->sliding, pIntNode->slidingUnit, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, pIntNode->precision), pIntNode->intervalUnit, + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->offset, pIntNode->intervalUnit, pIntNode->precision), pIntNode->intervalUnit, + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, pIntNode->precision), pIntNode->slidingUnit, + pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (pIntNode->pFill) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_FORMAT, qFillModeString(pIntNode->pFill->mode)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } + if (verbose) { + if (pIntNode->pFill) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_FORMAT, qFillModeString(pIntNode->pFill->mode)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } - if (pIntNode->window.node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (pIntNode->window.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } } break; } @@ -423,11 +495,13 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (pIntNode->window.node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (verbose) { + if (pIntNode->window.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } } break; } @@ -440,88 +514,48 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainRowCtx *ctx } -int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainRowCtx *ctx, char *tbuf, int32_t level) { +int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } int32_t code = 0; - QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, ctx, tbuf, level)); + QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, ctx, level)); SNode* pNode = NULL; FOREACH(pNode, pResNode->pChildren) { - QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, tbuf, level + 1)); + QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, level + 1)); } return TSDB_CODE_SUCCESS; } -int32_t qGenerateExplainResRowsFromNode(SExplainResNode *pResNode, SExplainRowCtx *ctx, int32_t level) { - if (NULL == pResNode) { - qError("explain res node is NULL"); - QRY_RET(TSDB_CODE_QRY_APP_ERROR); - } - - int32_t code = 0; - char *tbuf = taosMemoryMalloc(TSDB_EXPLAIN_RESULT_ROW_SIZE); - if (NULL == tbuf) { - qError("malloc size %d failed", TSDB_EXPLAIN_RESULT_ROW_SIZE); - QRY_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - QRY_ERR_JRET(qExplainResNodeToRows(pResNode, ctx, tbuf, level)); - -_return: - - taosMemoryFree(tbuf); - - QRY_RET(code); -} - -int32_t qAppendTaskExplainResRows(void **pRowCtx, void *plan, void *pExecTree, int32_t level) { +int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level) { SExplainResNode *node = NULL; int32_t code = 0; - struct SSubplan *pPlan = (struct SSubplan *)plan; - SExplainRowCtx *ctx = (SExplainRowCtx *)*pRowCtx; - - QRY_ERR_RET(qGenerateExplainResNodeTree(pPlan, pExecTree, &node)); + SExplainCtx *ctx = (SExplainCtx *)pCtx; - if (NULL == *pRowCtx) { - *pRowCtx = taosMemoryCalloc(1, sizeof(SExplainRowCtx)); - if (NULL == *pRowCtx) { - qError("calloc SExplainRowCtx failed"); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - ctx = (SExplainRowCtx *)*pRowCtx; - - SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); - if (NULL == rows) { - qError("taosArrayInit SQueryExplainRowInfo failed"); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - ctx->rows = rows; + SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId)); + if (NULL == group) { + qError("group %d not in groupHash", groupId); + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } + + QRY_ERR_RET(qGenerateExplainResNode(group->plan->pNode, group->execInfo, &node)); - QRY_ERR_JRET(qGenerateExplainResRowsFromNode(node, ctx, level)); + QRY_ERR_JRET(qExplainResNodeToRows(node, ctx, level)); _return: qFreeExplainResTree(node); - - if (code) { - qFreeExplainRowCtx(*pRowCtx); - *pRowCtx = NULL; - } QRY_RET(code); } -int32_t qGetExplainRspFromRowCtx(void *ctx, SRetrieveTableRsp **pRsp) { - SExplainRowCtx *pCtx = (SExplainRowCtx *)ctx; +int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { + SExplainCtx *pCtx = (SExplainCtx *)ctx; int32_t rowNum = taosArrayGetSize(pCtx->rows); if (rowNum <= 0) { qError("empty explain res rows"); diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 1b2c7d54e2..f19e822974 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -38,10 +38,12 @@ enum { SCH_WRITE, }; -typedef struct SSchExplainGroup { +//TODO MOVE TO LIB +typedef struct SExplainGroup { int32_t nodeNum; SSubplan *plan; -} SSchExplainGroup; + void *execInfo; //TODO +} SExplainGroup; typedef struct SSchTrans { void *transInst; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 4a84536d00..940aaed7c1 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -478,7 +478,14 @@ _return: int32_t schValidateAndBuildJobExplain(SQueryPlan *pDag, SSchJob *pJob) { int32_t code = 0; + SNodeListNode *plans = NULL; + int32_t taskNum = 0; + SExplainGroup *pGroup = NULL; + void *pCtx = NULL; + int32_t rootGroupId = 0; + pJob->queryId = pDag->queryId; + pJob->subPlans = pDag->pSubplans; if (pDag->numOfSubplans <= 0) { SCH_JOB_ELOG("invalid subplan num:%d", pDag->numOfSubplans); @@ -497,12 +504,7 @@ int32_t schValidateAndBuildJobExplain(SQueryPlan *pDag, SSchJob *pJob) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - pJob->subPlans = pDag->pSubplans; - - SNodeListNode *plans = NULL; - int32_t taskNum = 0; - SSchExplainGroup *pGroup = NULL; - void *rowCtx = NULL; + SCH_ERR_JRET(qInitExplainCtx(&pCtx, groupHash, pDag->explainInfo.verbose)); for (int32_t i = 0; i < levelNum; ++i) { plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); @@ -517,44 +519,44 @@ int32_t schValidateAndBuildJobExplain(SQueryPlan *pDag, SSchJob *pJob) { SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } + SSubplan *plan = NULL; for (int32_t n = 0; n < taskNum; ++n) { - SSubplan *plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); + plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); pGroup = taosHashGet(groupHash, &plan->id.groupId, sizeof(plan->id.groupId)); if (pGroup) { ++pGroup->nodeNum; continue; } - SSchExplainGroup group = {.nodeNum = 1, .plan = plan}; + SExplainGroup group = {.nodeNum = 1, .plan = plan, .execInfo = NULL}; if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { SCH_JOB_ELOG("taosHashPut to explainGroupHash failed, taskIdx:%d", n); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } } - void *pIter = taosHashIterate(groupHash, NULL); - while (pIter) { - pGroup = (SSchExplainGroup *)pIter; + if (0 == i) { + if (taskNum > 1) { + SCH_JOB_ELOG("invalid taskNum %d for level 0", taskNum); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } - SCH_ERR_JRET(qAppendTaskExplainResRows(&rowCtx, pGroup->plan, NULL, i)); - - pIter = taosHashIterate(groupHash, pIter); + rootGroupId = plan->id.groupId; } - taosHashClear(groupHash); - - SCH_JOB_DLOG("level initialized, taskNum:%d", taskNum); + SCH_JOB_DLOG("level %d group handled, taskNum:%d", i, taskNum); } + SCH_ERR_JRET(qAppendTaskExplainResRows(pCtx, rootGroupId, 0)); + SRetrieveTableRsp *pRsp = NULL; - SCH_ERR_JRET(qGetExplainRspFromRowCtx(rowCtx, &pRsp)); + SCH_ERR_JRET(qGetExplainRspFromCtx(pCtx, &pRsp)); pJob->resData = pRsp; _return: - taosHashCleanup(groupHash); - qFreeExplainRowCtx(rowCtx); + qFreeExplainCtx(pCtx); SCH_RET(code); } From b45f7cef2eb7129877f1e70ad2bdfe5275f6da20 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 31 Mar 2022 16:04:23 +0800 Subject: [PATCH 15/38] fix --- source/dnode/vnode/src/tq/tq.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index f34fa9e19c..82300f82cb 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -264,7 +264,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { fetchOffset = pReq->currentOffset + 1; } - printf("tmq poll vg %d req %ld %ld\n", pTq->pVnode->vgId, pReq->currentOffset, fetchOffset); + /*printf("tmq poll vg %d req %ld %ld\n", pTq->pVnode->vgId, pReq->currentOffset, fetchOffset);*/ SMqPollRsp rsp = { /*.consumerId = consumerId,*/ @@ -305,7 +305,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { /*pHead = pTopic->pReadhandle->pHead;*/ if (pHead->msgType == TDMT_VND_SUBMIT) { SSubmitReq* pCont = (SSubmitReq*)&pHead->body; - printf("from topic %s from consumer\n", pTopic->topicName, consumerId); + /*printf("from topic %s from consumer\n", pTopic->topicName, consumerId);*/ qTaskInfo_t task = pTopic->buffer.output[workerId].task; ASSERT(task); qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK); @@ -353,8 +353,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { pMsg->pCont = buf; pMsg->contLen = tlen; pMsg->code = 0; - printf("vg %d offset %ld msgType %d from epoch %d actual rsp\n", pTq->pVnode->vgId, fetchOffset, pHead->msgType, - pReq->epoch); + /*printf("vg %d offset %ld msgType %d from epoch %d actual rsp\n", pTq->pVnode->vgId, fetchOffset, + * pHead->msgType,*/ + /*pReq->epoch);*/ tmsgSendRsp(pMsg); taosMemoryFree(pHead); return 0; @@ -384,7 +385,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { pMsg->contLen = tlen; pMsg->code = 0; tmsgSendRsp(pMsg); - printf("vg %d offset %ld from epoch %d not rsp\n", pTq->pVnode->vgId, fetchOffset, pReq->epoch); + /*printf("vg %d offset %ld from epoch %d not rsp\n", pTq->pVnode->vgId, fetchOffset, pReq->epoch);*/ /*}*/ return 0; From ec97115b8350c14828587aeb5dc1a5498af122bb Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 31 Mar 2022 16:04:58 +0800 Subject: [PATCH 16/38] fix --- .clang-format | 1 + 1 file changed, 1 insertion(+) diff --git a/.clang-format b/.clang-format index 3ddd8b43f6..e58d518b3b 100644 --- a/.clang-format +++ b/.clang-format @@ -5,6 +5,7 @@ AccessModifierOffset: -1 AlignAfterOpenBracket: Align AlignConsecutiveAssignments: false AlignConsecutiveDeclarations: true +AlignConsecutiveMacros: true AlignEscapedNewlinesLeft: true AlignOperands: true AlignTrailingComments: true From fa354b85e5bb5374bc304800d3dc91b9346e8fdc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 31 Mar 2022 16:05:24 +0800 Subject: [PATCH 17/38] shm --- source/util/src/tprocess.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index f18e7d5777..71f819dbc1 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -140,14 +140,16 @@ static void taosProcDestroySem(SProcQueue *pQueue) { pQueue->sem = NULL; } } +#endif static void taosProcCleanupQueue(SProcQueue *pQueue) { +#if 0 if (pQueue != NULL) { taosProcDestroyMutex(pQueue); taosProcDestroySem(pQueue); } -} #endif +} static int32_t taosProcQueuePush(SProcQueue *pQueue, const char *pHead, int16_t rawHeadLen, const char *pBody, int32_t rawBodyLen, ProcFuncType ftype) { @@ -317,7 +319,7 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { pProc->pChildQueue = taosProcInitQueue(pCfg->name, pCfg->isChild, (char *)pCfg->shm.ptr + cstart, csize); pProc->pParentQueue = taosProcInitQueue(pCfg->name, pCfg->isChild, (char *)pCfg->shm.ptr + pstart, psize); if (pProc->pChildQueue == NULL || pProc->pParentQueue == NULL) { - // taosProcCleanupQueue(pProc->pChildQueue); + taosProcCleanupQueue(pProc->pChildQueue); taosMemoryFree(pProc); return NULL; } @@ -370,7 +372,7 @@ static void taosProcThreadLoop(SProcObj *pProc) { freeBodyFp = pProc->parentFreeBodyFp; } - uDebug("proc:%s, start to get msg from queue:%p, isChild:%d", pProc->name, pQueue, pProc->isChild); + uDebug("proc:%s, start to get msg from queue:%p", pProc->name, pQueue); while (1) { int32_t numOfMsgs = taosProcQueuePop(pQueue, &pHead, &headLen, &pBody, &bodyLen, &ftype, mallocHeadFp, freeHeadFp, @@ -406,7 +408,7 @@ int32_t taosProcRun(SProcObj *pProc) { static void taosProcStop(SProcObj *pProc) { if (!taosCheckPthreadValid(pProc->thread)) return; - uDebug("proc:%s, start to join thread:%" PRId64 ", isChild:%d", pProc->name, pProc->thread, pProc->isChild); + uDebug("proc:%s, start to join thread:%" PRId64, pProc->name, pProc->thread); SProcQueue *pQueue; if (pProc->isChild) { pQueue = pProc->pChildQueue; @@ -421,9 +423,9 @@ void taosProcCleanup(SProcObj *pProc) { if (pProc != NULL) { uDebug("proc:%s, start to clean up", pProc->name); taosProcStop(pProc); + taosProcCleanupQueue(pProc->pChildQueue); + taosProcCleanupQueue(pProc->pParentQueue); uDebug("proc:%s, is cleaned up", pProc->name); - // taosProcCleanupQueue(pProc->pChildQueue); - // taosProcCleanupQueue(pProc->pParentQueue); taosMemoryFree(pProc); } } From 847775c48d03df047d3ecff30cabb4089e227617 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 31 Mar 2022 16:10:32 +0800 Subject: [PATCH 18/38] [td-13039] refactor. --- source/libs/executor/inc/executil.h | 52 +++------ source/libs/executor/inc/executorimpl.h | 4 +- source/libs/executor/src/executil.c | 130 +++++++++------------- source/libs/executor/src/executorimpl.c | 142 +++++++++++++----------- 4 files changed, 149 insertions(+), 179 deletions(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 5d5e0cae88..c09b41f0f6 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -73,27 +73,12 @@ typedef struct SResultRowPosition { } SResultRowPosition; typedef struct SResultRowInfo { - SList *pRows; SResultRowPosition *pPosition; - SResultRow **pResult; // result list int32_t size; // number of result set int32_t capacity; // max capacity int32_t curPos; // current active result row index of pResult list } SResultRowInfo; -typedef struct SResultRowPool { - int32_t elemSize; - int32_t blockSize; - int32_t numOfElemPerBlock; - - struct { - int32_t blockIndex; - int32_t pos; - } position; - - SArray* pData; // SArray -} SResultRowPool; - struct STaskAttr; struct STaskRuntimeEnv; struct SUdfInfo; @@ -109,25 +94,33 @@ void resetResultRowInfo(struct STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* int32_t numOfClosedResultRows(SResultRowInfo* pResultRowInfo); void closeAllResultRows(SResultRowInfo* pResultRowInfo); -int32_t initResultRow(SResultRow *pResultRow); -void closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot); -bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot); -void clearResultRow(struct STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow); +void initResultRow(SResultRow *pResultRow); +void closeResultRow(SResultRow* pResultRow); +bool isResultRowClosed(SResultRow* pResultRow); struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset); -void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr); int32_t getRowNumForMultioutput(struct STaskAttr* pQueryAttr, bool topBottomQuery, bool stable); -static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { - assert(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); - return pResultRowInfo->pResult[slot]; +static FORCE_INLINE SResultRow *getResultRow(SDiskbasedBuf* pBuf, SResultRowInfo *pResultRowInfo, int32_t slot) { + ASSERT(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); + SResultRowPosition* pos = &pResultRowInfo->pPosition[slot]; + + SFilePage* bufPage = getBufPage(pBuf, pos->pageId); + SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset); + return pRow; +} + +static FORCE_INLINE SResultRow *getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos) { + SFilePage* bufPage = getBufPage(pBuf, pos->pageId); + SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset); + return pRow; } static FORCE_INLINE char* getPosInResultPage(struct STaskAttr* pQueryAttr, SFilePage* page, int32_t rowOffset, int32_t offset) { assert(rowOffset >= 0 && pQueryAttr != NULL); - + ASSERT(0); // int32_t numOfRows = (int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); // return ((char *)page->data) + rowOffset + offset * numOfRows; } @@ -139,22 +132,11 @@ static FORCE_INLINE char* getPosInResultPage_rv(SFilePage* page, int32_t rowOffs return (char*) page + rowOffset + offset * numOfRows; } -//bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); -//bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); - -__filter_func_t getFilterOperator(int32_t lowerOptr, int32_t upperOptr); - -SResultRow* getNewResultRow(SResultRowPool* p); - typedef struct { SArray* pResult; // SArray int32_t colId; } SStddevInterResult; -void interResToBinary(SBufferWriter* bw, SArray* pRes, int32_t tagLen); -SArray* interResFromBinary(const char* data, int32_t len); -void freeInterResult(void* param); - void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo); void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index ce0069fdfa..a92813812a 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -69,7 +69,7 @@ enum { typedef struct SResultRowCell { uint64_t groupId; - SResultRow* pRow; + SResultRowPosition pos; } SResultRowCell; /** @@ -277,8 +277,6 @@ typedef struct STaskRuntimeEnv { char* keyBuf; // window key buffer // The window result objects pool, all the resultRow Objects are allocated and managed by this object. char** prevRow; - SResultRowPool* pool; - SArray* prevResult; // intermediate result, SArray STSBuf* pTsBuf; // timestamp filter list STSCursor cur; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 93fbfe43ce..40e86c7840 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -53,15 +53,13 @@ int32_t getOutputInterResultBufSize(STaskAttr* pQueryAttr) { int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size) { pResultRowInfo->size = 0; - pResultRowInfo->curPos = -1; + pResultRowInfo->curPos = -1; pResultRowInfo->capacity = size; - - pResultRowInfo->pResult = taosMemoryCalloc(pResultRowInfo->capacity, POINTER_BYTES); pResultRowInfo->pPosition = taosMemoryCalloc(pResultRowInfo->capacity, sizeof(SResultRowPosition)); - if (pResultRowInfo->pResult == NULL || pResultRowInfo->pPosition == NULL) { + + if (pResultRowInfo->pPosition == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - return TSDB_CODE_SUCCESS; } @@ -71,17 +69,17 @@ void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) { } if (pResultRowInfo->capacity == 0) { - assert(pResultRowInfo->pResult == NULL); +// assert(pResultRowInfo->pResult == NULL); return; } for(int32_t i = 0; i < pResultRowInfo->size; ++i) { - if (pResultRowInfo->pResult[i]) { - taosMemoryFreeClear(pResultRowInfo->pResult[i]->key); - } +// if (pResultRowInfo->pResult[i]) { +// taosMemoryFreeClear(pResultRowInfo->pResult[i]->key); +// } } - taosMemoryFreeClear(pResultRowInfo->pResult); + taosMemoryFreeClear(pResultRowInfo->pPosition); } void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { @@ -90,8 +88,8 @@ void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow } for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - SResultRow *pWindowRes = pResultRowInfo->pResult[i]; - clearResultRow(pRuntimeEnv, pWindowRes); +// SResultRow *pWindowRes = pResultRowInfo->pResult[i]; +// clearResultRow(pRuntimeEnv, pWindowRes); int32_t groupIndex = 0; int64_t uid = 0; @@ -101,14 +99,13 @@ void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow } pResultRowInfo->size = 0; - pResultRowInfo->curPos = -1; } int32_t numOfClosedResultRows(SResultRowInfo *pResultRowInfo) { int32_t i = 0; - while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) { - ++i; - } +// while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) { +// ++i; +// } return i; } @@ -117,21 +114,22 @@ void closeAllResultRows(SResultRowInfo *pResultRowInfo) { assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - SResultRow* pRow = pResultRowInfo->pResult[i]; - if (pRow->closed) { - continue; - } +// ASSERT(0); +// SResultRow* pRow = pResultRowInfo->pResult[i]; +// if (pRow->closed) { +// continue; +// } - pRow->closed = true; +// pRow->closed = true; } } -bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot) { - return (getResultRow(pResultRowInfo, slot)->closed == true); +bool isResultRowClosed(SResultRow* pRow) { + return (pRow->closed == true); } -void closeResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { - getResultRow(pResultRowInfo, slot)->closed = true; +void closeResultRow(SResultRow* pResultRow) { + pResultRow->closed = true; } void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow) { @@ -181,29 +179,6 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { return rowSize; } -SResultRow* getNewResultRow(SResultRowPool* p) { - if (p == NULL) { - return NULL; - } - - void* ptr = NULL; - if (p->position.pos == 0) { - ptr = taosMemoryCalloc(1, p->blockSize); - taosArrayPush(p->pData, &ptr); - - } else { - size_t last = taosArrayGetSize(p->pData); - - void** pBlock = taosArrayGet(p->pData, last - 1); - ptr = ((char*) (*pBlock)) + p->elemSize * p->position.pos; - } - - p->position.pos = (p->position.pos + 1)%p->numOfElemPerBlock; - initResultRow(ptr); - - return ptr; -} - void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { assert(pGroupResInfo != NULL); @@ -261,8 +236,9 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { return (int32_t) taosArrayGetSize(pGroupResInfo->pRows); } -static int64_t getNumOfResultWindowRes(STaskRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow, int32_t* rowCellInfoOffset) { +static int64_t getNumOfResultWindowRes(STaskRuntimeEnv* pRuntimeEnv, SResultRowPosition *pos, int32_t* rowCellInfoOffset) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + ASSERT(0); for (int32_t j = 0; j < pQueryAttr->numOfOutput; ++j) { int32_t functionId = 0;//pQueryAttr->pExpr1[j].base.functionId; @@ -305,25 +281,26 @@ static int32_t tableResultComparFn(const void *pLeft, const void *pRight, void * return -1; } + ASSERT(0); STableQueryInfo** pList = supporter->pTableQueryInfo; - SResultRow* pWindowRes1 = pList[left]->resInfo.pResult[leftPos]; +// SResultRow* pWindowRes1 = pList[left]->resInfo.pResult[leftPos]; // SResultRow * pWindowRes1 = getResultRow(&(pList[left]->resInfo), leftPos); - TSKEY leftTimestamp = pWindowRes1->win.skey; +// TSKEY leftTimestamp = pWindowRes1->win.skey; // SResultRowInfo *pWindowResInfo2 = &(pList[right]->resInfo); // SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos); - SResultRow* pWindowRes2 = pList[right]->resInfo.pResult[rightPos]; - TSKEY rightTimestamp = pWindowRes2->win.skey; +// SResultRow* pWindowRes2 = pList[right]->resInfo.pResult[rightPos]; +// TSKEY rightTimestamp = pWindowRes2->win.skey; - if (leftTimestamp == rightTimestamp) { +// if (leftTimestamp == rightTimestamp) { return 0; - } +// } - if (supporter->order == TSDB_ORDER_ASC) { - return (leftTimestamp > rightTimestamp)? 1:-1; - } else { - return (leftTimestamp < rightTimestamp)? 1:-1; - } +// if (supporter->order == TSDB_ORDER_ASC) { +// return (leftTimestamp > rightTimestamp)? 1:-1; +// } else { +// return (leftTimestamp < rightTimestamp)? 1:-1; +// } } int32_t tsAscOrder(const void* p1, const void* p2) { @@ -331,11 +308,12 @@ int32_t tsAscOrder(const void* p1, const void* p2) { SResultRowCell* pc2 = (SResultRowCell*) p2; if (pc1->groupId == pc2->groupId) { - if (pc1->pRow->win.skey == pc2->pRow->win.skey) { - return 0; - } else { - return (pc1->pRow->win.skey < pc2->pRow->win.skey)? -1:1; - } + ASSERT(0); +// if (pc1->pRow->win.skey == pc2->pRow->win.skey) { +// return 0; +// } else { +// return (pc1->pRow->win.skey < pc2->pRow->win.skey)? -1:1; +// } } else { return (pc1->groupId < pc2->groupId)? -1:1; } @@ -346,11 +324,12 @@ int32_t tsDescOrder(const void* p1, const void* p2) { SResultRowCell* pc2 = (SResultRowCell*) p2; if (pc1->groupId == pc2->groupId) { - if (pc1->pRow->win.skey == pc2->pRow->win.skey) { - return 0; - } else { - return (pc1->pRow->win.skey < pc2->pRow->win.skey)? 1:-1; - } + ASSERT(0); +// if (pc1->pRow->win.skey == pc2->pRow->win.skey) { +// return 0; +// } else { +// return (pc1->pRow->win.skey < pc2->pRow->win.skey)? 1:-1; +// } } else { return (pc1->groupId < pc2->groupId)? -1:1; } @@ -384,13 +363,13 @@ static int32_t mergeIntoGroupResultImplRv(STaskRuntimeEnv *pRuntimeEnv, SGroupRe break; } - int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pResultRowCell->pRow, rowCellInfoOffset); + int64_t num = getNumOfResultWindowRes(pRuntimeEnv, &pResultRowCell->pos, rowCellInfoOffset); if (num <= 0) { continue; } - taosArrayPush(pGroupResInfo->pRows, &pResultRowCell->pRow); - pResultRowCell->pRow->numOfRows = (uint32_t) num; + taosArrayPush(pGroupResInfo->pRows, &pResultRowCell->pos); +// pResultRowCell->pRow->numOfRows = (uint32_t) num; } return TSDB_CODE_SUCCESS; @@ -449,9 +428,10 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv int32_t tableIndex = tMergeTreeGetChosenIndex(pTree); SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->resInfo; - SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]); + ASSERT(0); + SResultRow *pWindowRes = NULL;//getResultRow(pBuf, pWindowResInfo, cs.rowIndex[tableIndex]); - int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes, rowCellInfoOffset); + int64_t num = 0;//getNumOfResultWindowRes(pRuntimeEnv, pWindowRes, rowCellInfoOffset); if (num <= 0) { cs.rowIndex[tableIndex] += 1; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 6265fe6123..d97b3183f2 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -426,18 +426,10 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, jmp_buf env) newCapacity += 4; } - char* t = taosMemoryRealloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); - if (t == NULL) { - longjmp(env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - pResultRowInfo->pPosition = taosMemoryRealloc(pResultRowInfo->pPosition, newCapacity * sizeof(SResultRowPosition)); - pResultRowInfo->pResult = (SResultRow**)t; int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; - memset(&pResultRowInfo->pResult[pResultRowInfo->capacity], 0, POINTER_BYTES * inc); memset(&pResultRowInfo->pPosition[pResultRowInfo->capacity], 0, sizeof(SResultRowPosition)); - pResultRowInfo->capacity = (int32_t)newCapacity; } @@ -458,9 +450,8 @@ static bool chkResultRowFromKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pR if (p1 != NULL) { if (pResultRowInfo->size == 0) { existed = false; - assert(pResultRowInfo->curPos == -1); } else if (pResultRowInfo->size == 1) { - existed = (pResultRowInfo->pResult[0] == (*p1)); +// existed = (pResultRowInfo->pResult[0] == (*p1)); } else { // check if current pResultRowInfo contains the existed pResultRow SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid, pResultRowInfo); int64_t* index = @@ -479,6 +470,7 @@ static bool chkResultRowFromKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pR return p1 != NULL; } +#if 0 static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, int64_t tid, char* pData, int16_t bytes, bool masterscan, uint64_t tableGroupId) { bool existed = false; @@ -496,16 +488,16 @@ static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultR if (p1 != NULL) { if (pResultRowInfo->size == 0) { existed = false; - assert(pResultRowInfo->curPos == -1); +// assert(pResultRowInfo->curPos == -1); } else if (pResultRowInfo->size == 1) { - existed = (pResultRowInfo->pResult[0] == (*p1)); - pResultRowInfo->curPos = 0; +// existed = (pResultRowInfo->pResult[0] == (*p1)); +// pResultRowInfo->curPos = 0; } else { // check if current pResultRowInfo contains the existed pResultRow SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); if (index != NULL) { - pResultRowInfo->curPos = (int32_t)*index; +// pResultRowInfo->curPos = (int32_t)*index; existed = true; } else { existed = false; @@ -555,6 +547,7 @@ static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultR return pResultRowInfo->pResult[pResultRowInfo->curPos]; } +#endif SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize) { SFilePage* pData = NULL; @@ -599,65 +592,75 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, int64_t tid, char* pData, int16_t bytes, bool masterscan, uint64_t tableGroupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup) { - bool existed = false; + bool existInCurrentResusltRowInfo = false; SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, tableGroupId); - SResultRow** p1 = (SResultRow**)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); // in case of repeat scan/reverse scan, no new time window added. if (isIntervalQuery) { if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. - return (p1 != NULL) ? *p1 : NULL; + if (p1 != NULL) { + return getResultRowByPos(pResultBuf, p1); + } else { + return NULL; + } } if (p1 != NULL) { if (pResultRowInfo->size == 0) { - existed = false; + existInCurrentResusltRowInfo = false; // this time window created by other timestamp that does not belongs to current table. assert(pResultRowInfo->curPos == -1); } else if (pResultRowInfo->size == 1) { - existed = (pResultRowInfo->pResult[0] == (*p1)); - pResultRowInfo->curPos = 0; - } else { // check if current pResultRowInfo contains the existed pResultRow + ASSERT(0); +// existInCurrentResusltRowInfo = (pResultRowInfo->pResult[0] == (*p1)); + } else { // check if current pResultRowInfo contains the existInCurrentResusltRowInfo pResultRow SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo); int64_t* index = taosHashGet(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); if (index != NULL) { - pResultRowInfo->curPos = (int32_t)*index; - existed = true; + // TODO check the scan order for current opened time window +// pResultRowInfo->curPos = (int32_t)*index; + existInCurrentResusltRowInfo = true; } else { - existed = false; + existInCurrentResusltRowInfo = false; } } } } else { - // In case of group by column query, the required SResultRow object must be existed in the pResultRowInfo object. + // In case of group by column query, the required SResultRow object must be existInCurrentResusltRowInfo in the pResultRowInfo object. if (p1 != NULL) { - return *p1; + return getResultRowByPos(pResultBuf, p1); } } - if (!existed) { - prepareResultListBuffer(pResultRowInfo, pTaskInfo->env); - - SResultRow* pResult = NULL; - if (p1 == NULL) { - pResult = getNewResultRow_rv(pResultBuf, tableGroupId, pSup->resultRowSize); - int32_t ret = initResultRow(pResult); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - // add a new result set for a new group - taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); - SResultRowCell cell = {.groupId = tableGroupId, .pRow = pResult}; - taosArrayPush(pSup->pResultRowArrayList, &cell); - } else { - pResult = *p1; + SResultRow* pResult = NULL; + if (!existInCurrentResusltRowInfo) { + // 1. close current opened time window + if (pResultRowInfo->curPos != -1) { // todo extract function + SResultRowPosition* pos = &pResultRowInfo->pPosition[pResultRowInfo->curPos]; + SFilePage* pPage = getBufPage(pResultBuf, pos->pageId); + SResultRow* pRow = (SResultRow*)((char*)pPage + pos->offset); + closeResultRow(pRow); + releaseBufPage(pResultBuf, pPage); } + prepareResultListBuffer(pResultRowInfo, pTaskInfo->env); + if (p1 == NULL) { + pResult = getNewResultRow_rv(pResultBuf, tableGroupId, pSup->resultRowSize); + initResultRow(pResult); + + // add a new result set for a new group + SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; + taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos, POINTER_BYTES); + SResultRowCell cell = {.groupId = tableGroupId, .pos = pos}; + taosArrayPush(pSup->pResultRowArrayList, &cell); + } else { + pResult = getResultRowByPos(pResultBuf, p1); + } + + // 2. set the new time window to be the new active time window pResultRowInfo->curPos = pResultRowInfo->size; - pResultRowInfo->pPosition[pResultRowInfo->size] = - (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; - pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; + pResultRowInfo->pPosition[pResultRowInfo->size++] = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; int64_t index = pResultRowInfo->curPos; SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo); @@ -669,7 +672,7 @@ static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultR longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); } - return pResultRowInfo->pResult[pResultRowInfo->curPos]; + return pResult; } static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, TSKEY ekey, @@ -693,7 +696,7 @@ static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, T } // get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, +static STimeWindow getActiveTimeWindow(SDiskbasedBuf * pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t precision, STimeWindow* win) { STimeWindow w = {0}; @@ -701,7 +704,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo* pResultRowInfo, int64_t t getInitialStartTimeWindow(pInterval, precision, ts, &w, win->ekey, true); w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { - w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; + w = getResultRow(pBuf, pResultRowInfo, pResultRowInfo->curPos)->win; } if (w.skey > ts || w.ekey < ts) { @@ -730,7 +733,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo* pResultRowInfo, int64_t t // get the correct time window according to the handled timestamp static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo* pResultRowInfo, int64_t ts, STaskAttr* pQueryAttr) { STimeWindow w = {0}; - +#if 0 if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value // getInitialStartTimeWindow(pQueryAttr, ts, &w); @@ -742,7 +745,7 @@ static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo* pResultRowInfo, in w.ekey = w.skey + pQueryAttr->interval.interval - 1; } } else { - w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; + w = pRow->win; } /* @@ -752,6 +755,7 @@ static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo* pResultRowInfo, in if (w.ekey > pQueryAttr->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) { w.ekey = pQueryAttr->window.ekey; } +#endif return w; } @@ -816,8 +820,8 @@ static int32_t setResultOutputBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowI assert(win->skey <= win->ekey); SDiskbasedBuf* pResultBuf = pRuntimeEnv->pResultBuf; - SResultRow* pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char*)&win->skey, TSDB_KEYSIZE, - masterscan, tableGroupId); + SResultRow* pResultRow = NULL;//doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char*)&win->skey, TSDB_KEYSIZE, +// masterscan, tableGroupId); if (pResultRow == NULL) { *pResult = NULL; return TSDB_CODE_SUCCESS; @@ -909,9 +913,9 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se return forwardStep; } -static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey, bool ascQuery, - bool timeWindowInterpo) { +static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) { int64_t skey = TSKEY_INITIAL_VAL; +#if 0 int32_t i = 0; for (i = pResultRowInfo->size - 1; i >= 0; --i) { SResultRow* pResult = pResultRowInfo->pResult[i]; @@ -963,6 +967,7 @@ static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey pResultRowInfo->curPos = i + 1; // current not closed result object } } +#endif } static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, const STimeWindow* pWin, TSKEY lastKey, @@ -1551,14 +1556,14 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); tsCols = (int64_t*)pColDataInfo->pData; -// assert(tsCols[0] == pSDataBlock->info.window.skey && -// tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); +// assert(tsCols[0] == pSDataBlock->info.window.skey && tsCols[pSDataBlock->info.rows - 1] == +// pSDataBlock->info.window.ekey); } int32_t startPos = ascScan? 0 : (pSDataBlock->info.rows - 1); TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan); - STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, &pInfo->win); + STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, &pInfo->win); bool masterScan = true; SResultRow* pResult = NULL; @@ -1581,6 +1586,8 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe // prev time window not interpolation yet. int32_t curIndex = pResultRowInfo->curPos; + +#if 0 if (prevIndex != -1 && prevIndex < curIndex && pInfo->timeWindowInterpo) { for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. SResultRow* pRes = getResultRow(pResultRowInfo, j); @@ -1615,6 +1622,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } } +#endif // window start key interpolation doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, @@ -3430,10 +3438,8 @@ void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } } -// TODO fix this bug. -int32_t initResultRow(SResultRow* pResultRow) { +void initResultRow(SResultRow* pResultRow) { pResultRow->pEntryInfo = (struct SResultRowEntryInfo*)((char*)pResultRow + sizeof(SResultRow)); - return TSDB_CODE_SUCCESS; } /* @@ -3610,9 +3616,11 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD SFilePage* bufPage = getBufPage(pBuf, pPos->pageId); SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->offset); - if (!isResultRowClosed(pResultRowInfo, i)) { - continue; - } + + // TODO ignore the close status anyway. +// if (!isResultRowClosed(pRow)) { +// continue; +// } for (int32_t j = 0; j < numOfOutput; ++j) { pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset); @@ -3622,7 +3630,7 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD continue; } - if (pCtx[j].fpSet.process) { // TODO set the dummy function. + if (pCtx[j].fpSet.process) { // TODO set the dummy function, to avoid the check for null ptr. pCtx[j].fpSet.finalize(&pCtx[j]); } @@ -4132,7 +4140,7 @@ static void updateNumOfRowsInResultRows(SqlFunctionCtx* pCtx, int32_t numOfOutpu // if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { // return; // } - +#if 0 for (int32_t i = 0; i < pResultRowInfo->size; ++i) { SResultRow* pResult = pResultRowInfo->pResult[i]; @@ -4146,6 +4154,8 @@ static void updateNumOfRowsInResultRows(SqlFunctionCtx* pCtx, int32_t numOfOutpu pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes)); } } +#endif + } static int32_t compressQueryColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, int8_t compressed) { From be2ab1f4f82f3bbd2c051d8d8603ff1ac9f7257d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 31 Mar 2022 16:16:24 +0800 Subject: [PATCH 19/38] shm --- source/util/src/tprocess.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index 71f819dbc1..1d41bd4a48 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -224,7 +224,6 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int16_t *pHea taosThreadMutexLock(&pQueue->mutex); if (pQueue->total - pQueue->avail <= 0) { taosThreadMutexUnlock(&pQueue->mutex); - tsem_post(&pQueue->sem); terrno = TSDB_CODE_OUT_OF_SHM_MEM; return 0; } @@ -378,7 +377,7 @@ static void taosProcThreadLoop(SProcObj *pProc) { int32_t numOfMsgs = taosProcQueuePop(pQueue, &pHead, &headLen, &pBody, &bodyLen, &ftype, mallocHeadFp, freeHeadFp, mallocBodyFp, freeBodyFp); if (numOfMsgs == 0) { - uInfo("proc:%s, get no msg from queue:%p and exit the proc thread", pProc->name, pQueue); + uDebug("proc:%s, get no msg from queue:%p and exit the proc thread", pProc->name, pQueue); break; } else if (numOfMsgs < 0) { uTrace("proc:%s, get no msg from queue:%p since %s", pProc->name, pQueue, terrstr()); From 68f2911ce2657345166b8a11326516e03ba6467e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 31 Mar 2022 16:19:26 +0800 Subject: [PATCH 20/38] rename files --- source/dnode/mgmt/main/src/{dndInt.c => dndEnv.c} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename source/dnode/mgmt/main/src/{dndInt.c => dndEnv.c} (100%) diff --git a/source/dnode/mgmt/main/src/dndInt.c b/source/dnode/mgmt/main/src/dndEnv.c similarity index 100% rename from source/dnode/mgmt/main/src/dndInt.c rename to source/dnode/mgmt/main/src/dndEnv.c From bee82c6ff936da4c6b406f49e3d1d940ae029e96 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 31 Mar 2022 16:19:47 +0800 Subject: [PATCH 21/38] rename files --- source/dnode/mgmt/main/src/{dndObj.c => dndInt.c} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename source/dnode/mgmt/main/src/{dndObj.c => dndInt.c} (100%) diff --git a/source/dnode/mgmt/main/src/dndObj.c b/source/dnode/mgmt/main/src/dndInt.c similarity index 100% rename from source/dnode/mgmt/main/src/dndObj.c rename to source/dnode/mgmt/main/src/dndInt.c From 8ab075c342677a6354452c1af9b8e9170f1f945b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 31 Mar 2022 16:24:59 +0800 Subject: [PATCH 22/38] [td-14373]fix bug. --- source/dnode/mnode/impl/src/mndInfoSchema.c | 1 - source/dnode/mnode/impl/src/mndStb.c | 4 ---- 2 files changed, 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index ce08dfaaa3..b46920bc5e 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -89,7 +89,6 @@ static const SInfosTableSchema userStbsSchema[] = { {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_INT}, }; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 02e09252f9..196767462e 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1652,10 +1652,6 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32 *(int32_t *)pWrite = pStb->numOfTags; cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = 0; // number of tables - cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int64_t *)pWrite = pStb->updateTime; // number of tables cols++; From 810cfefddc57239176b51114919a1d1d2fc4935a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 31 Mar 2022 16:28:59 +0800 Subject: [PATCH 23/38] [td-13039] fix bug found by compiling. --- source/libs/executor/inc/executil.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index c09b41f0f6..3faf208c0e 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -106,13 +106,13 @@ static FORCE_INLINE SResultRow *getResultRow(SDiskbasedBuf* pBuf, SResultRowInfo ASSERT(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); SResultRowPosition* pos = &pResultRowInfo->pPosition[slot]; - SFilePage* bufPage = getBufPage(pBuf, pos->pageId); + SFilePage* bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId); SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset); return pRow; } static FORCE_INLINE SResultRow *getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos) { - SFilePage* bufPage = getBufPage(pBuf, pos->pageId); + SFilePage* bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId); SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset); return pRow; } From 781d31d39f71747043fca9c23702cdfc7a824af5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 31 Mar 2022 16:41:27 +0800 Subject: [PATCH 24/38] adjust log --- source/dnode/mgmt/main/src/dndExec.c | 2 +- source/util/src/tconfig.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/mgmt/main/src/dndExec.c b/source/dnode/mgmt/main/src/dndExec.c index 2999a30d67..c5d5707a99 100644 --- a/source/dnode/mgmt/main/src/dndExec.c +++ b/source/dnode/mgmt/main/src/dndExec.c @@ -276,8 +276,8 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) { } static int32_t dndRunInChildProcess(SDnode *pDnode) { - dInfo("dnode run in child process"); SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype]; + dInfo("%s run in child process", pWrapper->name); SMsgCb msgCb = dndCreateMsgcb(pWrapper); tmsgSetDefaultMsgCb(&msgCb); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 04061cbaf1..9101d3c7c8 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -590,12 +590,12 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { } int32_t cfgLoadFromEnvVar(SConfig *pConfig) { - uInfo("load from global env variables success"); + uInfo("load from global env variables not implemented yet"); return 0; } int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) { - uInfo("load from env file [%s] success", filepath); + uInfo("load from env file not implemented yet"); return 0; } @@ -649,11 +649,11 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { taosCloseFile(&pFile); if (line != NULL) taosMemoryFreeClear(line); - uInfo("load from cfg file [%s] success", filepath); + uInfo("load from cfg file %s success", filepath); return 0; } int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { - uInfo("load from apoll url [%s] success", url); + uInfo("load from apoll url not implemented yet"); return 0; } From d02850382c5d8704a742290ba0b447cfc4cdf1c7 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 31 Mar 2022 16:45:16 +0800 Subject: [PATCH 25/38] fix serialize --- source/dnode/vnode/src/vnd/vnodeWrite.c | 2 +- source/libs/scheduler/src/scheduler.c | 286 ++++++++++++------------ 2 files changed, 149 insertions(+), 139 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 0a6ed6bbe9..38eed46c9d 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -165,7 +165,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // } break; case TDMT_VND_SUBMIT: - printf("vnode %d write data %ld\n", pVnode->vgId, ver); + /*printf("vnode %d write data %ld\n", pVnode->vgId, ver);*/ if (pVnode->config.streamMode == 0) { if (tsdbInsertData(pVnode->pTsdb, (SSubmitReq *)ptr, NULL) < 0) { // TODO: handle error diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 5af13d97ca..61462f1dc3 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -75,10 +75,10 @@ void schFreeRpcCtx(SRpcCtx *pCtx) { SRpcCtxVal *ctxVal = (SRpcCtxVal *)pIter; (*ctxVal->freeFunc)(ctxVal->val); - + pIter = taosHashIterate(pCtx->args, pIter); } - + taosHashCleanup(pCtx->args); if (pCtx->brokenVal.freeFunc) { @@ -86,7 +86,7 @@ void schFreeRpcCtx(SRpcCtx *pCtx) { } } -void schFreeTask(SSchTask* pTask) { +void schFreeTask(SSchTask *pTask) { if (pTask->candidateAddrs) { taosArrayDestroy(pTask->candidateAddrs); } @@ -125,41 +125,47 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m case TDMT_SCH_LINK_BROKEN: return TSDB_CODE_SUCCESS; case TDMT_VND_QUERY_RSP: // query_rsp may be processed later than ready_rsp - if (lastMsgType != reqMsgType && -1 != lastMsgType && TDMT_VND_FETCH != lastMsgType) { - SCH_TASK_DLOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); + if (lastMsgType != reqMsgType && -1 != lastMsgType && TDMT_VND_FETCH != lastMsgType) { + SCH_TASK_DLOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), + TMSG_INFO(msgType)); } - + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_TASK_DLOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); + SCH_TASK_DLOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), + TMSG_INFO(msgType)); } - + SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); return TSDB_CODE_SUCCESS; case TDMT_VND_RES_READY_RSP: reqMsgType = TDMT_VND_QUERY; if (lastMsgType != reqMsgType && -1 != lastMsgType) { - SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", (lastMsgType > 0 ? TMSG_INFO(lastMsgType) : "null"), TMSG_INFO(msgType)); + SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", + (lastMsgType > 0 ? TMSG_INFO(lastMsgType) : "null"), TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); + SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), + TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); return TSDB_CODE_SUCCESS; case TDMT_VND_FETCH_RSP: - if (lastMsgType != reqMsgType && -1 != lastMsgType) { - SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); + if (lastMsgType != reqMsgType && -1 != lastMsgType) { + SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), + TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); + SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), + TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); return TSDB_CODE_SUCCESS; case TDMT_VND_CREATE_TABLE_RSP: @@ -171,12 +177,14 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m } if (lastMsgType != reqMsgType) { - SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); + SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), + TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); + SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), + TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -357,7 +365,7 @@ int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, void *handle) { SSchNodeInfo nodeInfo = {.addr = *addr, .handle = handle}; - + if (NULL == taosArrayPush(pTask->execNodes, &nodeInfo)) { SCH_TASK_ELOG("taosArrayPush nodeInfo to execNodes list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -587,7 +595,7 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - + SCH_TASK_WLOG("task already in failTask list, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -612,7 +620,7 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - + SCH_TASK_ELOG("task already in execTask list, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -631,7 +639,7 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) { int8_t status = 0; ++pTask->tryTimes; - + if (schJobNeedToStop(pJob, &status)) { *needRetry = false; SCH_TASK_DLOG("task no more retry cause of job status, job status:%s", jobTaskStatusStr(status)); @@ -643,7 +651,7 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo SCH_TASK_DLOG("task no more retry since reach max try times, tryTimes:%d", pTask->tryTimes); return TSDB_CODE_SUCCESS; } - + if (!NEED_SCHEDULER_RETRY_ERROR(errCode)) { *needRetry = false; SCH_TASK_DLOG("task no more retry cause of errCode, errCode:%x - %s", errCode, tstrerror(errCode)); @@ -654,7 +662,8 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo if (SCH_IS_DATA_SRC_TASK(pTask)) { if (pTask->tryTimes >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) { *needRetry = false; - SCH_TASK_DLOG("task no more retry since all ep tried, tryTimes:%d, epNum:%d", pTask->tryTimes, SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)); + SCH_TASK_DLOG("task no more retry since all ep tried, tryTimes:%d, epNum:%d", pTask->tryTimes, + SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)); return TSDB_CODE_SUCCESS; } } else { @@ -662,14 +671,15 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo if ((pTask->candidateIdx + 1) >= candidateNum) { *needRetry = false; - SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d", pTask->candidateIdx, candidateNum); + SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d", + pTask->candidateIdx, candidateNum); return TSDB_CODE_SUCCESS; } } *needRetry = true; SCH_TASK_DLOG("task need the %dth retry, errCode:%x - %s", pTask->tryTimes, errCode, tstrerror(errCode)); - + return TSDB_CODE_SUCCESS; } @@ -706,9 +716,8 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { memcpy(&hb->trans, trans, sizeof(*trans)); SCH_UNLOCK(SCH_WRITE, &hb->lock); - qDebug("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, instance:%p, handle:%p", - schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->transInst, - trans->transHandle); + qDebug("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, instance:%p, handle:%p", schMgmt.sId, + epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->transInst, trans->transHandle); return TSDB_CODE_SUCCESS; } @@ -730,15 +739,15 @@ void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode) { if (NEED_CLIENT_HANDLE_ERROR(origCode)) { return; } - + if (NEED_CLIENT_HANDLE_ERROR(errCode)) { atomic_store_32(&pJob->errCode, errCode); goto _return; } return; - -_return: + +_return: SCH_JOB_DLOG("job errCode updated to %x - %s", errCode, tstrerror(errCode)); } @@ -825,7 +834,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) } SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_FAILED); - + if (SCH_IS_WAIT_ALL_JOB(pJob)) { SCH_LOCK(SCH_WRITE, &pTask->level->lock); pTask->level->taskFailed++; @@ -833,9 +842,9 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) SCH_UNLOCK(SCH_WRITE, &pTask->level->lock); schUpdateJobErrCode(pJob, errCode); - + if (taskDone < pTask->level->taskNum) { - SCH_TASK_DLOG("need to wait other tasks, doneNum:%d, allNum:%d", taskDone, pTask->level->taskNum); + SCH_TASK_DLOG("need to wait other tasks, doneNum:%d, allNum:%d", taskDone, pTask->level->taskNum); SCH_RET(errCode); } } @@ -867,7 +876,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { int32_t parentNum = pTask->parents ? (int32_t)taosArrayGetSize(pTask->parents) : 0; if (parentNum == 0) { - int32_t taskDone = 0; + int32_t taskDone = 0; if (SCH_IS_WAIT_ALL_JOB(pJob)) { SCH_LOCK(SCH_WRITE, &pTask->level->lock); pTask->level->taskSucceed++; @@ -965,7 +974,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch int8_t status = 0; if (schJobNeedToStop(pJob, &status)) { - SCH_TASK_ELOG("rsp not processed cause of job status, job status:%s, rspCode:0x%x", jobTaskStatusStr(status), rspCode); + SCH_TASK_ELOG("rsp not processed cause of job status, job status:%s, rspCode:0x%x", jobTaskStatusStr(status), + rspCode); SCH_RET(atomic_load_32(&pJob->errCode)); } @@ -985,11 +995,11 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_JRET(rsp->code); } } - + taosArrayDestroy(batchRsp.rspList); } - } - + } + SCH_ERR_JRET(rspCode); SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); break; @@ -1011,21 +1021,21 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch break; } case TDMT_VND_QUERY_RSP: { - SQueryTableRsp rsp = {0}; - if (msg) { - tDeserializeSQueryTableRsp(msg, msgSize, &rsp); - SCH_ERR_JRET(rsp.code); - } - - SCH_ERR_JRET(rspCode); - - if (NULL == msg) { - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - - //SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); - - break; + SQueryTableRsp rsp = {0}; + if (msg) { + tDeserializeSQueryTableRsp(msg, msgSize, &rsp); + SCH_ERR_JRET(rsp.code); + } + + SCH_ERR_JRET(rspCode); + + if (NULL == msg) { + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + // SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); + + break; } case TDMT_VND_RES_READY_RSP: { SResReadyRsp *rsp = (SResReadyRsp *)msg; @@ -1088,14 +1098,14 @@ _return: } int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode) { - int32_t code = 0; + int32_t code = 0; SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - SSchTask *pTask = NULL; + SSchTask *pTask = NULL; SSchJob *pJob = schAcquireJob(pParam->refId); if (NULL == pJob) { qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "taosAcquireRef job failed, may be dropped, refId:%" PRIx64, - pParam->queryId, pParam->taskId, pParam->refId); + pParam->queryId, pParam->taskId, pParam->refId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); } @@ -1114,7 +1124,7 @@ int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, in pTask = *task; SCH_TASK_DLOG("rsp msg received, type:%s, handle:%p, code:%s", TMSG_INFO(msgType), pMsg->handle, tstrerror(rspCode)); - SCH_SET_TASK_HANDLE(pTask, pMsg->handle); + SCH_SET_TASK_HANDLE(pTask, pMsg->handle); SCH_ERR_JRET(schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode)); _return: @@ -1173,7 +1183,8 @@ int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { SCH_ERR_RET(schUpdateHbConnection(&rsp.epId, &trans)); int32_t taskNum = (int32_t)taosArrayGetSize(rsp.taskStatus); - qDebug("%d task status in hb rsp, nodeId:%d, fqdn:%s, port:%d", taskNum, rsp.epId.nodeId, rsp.epId.ep.fqdn, rsp.epId.ep.port); + qDebug("%d task status in hb rsp, nodeId:%d, fqdn:%s, port:%d", taskNum, rsp.epId.nodeId, rsp.epId.ep.fqdn, + rsp.epId.ep.port); for (int32_t i = 0; i < taskNum; ++i) { STaskStatus *taskStatus = taosArrayGet(rsp.taskStatus, i); @@ -1187,8 +1198,9 @@ int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { } // TODO - - SCH_JOB_DLOG("TID:0x%" PRIx64 " task status in server: %s", taskStatus->taskId, jobTaskStatusStr(taskStatus->status)); + + SCH_JOB_DLOG("TID:0x%" PRIx64 " task status in server: %s", taskStatus->taskId, + jobTaskStatusStr(taskStatus->status)); schReleaseJob(taskStatus->refId); } @@ -1208,7 +1220,7 @@ int32_t schHandleLinkBrokenCallback(void *param, const SDataBuf *pMsg, int32_t c if (head->isHbParam) { SSchHbCallbackParam *hbParam = (SSchHbCallbackParam *)param; - SSchTrans trans = {.transInst = hbParam->transport, .transHandle = NULL}; + SSchTrans trans = {.transInst = hbParam->transport, .transHandle = NULL}; SCH_ERR_RET(schUpdateHbConnection(&hbParam->nodeEpId, &trans)); SCH_ERR_RET(schBuildAndSendHbMsg(&hbParam->nodeEpId)); @@ -1219,7 +1231,6 @@ int32_t schHandleLinkBrokenCallback(void *param, const SDataBuf *pMsg, int32_t c return TSDB_CODE_SUCCESS; } - int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { switch (msgType) { case TDMT_VND_CREATE_TABLE: @@ -1258,8 +1269,8 @@ void schFreeRpcCtxVal(const void *arg) { if (NULL == arg) { return; } - - SMsgSendInfo* pMsgSendInfo = (SMsgSendInfo *)arg; + + SMsgSendInfo *pMsgSendInfo = (SMsgSendInfo *)arg; taosMemoryFreeClear(pMsgSendInfo->param); taosMemoryFreeClear(pMsgSendInfo); } @@ -1301,11 +1312,10 @@ int32_t schMakeHbCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) { return TSDB_CODE_SUCCESS; } - int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb) { - int32_t code = 0; - SMsgSendInfo* pMsgSendInfo = NULL; - + int32_t code = 0; + SMsgSendInfo *pMsgSendInfo = NULL; + pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); @@ -1318,17 +1328,17 @@ int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal * SCH_ERR_JRET(schMakeTaskCallbackParam(pJob, pTask, &pMsgSendInfo->param)); } - int32_t msgType = TDMT_SCH_LINK_BROKEN; + int32_t msgType = TDMT_SCH_LINK_BROKEN; __async_send_cb_fn_t fp = NULL; SCH_ERR_JRET(schGetCallbackFp(msgType, &fp)); - + pMsgSendInfo->fp = fp; brokenVal->msgType = msgType; brokenVal->val = pMsgSendInfo; brokenVal->clone = schCloneSMsgSendInfo; brokenVal->freeFunc = schFreeRpcCtxVal; - + return TSDB_CODE_SUCCESS; _return: @@ -1340,16 +1350,16 @@ _return: } int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { - int32_t code = 0; + int32_t code = 0; SSchTaskCallbackParam *param = NULL; - SMsgSendInfo* pMsgSendInfo = NULL; + SMsgSendInfo *pMsgSendInfo = NULL; pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pCtx->args) { SCH_TASK_ELOG("taosHashInit %d RpcCtx failed", 1); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); @@ -1362,7 +1372,7 @@ int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - int32_t msgType = TDMT_VND_RES_READY_RSP; + int32_t msgType = TDMT_VND_RES_READY_RSP; __async_send_cb_fn_t fp = NULL; SCH_ERR_JRET(schGetCallbackFp(TDMT_VND_RES_READY, &fp)); @@ -1370,7 +1380,7 @@ int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { param->refId = pJob->refId; param->taskId = SCH_TASK_ID(pTask); param->transport = pJob->transport; - + pMsgSendInfo->param = param; pMsgSendInfo->fp = fp; @@ -1394,11 +1404,11 @@ _return: } int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { - int32_t code = 0; + int32_t code = 0; SSchHbCallbackParam *param = NULL; - SMsgSendInfo* pMsgSendInfo = NULL; - SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); - SQueryNodeEpId epId = {0}; + SMsgSendInfo *pMsgSendInfo = NULL; + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; memcpy(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); @@ -1408,7 +1418,7 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SCH_TASK_ELOG("taosHashInit %d RpcCtx failed", 1); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); @@ -1421,13 +1431,13 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - int32_t msgType = TDMT_VND_QUERY_HEARTBEAT_RSP; + int32_t msgType = TDMT_VND_QUERY_HEARTBEAT_RSP; __async_send_cb_fn_t fp = NULL; SCH_ERR_JRET(schGetCallbackFp(TDMT_VND_QUERY_HEARTBEAT, &fp)); param->nodeEpId = epId; param->transport = pJob->transport; - + pMsgSendInfo->param = param; pMsgSendInfo->fp = fp; @@ -1450,19 +1460,18 @@ _return: SCH_RET(code); } - int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId, bool *exist) { - int32_t code = 0; + int32_t code = 0; SSchHbTrans hb = {0}; hb.trans.transInst = pJob->transport; - + SCH_ERR_RET(schMakeHbRpcCtx(pJob, pTask, &hb.rpcCtx)); code = taosHashPut(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId), &hb, sizeof(SSchHbTrans)); if (code) { schFreeRpcCtx(&hb.rpcCtx); - + if (HASH_NODE_EXIST(code)) { *exist = true; return TSDB_CODE_SUCCESS; @@ -1475,8 +1484,6 @@ int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId * return TSDB_CODE_SUCCESS; } - - int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHeader **pDst) { if (pSrc->isHbParam) { SSchHbCallbackParam *dst = taosMemoryMalloc(sizeof(SSchHbCallbackParam)); @@ -1496,16 +1503,16 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe qError("malloc SSchTaskCallbackParam failed"); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + memcpy(dst, pSrc, sizeof(*dst)); *pDst = (SSchCallbackParamHeader *)dst; - + return TSDB_CODE_SUCCESS; } int32_t schCloneSMsgSendInfo(void *src, void **dst) { SMsgSendInfo *pSrc = src; - int32_t code = 0; + int32_t code = 0; SMsgSendInfo *pDst = taosMemoryMalloc(sizeof(*pSrc)); if (NULL == pDst) { qError("malloc SMsgSendInfo for rpcCtx failed, len:%d", (int32_t)sizeof(*pSrc)); @@ -1520,7 +1527,7 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) { *dst = pDst; return TSDB_CODE_SUCCESS; - + _return: taosMemoryFreeClear(pDst); @@ -1531,7 +1538,7 @@ int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { int32_t code = 0; memcpy(&pDst->brokenVal, &pSrc->brokenVal, sizeof(pSrc->brokenVal)); pDst->brokenVal.val = NULL; - + SCH_ERR_RET(schCloneSMsgSendInfo(pSrc->brokenVal.val, &pDst->brokenVal.val)); pDst->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); @@ -1541,16 +1548,16 @@ int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { } SRpcCtxVal dst = {0}; - void *pIter = taosHashIterate(pSrc->args, NULL); + void *pIter = taosHashIterate(pSrc->args, NULL); while (pIter) { SRpcCtxVal *pVal = (SRpcCtxVal *)pIter; - int32_t *msgType = taosHashGetKey(pIter, NULL); + int32_t *msgType = taosHashGetKey(pIter, NULL); dst = *pVal; dst.val = NULL; - + SCH_ERR_JRET(schCloneSMsgSendInfo(pVal->val, &dst.val)); - + if (taosHashPut(pDst->args, msgType, sizeof(*msgType), &dst, sizeof(dst))) { qError("taosHashPut msg %d to rpcCtx failed", *msgType); (*dst.freeFunc)(dst.val); @@ -1568,8 +1575,8 @@ _return: SCH_RET(code); } - -int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet* epSet, int32_t msgType, void *msg, uint32_t msgSize, bool persistHandle, SRpcCtx *ctx) { +int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet *epSet, int32_t msgType, void *msg, + uint32_t msgSize, bool persistHandle, SRpcCtx *ctx) { int32_t code = 0; SSchTrans *trans = (SSchTrans *)transport; @@ -1601,11 +1608,11 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet* pMsgSendInfo->msgType = msgType; pMsgSendInfo->fp = fp; - qDebug("start to send %s msg to node[%d,%s,%d], refId:%" PRIx64 "instance:%p, handle:%p", - TMSG_INFO(msgType), ntohl(((SMsgHead *)msg)->vgId), epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, - pJob->refId, trans->transInst, trans->transHandle); - - int64_t transporterId = 0; + qDebug("start to send %s msg to node[%d,%s,%d], refId:%" PRIx64 "instance:%p, handle:%p", TMSG_INFO(msgType), + ntohl(((SMsgHead *)msg)->vgId), epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, pJob->refId, + trans->transInst, trans->transHandle); + + int64_t transporterId = 0; code = asyncSendMsgToServerExt(trans->transInst, epSet, &transporterId, pMsgSendInfo, persistHandle, ctx); if (code) { SCH_ERR_JRET(code); @@ -1623,18 +1630,19 @@ _return: int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId) { SSchedulerHbReq req = {0}; - int32_t code = 0; - SRpcCtx rpcCtx = {0}; - SSchTrans trans = {0}; - int32_t msgType = TDMT_VND_QUERY_HEARTBEAT; + int32_t code = 0; + SRpcCtx rpcCtx = {0}; + SSchTrans trans = {0}; + int32_t msgType = TDMT_VND_QUERY_HEARTBEAT; - req.header.vgId = htonl(nodeEpId->nodeId); + req.header.vgId = nodeEpId->nodeId; req.sId = schMgmt.sId; memcpy(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, nodeEpId, sizeof(SQueryNodeEpId)); if (NULL == hb) { - qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", nodeEpId->nodeId, nodeEpId->ep.fqdn, nodeEpId->ep.port); + qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", nodeEpId->nodeId, nodeEpId->ep.fqdn, + nodeEpId->ep.port); SCH_ERR_RET(code); } @@ -1642,9 +1650,9 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId) { code = schCloneHbRpcCtx(&hb->rpcCtx, &rpcCtx); memcpy(&trans, &hb->trans, sizeof(trans)); SCH_UNLOCK(SCH_WRITE, &hb->lock); - + SCH_ERR_RET(code); - + int32_t msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { qError("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); @@ -1655,7 +1663,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId) { qError("calloc hb req %d failed", msgSize); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { qError("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1684,17 +1692,18 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId) { pMsgSendInfo->msgInfo.handle = trans.transHandle; pMsgSendInfo->msgType = msgType; pMsgSendInfo->fp = fp; - - int64_t transporterId = 0; - SEpSet epSet = {.inUse = 0, .numOfEps = 1}; + + int64_t transporterId = 0; + SEpSet epSet = {.inUse = 0, .numOfEps = 1}; memcpy(&epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep)); - qDebug("start to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d", trans.transInst, trans.transHandle, nodeEpId->ep.fqdn, nodeEpId->ep.port); - + qDebug("start to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d", trans.transInst, trans.transHandle, + nodeEpId->ep.fqdn, nodeEpId->ep.port); + code = asyncSendMsgToServerExt(trans.transInst, &epSet, &transporterId, pMsgSendInfo, true, &rpcCtx); if (code) { - qError("fail to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d, error:%x - %s", - trans.transInst, trans.transHandle, nodeEpId->ep.fqdn, nodeEpId->ep.port, code, tstrerror(code)); + qError("fail to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d, error:%x - %s", trans.transInst, + trans.transHandle, nodeEpId->ep.fqdn, nodeEpId->ep.port, code, tstrerror(code)); SCH_ERR_JRET(code); } @@ -1712,12 +1721,12 @@ _return: int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { uint32_t msgSize = 0; - void *msg = NULL; - int32_t code = 0; - bool isCandidateAddr = false; - bool persistHandle = false; - SRpcCtx rpcCtx = {0}; - + void *msg = NULL; + int32_t code = 0; + bool isCandidateAddr = false; + bool persistHandle = false; + SRpcCtx rpcCtx = {0}; + if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); isCandidateAddr = true; @@ -1741,7 +1750,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, case TDMT_VND_QUERY: { SCH_ERR_RET(schMakeQueryRpcCtx(pJob, pTask, &rpcCtx)); - + uint32_t len = strlen(pJob->sql); msgSize = sizeof(SSubQueryMsg) + pTask->msgLen + len; msg = taosMemoryCalloc(1, msgSize); @@ -1822,7 +1831,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } case TDMT_VND_QUERY_HEARTBEAT: { SCH_ERR_RET(schMakeHbRpcCtx(pJob, pTask, &rpcCtx)); - + SSchedulerHbReq req = {0}; req.sId = schMgmt.sId; req.header.vgId = addr->nodeId; @@ -1856,7 +1865,8 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_SET_TASK_LASTMSG_TYPE(pTask, msgType); SSchTrans trans = {.transInst = pJob->transport, .transHandle = SCH_GET_TASK_HANDLE(pTask)}; - SCH_ERR_JRET(schAsyncSendMsg(pJob, pTask, &trans, &epSet, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL))); + SCH_ERR_JRET(schAsyncSendMsg(pJob, pTask, &trans, &epSet, msgType, msg, msgSize, persistHandle, + (rpcCtx.args ? &rpcCtx : NULL))); if (msgType == TDMT_VND_QUERY) { SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr, trans.transHandle)); @@ -1902,7 +1912,7 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("no need to launch task cause of job status, job status:%s", jobTaskStatusStr(status)); - + SCH_RET(atomic_load_32(&pJob->errCode)); } @@ -1911,7 +1921,7 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(schPushTaskToExecList(pJob, pTask)); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXECUTING); } - + SSubplan *plan = pTask->plan; if (NULL == pTask->msg) { // TODO add more detailed reason for failure @@ -2071,7 +2081,7 @@ void schFreeJobImpl(void *job) { taosArrayDestroy(pJob->levels); taosArrayDestroy(pJob->nodeList); - + taosMemoryFreeClear(pJob->resData); taosMemoryFreeClear(pJob); @@ -2212,7 +2222,7 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in pRes->code = atomic_load_32(&job->errCode); pRes->numOfRows = job->resNumOfRows; - + schReleaseJob(*pJob); return TSDB_CODE_SUCCESS; @@ -2400,7 +2410,7 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { SCH_JOB_ELOG("job failed or dropping, status:%s", jobTaskStatusStr(status)); SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); } - + if (pJob->resData && ((SRetrieveTableRsp *)pJob->resData)->completed) { SCH_ERR_JRET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_SUCCEED)); } From 651fb094076777aa87e127bdd48916246ba79065 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 31 Mar 2022 17:27:04 +0800 Subject: [PATCH 26/38] handle sigchild --- source/dnode/mgmt/main/exe/dndMain.c | 9 +-------- source/dnode/mgmt/main/src/dndExec.c | 10 +++++----- source/os/src/osProc.c | 3 +++ 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/source/dnode/mgmt/main/exe/dndMain.c b/source/dnode/mgmt/main/exe/dndMain.c index 61b480990d..0d2ddbfbbc 100644 --- a/source/dnode/mgmt/main/exe/dndMain.c +++ b/source/dnode/mgmt/main/exe/dndMain.c @@ -37,13 +37,6 @@ static void dndStopDnode(int signum, void *info, void *ctx) { } } -static void dndHandleChild(int signum, void *info, void *ctx) { - dInfo("sigchild received"); - if (global.pDnode != NULL) { - dndHandleEvent(global.pDnode, DND_EVENT_CHILD); - } -} - static void dndSetSignalHandle() { taosSetSignal(SIGTERM, dndStopDnode); taosSetSignal(SIGHUP, dndStopDnode); @@ -53,7 +46,7 @@ static void dndSetSignalHandle() { if (!tsMultiProcess) { } else if (global.ntype == DNODE || global.ntype == NODE_MAX) { - taosSetSignal(SIGCHLD, dndHandleChild); + taosIgnSignal(SIGCHLD); } else { taosKillChildOnParentStopped(); } diff --git a/source/dnode/mgmt/main/src/dndExec.c b/source/dnode/mgmt/main/src/dndExec.c index c5d5707a99..76614b2916 100644 --- a/source/dnode/mgmt/main/src/dndExec.c +++ b/source/dnode/mgmt/main/src/dndExec.c @@ -128,7 +128,7 @@ static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) { } pWrapper->procId = pid; - dInfo("node:%s, run in new process, pid:%d", pWrapper->name, pid); + dInfo("node:%s, continue running in new process:%d", pWrapper->name, pid); return 0; } @@ -263,13 +263,13 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) { if (!pWrapper->required) continue; if (pDnode->ntype == NODE_MAX) continue; - if (pWrapper->procId != 0 && !taosProcExists(pWrapper->procId)) { - dInfo("node:%s, process not exist, pid:%d", pWrapper->name, pWrapper->procId); + if (pWrapper->procId <= 0 || !taosProcExists(pWrapper->procId)) { + dInfo("node:%s, process:%d does not exist or is killed and needs to be restarted", pWrapper->name, pWrapper->procId); dndNewProc(pWrapper, n); } - - taosMsleep(100); } + + taosMsleep(100); } return 0; diff --git a/source/os/src/osProc.c b/source/os/src/osProc.c index 00ed650cbb..2d2174a4c8 100644 --- a/source/os/src/osProc.c +++ b/source/os/src/osProc.c @@ -23,6 +23,9 @@ int32_t taosNewProc(char **args) { int32_t pid = fork(); if (pid == 0) { args[0] = tsProcPath; + close(STDIN_FILENO); + close(STDOUT_FILENO); + close(STDERR_FILENO); return execvp(tsProcPath, args); } else { return pid; From b56ea026cb18ece0afdad9fc15ea54ed70076041 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 31 Mar 2022 17:36:21 +0800 Subject: [PATCH 27/38] fix --- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 40 +++++++++------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 304b3286fe..ac7c0ada90 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -99,7 +99,7 @@ int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet) { void tsdbCloseAndUnsetFSet(SReadH *pReadh) { tsdbResetReadFile(pReadh); } int tsdbLoadBlockIdx(SReadH *pReadh) { - SDFile * pHeadf = TSDB_READ_HEAD_FILE(pReadh); + SDFile *pHeadf = TSDB_READ_HEAD_FILE(pReadh); SBlockIdx blkIdx; ASSERT(taosArrayGetSize(pReadh->aBlkIdx) == 0); @@ -181,20 +181,13 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) { } SBlockIdx *pBlkIdx = taosArrayGet(pReadh->aBlkIdx, pReadh->cidx); - if (pBlkIdx->tid == TABLE_TID(pTable)) { - if (pBlkIdx->uid == TABLE_UID(pTable)) { - pReadh->pBlkIdx = pBlkIdx; - } else { - pReadh->pBlkIdx = NULL; - } - pReadh->cidx++; - break; - } else if (pBlkIdx->tid > TABLE_TID(pTable)) { - pReadh->pBlkIdx = NULL; + if (pBlkIdx->uid == TABLE_UID(pTable)) { + pReadh->pBlkIdx = pBlkIdx; break; } else { - pReadh->cidx++; + pReadh->pBlkIdx = NULL; } + pReadh->cidx++; } } else { pReadh->pBlkIdx = NULL; @@ -206,7 +199,7 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) { int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { ASSERT(pReadh->pBlkIdx != NULL); - SDFile * pHeadf = TSDB_READ_HEAD_FILE(pReadh); + SDFile *pHeadf = TSDB_READ_HEAD_FILE(pReadh); SBlockIdx *pBlkIdx = pReadh->pBlkIdx; if (tsdbSeekDFile(pHeadf, pBlkIdx->offset, SEEK_SET) < 0) { @@ -276,7 +269,8 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { return 0; } -int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds) { +int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, + int numOfColsIds) { ASSERT(pBlock->numOfSubBlocks > 0); int8_t update = pReadh->pRepo->config.update; @@ -539,9 +533,9 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat pDataCols->numOfRows = pBlock->numOfRows; // Recover the data - int ccol = 0; // loop iter for SBlockCol object - int dcol = 0; // loop iter for SDataCols object - int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows); + int ccol = 0; // loop iter for SBlockCol object + int dcol = 0; // loop iter for SDataCols object + int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows); SBlockCol *pBlockCol = NULL; while (dcol < pDataCols->numOfCols) { SDataCol *pDataCol = &(pDataCols->cols[dcol]); @@ -586,8 +580,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat } if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm, - pBlock->numOfRows, tBitmaps, tLenBitmap, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh), - (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) { + pBlock->numOfRows, tBitmaps, tLenBitmap, pDataCols->maxPoints, + TSDB_READ_COMP_BUF(pReadh), (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) { tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset); return -1; @@ -662,7 +656,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); ASSERT(colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID); - SDFile * pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); + SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); SBlockCol blockCol = {0}; tdResetDataCols(pDataCols); @@ -676,7 +670,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * int ccol = 0; for (int i = 0; i < numOfColIds; i++) { int16_t colId = colIds[i]; - SDataCol * pDataCol = NULL; + SDataCol *pDataCol = NULL; SBlockCol *pBlockCol = NULL; while (true) { @@ -740,7 +734,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol) { ASSERT(pDataCol->colId == pBlockCol->colId); - STsdb * pRepo = TSDB_READ_REPO(pReadh); + STsdb *pRepo = TSDB_READ_REPO(pReadh); STsdbCfg *pCfg = REPO_CFG(pRepo); int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows); @@ -757,7 +751,7 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc } } - int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + COMP_OVERFLOW_BYTES; + int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + COMP_OVERFLOW_BYTES; if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1; if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1; From 02e097dc5cd848e8a99d2d479ddadb04fd9a502a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 31 Mar 2022 17:36:51 +0800 Subject: [PATCH 28/38] [td-14426] fix bug. --- include/common/tcommon.h | 4 ++-- source/libs/executor/src/executorimpl.c | 6 +++--- source/libs/scalar/src/sclvector.c | 14 +++++++------- source/util/src/tcompare.c | 1 + 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 51eabb7d61..988bf3f4e8 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -198,8 +198,8 @@ typedef struct SGroupbyExpr { } SGroupbyExpr; enum { - FUNC_PARAM_TYPE_VALUE = 0, - FUNC_PARAM_TYPE_COLUMN, + FUNC_PARAM_TYPE_VALUE = 0x1, + FUNC_PARAM_TYPE_COLUMN= 0x2, }; typedef struct SFunctParam { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d97b3183f2..8e4e33c997 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1258,8 +1258,8 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, ASSERT(pCtx[i].input.pData[j] != NULL); } } - // setBlockStatisInfo(&pCtx[i], pBlock, pOperator->pExpr[i].base.pColumns); + // setBlockStatisInfo(&pCtx[i], pBlock, pOperator->pExpr[i].base.pColumns); // uint32_t flag = pOperator->pExpr[i].base.pParam[0].pCol->flag; // if (TSDB_COL_IS_NORMAL_COL(flag) /*|| (pCtx[i].functionId == FUNCTION_BLKINFO) || // (TSDB_COL_IS_TAG(flag) && pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)*/) { @@ -8771,8 +8771,8 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr; - pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; - pExp->base.pParam[0].pCol = createColumn(pTargetNode->dataBlockId, pTargetNode->slotId, pType); +// pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; +// pExp->base.pParam[0].pCol = createColumn(pTargetNode->dataBlockId, pTargetNode->slotId, pType); } else { ASSERT(0); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index f22f9a5c3c..33be65832c 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -720,7 +720,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i); double rx = getVectorDoubleValueFnRight(pRightCol->pData, i); - if (compareDoubleVal(&zero, &rx)) { + if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) { colDataAppend(pOutputCol, i, NULL, true); continue; } @@ -729,7 +729,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam } } else if (pLeft->numOfRows == 1) { double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, 0); - if (colDataIsNull_f(pLeftCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (colDataIsNull_f(pLeftCol->nullbitmap, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value // TODO set numOfRows NULL value } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { @@ -739,7 +739,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam } double rx = getVectorDoubleValueFnRight(pRightCol->pData, i); - if (compareDoubleVal(&zero, &rx)) { + if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { colDataAppend(pOutputCol, i, NULL, true); continue; } @@ -749,17 +749,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam } } else if (pRight->numOfRows == 1) { double rx = getVectorDoubleValueFnRight(pRightCol->pData, 0); - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (colDataIsNull_f(pRightCol->nullbitmap, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value // TODO set numOfRows NULL value } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { - if (colDataIsNull_f(pRightCol->nullbitmap, i)) { + if (colDataIsNull_f(pLeftCol->nullbitmap, i)) { colDataAppend(pOutputCol, i, NULL, true); continue; } - double lx = getVectorDoubleValueFnLeft(pRightCol->pData, i); - if (compareDoubleVal(&zero, &lx)) { + double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i); + if (isnan(lx) || isinf(lx)) { colDataAppend(pOutputCol, i, NULL, true); continue; } diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index c98f6eb9be..7f9a3ff593 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -173,6 +173,7 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) { if (isnan(p2)) { return 1; } + if (FLT_EQUAL(p1, p2)) { return 0; } From d89ab513758f652c776b886a390a86cce21db343 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 31 Mar 2022 17:38:47 +0800 Subject: [PATCH 29/38] minor changes --- source/dnode/mgmt/main/src/dndExec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/main/src/dndExec.c b/source/dnode/mgmt/main/src/dndExec.c index 76614b2916..fdc6125fb0 100644 --- a/source/dnode/mgmt/main/src/dndExec.c +++ b/source/dnode/mgmt/main/src/dndExec.c @@ -264,7 +264,7 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) { if (pDnode->ntype == NODE_MAX) continue; if (pWrapper->procId <= 0 || !taosProcExists(pWrapper->procId)) { - dInfo("node:%s, process:%d does not exist or is killed and needs to be restarted", pWrapper->name, pWrapper->procId); + dInfo("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId); dndNewProc(pWrapper, n); } } From 869b0b99a9c335ecc8a28b1afd8e1b90c0236752 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 31 Mar 2022 17:59:41 +0800 Subject: [PATCH 30/38] [td-13039] fix bug. --- source/libs/executor/src/executorimpl.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 8e4e33c997..169e47eb33 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3455,7 +3455,9 @@ void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t SqlFunctionCtx* pCtx = pInfo->pCtx; SSDataBlock* pDataBlock = pInfo->pRes; int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; + SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; + initResultRowInfo(pResultRowInfo, 16); int64_t tid = 0; int64_t groupId = 0; @@ -7595,10 +7597,10 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, const char* pKey) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); - pAggSup->keyBuf = taosMemoryCalloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES); + pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); + pAggSup->keyBuf = taosMemoryCalloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES); pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); - pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); + pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); if (pAggSup->keyBuf == NULL || pAggSup->pResultRowArrayList == NULL || pAggSup->pResultRowListSet == NULL || From 0608824186c025e294ee0aa3384ddbc4f66c44c8 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 31 Mar 2022 18:02:58 +0800 Subject: [PATCH 31/38] feature/qnode --- include/common/ttime.h | 1 + include/libs/command/command.h | 5 + include/libs/nodes/plannodes.h | 2 +- include/libs/qcom/query.h | 10 +- source/common/src/ttime.c | 12 ++ source/libs/command/inc/commandInt.h | 95 +++++++++++++++ .../queryExplain.c => command/src/explain.c} | 112 ++++++++++++++++-- source/libs/qcom/inc/queryInt.h | 71 ----------- source/libs/scheduler/CMakeLists.txt | 2 +- source/libs/scheduler/inc/schedulerInt.h | 7 -- source/libs/scheduler/src/scheduler.c | 95 +-------------- 11 files changed, 227 insertions(+), 185 deletions(-) create mode 100644 source/libs/command/inc/commandInt.h rename source/libs/{qcom/src/queryExplain.c => command/src/explain.c} (85%) diff --git a/include/common/ttime.h b/include/common/ttime.h index 2209cc998f..306f54bedb 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -60,6 +60,7 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth); void deltaToUtcInitOnce(); +char getPrecisionUnit(int32_t precision); int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit); diff --git a/include/libs/command/command.h b/include/libs/command/command.h index 699c2f9792..7e58d39692 100644 --- a/include/libs/command/command.h +++ b/include/libs/command/command.h @@ -15,5 +15,10 @@ #include "cmdnodes.h" #include "tmsg.h" +#include "plannodes.h" int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp); + +int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp); + + diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index f41e049196..37163f60dd 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -253,7 +253,7 @@ typedef struct SIntervalPhysiNode { int64_t sliding; int8_t intervalUnit; int8_t slidingUnit; - uint8_t precision; + uint8_t precision; SFillNode* pFill; } SIntervalPhysiNode; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 5908b0b875..bb550e75e8 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -174,11 +174,6 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); char *jobTaskStatusStr(int32_t status); -int32_t qInitExplainCtx(void **pCtx, SHashObj *groupHash, bool verbose); -int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level); -int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp); -void qFreeExplainCtx(void *ctx); - SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen); @@ -241,6 +236,11 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t } \ } while (0) +#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) + + #ifdef __cplusplus } #endif diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 23b19b55e7..a65352f2b9 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -361,6 +361,18 @@ int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec) { return 0; } +char getPrecisionUnit(int32_t precision) { + static char units[3] = {TIME_UNIT_MILLISECOND, TIME_UNIT_MICROSECOND, TIME_UNIT_NANOSECOND}; + switch (precision) { + case TSDB_TIME_PRECISION_MILLI: + case TSDB_TIME_PRECISION_MICRO: + case TSDB_TIME_PRECISION_NANO: + return units[precision]; + default: + return 0; + } +} + int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) { assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || fromPrecision == TSDB_TIME_PRECISION_NANO); diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h new file mode 100644 index 0000000000..5915aa7f36 --- /dev/null +++ b/source/libs/command/inc/commandInt.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_QUERY_INT_H_ +#define _TD_QUERY_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif +#include "nodes.h" +#include "plannodes.h" +#include "ttime.h" + +#define EXPLAIN_MAX_GROUP_NUM 100 + +//newline area +#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d" +#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d" +#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d" +#define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d" +#define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d" +#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d" +#define EXPLAIN_EXCHANGE_FORMAT "Data Exchange %d:1 width=%d" +#define EXPLAIN_SORT_FORMAT "Sort on %d Column(s) width=%d" +#define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d" +#define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d" +#define EXPLAIN_ORDER_FORMAT "Order: %s" +#define EXPLAIN_FILTER_FORMAT "Filter: " +#define EXPLAIN_FILL_FORMAT "Fill: %s" +#define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: " +#define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]" + +//append area +#define EXPLAIN_GROUPS_FORMAT " groups=%d" +#define EXPLAIN_WIDTH_FORMAT " width=%d" +#define EXPLAIN_LOOPS_FORMAT " loops=%d" +#define EXPLAIN_REVERSE_FORMAT " reverse=%d" + +typedef struct SExplainGroup { + int32_t nodeNum; + SSubplan *plan; + void *execInfo; //TODO +} SExplainGroup; + +typedef struct SExplainResNode { + SNodeList* pChildren; + SPhysiNode* pNode; + void* pExecInfo; +} SExplainResNode; + +typedef struct SQueryExplainRowInfo { + int32_t level; + int32_t len; + char *buf; +} SQueryExplainRowInfo; + +typedef struct SExplainCtx { + int32_t totalSize; + bool verbose; + char *tbuf; + SArray *rows; + SHashObj *groupHash; +} SExplainCtx; + +#define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending") +#define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") + +#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u))) + +#define EXPLAIN_ROW_NEW(level, ...) \ + do { \ + tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", (isVerboseLine ? "" : "-> ")); \ + tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \ + } while (0) + +#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__) +#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; isVerboseLine = true; } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_QUERY_INT_H_*/ diff --git a/source/libs/qcom/src/queryExplain.c b/source/libs/command/src/explain.c similarity index 85% rename from source/libs/qcom/src/queryExplain.c rename to source/libs/command/src/explain.c index af092e842e..5fa12f5ebe 100644 --- a/source/libs/qcom/src/queryExplain.c +++ b/source/libs/command/src/explain.c @@ -13,11 +13,12 @@ * along with this program. If not, see . */ -#include "queryInt.h" #include "query.h" #include "plannodes.h" +#include "commandInt.h" int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes); +int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level); void qFreeExplainResTree(SExplainResNode *res) { @@ -265,6 +266,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i bool isVerboseLine = false; char *tbuf = ctx->tbuf; bool verbose = ctx->verbose; + int32_t filterLen = 0; SPhysiNode* pNode = pResNode->pNode; if (NULL == pNode) { qError("pyhsical node in explain res node is NULL"); @@ -317,7 +319,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (pTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -356,7 +359,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pPrjNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -375,13 +379,15 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pJoinNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -404,7 +410,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pAggNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -429,7 +436,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pExchNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -450,7 +458,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pSortNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -461,7 +470,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, pIntNode->precision), pIntNode->intervalUnit, - INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->offset, pIntNode->intervalUnit, pIntNode->precision), pIntNode->intervalUnit, + pIntNode->offset, getPrecisionUnit(pIntNode->precision), INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, pIntNode->precision), pIntNode->slidingUnit, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { @@ -479,7 +488,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (pIntNode->window.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -498,7 +508,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pIntNode->window.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); + tlen += filterLen; EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -595,5 +606,84 @@ int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { return TSDB_CODE_SUCCESS; } +int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) { + int32_t code = 0; + SNodeListNode *plans = NULL; + int32_t taskNum = 0; + SExplainGroup *pGroup = NULL; + void *pCtx = NULL; + int32_t rootGroupId = 0; + + if (pDag->numOfSubplans <= 0) { + qError("invalid subplan num:%d", pDag->numOfSubplans); + QRY_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans); + if (levelNum <= 0) { + qError("invalid level num:%d", levelNum); + QRY_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SHashObj *groupHash = taosHashInit(EXPLAIN_MAX_GROUP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == groupHash) { + qError("groupHash %d failed", EXPLAIN_MAX_GROUP_NUM); + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + QRY_ERR_JRET(qInitExplainCtx(&pCtx, groupHash, pDag->explainInfo.verbose)); + + for (int32_t i = 0; i < levelNum; ++i) { + plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); + if (NULL == plans) { + qError("empty level plan, level:%d", i); + QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + taskNum = (int32_t)LIST_LENGTH(plans->pNodeList); + if (taskNum <= 0) { + qError("invalid level plan number:%d, level:%d", taskNum, i); + QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SSubplan *plan = NULL; + for (int32_t n = 0; n < taskNum; ++n) { + plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); + pGroup = taosHashGet(groupHash, &plan->id.groupId, sizeof(plan->id.groupId)); + if (pGroup) { + ++pGroup->nodeNum; + continue; + } + + SExplainGroup group = {.nodeNum = 1, .plan = plan, .execInfo = NULL}; + if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { + qError("taosHashPut to explainGroupHash failed, taskIdx:%d", n); + QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + if (0 == i) { + if (taskNum > 1) { + qError("invalid taskNum %d for level 0", taskNum); + QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + rootGroupId = plan->id.groupId; + } + + qDebug("level %d group handled, taskNum:%d", i, taskNum); + } + + QRY_ERR_JRET(qAppendTaskExplainResRows(pCtx, rootGroupId, 0)); + + QRY_ERR_JRET(qGetExplainRspFromCtx(pCtx, pRsp)); + +_return: + + qFreeExplainCtx(pCtx); + + QRY_RET(code); +} + diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h index 06d5b07dc7..f120bf26ce 100644 --- a/source/libs/qcom/inc/queryInt.h +++ b/source/libs/qcom/inc/queryInt.h @@ -19,77 +19,6 @@ #ifdef __cplusplus extern "C" { #endif -#include "nodes.h" -#include "plannodes.h" - -//newline area -#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d" -#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d" -#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d" -#define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d" -#define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d" -#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d" -#define EXPLAIN_EXCHANGE_FORMAT "Data Exchange %d:1 width=%d" -#define EXPLAIN_SORT_FORMAT "Sort on %d Column(s) width=%d" -#define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d" -#define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d" -#define EXPLAIN_ORDER_FORMAT "Order: %s" -#define EXPLAIN_FILTER_FORMAT "Filter: " -#define EXPLAIN_FILL_FORMAT "Fill: %s" -#define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: " -#define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]" - -//append area -#define EXPLAIN_GROUPS_FORMAT " groups=%d" -#define EXPLAIN_WIDTH_FORMAT " width=%d" -#define EXPLAIN_LOOPS_FORMAT " loops=%d" -#define EXPLAIN_REVERSE_FORMAT " reverse=%d" - -//TODO MOVE TO LIB -typedef struct SExplainGroup { - int32_t nodeNum; - SSubplan *plan; - void *execInfo; //TODO -} SExplainGroup; - - -typedef struct SExplainResNode { - SNodeList* pChildren; - SPhysiNode* pNode; - void* pExecInfo; -} SExplainResNode; - -typedef struct SQueryExplainRowInfo { - int32_t level; - int32_t len; - char *buf; -} SQueryExplainRowInfo; - -typedef struct SExplainCtx { - int32_t totalSize; - bool verbose; - char *tbuf; - SArray *rows; - SHashObj *groupHash; -} SExplainCtx; - -#define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending") -#define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") - -#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u))) - -#define EXPLAIN_ROW_NEW(level, ...) \ - do { \ - tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", (isVerboseLine ? "" : "-> ")); \ - tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \ - } while (0) - -#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__) -#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; isVerboseLine = true; } while (0) - -#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) -#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) -#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) #ifdef __cplusplus diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index a4a299317c..1a62c7d89d 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -9,7 +9,7 @@ target_include_directories( target_link_libraries( scheduler - PUBLIC os util nodes planner qcom common catalog transport + PUBLIC os util nodes planner qcom common catalog transport command ) if(${BUILD_TEST}) diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index f19e822974..22bd039219 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -38,13 +38,6 @@ enum { SCH_WRITE, }; -//TODO MOVE TO LIB -typedef struct SExplainGroup { - int32_t nodeNum; - SSubplan *plan; - void *execInfo; //TODO -} SExplainGroup; - typedef struct SSchTrans { void *transInst; void *transHandle; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 940aaed7c1..7c24a25f65 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -19,6 +19,7 @@ #include "tmsg.h" #include "tref.h" #include "trpc.h" +#include "command.h" SSchedulerMgmt schMgmt = {0}; @@ -476,92 +477,6 @@ _return: SCH_RET(code); } -int32_t schValidateAndBuildJobExplain(SQueryPlan *pDag, SSchJob *pJob) { - int32_t code = 0; - SNodeListNode *plans = NULL; - int32_t taskNum = 0; - SExplainGroup *pGroup = NULL; - void *pCtx = NULL; - int32_t rootGroupId = 0; - - pJob->queryId = pDag->queryId; - pJob->subPlans = pDag->pSubplans; - - if (pDag->numOfSubplans <= 0) { - SCH_JOB_ELOG("invalid subplan num:%d", pDag->numOfSubplans); - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } - - int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans); - if (levelNum <= 0) { - SCH_JOB_ELOG("invalid level num:%d", levelNum); - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } - - SHashObj *groupHash = taosHashInit(SCHEDULE_DEFAULT_MAX_TASK_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - if (NULL == groupHash) { - SCH_JOB_ELOG("groupHash %d failed", SCHEDULE_DEFAULT_MAX_TASK_NUM); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - SCH_ERR_JRET(qInitExplainCtx(&pCtx, groupHash, pDag->explainInfo.verbose)); - - for (int32_t i = 0; i < levelNum; ++i) { - plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); - if (NULL == plans) { - SCH_JOB_ELOG("empty level plan, level:%d", i); - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - - taskNum = (int32_t)LIST_LENGTH(plans->pNodeList); - if (taskNum <= 0) { - SCH_JOB_ELOG("invalid level plan number:%d, level:%d", taskNum, i); - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - - SSubplan *plan = NULL; - for (int32_t n = 0; n < taskNum; ++n) { - plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); - pGroup = taosHashGet(groupHash, &plan->id.groupId, sizeof(plan->id.groupId)); - if (pGroup) { - ++pGroup->nodeNum; - continue; - } - - SExplainGroup group = {.nodeNum = 1, .plan = plan, .execInfo = NULL}; - if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { - SCH_JOB_ELOG("taosHashPut to explainGroupHash failed, taskIdx:%d", n); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } - - if (0 == i) { - if (taskNum > 1) { - SCH_JOB_ELOG("invalid taskNum %d for level 0", taskNum); - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - - rootGroupId = plan->id.groupId; - } - - SCH_JOB_DLOG("level %d group handled, taskNum:%d", i, taskNum); - } - - SCH_ERR_JRET(qAppendTaskExplainResRows(pCtx, rootGroupId, 0)); - - SRetrieveTableRsp *pRsp = NULL; - SCH_ERR_JRET(qGetExplainRspFromCtx(pCtx, &pRsp)); - - pJob->resData = pRsp; - -_return: - - qFreeExplainCtx(pCtx); - - SCH_RET(code); -} - - int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { if (NULL != pTask->candidateAddrs) { return TSDB_CODE_SUCCESS; @@ -2250,7 +2165,7 @@ _return: SCH_RET(code); } -int32_t schStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, +int32_t schExecStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, bool syncSchedule) { qDebug("QID:0x%" PRIx64 " job started", pDag->queryId); @@ -2264,8 +2179,10 @@ int32_t schStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDag, i pJob->sql = sql; pJob->attr.queryJob = true; pJob->attr.explainMode = pDag->explainInfo.mode; + pJob->queryId = pDag->queryId; + pJob->subPlans = pDag->pSubplans; - SCH_ERR_JRET(schValidateAndBuildJobExplain(pDag, pJob)); + SCH_ERR_JRET(qExecStaticExplain(pDag, (SRetrieveTableRsp **)&pJob->resData)); int64_t refId = taosAddRef(schMgmt.jobRef, pJob); if (refId < 0) { @@ -2345,7 +2262,7 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in } if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { - SCH_ERR_RET(schStaticExplain(transport, nodeList, pDag, pJob, sql, true)); + SCH_ERR_RET(schExecStaticExplain(transport, nodeList, pDag, pJob, sql, true)); } else { SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true)); } From 6256e6f288cdeb56b6a3de8087739633dd2390cf Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 31 Mar 2022 18:05:13 +0800 Subject: [PATCH 32/38] add fuzzy search --- source/libs/index/inc/indexFstRegex.h | 3 --- source/libs/index/inc/indexFstSparse.h | 6 ++--- source/libs/index/src/indexFstSparse.c | 37 ++++++++++++-------------- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/source/libs/index/inc/indexFstRegex.h b/source/libs/index/inc/indexFstRegex.h index a80e768feb..8fb5455336 100644 --- a/source/libs/index/inc/indexFstRegex.h +++ b/source/libs/index/inc/indexFstRegex.h @@ -67,9 +67,6 @@ uint32_t regexAutomStart(FstRegex *regex); bool regexAutomIsMatch(FstRegex *regex, uint32_t state); bool regexAutomCanMatch(FstRegex *regex, uint32_t state, bool null); bool regexAutomAccept(FstRegex *regex, uint32_t state, uint8_t byte, uint32_t *result); -// void regexSetup(FstRegex *regex, uint32_t size, const char *str); - -// uint32_t regexStart() #ifdef __cplusplus } diff --git a/source/libs/index/inc/indexFstSparse.h b/source/libs/index/inc/indexFstSparse.h index 69b33c82d9..665fb2ba5c 100644 --- a/source/libs/index/inc/indexFstSparse.h +++ b/source/libs/index/inc/indexFstSparse.h @@ -23,9 +23,9 @@ extern "C" { #endif typedef struct FstSparseSet { - SArray *dense; - SArray *sparse; - int32_t size; + uint32_t *dense; + uint32_t *sparse; + int32_t size; } FstSparseSet; FstSparseSet *sparSetCreate(int32_t sz); diff --git a/source/libs/index/src/indexFstSparse.c b/source/libs/index/src/indexFstSparse.c index 9d228e71ff..e8ab3be2fe 100644 --- a/source/libs/index/src/indexFstSparse.c +++ b/source/libs/index/src/indexFstSparse.c @@ -21,47 +21,44 @@ FstSparseSet *sparSetCreate(int32_t sz) { return NULL; } - ss->dense = taosArrayInit(sz, sizeof(uint32_t)); - ss->sparse = taosArrayInit(sz, sizeof(uint32_t)); - ss->size = sz; + ss->dense = (uint32_t *)taosMemoryCalloc(sz, sizeof(uint32_t)); + ss->sparse = (uint32_t *)taosMemoryCalloc(sz, sizeof(uint32_t)); + ss->size = 0; return ss; } void sparSetDestroy(FstSparseSet *ss) { if (ss == NULL) { return; } - taosArrayDestroy(ss->dense); - taosArrayDestroy(ss->sparse); + taosMemoryFree(ss->dense); + taosMemoryFree(ss->sparse); taosMemoryFree(ss); } -uint32_t sparSetLen(FstSparseSet *ss) { return ss == NULL ? 0 : ss->size; } +uint32_t sparSetLen(FstSparseSet *ss) { + // Get occupied size + return ss == NULL ? 0 : ss->size; +} uint32_t sparSetAdd(FstSparseSet *ss, uint32_t ip) { if (ss == NULL) { return 0; } uint32_t i = ss->size; - taosArraySet(ss->dense, i, &ip); - taosArraySet(ss->sparse, ip, &i); + ss->dense[i] = ip; + ss->sparse[ip] = i; ss->size += 1; return i; } uint32_t sparSetGet(FstSparseSet *ss, uint32_t i) { - if (i >= taosArrayGetSize(ss->dense)) { - return 0; - } - uint32_t *v = taosArrayGet(ss->dense, i); - return *v; + // check later + return ss->dense[i]; } bool sparSetContains(FstSparseSet *ss, uint32_t ip) { - if (ip >= taosArrayGetSize(ss->sparse)) { + uint32_t i = ss->sparse[ip]; + if (i < ss->size && ss->dense[i] == ip) { + return true; + } else { return false; } - uint32_t i = *(uint32_t *)taosArrayGet(ss->sparse, ip); - if (i >= taosArrayGetSize(ss->dense)) { - return false; - } - uint32_t v = *(uint32_t *)taosArrayGet(ss->dense, i); - return v == ip; } void sparSetClear(FstSparseSet *ss) { if (ss == NULL) { From e21ba1637ecca0e1ee95849ecb070367afc8373a Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 31 Mar 2022 18:39:11 +0800 Subject: [PATCH 33/38] revert uid --- source/dnode/vnode/src/inc/tsdbCommit.h | 2 +- source/dnode/vnode/src/inc/tsdbDef.h | 2 +- source/dnode/vnode/src/meta/metaTbUid.c | 2 +- source/dnode/vnode/src/tq/tqRead.c | 1 + source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 40 +++++++++++++--------- 7 files changed, 29 insertions(+), 22 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdbCommit.h b/source/dnode/vnode/src/inc/tsdbCommit.h index 699aeaa133..2c6dd75e03 100644 --- a/source/dnode/vnode/src/inc/tsdbCommit.h +++ b/source/dnode/vnode/src/inc/tsdbCommit.h @@ -74,4 +74,4 @@ int tsdbApplyRtn(STsdbRepo *pRepo); } #endif -#endif /* _TD_TSDB_COMMIT_H_ */ \ No newline at end of file +#endif /* _TD_TSDB_COMMIT_H_ */ diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h index 02aba95517..3e42cb627a 100644 --- a/source/dnode/vnode/src/inc/tsdbDef.h +++ b/source/dnode/vnode/src/inc/tsdbDef.h @@ -79,4 +79,4 @@ static FORCE_INLINE STSchema *tsdbGetTableSchemaImpl(STable *pTable, bool lock, } #endif -#endif /*_TD_TSDB_DEF_H_*/ \ No newline at end of file +#endif /*_TD_TSDB_DEF_H_*/ diff --git a/source/dnode/vnode/src/meta/metaTbUid.c b/source/dnode/vnode/src/meta/metaTbUid.c index b488630024..1f57d1396a 100644 --- a/source/dnode/vnode/src/meta/metaTbUid.c +++ b/source/dnode/vnode/src/meta/metaTbUid.c @@ -27,5 +27,5 @@ void metaCloseUidGnrt(SMeta *pMeta) { /* TODO */ tb_uid_t metaGenerateUid(SMeta *pMeta) { // Generate a new table UID - return tGenIdPI64(); + return tGenIdPI32(); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 344b4fa655..61538055f2 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -168,6 +168,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { break; } if (colDataAppend(pColData, curRow, sVal.val, false) < 0) { + /*if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) {*/ taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock); return NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 3e0b03f331..d5e9b55a71 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1394,7 +1394,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF tsdbDebug("vgId:%d uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64 " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, - REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, + REPO_ID(pRepo), TABLE_UID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bac5255d17..cae6d5ec1a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -4068,4 +4068,4 @@ void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *re //apply the hierarchical filter expression to every node in skiplist to find the qualified nodes applyFilterToSkipListNode(pSkipList, pExpr, result, param); } -#endif \ No newline at end of file +#endif diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index ac7c0ada90..304b3286fe 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -99,7 +99,7 @@ int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet) { void tsdbCloseAndUnsetFSet(SReadH *pReadh) { tsdbResetReadFile(pReadh); } int tsdbLoadBlockIdx(SReadH *pReadh) { - SDFile *pHeadf = TSDB_READ_HEAD_FILE(pReadh); + SDFile * pHeadf = TSDB_READ_HEAD_FILE(pReadh); SBlockIdx blkIdx; ASSERT(taosArrayGetSize(pReadh->aBlkIdx) == 0); @@ -181,13 +181,20 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) { } SBlockIdx *pBlkIdx = taosArrayGet(pReadh->aBlkIdx, pReadh->cidx); - if (pBlkIdx->uid == TABLE_UID(pTable)) { - pReadh->pBlkIdx = pBlkIdx; + if (pBlkIdx->tid == TABLE_TID(pTable)) { + if (pBlkIdx->uid == TABLE_UID(pTable)) { + pReadh->pBlkIdx = pBlkIdx; + } else { + pReadh->pBlkIdx = NULL; + } + pReadh->cidx++; + break; + } else if (pBlkIdx->tid > TABLE_TID(pTable)) { + pReadh->pBlkIdx = NULL; break; } else { - pReadh->pBlkIdx = NULL; + pReadh->cidx++; } - pReadh->cidx++; } } else { pReadh->pBlkIdx = NULL; @@ -199,7 +206,7 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) { int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { ASSERT(pReadh->pBlkIdx != NULL); - SDFile *pHeadf = TSDB_READ_HEAD_FILE(pReadh); + SDFile * pHeadf = TSDB_READ_HEAD_FILE(pReadh); SBlockIdx *pBlkIdx = pReadh->pBlkIdx; if (tsdbSeekDFile(pHeadf, pBlkIdx->offset, SEEK_SET) < 0) { @@ -269,8 +276,7 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { return 0; } -int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, - int numOfColsIds) { +int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds) { ASSERT(pBlock->numOfSubBlocks > 0); int8_t update = pReadh->pRepo->config.update; @@ -533,9 +539,9 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat pDataCols->numOfRows = pBlock->numOfRows; // Recover the data - int ccol = 0; // loop iter for SBlockCol object - int dcol = 0; // loop iter for SDataCols object - int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows); + int ccol = 0; // loop iter for SBlockCol object + int dcol = 0; // loop iter for SDataCols object + int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows); SBlockCol *pBlockCol = NULL; while (dcol < pDataCols->numOfCols) { SDataCol *pDataCol = &(pDataCols->cols[dcol]); @@ -580,8 +586,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat } if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm, - pBlock->numOfRows, tBitmaps, tLenBitmap, pDataCols->maxPoints, - TSDB_READ_COMP_BUF(pReadh), (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) { + pBlock->numOfRows, tBitmaps, tLenBitmap, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh), + (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) { tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset); return -1; @@ -656,7 +662,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); ASSERT(colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID); - SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); + SDFile * pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); SBlockCol blockCol = {0}; tdResetDataCols(pDataCols); @@ -670,7 +676,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * int ccol = 0; for (int i = 0; i < numOfColIds; i++) { int16_t colId = colIds[i]; - SDataCol *pDataCol = NULL; + SDataCol * pDataCol = NULL; SBlockCol *pBlockCol = NULL; while (true) { @@ -734,7 +740,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol) { ASSERT(pDataCol->colId == pBlockCol->colId); - STsdb *pRepo = TSDB_READ_REPO(pReadh); + STsdb * pRepo = TSDB_READ_REPO(pReadh); STsdbCfg *pCfg = REPO_CFG(pRepo); int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows); @@ -751,7 +757,7 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc } } - int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + COMP_OVERFLOW_BYTES; + int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + COMP_OVERFLOW_BYTES; if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1; if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1; From 6c853aebab8f80eb3c2092ba2cf5e1c426ba34fa Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 31 Mar 2022 18:50:50 +0800 Subject: [PATCH 34/38] use single vgroup test --- tests/script/tsim/tmq/basic1.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index 26a5abbfcf..cfcb1ac992 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -35,7 +35,7 @@ sql connect $dbNamme = d0 print =============== create database , vgroup 1 -sql create database $dbNamme vgroups 2 +sql create database $dbNamme vgroups 1 sql show databases print $data00 $data01 $data02 if $rows != 2 then From 08eef9351d0ecd186fbbe8a62bedfd217a5fbb17 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 1 Apr 2022 09:11:07 +0800 Subject: [PATCH 35/38] [add tmq cases] --- tests/script/tsim/tmq/multiTopic.sim | 224 +++++++++++++++++++++++ tests/script/tsim/tmq/oneTopic.sim | 264 +++++++++++++++++++++++++++ 2 files changed, 488 insertions(+) create mode 100644 tests/script/tsim/tmq/multiTopic.sim create mode 100644 tests/script/tsim/tmq/oneTopic.sim diff --git a/tests/script/tsim/tmq/multiTopic.sim b/tests/script/tsim/tmq/multiTopic.sim new file mode 100644 index 0000000000..cd977e5909 --- /dev/null +++ b/tests/script/tsim/tmq/multiTopic.sim @@ -0,0 +1,224 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## +######## This test case include scene2 and scene4 +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +$loop_cnt = 0 +$vgroups = 1 +$dbNamme = d0 +loop_vgroups: +print =============== create database $dbNamme vgroups $vgroups +sql create database $dbNamme vgroups $vgroups +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 + +if $loop_cnt == 0 then + if $rows != 2 then + return -1 + endi + if $data02 != 1 then # vgroups + print vgroups: $data02 + return -1 + endi +else + if $rows != 3 then + return -1 + endi + if $data00 == d1 then + if $data02 != 4 then # vgroups + print vgroups: $data02 + return -1 + endi + else + if $data12 != 4 then # vgroups + print vgroups: $data12 + return -1 + endi + endi +endi + +sql use $dbNamme + +print =============== create super table +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +$tbPrefix = ct +$tbNum = 100 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $i = $i + 1 +endw + +print =============== create normal table +sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + +print =============== create multi topics. notes: now only support: +print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb +print =============== will support: * from stb + +sql create topic topic_stb_column as select ts, c1, c3 from stb +#sql create topic topic_stb_all as select * from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +sql create topic topic_ctb_column as select ts, c1, c3 from ct0 +sql create topic topic_ctb_all as select * from ct0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 + +sql create topic topic_ntb_column as select ts, c1, c3 from ntb +sql create topic topic_ntb_all as select * from ntb +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb + +sql show tables +if $rows != 101 then + return -1 +endi + +print =============== insert data +$rowNum = 100 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + + $x = 0 + while $x < $rowNum + $c = $x / 10 + $c = $c * 10 + $c = $x - $c + + $binary = ' . binary + $binary = $binary . $c + $binary = $binary . ' + + sql insert into $tb values ($tstart , $c , $x , $binary ) + sql insert into ntb values ($tstart , $c , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw + +#root@trd02 /home $ tmq_sim --help +# -c Configuration directory, default is +# -d The name of the database for cosumer, no default +# -t The topic string for cosumer, no default +# -k The key-value string for cosumer, no default +# -g showMsgFlag, default is 0 +# + +$totalMsgCnt = $rowNum * $tbNum +print inserted totalMsgCnt: $totalMsgCnt + +# supported key: +# group.id: +# enable.auto.commit: +# auto.offset.reset: +# td.connect.ip: +# td.connect.user:root +# td.connect.pass:taosdata +# td.connect.port:6030 +# td.connect.db:db + +$numOfTopics = 2 +$totalMsgCntOfmultiTopics = $totalMsgCnt * $numOfTopics +$expect_result = @{consume success: @ +$expect_result = $expect_result . $totalMsgCntOfmultiTopics +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 20000, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +$numOfTopics = 3 +$totalMsgCntOfmultiTopics = $rowNum * $numOfTopics +$expect_result = @{consume success: @ +$expect_result = $expect_result . $totalMsgCntOfmultiTopics +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 300, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +$numOfTopics = 3 +$totalMsgCntOfmultiTopics = $totalMsgCnt * $numOfTopics +$expect_result = @{consume success: @ +$expect_result = $expect_result . $totalMsgCntOfmultiTopics +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 30000, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +if $loop_cnt == 0 then + $loop_cnt = 1 + $vgroups = 4 + $dbNamme = d1 + goto loop_vgroups +endi + + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/oneTopic.sim b/tests/script/tsim/tmq/oneTopic.sim new file mode 100644 index 0000000000..8e8d00977c --- /dev/null +++ b/tests/script/tsim/tmq/oneTopic.sim @@ -0,0 +1,264 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## +######## This test case include scene1 and scene3 +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +$loop_cnt = 0 +$vgroups = 1 +$dbNamme = d0 +loop_vgroups: +print =============== create database $dbNamme vgroups $vgroups +sql create database $dbNamme vgroups $vgroups +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 + +if $loop_cnt == 0 then + if $rows != 2 then + return -1 + endi + if $data02 != 1 then # vgroups + print vgroups: $data02 + return -1 + endi +else + if $rows != 3 then + return -1 + endi + if $data00 == d1 then + if $data02 != 4 then # vgroups + print vgroups: $data02 + return -1 + endi + else + if $data12 != 4 then # vgroups + print vgroups: $data12 + return -1 + endi + endi +endi + +sql use $dbNamme + +print =============== create super table +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +$tbPrefix = ct +$tbNum = 100 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $i = $i + 1 +endw + +print =============== create normal table +sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + +print =============== create multi topics. notes: now only support: +print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb +print =============== will support: * from stb + +sql create topic topic_stb_column as select ts, c1, c3 from stb +#sql create topic topic_stb_all as select * from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +sql create topic topic_ctb_column as select ts, c1, c3 from ct0 +sql create topic topic_ctb_all as select * from ct0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 + +sql create topic topic_ntb_column as select ts, c1, c3 from ntb +sql create topic topic_ntb_all as select * from ntb +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb + +sql show tables +if $rows != 101 then + return -1 +endi + +print =============== insert data +$rowNum = 100 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + + $x = 0 + while $x < $rowNum + $c = $x / 10 + $c = $c * 10 + $c = $x - $c + + $binary = ' . binary + $binary = $binary . $c + $binary = $binary . ' + + sql insert into $tb values ($tstart , $c , $x , $binary ) + sql insert into ntb values ($tstart , $c , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw + +#root@trd02 /home $ tmq_sim --help +# -c Configuration directory, default is +# -d The name of the database for cosumer, no default +# -t The topic string for cosumer, no default +# -k The key-value string for cosumer, no default +# -g showMsgFlag, default is 0 +# + +$totalMsgCnt = $rowNum * $tbNum +print inserted totalMsgCnt: $totalMsgCnt + +# supported key: +# group.id: +# enable.auto.commit: +# auto.offset.reset: +# td.connect.ip: +# td.connect.user:root +# td.connect.pass:taosdata +# td.connect.port:6030 +# td.connect.db:db + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $totalMsgCnt +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 10000, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" +#print cmd result----> $system_content +##if $system_content != @{consume success: 10000, 0}@ then +#if $system_content != $expect_result then +# return -1 +#endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 10000, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $rowNum +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 100, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 100, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 100, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $totalMsgCnt +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 10000, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 10000, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 10000, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +if $loop_cnt == 0 then + $loop_cnt = 1 + $vgroups = 4 + $dbNamme = d1 + goto loop_vgroups +endi + + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT From d9091f74db74ec42bbaa277c40249d6f02119484 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 1 Apr 2022 09:55:11 +0800 Subject: [PATCH 36/38] fix catalog exit issue --- source/libs/catalog/src/catalog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index e87fdba71d..a3019d8d56 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2304,8 +2304,6 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - CTG_ERR_RET(ctgStartUpdateThread()); - tsem_init(&gCtgMgmt.queue.reqSem, 0, 0); tsem_init(&gCtgMgmt.queue.rspSem, 0, 0); @@ -2316,6 +2314,8 @@ int32_t catalogInit(SCatalogCfg *cfg) { } gCtgMgmt.queue.tail = gCtgMgmt.queue.head; + CTG_ERR_RET(ctgStartUpdateThread()); + qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); return TSDB_CODE_SUCCESS; From b449369265894a3fdcb8070e8bab040db50bdcf9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 1 Apr 2022 10:29:20 +0800 Subject: [PATCH 37/38] feature/qnode --- source/libs/catalog/src/catalog.c | 28 ++++++++++++---- source/libs/command/inc/commandInt.h | 6 ++-- source/libs/command/src/explain.c | 44 ++++++++++++------------- source/libs/nodes/src/nodesToSQLFuncs.c | 35 ++++++++++---------- source/libs/nodes/src/nodesUtilFuncs.c | 5 +-- 5 files changed, 65 insertions(+), 53 deletions(-) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index db772000b5..e1f5332899 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2011,7 +2011,9 @@ void* ctgUpdateThreadFunc(void* param) { CTG_LOCK(CTG_READ, &gCtgMgmt.lock); while (true) { - tsem_wait(&gCtgMgmt.queue.reqSem); + if (tsem_wait(&gCtgMgmt.queue.reqSem)) { + qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + } if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { tsem_post(&gCtgMgmt.queue.rspSem); @@ -2175,10 +2177,15 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - CTG_ERR_RET(ctgStartUpdateThread()); - - tsem_init(&gCtgMgmt.queue.reqSem, 0, 0); - tsem_init(&gCtgMgmt.queue.rspSem, 0, 0); + if (tsem_init(&gCtgMgmt.queue.reqSem, 0, 0)) { + qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR); + } + + if (tsem_init(&gCtgMgmt.queue.rspSem, 0, 0)) { + qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR); + } gCtgMgmt.queue.head = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == gCtgMgmt.queue.head) { @@ -2187,6 +2194,8 @@ int32_t catalogInit(SCatalogCfg *cfg) { } gCtgMgmt.queue.tail = gCtgMgmt.queue.head; + CTG_ERR_RET(ctgStartUpdateThread()); + qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); return TSDB_CODE_SUCCESS; @@ -2718,8 +2727,13 @@ void catalogDestroy(void) { atomic_store_8((int8_t*)&gCtgMgmt.exit, true); - tsem_post(&gCtgMgmt.queue.reqSem); - tsem_post(&gCtgMgmt.queue.rspSem); + if (tsem_post(&gCtgMgmt.queue.reqSem)) { + qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + } + + if (tsem_post(&gCtgMgmt.queue.rspSem)) { + qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + } while (CTG_IS_LOCKED(&gCtgMgmt.lock)) { taosUsleep(1); diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index 5915aa7f36..cc3e1d8b55 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -26,9 +26,9 @@ extern "C" { #define EXPLAIN_MAX_GROUP_NUM 100 //newline area -#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d" -#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d" -#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d" +#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d width=%d" +#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d width=%d" +#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d width=%d" #define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d" #define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d" #define EXPLAIN_AGG_FORMAT "Aggragate functions=%d" diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 5fa12f5ebe..847a863e76 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -266,7 +266,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i bool isVerboseLine = false; char *tbuf = ctx->tbuf; bool verbose = ctx->verbose; - int32_t filterLen = 0; SPhysiNode* pNode = pResNode->pNode; if (NULL == pNode) { qError("pyhsical node in explain res node is NULL"); @@ -276,7 +275,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname, pTagScanNode->pScanCols->length); + EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname, pTagScanNode->pScanCols->length, pTagScanNode->node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -297,7 +296,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname, pTblScanNode->scan.pScanCols->length); + EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname, pTblScanNode->scan.pScanCols->length, pTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -319,8 +318,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (pTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -329,7 +327,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname, pSTblScanNode->scan.pScanCols->length); + EXPLAIN_ROW_NEW(level, EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname, pSTblScanNode->scan.pScanCols->length, pSTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); } @@ -343,7 +341,15 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (pSTblScanNode->scan.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + } break; } @@ -359,8 +365,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pPrjNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -379,15 +384,13 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pJoinNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -410,8 +413,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pAggNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -436,8 +438,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pExchNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -458,8 +459,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pSortNode->node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -488,8 +488,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (pIntNode->window.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -508,8 +507,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { if (pIntNode->window.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, &filterLen)); - tlen += filterLen; + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index fa941cfe9c..68f38bb6a7 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -30,18 +30,17 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { switch (pNode->type) { case QUERY_NODE_COLUMN: { SColumnNode *colNode = (SColumnNode *)pNode; - *len = 0; if (colNode->dbName[0]) { - *len += snprintf(buf, bufSize - *len, "`%s`.", colNode->dbName); + *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->dbName); } if (colNode->tableAlias[0]) { - *len += snprintf(buf, bufSize - *len, "`%s`.", colNode->tableAlias); + *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableAlias); } else if (colNode->tableName[0]) { - *len += snprintf(buf, bufSize - *len, "`%s`.", colNode->tableName); + *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableName); } - *len += snprintf(buf, bufSize - *len, "`%s`", colNode->colName); + *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->colName); return TSDB_CODE_SUCCESS; } @@ -53,14 +52,14 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - *len += snprintf(buf, bufSize - *len, "%s", t); + *len += snprintf(buf + *len, bufSize - *len, "%s", t); taosMemoryFree(t); return TSDB_CODE_SUCCESS; } case QUERY_NODE_OPERATOR: { SOperatorNode* pOpNode = (SOperatorNode*)pNode; - *len += snprintf(buf, bufSize - *len, "("); + *len += snprintf(buf + *len, bufSize - *len, "("); if (pOpNode->pLeft) { NODES_ERR_RET(nodesNodeToSQL(pOpNode->pLeft, buf, bufSize, len)); } @@ -70,13 +69,13 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - *len += snprintf(buf, bufSize - *len, " %s ", gOperatorStr[pOpNode->opType]); + *len += snprintf(buf + *len, bufSize - *len, " %s ", gOperatorStr[pOpNode->opType]); if (pOpNode->pRight) { NODES_ERR_RET(nodesNodeToSQL(pOpNode->pRight, buf, bufSize, len)); } - *len += snprintf(buf, bufSize - *len, ")"); + *len += snprintf(buf + *len, bufSize - *len, ")"); return TSDB_CODE_SUCCESS; } @@ -85,17 +84,17 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { SNode* node = NULL; bool first = true; - *len += snprintf(buf, bufSize - *len, "("); + *len += snprintf(buf + *len, bufSize - *len, "("); FOREACH(node, pLogicNode->pParameterList) { if (!first) { - *len += snprintf(buf, bufSize - *len, " %s ", gLogicConditionStr[pLogicNode->condType]); + *len += snprintf(buf + *len, bufSize - *len, " %s ", gLogicConditionStr[pLogicNode->condType]); } NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); first = false; } - *len += snprintf(buf, bufSize - *len, ")"); + *len += snprintf(buf + *len, bufSize - *len, ")"); return TSDB_CODE_SUCCESS; } @@ -104,17 +103,17 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { SNode* node = NULL; bool first = true; - *len += snprintf(buf, bufSize - *len, "%s(", pFuncNode->functionName); + *len += snprintf(buf + *len, bufSize - *len, "%s(", pFuncNode->functionName); FOREACH(node, pFuncNode->pParameterList) { if (!first) { - *len += snprintf(buf, bufSize - *len, ", "); + *len += snprintf(buf + *len, bufSize - *len, ", "); } NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); first = false; } - *len += snprintf(buf, bufSize - *len, ")"); + *len += snprintf(buf + *len, bufSize - *len, ")"); return TSDB_CODE_SUCCESS; } @@ -123,17 +122,17 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { SNode* node = NULL; bool first = true; - *len += snprintf(buf, bufSize - *len, "("); + *len += snprintf(buf + *len, bufSize - *len, "("); FOREACH(node, pListNode->pNodeList) { if (!first) { - *len += snprintf(buf, bufSize - *len, ", "); + *len += snprintf(buf + *len, bufSize - *len, ", "); } NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len)); first = false; } - *len += snprintf(buf, bufSize - *len, ")"); + *len += snprintf(buf + *len, bufSize - *len, ")"); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index de46c5a48b..732d9acfbe 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -839,12 +839,13 @@ char* nodesGetStrValueFromNode(SValueNode *pNode) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { - void *buf = taosMemoryMalloc(varDataLen(pNode->datum.p) + 1); + int32_t bufSize = varDataLen(pNode->datum.p) + 2 + 1; + void *buf = taosMemoryMalloc(bufSize); if (NULL == buf) { return NULL; } - strncpy(buf, varDataVal(pNode->datum.p), varDataLen(pNode->datum.p) + 1); + snprintf(buf, bufSize, "'%s'", varDataVal(pNode->datum.p)); return buf; } default: From 1bf4b37591d22fa2014f69fec9da7c6f39330eb5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 1 Apr 2022 11:19:21 +0800 Subject: [PATCH 38/38] feature/qnode --- source/libs/command/inc/commandInt.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index cc3e1d8b55..771baca2ab 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -79,10 +79,14 @@ typedef struct SExplainCtx { #define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u))) -#define EXPLAIN_ROW_NEW(level, ...) \ - do { \ - tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", (isVerboseLine ? "" : "-> ")); \ - tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \ +#define EXPLAIN_ROW_NEW(level, ...) \ + do { \ + if (isVerboseLine) { \ + tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s", (level) * 2 + 3, ""); \ + } else { \ + tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", "-> "); \ + } \ + tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__); \ } while (0) #define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__)