From 03a9679f975d085d7ae21fc9996697b786b67af5 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Apr 2022 19:40:50 +0800 Subject: [PATCH 01/22] fix(query): translate var type from value node to SVariant should include var header --- source/libs/nodes/src/nodesUtilFuncs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 758f9d5d6f..d4457a056e 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1226,7 +1226,7 @@ void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: - pVal->pz = pNode->datum.p + VARSTR_HEADER_SIZE; + pVal->pz = pNode->datum.p; break; case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_DECIMAL: From 553f0aa11df7035e583d65fcc27ec56e52fd4b00 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 16 Apr 2022 12:07:28 +0000 Subject: [PATCH 02/22] refact vnode --- source/dnode/vnode/src/inc/tsdb.h | 10 +++------- source/dnode/vnode/src/tsdb/tsdbCommit.c | 6 +++--- source/dnode/vnode/src/tsdb/tsdbFS.c | 8 ++++---- source/dnode/vnode/src/tsdb/tsdbFile.c | 8 ++++---- source/dnode/vnode/src/tsdb/tsdbMain.c | 20 +++++++------------- source/dnode/vnode/src/tsdb/tsdbRead.c | 10 +++++----- source/dnode/vnode/src/tsdb/tsdbSma.c | 16 ++++++++-------- source/dnode/vnode/src/vnd/vnodeOpen.c | 3 +-- 8 files changed, 35 insertions(+), 46 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index d0e005f990..5a9576f0c1 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -43,10 +43,8 @@ typedef struct STable { #define TABLE_TID(t) (t)->tid #define TABLE_UID(t) (t)->uid -STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, - STfs *pTfs); +STsdb *tsdbOpen(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF); void tsdbClose(STsdb *); -void tsdbRemove(const char *path); int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp); int tsdbPrepareCommit(STsdb *pTsdb); int tsdbCommit(STsdb *pTsdb); @@ -167,16 +165,14 @@ struct STsdb { SRtn rtn; SMemAllocatorFactory *pmaf; STsdbFS *fs; - SMeta *pMeta; - STfs *pTfs; SSmaEnvs smaEnvs; }; #define REPO_ID(r) ((r)->vgId) #define REPO_CFG(r) (&(r)->config) #define REPO_FS(r) ((r)->fs) -#define REPO_META(r) ((r)->pMeta) -#define REPO_TFS(r) ((r)->pTfs) +#define REPO_META(r) ((r)->pVnode->pMeta) +#define REPO_TFS(r) ((r)->pVnode->pTfs) #define IS_REPO_LOCKED(r) ((r)->repoLocked) #define REPO_TSMA_NUM(r) ((r)->smaEnvs.nTSma) #define REPO_RSMA_NUM(r) ((r)->smaEnvs.nRSma) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index a1edb7cd9c..25ef18cb36 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -100,7 +100,7 @@ int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { level = tsdbGetFidLevel(pSet->fid, pRtn); - if (tfsAllocDisk(pRepo->pTfs, level, &did) < 0) { + if (tfsAllocDisk(pRepo->pVnode->pTfs, level, &did) < 0) { terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; return -1; } @@ -427,7 +427,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) { pCommitIter->pTable = (STable *)taosMemoryMalloc(sizeof(STable)); pCommitIter->pTable->uid = pTbData->uid; pCommitIter->pTable->tid = pTbData->uid; - pCommitIter->pTable->pSchema = metaGetTbTSchema(pRepo->pMeta, pTbData->uid, 0); + pCommitIter->pTable->pSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0); } return 0; @@ -459,7 +459,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith); - if (tfsAllocDisk(pRepo->pTfs, tsdbGetFidLevel(fid, &(pCommith->rtn)), &did) < 0) { + if (tfsAllocDisk(REPO_TFS(pRepo), tsdbGetFidLevel(fid, &(pCommith->rtn)), &did) < 0) { terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 866c02cbb3..0012a50ccf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -644,7 +644,7 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) { } static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) { - snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(pRepo->pTfs), pRepo->vgId, + snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), pRepo->vgId, tsdbTxnFname[ftype]); } @@ -912,7 +912,7 @@ static int tsdbScanRootDir(STsdb *pRepo) { const STfsFile *pf; tsdbGetRootDir(REPO_ID(pRepo), rootDir); - STfsDir *tdir = tfsOpendir(pRepo->pTfs, rootDir); + STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), rootDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); return -1; @@ -946,7 +946,7 @@ static int tsdbScanDataDir(STsdb *pRepo) { const STfsFile *pf; tsdbGetDataDir(REPO_ID(pRepo), dataDir); - STfsDir *tdir = tfsOpendir(pRepo->pTfs, dataDir); + STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), dataDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); return -1; @@ -1128,7 +1128,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { return -1; } - tdir = tfsOpendir(pRepo->pTfs, dataDir); + tdir = tfsOpendir(REPO_TFS(pRepo), dataDir); if (tdir == NULL) { tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 8e5a6afc4d..2fbe819476 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -311,7 +311,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t pDFile->info.fver = tsdbGetDFSVersion(ftype); tsdbGetFilename(pRepo->vgId, fid, ver, ftype, fname); - tfsInitFile(pRepo->pTfs, &(pDFile->f), did, fname); + tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname); } void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile) { @@ -330,7 +330,7 @@ int tsdbEncodeSDFile(void **buf, SDFile *pDFile) { void *tsdbDecodeSDFile(STsdb *pRepo, void *buf, SDFile *pDFile) { buf = tsdbDecodeDFInfo(buf, &(pDFile->info)); - buf = tfsDecodeFile(pRepo->pTfs, buf, &(pDFile->f)); + buf = tfsDecodeFile(REPO_TFS(pRepo), buf, &(pDFile->f)); TSDB_FILE_SET_CLOSED(pDFile); return buf; @@ -365,7 +365,7 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T if (errno == ENOENT) { // Try to create directory recursively char *s = strdup(TSDB_FILE_REL_NAME(pDFile)); - if (tfsMkdirRecurAt(pRepo->pTfs, taosDirName(s), TSDB_FILE_DID(pDFile)) < 0) { + if (tfsMkdirRecurAt(REPO_TFS(pRepo), taosDirName(s), TSDB_FILE_DID(pDFile)) < 0) { taosMemoryFreeClear(s); return -1; } @@ -443,7 +443,7 @@ int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) { } static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) { - SDFile df; + SDFile df; tsdbInitDFileEx(&df, pDFile); diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index 8543cde046..dd8723366d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -15,14 +15,12 @@ #include "vnodeInt.h" -static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, - SMeta *pMeta, STfs *pTfs); +static STsdb *tsdbNew(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF); static void tsdbFree(STsdb *pTsdb); static int tsdbOpenImpl(STsdb *pTsdb); static void tsdbCloseImpl(STsdb *pTsdb); -STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, - STfs *pTfs) { +STsdb *tsdbOpen(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF) { STsdb *pTsdb = NULL; // Set default TSDB Options @@ -37,7 +35,7 @@ STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAl } // Create the handle - pTsdb = tsdbNew(path, vgId, pTsdbCfg, pMAF, pMeta, pTfs); + pTsdb = tsdbNew(path, pVnode, pTsdbCfg, pMAF); if (pTsdb == NULL) { // TODO: handle error return NULL; @@ -61,11 +59,8 @@ void tsdbClose(STsdb *pTsdb) { } } -void tsdbRemove(const char *path) { taosRemoveDir(path); } - /* ------------------------ STATIC METHODS ------------------------ */ -static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, - SMeta *pMeta, STfs *pTfs) { +static STsdb *tsdbNew(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF) { STsdb *pTsdb = NULL; pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb)); @@ -75,11 +70,10 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, } pTsdb->path = strdup(path); - pTsdb->vgId = vgId; + pTsdb->vgId = TD_VID(pVnode); + pTsdb->pVnode = pVnode; tsdbOptionsCopy(&(pTsdb->config), pTsdbCfg); pTsdb->pmaf = pMAF; - pTsdb->pMeta = pMeta; - pTsdb->pTfs = pTfs; pTsdb->fs = tsdbNewFS(pTsdbCfg); return pTsdb; @@ -156,7 +150,7 @@ int tsdbUnlockRepo(STsdb *pTsdb) { #define IS_VALID_PRECISION(precision) \ (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO)) -#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP +#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP #define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 1314ef7d1e..f8eed6c391 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -986,7 +986,7 @@ static int32_t loadBlockInfo(STsdbReadHandle* pTsdbReadHandle, int32_t index, in pCheckInfo->numOfBlocks = 0; STable table = {.uid = pCheckInfo->tableId, .tid = pCheckInfo->tableId}; - table.pSchema = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, pCheckInfo->tableId, 0); + table.pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 0); if (tsdbSetReadTable(&pTsdbReadHandle->rhelper, &table) != TSDB_CODE_SUCCESS) { code = terrno; @@ -1091,7 +1091,7 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl int32_t slotIndex) { int64_t st = taosGetTimestampUs(); - STSchema* pSchema = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, pCheckInfo->tableId, 0); + STSchema* pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 0); int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pSchema); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p failed to malloc buf for pDataCols, %s", pTsdbReadHandle, pTsdbReadHandle->idStr); @@ -1483,7 +1483,7 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit int32_t numOfColsOfRow1 = 0; if (pSchema1 == NULL) { - pSchema1 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, TD_ROW_SVER(row1)); + pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1)); } if (isRow1DataRow) { @@ -1496,7 +1496,7 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit if (row2) { isRow2DataRow = TD_IS_TP_ROW(row2); if (pSchema2 == NULL) { - pSchema2 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, TD_ROW_SVER(row2)); + pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2)); } if (isRow2DataRow) { numOfColsOfRow2 = schemaNCols(pSchema2); @@ -2514,7 +2514,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int win->ekey = key; if (rv != TD_ROW_SVER(row)) { - pSchema = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, pCheckInfo->tableId, 0); + pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 0); rv = TD_ROW_SVER(row); } mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index b98fe8936a..4b1c213cdd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -260,7 +260,7 @@ static void poolFree(void *arg, void *ptr) { int32_t tsdbInitSma(STsdb *pTsdb) { // tSma - int32_t numOfTSma = taosArrayGetSize(metaGetSmaTbUids(pTsdb->pMeta, false)); + int32_t numOfTSma = taosArrayGetSize(metaGetSmaTbUids(REPO_META(pTsdb), false)); if (numOfTSma > 0) { atomic_store_16(&REPO_TSMA_NUM(pTsdb), (int16_t)numOfTSma); } @@ -348,7 +348,7 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) } char aname[TSDB_FILENAME_LEN] = {0}; - tfsAbsoluteName(pTsdb->pTfs, did, path, aname); + tfsAbsoluteName(REPO_TFS(pTsdb), did, path, aname); if (tsdbOpenDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) { tsdbFreeSmaEnv(pEnv); return NULL; @@ -519,14 +519,14 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { char rname[TSDB_FILENAME_LEN] = {0}; SDiskID did = {0}; - tfsAllocDisk(pTsdb->pTfs, TFS_PRIMARY_LEVEL, &did); + tfsAllocDisk(REPO_TFS(pTsdb), TFS_PRIMARY_LEVEL, &did); if (did.level < 0 || did.id < 0) { tsdbUnlockRepo(pTsdb); return TSDB_CODE_FAILED; } tsdbGetSmaDir(REPO_ID(pTsdb), smaType, rname); - if (tfsMkdirRecurAt(pTsdb->pTfs, rname, did) != TSDB_CODE_SUCCESS) { + if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rname, did) != TSDB_CODE_SUCCESS) { tsdbUnlockRepo(pTsdb); return TSDB_CODE_FAILED; } @@ -557,7 +557,7 @@ static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t } // cache smaMeta - STSma *pSma = metaGetSmaInfoByIndex(pTsdb->pMeta, indexUid, true); + STSma *pSma = metaGetSmaInfoByIndex(REPO_META(pTsdb), indexUid, true); if (pSma == NULL) { terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; taosHashCleanup(pItem->expiredWindows); @@ -613,7 +613,7 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers return TSDB_CODE_SUCCESS; } - if (!pTsdb->pMeta) { + if (!REPO_META(pTsdb)) { terrno = TSDB_CODE_INVALID_PTR; return TSDB_CODE_FAILED; } @@ -1583,7 +1583,7 @@ int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) { // record current timezone of server side vCreateSmaReq.tSma.timezoneInt = tsTimezone; - if (metaCreateTSma(pTsdb->pMeta, &vCreateSmaReq) < 0) { + if (metaCreateTSma(REPO_META(pTsdb), &vCreateSmaReq) < 0) { // TODO: handle error tdDestroyTSma(&vCreateSmaReq.tSma); return -1; @@ -1610,7 +1610,7 @@ int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) { // } // - if (metaDropTSma(pTsdb->pMeta, vDropSmaReq.indexUid) < 0) { + if (metaDropTSma(REPO_META(pTsdb), vDropSmaReq.indexUid) < 0) { // TODO: handle error return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 5d4b07ec18..9e4aa714e2 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -96,8 +96,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // open tsdb sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TSDB_DIR); - pVnode->pTsdb = - tsdbOpen(tdir, TD_VID(pVnode), &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta, pVnode->pTfs); + pVnode->pTsdb = tsdbOpen(tdir, pVnode, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode)); if (pVnode->pTsdb == NULL) { vError("vgId: %d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; From 8f1f47c62a7e410055387b93d0f8f377c29d38db Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 16 Apr 2022 12:34:24 +0000 Subject: [PATCH 03/22] more --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b62bd62d9c..3044348648 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ compile_commands.json .tasks .vimspector.json .vscode/ +.vscode .idea/ cmake-build-debug/ cmake-build-release/ From 7429f37c0cf523562ff0aaaec234d4ce2c4ff1f2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Apr 2022 10:46:07 +0800 Subject: [PATCH 04/22] fix(query): fix bug in calculating the aggregate function with constant numeric value as input parameter. --- source/libs/executor/inc/executorimpl.h | 10 +- source/libs/executor/src/executorimpl.c | 439 +++++++----------------- 2 files changed, 134 insertions(+), 315 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 280d3da614..fe230f04b4 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -220,14 +220,9 @@ typedef struct SExecTaskInfo { } SExecTaskInfo; typedef struct STaskRuntimeEnv { - - jmp_buf env; STaskAttr* pQueryAttr; uint32_t status; // query status - void* qinfo; uint8_t scanFlag; // denotes reversed scan of data or not - void* pTsdbReadHandle; - bool enableGroupData; SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file SHashObj* pResultRowHashTable; // quick locate the window object for each result SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not @@ -235,12 +230,10 @@ typedef struct STaskRuntimeEnv { char* keyBuf; // window key buffer // The window result objects pool, all the resultRow Objects are allocated and managed by this object. char** prevRow; - SArray* prevResult; // intermediate result, SArray STSBuf* pTsBuf; // timestamp filter list STSCursor cur; char* tagVal; // tag value of current data block - struct SScalarFunctionSupport* scalarSup; STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure struct SOperatorInfo* proot; SGroupResInfo groupResInfo; @@ -266,7 +259,6 @@ typedef struct SOperatorInfo { char* name; // name, used to show the query execution plan void* info; // extension attribution SExprInfo* pExpr; - STaskRuntimeEnv* pRuntimeEnv; // todo remove it SExecTaskInfo* pTaskInfo; SOperatorCostInfo cost; SResultInfo resultInfo; @@ -291,7 +283,7 @@ typedef struct { typedef enum { EX_SOURCE_DATA_NOT_READY = 0x1, - EX_SOURCE_DATA_READY = 0x2, + EX_SOURCE_DATA_READY = 0x2, EX_SOURCE_DATA_EXHAUSTED = 0x3, } EX_SOURCE_STATUS; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 11c6a08c00..85516e14ad 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -186,10 +186,9 @@ static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t o static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); static bool functionNeedToExecute(SqlFunctionCtx* pCtx); -static void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pSDataBlock, SColumn* pColumn); +static void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pSDataBlock); static void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo); -static bool hasMainOutput(STaskAttr* pQueryAttr); static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols); @@ -456,7 +455,6 @@ static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultR if (p1 != NULL) { if (pResultRowInfo->size == 0) { existInCurrentResusltRowInfo = false; // this time window created by other timestamp that does not belongs to current table. -// assert(pResultRowInfo->curPos == -1); } else if (pResultRowInfo->size == 1) { SResultRowPosition* p = &pResultRowInfo->pPosition[0]; existInCurrentResusltRowInfo = (p->pageId == p1->pageId && p->offset == p1->offset); @@ -465,7 +463,6 @@ static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultR int64_t* index = taosHashGet(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); if (index != NULL) { // TODO check the scan order for current opened time window -// pResultRowInfo->curPos = (int32_t)*index; existInCurrentResusltRowInfo = true; } else { existInCurrentResusltRowInfo = false; @@ -505,7 +502,6 @@ static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultR } // 2. set the new time window to be the new active time window -// pResultRowInfo->curPos = pResultRowInfo->size; pResultRowInfo->pPosition[pResultRowInfo->size++] = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; pResultRowInfo->cur = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, uid, pResultRowInfo); @@ -1035,13 +1031,13 @@ static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols, int32_t rows, return ts; } -static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); +static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { pCtx[i].order = order; pCtx[i].size = pBlock->info.rows; - setBlockStatisInfo(&pCtx[i], &pOperator->pExpr[i], pBlock, NULL /*&pOperator->pExpr[i].base.colInfo*/); + setBlockStatisInfo(&pCtx[i], &pOperator->pExpr[i], pBlock); } } @@ -1053,22 +1049,64 @@ void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlo } } -static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { +static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t type, int32_t paramIndex, int32_t numOfRows) { + SColumnInfoData* pColInfo = NULL; + if (pInput->pData[paramIndex] == NULL) { + pColInfo = taosMemoryCalloc(1, sizeof(SColumnInfoData)); + if (pColInfo == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + // Set the correct column info (data type and bytes) + pColInfo->info.type = type; + pColInfo->info.bytes = tDataTypes[type].bytes; + + pInput->pData[paramIndex] = pColInfo; + } + + ASSERT(!IS_VAR_DATA_TYPE(type)); + colInfoDataEnsureCapacity(pColInfo, numOfRows); + + if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) { + int64_t v = pFuncParam->param.i; + for(int32_t i = 0; i < numOfRows; ++i) { + colDataAppendInt64(pColInfo, i, &v); + } + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + double v = pFuncParam->param.d; + for(int32_t i = 0; i < numOfRows; ++i) { + colDataAppendDouble(pColInfo, i, &v); + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { + int32_t code = TSDB_CODE_SUCCESS; + for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { pCtx[i].order = order; pCtx[i].size = pBlock->info.rows; pCtx[i].currentStage = MAIN_SCAN; + SInputColumnInfoData* pInput = &pCtx[i].input; + SExprInfo* pOneExpr = &pOperator->pExpr[i]; for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) { SFunctParam *pFuncParam = &pOneExpr->base.pParam[j]; if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) { int32_t slotId = pFuncParam->pCol->slotId; - pCtx[i].input.pData[j] = taosArrayGet(pBlock->pDataBlock, slotId); - pCtx[i].input.totalRows = pBlock->info.rows; - pCtx[i].input.numOfRows = pBlock->info.rows; - pCtx[i].input.startRowIndex = 0; - ASSERT(pCtx[i].input.pData[j] != NULL); + pInput->pData[j] = taosArrayGet(pBlock->pDataBlock, slotId); + pInput->totalRows = pBlock->info.rows; + pInput->numOfRows = pBlock->info.rows; + pInput->startRowIndex = 0; + ASSERT(pInput->pData[j] != NULL); + } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { + code = doCreateConstantValColumnInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } @@ -1111,6 +1149,8 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, // } // } } + + return code; } static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunctionCtx* pCtx) { @@ -1200,7 +1240,6 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { - STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; SExprInfo* pExpr = pOperator->pExpr; SqlFunctionCtx* pCtx = pInfo->pCtx; @@ -1220,7 +1259,7 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, double v1 = 0, v2 = 0, v = 0; if (prevRowIndex == -1) { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pRuntimeEnv->prevRow[index]); +// GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pRuntimeEnv->prevRow[index]); } else { GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); } @@ -1237,7 +1276,7 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { if (prevRowIndex == -1) { - pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index]; +// pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index]; } else { pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes; } @@ -1507,86 +1546,6 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe // updateResultRowInfoActiveIndex(pResultRowInfo, &pInfo->win, pRuntimeEnv->current->lastKey, true, false); } -static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, - int32_t tableGroupId) { - STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*)pOperatorInfo->info; - - STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - int32_t numOfOutput = pOperatorInfo->numOfOutput; - - int32_t step = 1;//GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - bool ascQuery = true; - - TSKEY* tsCols = NULL; - if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); - tsCols = (int64_t*)pColDataInfo->pData; - assert(tsCols[0] == pSDataBlock->info.window.skey && - tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); - } - - int32_t startPos = ascQuery ? 0 : (pSDataBlock->info.rows - 1); - TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascQuery); - - STimeWindow win = {0};//getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); - bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); - - SResultRow* pResult = NULL; - int32_t forwardStep = 0; - int32_t ret = 0; - STimeWindow preWin = win; - - while (1) { - // null data, failed to allocate more memory buffer -// ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, -// tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - TSKEY ekey = 0;//reviseWindowEkey(pQueryAttr, &win); - // forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, - // binarySearchForKey, true); - - // window start(end) key interpolation - // doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, - // forwardStep); doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, ascQuery ? &win : &preWin, startPos, - // forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); - preWin = win; - - int32_t prevEndPos = (forwardStep - 1) * step + startPos; - // startPos = getNextQualifiedWindow(pQueryAttr, &win, &pSDataBlock->info, tsCols, binarySearchForKey, - // prevEndPos); - if (startPos < 0) { -// if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) { -// int32_t code = -// setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, -// tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset); -// if (code != TSDB_CODE_SUCCESS || pResult == NULL) { -// longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); -// } -// -// startPos = pSDataBlock->info.rows - 1; - - // window start(end) key interpolation - // doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, - // forwardStep); doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, ascQuery ? &win : &preWin, startPos, - // forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); -// } - - break; - } - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - -// if (pQueryAttr->timeWindowInterpo) { -// int32_t rowIndex = ascQuery ? (pSDataBlock->info.rows - 1) : 0; - // saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); -// } - - // updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); -} - static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts) { pRowSup->win.ekey = ts; pRowSup->prevTs = ts; @@ -1730,30 +1689,85 @@ static bool functionNeedToExecute(SqlFunctionCtx* pCtx) { return true; } -void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pBlock, SColumn* pColumn) { - if (pBlock->pBlockAgg != NULL /*&& TSDB_COL_IS_NORMAL_COL(pColumn->flag)*/) { +static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t type, int32_t paramIndex, int32_t numOfRows) { + if (pInput->pData[paramIndex] == NULL) { + pInput->pData[paramIndex] = taosMemoryCalloc(1, sizeof(SColumnInfoData)); + if (pInput->pData[paramIndex] == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + // Set the correct column info (data type and bytes) + pInput->pData[paramIndex]->info.type = type; + pInput->pData[paramIndex]->info.bytes = tDataTypes[type].bytes; + } + + SColumnDataAgg* da = NULL; + if (pInput->pColumnDataAgg[paramIndex] == NULL) { + da = taosMemoryCalloc(1, sizeof(SColumnDataAgg)); + pInput->pColumnDataAgg[paramIndex] = da; + if (da == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } else { + da = pInput->pColumnDataAgg[paramIndex]; + } + + ASSERT(!IS_VAR_DATA_TYPE(type)); + + if (type == TSDB_DATA_TYPE_BIGINT) { + int64_t v = pFuncParam->param.i; + *da = (SColumnDataAgg) {.numOfNull = 0, .min = v, .max = v, .maxIndex = 0, .minIndex = 0, .sum = v * numOfRows}; + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + double v = pFuncParam->param.d; + *da = (SColumnDataAgg) {.numOfNull = 0, .maxIndex = 0, .minIndex = 0}; + + *(double*) &da->min = v; + *(double*) &da->max = v; + *(double*) &da->sum = v * numOfRows; + } else if (type == TSDB_DATA_TYPE_BOOL) { // todo validate this data type + bool v = pFuncParam->param.i; + + *da = (SColumnDataAgg) {.numOfNull = 0, .maxIndex = 0, .minIndex = 0}; + *(bool*) &da->min = 0; + *(bool*) &da->max = v; + *(bool*) &da->sum = v * numOfRows; + } else if (type == TSDB_DATA_TYPE_TIMESTAMP) { + // do nothing + } else { + ASSERT(0); + } + + return TSDB_CODE_SUCCESS; +} + +void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pBlock) { + int32_t numOfRows = pBlock->info.rows; + + SInputColumnInfoData* pInput = &pCtx->input; + pInput->numOfRows = numOfRows; + pInput->totalRows = numOfRows; + + if (pBlock->pBlockAgg != NULL) { + pInput->colDataAggIsSet = true; + for (int32_t j = 0; j < pExprInfo->base.numOfParams; ++j) { SFunctParam* pFuncParam = &pExprInfo->base.pParam[j]; + if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) { int32_t slotId = pFuncParam->pCol->slotId; - SInputColumnInfoData* pInput = &pCtx->input; - pInput->pColumnDataAgg[j] = &pBlock->pBlockAgg[slotId]; - pInput->colDataAggIsSet = true; - pInput->numOfRows = pBlock->info.rows; - pInput->totalRows = pBlock->info.rows; // Here we set the column info data since the data type for each column data is required, but // the data in the corresponding SColumnInfoData will not be used. pInput->pData[j] = taosArrayGet(pBlock->pDataBlock, slotId); + } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { + doCreateConstantValColumnAggInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows); } } } else { - pCtx->input.colDataAggIsSet = false; + pInput->colDataAggIsSet = false; } -// pCtx->hasNull = hasNull(pColumn, pAgg); - // set the statistics data for primary time stamp column // if (pCtx->functionId == FUNCTION_SPREAD && pColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { // pCtx->isAggSet = true; @@ -2224,33 +2238,6 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI return false; } -static int32_t doTSJoinFilter(STaskRuntimeEnv* pRuntimeEnv, TSKEY key, bool ascQuery) { - STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); - -#if defined(_DEBUG_VIEW) - printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%" PRIu64 - ", query order:%d, ts order:%d, traverse:%d, index:%d\n", - elem.ts, key, elem.tag.i, pQueryAttr->order.order, pRuntimeEnv->pTsBuf->tsOrder, - pRuntimeEnv->pTsBuf->cur.order, pRuntimeEnv->pTsBuf->cur.tsIndex); -#endif - - if (ascQuery) { - if (key < elem.ts) { - return TS_JOIN_TS_NOT_EQUALS; - } else if (key > elem.ts) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); - } - } else { - if (key > elem.ts) { - return TS_JOIN_TS_NOT_EQUALS; - } else if (key < elem.ts) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); - } - } - - return TS_JOIN_TS_EQUAL; -} - void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p) { int32_t len = 0; int32_t start = 0; @@ -2300,54 +2287,6 @@ void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p) { } } -void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) { - int32_t numOfRows = pBlock->info.rows; - - int8_t* p = NULL; - bool all = true; - - if (pRuntimeEnv->pTsBuf != NULL) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); - - TSKEY* k = (TSKEY*)pColInfoData->pData; - for (int32_t i = 0; i < numOfRows; ++i) { - int32_t offset = ascQuery ? i : (numOfRows - i - 1); - int32_t ret = doTSJoinFilter(pRuntimeEnv, k[offset], ascQuery); - if (ret == TS_JOIN_TAG_NOT_EQUALS) { - break; - } else if (ret == TS_JOIN_TS_NOT_EQUALS) { - all = false; - continue; - } else { - assert(ret == TS_JOIN_TS_EQUAL); - p[offset] = true; - } - - if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { - break; - } - } - - // save the cursor status - // pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); - } else { - // all = filterExecute(pRuntimeEnv->pQueryAttr->pFilters, numOfRows, &p, pBlock->pBlockAgg, - // pRuntimeEnv->pQueryAttr->numOfCols); - } - - if (!all) { - if (p) { - doCompactSDataBlock(pBlock, numOfRows, p); - } else { - pBlock->info.rows = 0; - pBlock->pBlockAgg = NULL; // clean the block statistics info - } - } - - taosMemoryFreeClear(p); -} - static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId); static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant* tag, int16_t type, int16_t bytes); @@ -2936,18 +2875,6 @@ void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbased } } -static bool hasMainOutput(STaskAttr* pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); - - if (functionId != FUNCTION_TS && functionId != FUNCTION_TAG && functionId != FUNCTION_TAGPRJ) { - return true; - } - } - - return false; -} - STableQueryInfo* createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow win) { STableQueryInfo* pTableQueryInfo = buf; pTableQueryInfo->lastKey = win.skey; @@ -3111,48 +3038,6 @@ void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprI // } } -int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo* pTableQueryInfo) { - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - assert(pRuntimeEnv->pTsBuf != NULL); -#if 0 - // both the master and supplement scan needs to set the correct ts comp start position - if (pTableQueryInfo->cur.vgroupIndex == -1) { - taosVariantAssign(&pTableQueryInfo->tag, pTag); - - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQueryAttr->vgId, &pTableQueryInfo->tag); - - // failed to find data with the specified tag value and vnodeId - if (!tsBufIsValidElem(&elem)) { - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - //qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_TASKID(pRuntimeEnv), pTag->pz); - } else { - //qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_TASKID(pRuntimeEnv), pTag->i); - } - - return -1; - } - - // Keep the cursor info of current table - pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } else { - //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } - - } else { - tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } else { - //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } - } -#endif - return 0; -} - /* * There are two cases to handle: * @@ -3323,19 +3208,15 @@ int32_t doFillTimeIntervalGapsInResults(struct SFillInfo* pFillInfo, SSDataBlock return pOutput->info.rows; } -void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType) { +void publishOperatorProfEvent(SOperatorInfo* pOperator, EQueryProfEventType eventType) { SQueryProfEvent event = {0}; event.eventType = eventType; event.eventTime = taosGetTimestampUs(); - event.operatorType = operatorInfo->operatorType; - - if (operatorInfo->pRuntimeEnv) { - // SQInfo* pQInfo = operatorInfo->pRuntimeEnv->qinfo; - // if (pQInfo->summary.queryProfEvents) { - // taosArrayPush(pQInfo->summary.queryProfEvents, &event); - // } - } + event.operatorType = pOperator->operatorType; +// if (pQInfo->summary.queryProfEvents) { +// taosArrayPush(pQInfo->summary.queryProfEvents, &event); +// } } void publishQueryAbortEvent(SExecTaskInfo* pTaskInfo, int32_t code) { @@ -5313,7 +5194,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo *pOperator, bool* newgroup) { // setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order); - hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0); +// hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0); } // restore the value @@ -5387,57 +5268,6 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; } -static SSDataBlock* doAllSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - - if (pOperator->status == OP_RES_TO_RETURN) { - // copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pIntervalInfo->binfo.pRes; - } - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - // the pDataBlock are always the same one, no need to call this again - STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; - - // setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); -// setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order); -// setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); - -// hashAllIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); - } - - pOperator->status = OP_RES_TO_RETURN; -// pQueryAttr->order.order = order; // TODO : restore the order - setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - - int64_t st = taosGetTimestampUs(); - // copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pIntervalInfo->binfo.pRes; -} - static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SOptrBasicInfo* pBInfo = &pInfo->binfo; @@ -6480,8 +6310,9 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* pExp->pExpr->_function.num = 1; pExp->pExpr->_function.functionId = -1; + int32_t type = nodeType(pTargetNode->pExpr); // it is a project query, or group by column - if (nodeType(pTargetNode->pExpr) == QUERY_NODE_COLUMN) { + if (type == QUERY_NODE_COLUMN) { pExp->pExpr->nodeType = QUERY_NODE_COLUMN; SColumnNode* pColNode = (SColumnNode*)pTargetNode->pExpr; @@ -6492,7 +6323,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pColNode->colName); pExp->base.pParam[0].pCol = createColumn(pColNode->dataBlockId, pColNode->slotId, pType); pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; - } else if (nodeType(pTargetNode->pExpr) == QUERY_NODE_VALUE) { + } else if (type == QUERY_NODE_VALUE) { pExp->pExpr->nodeType = QUERY_NODE_VALUE; SValueNode* pValNode = (SValueNode*)pTargetNode->pExpr; @@ -6503,7 +6334,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pValNode->node.aliasName); pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE; valueNodeToVariant(pValNode, &pExp->base.pParam[0].param); - } else if (nodeType(pTargetNode->pExpr) == QUERY_NODE_FUNCTION) { + } else if (type == QUERY_NODE_FUNCTION) { pExp->pExpr->nodeType = QUERY_NODE_FUNCTION; SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; @@ -6514,14 +6345,13 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* pExp->pExpr->_function.pFunctNode = pFuncNode; strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, tListLen(pExp->pExpr->_function.functionName)); - // TODO: value parameter needs to be handled int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList); pExp->base.pParam = taosMemoryCalloc(numOfParam, sizeof(SFunctParam)); pExp->base.numOfParams = numOfParam; for (int32_t j = 0; j < numOfParam; ++j) { - SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j); + SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j); if (p1->type == QUERY_NODE_COLUMN) { SColumnNode* pcn = (SColumnNode*) p1; @@ -6530,9 +6360,10 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* } else if (p1->type == QUERY_NODE_VALUE) { SValueNode* pvn = (SValueNode*)p1; pExp->base.pParam[j].type = FUNC_PARAM_TYPE_VALUE; + valueNodeToVariant(pvn, &pExp->base.pParam[j].param); } } - } else if (nodeType(pTargetNode->pExpr) == QUERY_NODE_OPERATOR) { + } else if (type == QUERY_NODE_OPERATOR) { pExp->pExpr->nodeType = QUERY_NODE_OPERATOR; SOperatorNode* pNode = (SOperatorNode*)pTargetNode->pExpr; @@ -6541,11 +6372,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* SDataType* pType = &pNode->node.resType; pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pNode->node.aliasName); - pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr; - -// pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; -// pExp->base.pParam[0].pCol = createColumn(pTargetNode->dataBlockId, pTargetNode->slotId, pType); } else { ASSERT(0); } From f56c354c00a33909817c5c786b652da8f68e872a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Apr 2022 10:58:40 +0800 Subject: [PATCH 05/22] fix: startup is slow in valgrind mode --- tests/tsim/src/simExe.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 5a18084fff..a4db0eff13 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -18,7 +18,7 @@ void simLogSql(char *sql, bool useSharp) { static TdFilePtr pFile = NULL; - char filename[256]; + char filename[256]; sprintf(filename, "%s/sim.sql", simScriptDir); if (pFile == NULL) { // fp = fopen(filename, "w"); @@ -305,7 +305,8 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { return true; } -void simReplaceStr(char *buf, char *src, char *dst) { +bool simReplaceStr(char *buf, char *src, char *dst) { + bool replaced = false; char *begin = strstr(buf, src); if (begin != NULL) { int32_t srcLen = (int32_t)strlen(src); @@ -320,13 +321,16 @@ void simReplaceStr(char *buf, char *src, char *dst) { } memcpy(begin, dst, dstLen); + replaced = true; } simInfo("system cmd is %s", buf); + return replaced; } bool simExecuteSystemCmd(SScript *script, char *option) { char buf[4096] = {0}; + bool replaced = false; #ifndef WINDOWS sprintf(buf, "cd %s; ", simScriptDir); @@ -341,7 +345,7 @@ bool simExecuteSystemCmd(SScript *script, char *option) { } if (useValgrind) { - simReplaceStr(buf, "exec.sh", "exec.sh -v"); + replaced = simReplaceStr(buf, "exec.sh", "exec.sh -v"); } simLogSql(buf, true); @@ -359,6 +363,11 @@ bool simExecuteSystemCmd(SScript *script, char *option) { sprintf(script->system_exit_code, "%d", code); script->linePos++; + if (replaced && strstr(buf, "start") != NULL) { + simInfo("====> startup is slow in valgrind mode, so sleep 5 seconds after exec.sh -s start"); + taosMsleep(5000); + } + return true; } @@ -774,7 +783,7 @@ bool simExecuteSqlSlowCmd(SScript *script, char *rest) { bool simExecuteRestfulCmd(SScript *script, char *rest) { TdFilePtr pFile = NULL; - char filename[256]; + char filename[256]; sprintf(filename, "%s/tmp.sql", simScriptDir); // fp = fopen(filename, "w"); pFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); From e91233f5ebe477f7172d82e4aaddaad68bef74b2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Apr 2022 11:01:07 +0800 Subject: [PATCH 06/22] fix(query): set the correct flag to denote the initialization status. --- source/libs/executor/src/executorimpl.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 85516e14ad..425696ced0 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2914,8 +2914,12 @@ void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, continue; } - if (!pResInfo->initialized && pCtx[i].functionId != -1) { - pCtx[i].fpSet.init(&pCtx[i], pResInfo); + if (!pResInfo->initialized) { + if (pCtx[i].functionId != -1) { + pCtx[i].fpSet.init(&pCtx[i], pResInfo); + } else { + pResInfo->initialized = true; + } } } } From 6105dfc934df8c0fbd0fccf684a74043a010d6a9 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 18 Apr 2022 11:39:08 +0800 Subject: [PATCH 07/22] refactor(query): refactor concat/concat_ws and param check --- source/libs/function/src/builtins.c | 42 ++++++++++++++++++----------- source/libs/scalar/src/sclfunc.c | 40 +++++++-------------------- 2 files changed, 36 insertions(+), 46 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index bba5145812..9766a658fd 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -231,33 +231,45 @@ static int32_t translateLength(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return TSDB_CODE_SUCCESS; } -static int32_t translateConcatImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, int32_t minParaNum, int32_t maxParaNum, int32_t primaryParaNo) { +static int32_t translateConcatImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, int32_t minParaNum, int32_t maxParaNum, bool hasSep) { int32_t paraNum = LIST_LENGTH(pFunc->pParameterList); if (paraNum < minParaNum || paraNum > maxParaNum) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - uint8_t resultType = TSDB_DATA_TYPE_NCHAR; + uint8_t resultType = TSDB_DATA_TYPE_BINARY; int32_t resultBytes = 0; int32_t sepBytes = 0; - for (int32_t i = 0; i < LIST_LENGTH(pFunc->pParameterList); ++i) { + + /* For concat/concat_ws function, if params have NCHAR type, promote the final result to NCHAR */ + for (int32_t i = 0; i < paraNum; ++i) { SNode* pPara = nodesListGetNode(pFunc->pParameterList, i); uint8_t paraType = ((SExprNode*)pPara)->resType.type; - int32_t paraBytes = ((SExprNode*)pPara)->resType.bytes; if (!IS_VAR_DATA_TYPE(paraType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - if (i < primaryParaNo) { - sepBytes = paraBytes; - continue; + if (TSDB_DATA_TYPE_NCHAR == paraType) { + resultType = paraType; } - if (TSDB_DATA_TYPE_BINARY == paraType) { - resultType = TSDB_DATA_TYPE_BINARY; - } - resultBytes += paraBytes; } - if (sepBytes > 0) { - resultBytes += sepBytes * (paraNum - 2); + + for (int32_t i = 0; i < paraNum; ++i) { + SNode* pPara = nodesListGetNode(pFunc->pParameterList, i); + uint8_t paraType = ((SExprNode*)pPara)->resType.type; + int32_t paraBytes = ((SExprNode*)pPara)->resType.bytes; + int32_t factor = 1; + if (TSDB_DATA_TYPE_NCHAR == resultType && TSDB_DATA_TYPE_VARCHAR == paraType) { + factor *= TSDB_NCHAR_SIZE; + } + resultBytes += paraBytes * factor; + + if (i == 0) { + sepBytes = paraBytes * factor; + } + } + + if (hasSep) { + resultBytes += sepBytes * (paraNum - 3); } pFunc->node.resType = (SDataType) { .bytes = resultBytes, .type = resultType }; @@ -265,11 +277,11 @@ static int32_t translateConcatImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t } static int32_t translateConcat(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - return translateConcatImpl(pFunc, pErrBuf, len, 2, 8, 0); + return translateConcatImpl(pFunc, pErrBuf, len, 2, 8, false); } static int32_t translateConcatWs(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - return translateConcatImpl(pFunc, pErrBuf, len, 3, 9, 1); + return translateConcatImpl(pFunc, pErrBuf, len, 3, 9, true); } static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 97cc80b946..84d47adc29 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -313,8 +313,8 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP return TSDB_CODE_SUCCESS; } -static int32_t concatCopyHelper(const char *input, char *output, bool hasNcharCol, int32_t type, int16_t *dataLen) { - if (hasNcharCol && type == TSDB_DATA_TYPE_VARCHAR) { +static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, int16_t *dataLen) { + if (hasNchar && type == TSDB_DATA_TYPE_VARCHAR) { TdUcs4 *newBuf = taosMemoryCalloc((varDataLen(input) + 1) * TSDB_NCHAR_SIZE, 1); bool ret = taosMbsToUcs4(varDataVal(input), varDataLen(input), newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, NULL); if (!ret) { @@ -345,10 +345,6 @@ static int32_t getNumOfNullEntries(SColumnInfoData *pColumnInfoData, int32_t num } int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - if (inputNum < 2 || inputNum > 8) { // concat accpet 2-8 input strings - return TSDB_CODE_FAILED; - } - SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); SColumnInfoData *pOutputData = pOutput->columnData; char **input = taosMemoryCalloc(inputNum, POINTER_BYTES); @@ -356,15 +352,8 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t inputLen = 0; int32_t numOfRows = 0; - bool hasNcharCol = false; + bool hasNchar = (GET_PARAM_TYPE(pOutput) == TSDB_DATA_TYPE_NCHAR) ? true : false; for (int32_t i = 0; i < inputNum; ++i) { - int32_t type = GET_PARAM_TYPE(&pInput[i]); - if (!IS_VAR_DATA_TYPE(type)) { - return TSDB_CODE_FAILED; - } - if (type == TSDB_DATA_TYPE_NCHAR) { - hasNcharCol = true; - } if (pInput[i].numOfRows > numOfRows) { numOfRows = pInput[i].numOfRows; } @@ -373,7 +362,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu pInputData[i] = pInput[i].columnData; input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[0]; int32_t factor = 1; - if (hasNcharCol && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) { + if (hasNchar && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) { factor = TSDB_NCHAR_SIZE; } @@ -405,7 +394,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int16_t dataLen = 0; for (int32_t i = 0; i < inputNum; ++i) { - int32_t ret = concatCopyHelper(input[i], output, hasNcharCol, GET_PARAM_TYPE(&pInput[i]), &dataLen); + int32_t ret = concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -428,10 +417,6 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - if (inputNum < 3 || inputNum > 9) { // concat accpet 3-9 input strings including the separator - return TSDB_CODE_FAILED; - } - SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); SColumnInfoData *pOutputData = pOutput->columnData; char **input = taosMemoryCalloc(inputNum, POINTER_BYTES); @@ -439,15 +424,8 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t inputLen = 0; int32_t numOfRows = 0; - bool hasNcharCol = false; + bool hasNchar = (GET_PARAM_TYPE(pOutput) == TSDB_DATA_TYPE_NCHAR) ? true : false; for (int32_t i = 1; i < inputNum; ++i) { - int32_t type = GET_PARAM_TYPE(&pInput[i]); - if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i]))) { - return TSDB_CODE_FAILED; - } - if (type == TSDB_DATA_TYPE_NCHAR) { - hasNcharCol = true; - } if (pInput[i].numOfRows > numOfRows) { numOfRows = pInput[i].numOfRows; } @@ -456,7 +434,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p pInputData[i] = pInput[i].columnData; input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[0]; int32_t factor = 1; - if (hasNcharCol && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) { + if (hasNchar && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) { factor = TSDB_NCHAR_SIZE; } @@ -487,7 +465,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p continue; } - int32_t ret = concatCopyHelper(input[i], output, hasNcharCol, GET_PARAM_TYPE(&pInput[i]), &dataLen); + int32_t ret = concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -499,7 +477,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p if (i < inputNum - 1) { //insert the separator char *sep = pInputData[0]->pData; - int32_t ret = concatCopyHelper(sep, output, hasNcharCol, GET_PARAM_TYPE(&pInput[0]), &dataLen); + int32_t ret = concatCopyHelper(sep, output, hasNchar, GET_PARAM_TYPE(&pInput[0]), &dataLen); if (ret != TSDB_CODE_SUCCESS) { return ret; } From 649a9ac230fa725c9d340a0b3f34994cd413440c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Apr 2022 11:42:39 +0800 Subject: [PATCH 08/22] fix: memory overwrite while process nchar in sim script --- tests/tsim/inc/simInt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tsim/inc/simInt.h b/tests/tsim/inc/simInt.h index c8b13736c7..24119b1433 100644 --- a/tests/tsim/inc/simInt.h +++ b/tests/tsim/inc/simInt.h @@ -33,7 +33,7 @@ #define MAX_BACKGROUND_SCRIPT_NUM 10 #define MAX_FILE_NAME_LEN 256 #define MAX_ERROR_LEN 1024 -#define MAX_QUERY_VALUE_LEN 40 +#define MAX_QUERY_VALUE_LEN 10240 #define MAX_QUERY_COL_NUM 20 #define MAX_QUERY_ROW_NUM 20 #define MAX_SYSTEM_RESULT_LEN 2048 From 781fbce942a79f47a4a3b5ddab1e3446d7f608ae Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Apr 2022 12:07:04 +0800 Subject: [PATCH 09/22] fix(query): fix bug caused by new completed query function. --- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executorimpl.c | 34 +++++++++++++----------- source/libs/executor/src/groupoperator.c | 2 +- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index fe230f04b4..5ef1754913 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -674,7 +674,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReaderHandle* pReaderHandle, SExprInfo void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, int32_t numOfOutput, SArray* pPseudoList); -void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); +void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol); void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput); void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 425696ced0..ed2454faae 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1031,7 +1031,7 @@ static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols, int32_t rows, return ts; } -static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); +static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol); static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { @@ -1041,11 +1041,11 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pC } } -void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { +void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol) { if (pBlock->pBlockAgg != NULL) { doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); } else { - doSetInputDataBlock(pOperator, pCtx, pBlock, order); + doSetInputDataBlock(pOperator, pCtx, pBlock, order, createDummyCol); } } @@ -1082,7 +1082,7 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc return TSDB_CODE_SUCCESS; } -static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { +static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol) { int32_t code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { @@ -1103,9 +1103,11 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pInput->startRowIndex = 0; ASSERT(pInput->pData[j] != NULL); } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { - code = doCreateConstantValColumnInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows); - if (code != TSDB_CODE_SUCCESS) { - return code; + if (createDummyCol) { + code = doCreateConstantValColumnInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } } @@ -4472,7 +4474,7 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) { break; } - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pDataBlock, TSDB_ORDER_ASC); + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pDataBlock, TSDB_ORDER_ASC, true); // updateOutputBuf(&pInfo->binfo, &pAggInfo->bufCapacity, pBlock->info.rows * pAggInfo->resultRowFactor, // pOperator->pRuntimeEnv, true); doMergeImpl(pOperator, pOperator->numOfOutput, pDataBlock); @@ -4745,7 +4747,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { // the pDataBlock are always the same one, no need to call this again setExecutionContext(pOperator->numOfOutput, pBlock->info.groupId, pTaskInfo, pAggInfo); - setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, true); doAggregateImpl(pOperator, 0, pInfo->pCtx); #if 0 // test for encode/decode result info @@ -4971,7 +4973,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) // } // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC); + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC, false); blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfOutput, pProjectInfo->pPseudoColInfo); @@ -5057,7 +5059,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order); + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); @@ -5152,7 +5154,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator, bool* newgroup // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order); + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); } @@ -5197,7 +5199,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo *pOperator, bool* newgroup) { // setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order); + setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, true); // hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0); } @@ -5247,7 +5249,7 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup // the pDataBlock are always the same one, no need to call this again // setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, TSDB_ORDER_ASC); + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, TSDB_ORDER_ASC, true); STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); @@ -5375,7 +5377,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { break; } - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order); + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); doStateWindowAggImpl(pOperator, pInfo, pBlock); } @@ -5423,7 +5425,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) } // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order); + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); doSessionWindowAggImpl(pOperator, pInfo, pBlock); } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index d58bcd1162..8739371dd9 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -284,7 +284,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou } // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order); + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); // there is an scalar expression that needs to be calculated right before apply the group aggregation. if (pInfo->pScalarExprInfo != NULL) { From 715787286cfadaa2f51a38901ee778d5215b9154 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Apr 2022 14:24:23 +0800 Subject: [PATCH 10/22] ehn(query): set the parameters for system database to be NULL by default when responsing "show databases" command. --- source/dnode/mnode/impl/src/mndDb.c | 207 +++++++++++++++------------- 1 file changed, 112 insertions(+), 95 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 13fbcdf9b3..9889743a82 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1354,7 +1354,7 @@ char *mndGetDbStr(char *src) { return pos; } -static void dumpDbInfoData(SSDataBlock* pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables) { +static void dumpDbInfoData(SSDataBlock* pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables, bool sysDb) { int32_t cols = 0; char* buf = taosMemoryMalloc(pShow->bytes[cols]); @@ -1366,100 +1366,117 @@ static void dumpDbInfoData(SSDataBlock* pBlock, SDbObj *pDb, SShowObj *pShow, in ASSERT(0); } - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, buf, false); - taosMemoryFree(buf); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->createdTime, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfVgroups, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&numOfTables, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.replications, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.quorum, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.daysPerFile, false); - - char tmp[128] = {0}; - int32_t len = 0; - if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { - len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%d,%d,%d", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, pDb->cfg.daysToKeep0); - } else { - len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%d,%d,%d", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2); - } - - varDataSetLen(tmp, len); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)tmp, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheBlockSize, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.totalBlocks, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.minRows, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.maxRows, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walLevel, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.fsyncPeriod, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.compression, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLastRow, false); - - char *prec = NULL; - switch (pDb->cfg.precision) { - case TSDB_TIME_PRECISION_MILLI: - prec = TSDB_TIME_PRECISION_MILLI_STR; - break; - case TSDB_TIME_PRECISION_MICRO: - prec = TSDB_TIME_PRECISION_MICRO_STR; - break; - case TSDB_TIME_PRECISION_NANO: - prec = TSDB_TIME_PRECISION_NANO_STR; - break; - default: - prec = "none"; - break; - } - - char t[10] = {0}; - STR_WITH_SIZE_TO_VARSTR(t, prec, 2); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)t, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.ttl, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.singleSTable, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.streamMode, false); - char *status = "ready"; - char b[24] = {0}; + char b[24] = {0}; STR_WITH_SIZE_TO_VARSTR(b, status, strlen(status)); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)b, false); + if (sysDb) { + for(int32_t i = 0; i < pShow->numOfColumns; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + if (i == 0) { + colDataAppend(pColInfo, rows, buf, false); + } else if (i == 3) { + colDataAppend(pColInfo, rows, (const char *)&numOfTables, false); + } else if (i == 20) { + colDataAppend(pColInfo, rows, b, false); + } else { + colDataAppendNULL(pColInfo, rows); + } + } + } else { + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, buf, false); + taosMemoryFree(buf); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->createdTime, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfVgroups, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&numOfTables, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.replications, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.quorum, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.daysPerFile, false); + + char tmp[128] = {0}; + int32_t len = 0; + if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { + len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%d,%d,%d", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, + pDb->cfg.daysToKeep0); + } else { + len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%d,%d,%d", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1, + pDb->cfg.daysToKeep2); + } + + varDataSetLen(tmp, len); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)tmp, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheBlockSize, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.totalBlocks, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.minRows, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.maxRows, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walLevel, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.fsyncPeriod, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.compression, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLastRow, false); + + char *prec = NULL; + switch (pDb->cfg.precision) { + case TSDB_TIME_PRECISION_MILLI: + prec = TSDB_TIME_PRECISION_MILLI_STR; + break; + case TSDB_TIME_PRECISION_MICRO: + prec = TSDB_TIME_PRECISION_MICRO_STR; + break; + case TSDB_TIME_PRECISION_NANO: + prec = TSDB_TIME_PRECISION_NANO_STR; + break; + default: + prec = "none"; + break; + } + + char t[10] = {0}; + STR_WITH_SIZE_TO_VARSTR(t, prec, 2); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)t, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.ttl, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.singleSTable, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.streamMode, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + colDataAppend(pColInfo, rows, (const char *)b, false); + } // pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); // *(int8_t *)pWrite = pDb->cfg.update; @@ -1508,13 +1525,13 @@ static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlo if (!pShow->sysDbRsp) { SDbObj infoschemaDb = {0}; setInformationSchemaDbCfg(&infoschemaDb); - dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14); + dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true); numOfRows += 1; SDbObj perfschemaDb = {0}; setPerfSchemaDbCfg(&perfschemaDb); - dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3); + dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true); numOfRows += 1; pShow->sysDbRsp = true; @@ -1529,7 +1546,7 @@ static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlo int32_t numOfTables = 0; sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL); - dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables); + dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false); numOfRows++; sdbRelease(pSdb, pDb); } From 595d2ce352b2412013d05f812141c52ddd5c2762 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Apr 2022 14:37:45 +0800 Subject: [PATCH 11/22] enh: support variables like $data[0][1] and $data(db)[0] in tsim scripts --- tests/script/tsim/db/alter_option.sim | 2 +- tests/script/tsim/db/basic6.sim | 1 - tests/script/tsim/user/basic1.sim | 38 ++++++------- tests/tsim/inc/simInt.h | 4 +- tests/tsim/src/simExe.c | 77 ++++++++++++++++++++++++--- 5 files changed, 92 insertions(+), 30 deletions(-) diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index e9795bd8d2..525de9c5c4 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -72,7 +72,7 @@ print rows: $rows print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 print ====> dataX_db -print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +print $data(db)[0] $data(db)[1] $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db if $rows != 3 then return -1 diff --git a/tests/script/tsim/db/basic6.sim b/tests/script/tsim/db/basic6.sim index 4d0a1e9ee9..f682dcc816 100644 --- a/tests/script/tsim/db/basic6.sim +++ b/tests/script/tsim/db/basic6.sim @@ -17,7 +17,6 @@ print =============== step1 # quorum presicion sql create database $db vgroups 8 replica 1 days 2880 keep 3650 cache 32 blocks 12 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachelast 2 precision 'us' sql show databases -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 if $rows != 3 then diff --git a/tests/script/tsim/user/basic1.sim b/tests/script/tsim/user/basic1.sim index 7af5ba8d00..06a52c6604 100644 --- a/tests/script/tsim/user/basic1.sim +++ b/tests/script/tsim/user/basic1.sim @@ -9,9 +9,9 @@ if $rows != 1 then return -1 endi -print $data00 $data01 $data02 -print $data10 $data11 $data22 -print $data20 $data11 $data22 +print $data[0][0] $data[0][1] $data[0][2] +print $data[1][0] $data[1][1] $data[1][2] +print $data[2][0] $data[1][2] $data[2][2] sql_error show accounts; sql_error create account a pass "a" @@ -25,10 +25,10 @@ if $rows != 2 then return -1 endi -print $data00 $data01 $data02 -print $data10 $data11 $data12 -print $data20 $data11 $data22 -print $data30 $data31 $data32 +print $data[0][0] $data[0][1] $data[0][2] +print $data[1][0] $data[1][1] $data[1][2] +print $data[2][0] $data[1][2] $data[2][2] +print $data[3][0] $data[3][1] $data[3][2] print =============== create user2 sql create user user2 PASS 'user2' @@ -37,10 +37,10 @@ if $rows != 3 then return -1 endi -print $data00 $data01 $data02 -print $data10 $data11 $data12 -print $data20 $data11 $data22 -print $data30 $data31 $data32 +print $data[0][0] $data[0][1] $data[0][2] +print $data[1][0] $data[1][1] $data[1][2] +print $data[2][0] $data[1][2] $data[2][2] +print $data[3][0] $data[3][1] $data[3][2] print $data40 $data41 $data42 print =============== drop user1 @@ -50,10 +50,10 @@ if $rows != 2 then return -1 endi -print $data00 $data01 $data02 -print $data10 $data11 $data22 -print $data20 $data11 $data22 -print $data30 $data31 $data32 +print $data[0][0] $data[0][1] $data[0][2] +print $data[1][0] $data[1][1] $data[1][2] +print $data[2][0] $data[1][2] $data[2][2] +print $data[3][0] $data[3][1] $data[3][2] print =============== restart taosd system sh/exec.sh -n dnode1 -s stop @@ -66,9 +66,9 @@ if $rows != 2 then return -1 endi -print $data00 $data01 $data02 -print $data10 $data11 $data22 -print $data20 $data11 $data22 -print $data30 $data31 $data32 +print $data[0][0] $data[0][1] $data[0][2] +print $data[1][0] $data[1][1] $data[1][2] +print $data[2][0] $data[1][2] $data[2][2] +print $data[3][0] $data[3][1] $data[3][2] system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/tsim/inc/simInt.h b/tests/tsim/inc/simInt.h index c8b13736c7..5ea372dee1 100644 --- a/tests/tsim/inc/simInt.h +++ b/tests/tsim/inc/simInt.h @@ -34,8 +34,8 @@ #define MAX_FILE_NAME_LEN 256 #define MAX_ERROR_LEN 1024 #define MAX_QUERY_VALUE_LEN 40 -#define MAX_QUERY_COL_NUM 20 -#define MAX_QUERY_ROW_NUM 20 +#define MAX_QUERY_COL_NUM 100 +#define MAX_QUERY_ROW_NUM 100 #define MAX_SYSTEM_RESULT_LEN 2048 #define MAX_VAR_LEN 100 #define MAX_VAR_NAME_LEN 32 diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 5a18084fff..af612603a3 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -18,7 +18,7 @@ void simLogSql(char *sql, bool useSharp) { static TdFilePtr pFile = NULL; - char filename[256]; + char filename[256]; sprintf(filename, "%s/sim.sql", simScriptDir); if (pFile == NULL) { // fp = fopen(filename, "w"); @@ -49,6 +49,41 @@ char *simParseHostName(char *varName) { return hostName; } +static void simFindFirstNum(const char *begin, int32_t beginLen, int32_t *num) { + if (beginLen <= 5) { + *num = 0; + } else { + *num = atoi(begin + 5); + } +} + +static void simFindSecondNum(const char *begin, int32_t beginLen, int32_t *num) { + const char *number = strstr(begin, "]["); + if (number == NULL) { + *num = 0; + } else { + *num = atoi(number + 2); + } +} + +static void simFindFirstKeyVal(const char *begin, int32_t beginLen, char *key, int32_t keyLen) { + key[0] = 0; + for (int32_t i = 5; i < beginLen && i - 5 < keyLen; ++i) { + if (begin[i] != 0 && begin[i] != ']' && begin[i] != ')') { + key[i - 5] = begin[i]; + } + } +} + +static void simFindSecondKeyNum(const char *begin, int32_t beginLen, int32_t *num) { + const char *number = strstr(begin, ")["); + if (number == NULL) { + *num = 0; + } else { + *num = atoi(number + 2); + } +} + char *simGetVariable(SScript *script, char *varName, int32_t varLen) { if (strncmp(varName, "hostname", 8) == 0) { return simParseHostName(varName); @@ -66,13 +101,40 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { if (strncmp(varName, "system_content", varLen) == 0) return script->system_ret_content; - // variable like data2_192.168.0.1 if (strncmp(varName, "data", 4) == 0) { if (varLen < 6) { return "null"; } - if (varName[5] == '_') { + int32_t row = 0; + int32_t col = 0; + char keyVal[1024] = {0}; + int32_t keyLen = 1024; + + if (varName[4] == '[') { + // $data[0][1] + simFindFirstNum(varName, varLen, &row); + simFindSecondNum(varName, varLen, &col); + if (row < 0 || row >= MAX_QUERY_ROW_NUM) { + return "null"; + } + if (col < 0 || col >= MAX_QUERY_COL_NUM) { + return "null"; + } + simDebug("script:%s, data[%d][%d]=%s", script->fileName, row, col, script->data[row][col]); + return script->data[row][col]; + } else if (varName[4] == '(') { + // $data(db)[0] + simFindFirstKeyVal(varName, varLen, keyVal, keyLen); + simFindSecondKeyNum(varName, varLen, &col); + for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) { + if (strncmp(keyVal, script->data[i][0], keyLen) == 0) { + simDebug("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]); + return script->data[i][col]; + } + } + } else if (varName[5] == '_') { + // data2_db int32_t col = varName[4] - '0'; if (col < 0 || col >= MAX_QUERY_COL_NUM) { return "null"; @@ -90,6 +152,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { } return "null"; } else if (varName[6] == '_') { + // data21_db int32_t col = (varName[4] - '0') * 10 + (varName[5] - '0'); if (col < 0 || col >= MAX_QUERY_COL_NUM) { return "null"; @@ -107,6 +170,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { } return "null"; } else { + // $data00 int32_t row = varName[4] - '0'; int32_t col = varName[5] - '0'; if (row < 0 || row >= MAX_QUERY_ROW_NUM) { @@ -119,6 +183,8 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { simDebug("script:%s, data[%d][%d]=%s", script->fileName, row, col, script->data[row][col]); return script->data[row][col]; } + + return "null"; } for (int32_t i = 0; i < script->varLen; ++i) { @@ -127,9 +193,6 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { continue; } if (strncmp(varName, var->varName, varLen) == 0) { - // if (strlen(var->varValue) != 0) - // simDebug("script:%s, var:%s, value:%s", script->fileName, - // var->varName, var->varValue); return var->varValue; } } @@ -774,7 +837,7 @@ bool simExecuteSqlSlowCmd(SScript *script, char *rest) { bool simExecuteRestfulCmd(SScript *script, char *rest) { TdFilePtr pFile = NULL; - char filename[256]; + char filename[256]; sprintf(filename, "%s/tmp.sql", simScriptDir); // fp = fopen(filename, "w"); pFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); From 5f6eac8e474b671262756a5fbe9264c1efd06934 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Apr 2022 14:46:11 +0800 Subject: [PATCH 12/22] refactor(cluster): replace the keyword master/slave with LEADER FOLLOWER --- source/dnode/mnode/impl/src/mndMnode.c | 8 ++++---- tests/script/tsim/bnode/basic1.sim | 4 ++-- tests/script/tsim/dnode/basic1.sim | 4 ++-- tests/script/tsim/mnode/basic1.sim | 4 ++-- tests/script/tsim/qnode/basic1.sim | 4 ++-- tests/script/tsim/query/explain.sim | 2 +- tests/script/tsim/snode/basic1.sim | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 6be7097e6a..cd3cec6ec4 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -78,13 +78,13 @@ void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj) { const char *mndGetRoleStr(int32_t showType) { switch (showType) { case TAOS_SYNC_STATE_FOLLOWER: - return "unsynced"; + return "FOLLOWER"; case TAOS_SYNC_STATE_CANDIDATE: - return "slave"; + return "CANDIDATE"; case TAOS_SYNC_STATE_LEADER: - return "master"; + return "LEADER"; default: - return "undefined"; + return "ERROR"; } } diff --git a/tests/script/tsim/bnode/basic1.sim b/tests/script/tsim/bnode/basic1.sim index 0ce0c2bc5d..b1db6efc72 100644 --- a/tests/script/tsim/bnode/basic1.sim +++ b/tests/script/tsim/bnode/basic1.sim @@ -24,7 +24,7 @@ if $data00 != 1 then return -1 endi -if $data02 != master then +if $data02 != LEADER then return -1 endi @@ -71,7 +71,7 @@ if $data00 != 1 then return -1 endi -if $data02 != master then +if $data02 != LEADER then return -1 endi diff --git a/tests/script/tsim/dnode/basic1.sim b/tests/script/tsim/dnode/basic1.sim index 51399e9050..d49dba60f3 100644 --- a/tests/script/tsim/dnode/basic1.sim +++ b/tests/script/tsim/dnode/basic1.sim @@ -29,7 +29,7 @@ if $data00 != 1 then return -1 endi -if $data02 != master then +if $data02 != LEADER then return -1 endi @@ -76,7 +76,7 @@ if $data00 != 1 then return -1 endi -if $data02 != master then +if $data02 != LEADER then return -1 endi diff --git a/tests/script/tsim/mnode/basic1.sim b/tests/script/tsim/mnode/basic1.sim index e3d27d0c13..235889ece6 100644 --- a/tests/script/tsim/mnode/basic1.sim +++ b/tests/script/tsim/mnode/basic1.sim @@ -24,7 +24,7 @@ if $data00 != 1 then return -1 endi -if $data02 != master then +if $data02 != LEADER then return -1 endi @@ -71,7 +71,7 @@ if $data00 != 1 then return -1 endi -if $data02 != master then +if $data02 != LEADER then return -1 endi diff --git a/tests/script/tsim/qnode/basic1.sim b/tests/script/tsim/qnode/basic1.sim index 3b478cfa46..2351403909 100644 --- a/tests/script/tsim/qnode/basic1.sim +++ b/tests/script/tsim/qnode/basic1.sim @@ -24,7 +24,7 @@ if $data00 != 1 then return -1 endi -if $data02 != master then +if $data02 != LEADER then return -1 endi @@ -71,7 +71,7 @@ if $data00 != 1 then return -1 endi -if $data02 != master then +if $data02 != LEADER then return -1 endi diff --git a/tests/script/tsim/query/explain.sim b/tests/script/tsim/query/explain.sim index 5119f58ec3..638109d510 100644 --- a/tests/script/tsim/query/explain.sim +++ b/tests/script/tsim/query/explain.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 2 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 -print ========= start dnode1 as master +print ========= start dnode1 as LEADER system sh/exec.sh -n dnode1 -s start sleep 2000 sql connect diff --git a/tests/script/tsim/snode/basic1.sim b/tests/script/tsim/snode/basic1.sim index 3b478cfa46..2351403909 100644 --- a/tests/script/tsim/snode/basic1.sim +++ b/tests/script/tsim/snode/basic1.sim @@ -24,7 +24,7 @@ if $data00 != 1 then return -1 endi -if $data02 != master then +if $data02 != LEADER then return -1 endi @@ -71,7 +71,7 @@ if $data00 != 1 then return -1 endi -if $data02 != master then +if $data02 != LEADER then return -1 endi From 5569c846245454d196c843656bcf91e79c7fffc7 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 18 Apr 2022 15:06:13 +0800 Subject: [PATCH 13/22] bsma integrate --- include/common/trow.h | 21 +++-- include/common/ttypes.h | 4 +- source/common/src/trow.c | 55 ++++++++++-- source/common/src/ttypes.c | 102 +++++++++++------------ source/dnode/vnode/src/tsdb/tsdbCommit.c | 31 +++++-- 5 files changed, 140 insertions(+), 73 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 9a09bd3c8f..dbbdef84a0 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -48,9 +48,12 @@ extern "C" { #define TD_VTYPE_NONE 0x02U // none or unknown/undefined #define TD_VTYPE_MAX 0x03U // -#define TD_VTYPE_NORM_BYTE 0x0U -#define TD_VTYPE_NULL_BYTE 0x55U -#define TD_VTYPE_NONE_BYTE 0xAAU +#define TD_VTYPE_NORM_BYTE_I 0x0U +#define TD_VTYPE_NULL_BYTE_I 0xFFU + +#define TD_VTYPE_NORM_BYTE_II 0x0U +#define TD_VTYPE_NULL_BYTE_II 0x55U +#define TD_VTYPE_NONE_BYTE_II 0xAAU #define TD_ROWS_ALL_NORM 0x00U #define TD_ROWS_NULL_NORM 0x01U @@ -224,8 +227,10 @@ static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TD int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode); static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); -static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode); -static FORCE_INLINE bool tdIsBitmapValTypeNormII(const void *pBitmap, int16_t idx); +static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, + int8_t bitmapMode); +static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode); +bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode); int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints, int8_t bitmapMode); static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, @@ -233,7 +238,7 @@ static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowVa static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId); -int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols); +int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols); /** * @brief @@ -327,9 +332,9 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, return TSDB_CODE_SUCCESS; } -static FORCE_INLINE bool tdIsBitmapValTypeNormII(const void *pBitmap, int16_t idx) { +static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) { TDRowValT valType = 0; - tdGetBitmapValTypeII(pBitmap, idx, &valType); + tdGetBitmapValType(pBitmap, idx, &valType, bitmapMode); if (tdValTypeIsNorm(valType)) { return true; } diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 1a09368bd3..7f2e59ea85 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -248,8 +248,8 @@ typedef struct tDataTypeDescriptor { int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); - void (*statisFunc)(const void* pBitmap, const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex, - int16_t *maxindex, int16_t *numofnull); + void (*statisFunc)(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numofrow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull); } tDataTypeDescriptor; extern tDataTypeDescriptor tDataTypes[15]; diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 7392b0bf8b..a1a2d236f9 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -16,10 +16,19 @@ #define _DEFAULT_SOURCE #include "trow.h" -const uint8_t tdVTypeByte[3] = { - TD_VTYPE_NORM_BYTE, // TD_VTYPE_NORM - TD_VTYPE_NONE_BYTE, // TD_VTYPE_NONE - TD_VTYPE_NULL_BYTE, // TD_VTYPE_NULL +const uint8_t tdVTypeByte[2][3] = {{ + // 2 bits + TD_VTYPE_NORM_BYTE_II, + TD_VTYPE_NONE_BYTE_II, + TD_VTYPE_NULL_BYTE_II, + }, + { + // 1 bit + TD_VTYPE_NORM_BYTE_I, + TD_VTYPE_NULL_BYTE_I, + TD_VTYPE_NULL_BYTE_I, // padding + } + }; // declaration @@ -266,21 +275,53 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBit // } // } +/** + * @brief Set bitmap area by byte preferentially and then by bit. + * + * @param pBitmap + * @param nEle + * @param valType + * @param bitmapMode 0 for 2 bits, 1 for 1 bit + * @return int32_t + */ int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode) { TASSERT(valType < TD_VTYPE_MAX); - int16_t nBytes = nEle / TD_VTYPE_PARTS; + int32_t nBytes = (bitmapMode == 0 ? nEle / TD_VTYPE_PARTS : nEle / TD_VTYPE_PARTS_I); + uint8_t vTypeByte = tdVTypeByte[bitmapMode][valType]; for (int i = 0; i < nBytes; ++i) { - *(uint8_t *)pBitmap = tdVTypeByte[valType]; + *(uint8_t *)pBitmap = vTypeByte; pBitmap = POINTER_SHIFT(pBitmap, 1); } - int16_t nLeft = nEle - nBytes * TD_VTYPE_BITS; + int32_t nLeft = nEle - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I); for (int j = 0; j < nLeft; ++j) { tdSetBitmapValType(pBitmap, j, valType, bitmapMode); } return TSDB_CODE_SUCCESS; } +bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode) { + int32_t nBytes = (bitmapMode == 0 ? numOfBits / TD_VTYPE_PARTS : numOfBits / TD_VTYPE_PARTS_I); + uint8_t vTypeByte = tdVTypeByte[bitmapMode][TD_VTYPE_NORM]; + for (int i = 0; i < nBytes; ++i) { + if (*((uint8_t *)pBitmap) != vTypeByte) { + return false; + } + pBitmap = POINTER_SHIFT(pBitmap, 1); + } + + int32_t nLeft = numOfBits - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I); + + for (int j = 0; j < nLeft; ++j) { + uint8_t vType; + tdGetBitmapValType(pBitmap, j, &vType, bitmapMode); + if (vType != TD_VTYPE_NORM) { + return false; + } + } + return true; +} + static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBitmap, int8_t bitmapMode) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->dataOff[index] = pCol->len; diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index d4371e11f7..cb01a17273 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -50,8 +50,8 @@ const int32_t TYPE_BYTES[15] = { } \ } while (0) -static void getStatics_bool(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_bool(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int8_t *data = (int8_t *)pData; *min = INT64_MAX; *max = INT64_MIN; @@ -62,7 +62,7 @@ static void getStatics_bool(const void *pBitmap, const void *pData, int32_t numO for (int32_t i = 0; i < numOfRow; ++i) { // if (data[i] == TSDB_DATA_BOOL_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -71,8 +71,8 @@ static void getStatics_bool(const void *pBitmap, const void *pData, int32_t numO } } -static void getStatics_i8(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_i8(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int8_t *data = (int8_t *)pData; *min = INT64_MAX; *max = INT64_MIN; @@ -83,7 +83,7 @@ static void getStatics_i8(const void *pBitmap, const void *pData, int32_t numOfR for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -92,8 +92,8 @@ static void getStatics_i8(const void *pBitmap, const void *pData, int32_t numOfR } } -static void getStatics_u8(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_u8(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint8_t *data = (uint8_t *)pData; uint64_t _min = UINT64_MAX; uint64_t _max = 0; @@ -106,7 +106,7 @@ static void getStatics_u8(const void *pBitmap, const void *pData, int32_t numOfR for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint8_t)data[i]) == TSDB_DATA_UTINYINT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -119,8 +119,8 @@ static void getStatics_u8(const void *pBitmap, const void *pData, int32_t numOfR *sum = _sum; } -static void getStatics_i16(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_i16(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int16_t *data = (int16_t *)pData; *min = INT64_MAX; *max = INT64_MIN; @@ -131,7 +131,7 @@ static void getStatics_i16(const void *pBitmap, const void *pData, int32_t numOf for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -140,8 +140,8 @@ static void getStatics_i16(const void *pBitmap, const void *pData, int32_t numOf } } -static void getStatics_u16(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_u16(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint16_t *data = (uint16_t *)pData; uint64_t _min = UINT64_MAX; uint64_t _max = 0; @@ -154,7 +154,7 @@ static void getStatics_u16(const void *pBitmap, const void *pData, int32_t numOf for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint16_t)data[i]) == TSDB_DATA_USMALLINT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -167,8 +167,8 @@ static void getStatics_u16(const void *pBitmap, const void *pData, int32_t numOf *sum = _sum; } -static void getStatics_i32(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_i32(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int32_t *data = (int32_t *)pData; *min = INT64_MAX; *max = INT64_MIN; @@ -179,7 +179,7 @@ static void getStatics_i32(const void *pBitmap, const void *pData, int32_t numOf for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -188,8 +188,8 @@ static void getStatics_i32(const void *pBitmap, const void *pData, int32_t numOf } } -static void getStatics_u32(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_u32(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint32_t *data = (uint32_t *)pData; uint64_t _min = UINT64_MAX; uint64_t _max = 0; @@ -202,7 +202,7 @@ static void getStatics_u32(const void *pBitmap, const void *pData, int32_t numOf for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint32_t)data[i]) == TSDB_DATA_UINT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -215,8 +215,8 @@ static void getStatics_u32(const void *pBitmap, const void *pData, int32_t numOf *sum = _sum; } -static void getStatics_i64(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_i64(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *data = (int64_t *)pData; *min = INT64_MAX; *max = INT64_MIN; @@ -227,7 +227,7 @@ static void getStatics_i64(const void *pBitmap, const void *pData, int32_t numOf for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -236,8 +236,8 @@ static void getStatics_i64(const void *pBitmap, const void *pData, int32_t numOf } } -static void getStatics_u64(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_u64(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint64_t *data = (uint64_t *)pData; uint64_t _min = UINT64_MAX; uint64_t _max = 0; @@ -250,7 +250,7 @@ static void getStatics_u64(const void *pBitmap, const void *pData, int32_t numOf for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint64_t)data[i]) == TSDB_DATA_UBIGINT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -263,8 +263,8 @@ static void getStatics_u64(const void *pBitmap, const void *pData, int32_t numOf *sum = _sum; } -static void getStatics_f(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_f(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { float *data = (float *)pData; float fmin = FLT_MAX; float fmax = -FLT_MAX; @@ -276,7 +276,7 @@ static void getStatics_f(const void *pBitmap, const void *pData, int32_t numOfRo for (int32_t i = 0; i < numOfRow; ++i) { // if ((*(uint32_t *)&(data[i])) == TSDB_DATA_FLOAT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -300,8 +300,8 @@ static void getStatics_f(const void *pBitmap, const void *pData, int32_t numOfRo SET_DOUBLE_VAL(min, fmin); } -static void getStatics_d(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_d(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { double *data = (double *)pData; double dmin = DBL_MAX; double dmax = -DBL_MAX; @@ -313,7 +313,7 @@ static void getStatics_d(const void *pBitmap, const void *pData, int32_t numOfRo for (int32_t i = 0; i < numOfRow; ++i) { // if ((*(uint64_t *)&(data[i])) == TSDB_DATA_DOUBLE_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -337,14 +337,14 @@ static void getStatics_d(const void *pBitmap, const void *pData, int32_t numOfRo SET_DOUBLE_PTR(min, &dmin); } -static void getStatics_bin(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_bin(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { const char *data = pData; assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { // if (isNull(data, TSDB_DATA_TYPE_BINARY)) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; } @@ -358,14 +358,14 @@ static void getStatics_bin(const void *pBitmap, const void *pData, int32_t numOf *maxIndex = 0; } -static void getStatics_nchr(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_nchr(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { const char *data = pData; assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { // if (isNull(data, TSDB_DATA_TYPE_NCHAR)) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; } @@ -405,19 +405,19 @@ tDataTypeDescriptor tDataTypes[15] = { }; char tTokenTypeSwitcher[13] = { - TSDB_DATA_TYPE_NULL, // no type - TSDB_DATA_TYPE_BINARY, // TK_ID - TSDB_DATA_TYPE_BOOL, // TK_BOOL - TSDB_DATA_TYPE_BIGINT, // TK_TINYINT - TSDB_DATA_TYPE_BIGINT, // TK_SMALLINT - TSDB_DATA_TYPE_BIGINT, // TK_INTEGER - TSDB_DATA_TYPE_BIGINT, // TK_BIGINT - TSDB_DATA_TYPE_DOUBLE, // TK_FLOAT - TSDB_DATA_TYPE_DOUBLE, // TK_DOUBLE - TSDB_DATA_TYPE_BINARY, // TK_STRING - TSDB_DATA_TYPE_BIGINT, // TK_TIMESTAMP + TSDB_DATA_TYPE_NULL, // no type + TSDB_DATA_TYPE_BINARY, // TK_ID + TSDB_DATA_TYPE_BOOL, // TK_BOOL + TSDB_DATA_TYPE_BIGINT, // TK_TINYINT + TSDB_DATA_TYPE_BIGINT, // TK_SMALLINT + TSDB_DATA_TYPE_BIGINT, // TK_INTEGER + TSDB_DATA_TYPE_BIGINT, // TK_BIGINT + TSDB_DATA_TYPE_DOUBLE, // TK_FLOAT + TSDB_DATA_TYPE_DOUBLE, // TK_DOUBLE + TSDB_DATA_TYPE_BINARY, // TK_STRING + TSDB_DATA_TYPE_BIGINT, // TK_TIMESTAMP TSDB_DATA_TYPE_VARCHAR, // TK_BINARY - TSDB_DATA_TYPE_NCHAR, // TK_NCHAR + TSDB_DATA_TYPE_NCHAR, // TK_NCHAR }; float floatMin = -FLT_MAX, floatMax = FLT_MAX; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index cb4937d758..09616a8969 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1201,6 +1201,23 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { } } +/** + * @brief Write SDataCols to data file. + * + * @param pRepo + * @param pTable + * @param pDFile + * @param pDFileAggr + * @param pDataCols The pDataCols would be generated from mem/imem directly with 2 bits bitmap or from tsdbRead + * interface with 1 bit bitmap. + * @param pBlock + * @param isLast + * @param isSuper + * @param ppBuf + * @param ppCBuf + * @param ppExBuf + * @return int + */ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols, SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) { STsdbCfg *pCfg = REPO_CFG(pRepo); @@ -1244,14 +1261,15 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF pBlockCol->type = pDataCol->type; pAggrBlkCol->colId = pDataCol->colId; - if (tDataTypes[pDataCol->type].statisFunc) { + if (isSuper && pColumn->sma && tDataTypes[pDataCol->type].statisFunc) { #if 0 (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), &(pBlockCol->numOfNull)); #endif - (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pBitmap, pDataCol->pData, rowsToWrite, &(pAggrBlkCol->min), &(pAggrBlkCol->max), - &(pAggrBlkCol->sum), &(pAggrBlkCol->minIndex), &(pAggrBlkCol->maxIndex), + (*tDataTypes[pDataCol->type].statisFunc)(pDataCols->bitmapMode, pDataCol->pBitmap, pDataCol->pData, rowsToWrite, + &(pAggrBlkCol->min), &(pAggrBlkCol->max), &(pAggrBlkCol->sum), + &(pAggrBlkCol->minIndex), &(pAggrBlkCol->maxIndex), &(pAggrBlkCol->numOfNull)); if (pAggrBlkCol->numOfNull == 0) { @@ -1259,13 +1277,16 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF } else { TD_SET_COL_ROWS_MISC(pBlockCol); } + } else if (tdIsBitmapBlkNorm(pDataCol->pBitmap, rowsToWrite, pDataCols->bitmapMode)) { + // check if all rows normal + TD_SET_COL_ROWS_NORM(pBlockCol); } else { TD_SET_COL_ROWS_MISC(pBlockCol); } ++nColsNotAllNull; - if (pColumn->sma) { + if (isSuper && pColumn->sma) { ++nColsOfBlockSma; } } @@ -1277,7 +1298,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF uint32_t toffset = 0; int32_t tsize = (int32_t)tsdbBlockStatisSize(nColsNotAllNull, SBlockVerLatest); int32_t lsize = tsize; - uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsNotAllNull, SBlockVerLatest); + uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsOfBlockSma, SBlockVerLatest); int32_t keyLen = 0; int32_t nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite); int32_t sBitmaps = isSuper ? (int32_t)TD_BITMAP_BYTES_I(rowsToWrite) : nBitmaps; From 5ca1f7922b701966cb981be8523041c5ea73b4ed Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Apr 2022 16:05:06 +0800 Subject: [PATCH 14/22] style: format file --- source/dnode/mgmt/mgmt_mnode/src/mmFile.c | 2 +- source/dnode/mnode/impl/src/mndBnode.c | 10 +++++----- source/dnode/mnode/impl/src/mndMnode.c | 10 +++++----- source/dnode/mnode/impl/src/mndQnode.c | 8 ++++---- source/dnode/mnode/impl/src/mndSnode.c | 8 ++++---- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index baf16f591d..75c48e79eb 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -121,7 +121,7 @@ int32_t mmWriteFile(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq, bool deploye int32_t len = 0; int32_t maxLen = 4096; char *content = taosMemoryCalloc(1, maxLen + 1); - + len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"mnodes\": [{\n"); diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index f7c4a6c225..168bd1db76 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -33,7 +33,7 @@ static int32_t mndProcessCreateBnodeReq(SNodeMsg *pReq); static int32_t mndProcessCreateBnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessDropBnodeReq(SNodeMsg *pReq); static int32_t mndProcessDropBnodeRsp(SNodeMsg *pRsp); -static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter); int32_t mndInitBnode(SMnode *pMnode) { @@ -437,7 +437,7 @@ static int32_t mndProcessDropBnodeRsp(SNodeMsg *pRsp) { return 0; } -static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -449,8 +449,8 @@ static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p if (pShow->pIter == NULL) break; cols = 0; - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) &pObj->id, false); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); char buf[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(buf, pObj->pDnode->ep, pShow->bytes[cols]); @@ -459,7 +459,7 @@ static int32_t mndRetrieveBnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p colDataAppend(pColInfo, numOfRows, buf, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) &pObj->createdTime, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pObj->createdTime, false); numOfRows++; sdbRelease(pSdb, pObj); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 6be7097e6a..0d734d8afe 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -615,7 +615,7 @@ static int32_t mndProcessDropMnodeRsp(SNodeMsg *pRsp) { return 0; } -static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -628,8 +628,8 @@ static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p if (pShow->pIter == NULL) break; cols = 0; - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) &pObj->id, false); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); char b1[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(b1, pObj->pDnode->ep, pShow->bytes[cols]); @@ -638,11 +638,11 @@ static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p colDataAppend(pColInfo, numOfRows, b1, false); const char *roles = mndGetRoleStr(pObj->role); - char* b2 = taosMemoryCalloc(1, strlen(roles) + VARSTR_HEADER_SIZE); + char *b2 = taosMemoryCalloc(1, strlen(roles) + VARSTR_HEADER_SIZE); STR_WITH_MAXSIZE_TO_VARSTR(b2, roles, pShow->bytes[cols]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) b2, false); + colDataAppend(pColInfo, numOfRows, (const char *)b2, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pObj->roleTime, false); diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 3b622795cb..96dba9baf6 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -34,7 +34,7 @@ static int32_t mndProcessCreateQnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessDropQnodeReq(SNodeMsg *pReq); static int32_t mndProcessDropQnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessQnodeListReq(SNodeMsg *pReq); -static int32_t mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextQnode(SMnode *pMnode, void *pIter); int32_t mndInitQnode(SMnode *pMnode) { @@ -497,7 +497,7 @@ static int32_t mndProcessDropQnodeRsp(SNodeMsg *pRsp) { return 0; } -static int32_t mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -510,8 +510,8 @@ static int32_t mndRetrieveQnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p if (pShow->pIter == NULL) break; cols = 0; - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*)&pObj->id, false); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); char ep[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(ep, pObj->pDnode->ep, pShow->bytes[cols]); diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 686cf1400d..b7d0ed8f5a 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -33,7 +33,7 @@ static int32_t mndProcessCreateSnodeReq(SNodeMsg *pReq); static int32_t mndProcessCreateSnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessDropSnodeReq(SNodeMsg *pReq); static int32_t mndProcessDropSnodeRsp(SNodeMsg *pRsp); -static int32_t mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSnode(SMnode *pMnode, void *pIter); int32_t mndInitSnode(SMnode *pMnode) { @@ -447,7 +447,7 @@ static int32_t mndProcessDropSnodeRsp(SNodeMsg *pRsp) { return 0; } -static int32_t mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -459,8 +459,8 @@ static int32_t mndRetrieveSnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p if (pShow->pIter == NULL) break; cols = 0; - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*)&pObj->id, false); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); char ep[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(ep, pObj->pDnode->ep, pShow->bytes[cols]); From 865bc40dd84d6b16ab2d981119d25419d0a9a293 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 18 Apr 2022 16:09:03 +0800 Subject: [PATCH 15/22] [test: add python test cases] --- tests/system-test/fulltest.sh | 17 +++ tests/system-test/test.py | 198 ++++++++++++++++++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 tests/system-test/fulltest.sh create mode 100644 tests/system-test/test.py diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh new file mode 100644 index 0000000000..ce15beae6e --- /dev/null +++ b/tests/system-test/fulltest.sh @@ -0,0 +1,17 @@ +python3 ./test.py -f 2-query/function_elapsed.py + + + + + + + + + + + + + + + + diff --git a/tests/system-test/test.py b/tests/system-test/test.py new file mode 100644 index 0000000000..31afd027ec --- /dev/null +++ b/tests/system-test/test.py @@ -0,0 +1,198 @@ +#!/usr/bin/python +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### +# install pip +# pip install src/connector/python/ + +# -*- coding: utf-8 -*- +import sys +import getopt +import subprocess +import time +from distutils.log import warn as printf +from fabric2 import Connection +sys.path.append("../pytest") +from util.log import * +from util.dnodes import * +from util.cases import * + +import taos + + +if __name__ == "__main__": + + fileName = "all" + deployPath = "" + masterIp = "" + testCluster = False + valgrind = 0 + logSql = True + stop = 0 + restart = False + windows = 0 + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrw', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'windows']) + for key, value in opts: + if key in ['-h', '--help']: + tdLog.printNoPrefix( + 'A collection of test cases written using Python') + tdLog.printNoPrefix('-f Name of test case file written by Python') + tdLog.printNoPrefix('-p Deploy Path for Simulator') + tdLog.printNoPrefix('-m Master Ip for Simulator') + tdLog.printNoPrefix('-l logSql Flag') + tdLog.printNoPrefix('-s stop All dnodes') + tdLog.printNoPrefix('-c Test Cluster Flag') + tdLog.printNoPrefix('-g valgrind Test Flag') + tdLog.printNoPrefix('-r taosd restart test') + tdLog.printNoPrefix('-w taos on windows') + sys.exit(0) + + if key in ['-r', '--restart']: + restart = True + + if key in ['-f', '--file']: + fileName = value + + if key in ['-p', '--path']: + deployPath = value + + if key in ['-m', '--master']: + masterIp = value + + if key in ['-l', '--logSql']: + if (value.upper() == "TRUE"): + logSql = True + elif (value.upper() == "FALSE"): + logSql = False + else: + tdLog.printNoPrefix("logSql value %s is invalid" % logSql) + sys.exit(0) + + if key in ['-c', '--cluster']: + testCluster = True + + if key in ['-g', '--valgrind']: + valgrind = 1 + + if key in ['-s', '--stop']: + stop = 1 + + if key in ['-w', '--windows']: + windows = 1 + + if (stop != 0): + if (valgrind == 0): + toBeKilled = "taosd" + else: + toBeKilled = "valgrind.bin" + + killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled + + psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled + processID = subprocess.check_output(psCmd, shell=True) + + while(processID): + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output(psCmd, shell=True) + + for port in range(6030, 6041): + usePortPID = "lsof -i tcp:%d | grep LISTEn | awk '{print $2}'" % port + processID = subprocess.check_output(usePortPID, shell=True) + + if processID: + killCmd = "kill -TERM %s" % processID + os.system(killCmd) + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if valgrind: + time.sleep(2) + + tdLog.info('stop All dnodes') + + if masterIp == "": + host = '127.0.0.1' + else: + host = masterIp + + tdLog.info("Procedures for tdengine deployed in %s" % (host)) + if windows: + tdCases.logSql(logSql) + tdLog.info("Procedures for testing self-deployment") + td_clinet = TDSimClient("C:\\TDengine") + td_clinet.deploy() + remote_conn = Connection("root@%s"%host) + with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'): + remote_conn.run("python3 ./test.py") + conn = taos.connect( + host="%s"%(host), + config=td_clinet.cfgDir) + tdCases.runOneWindows(conn, fileName) + else: + tdDnodes.init(deployPath) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + tdDnodes.stopAll() + is_test_framework = 0 + key_word = 'tdCases.addLinux' + try: + if key_word in open(fileName).read(): + is_test_framework = 1 + except: + pass + if is_test_framework: + moduleName = fileName.replace(".py", "").replace("/", ".") + uModule = importlib.import_module(moduleName) + try: + ucase = uModule.TDTestCase() + tdDnodes.deploy(1,ucase.updatecfgDict) + except : + tdDnodes.deploy(1,{}) + else: + pass + tdDnodes.deploy(1,{}) + tdDnodes.start(1) + + + + tdCases.logSql(logSql) + + if testCluster: + tdLog.info("Procedures for testing cluster") + if fileName == "all": + tdCases.runAllCluster() + else: + tdCases.runOneCluster(fileName) + else: + tdLog.info("Procedures for testing self-deployment") + conn = taos.connect( + host, + config=tdDnodes.getSimCfgPath()) + if fileName == "all": + tdCases.runAllLinux(conn) + else: + tdCases.runOneLinux(conn, fileName) + if restart: + if fileName == "all": + tdLog.info("not need to query ") + else: + sp = fileName.rsplit(".", 1) + if len(sp) == 2 and sp[1] == "py": + tdDnodes.stopAll() + tdDnodes.start(1) + time.sleep(1) + conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) + tdLog.info("Procedures for tdengine deployed in %s" % (host)) + tdLog.info("query test after taosd restart") + tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py") + else: + tdLog.info("not need to query") + conn.close() From 485adffc1f7be1ac3685aa002c6fd1f8d683ad14 Mon Sep 17 00:00:00 2001 From: cpwu Date: Mon, 18 Apr 2022 16:34:05 +0800 Subject: [PATCH 16/22] add between python case --- tests/system-test/2-query/between.py | 193 +++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 tests/system-test/2-query/between.py diff --git a/tests/system-test/2-query/between.py b/tests/system-test/2-query/between.py new file mode 100644 index 0000000000..8a5260bbaa --- /dev/null +++ b/tests/system-test/2-query/between.py @@ -0,0 +1,193 @@ +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): # sourcery skip: extract-duplicate-method + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + tdSql.execute( + '''create table if not exists supt + (ts timestamp, c1 int, c2 float, c3 bigint, c4 double, c5 smallint, c6 tinyint) + tags(location binary(64), type int, isused bool , family nchar(64))''' + ) + tdSql.execute("create table t1 using supt tags('beijing', 1, 1, '自行车')") + tdSql.execute("create table t2 using supt tags('shanghai', 2, 0, '拖拉机')") + + tdLog.printNoPrefix("==========step2:insert data") + for i in range(10): + tdSql.execute( + f"insert into t1 values (now()+{i}m, {32767+i}, {20.0+i/10}, {2**31+i}, {3.4*10**38+i/10}, {127+i}, {i})" + ) + tdSql.execute( + f"insert into t2 values (now()-{i}m, {-32767-i}, {20.0-i/10}, {-i-2**31}, {-i/10-3.4*10**38}, {-127-i}, {-i})" + ) + tdSql.execute( + f"insert into t1 values (now()+11m, {2**31-1}, {pow(10,37)*34}, {pow(2,63)-1}, {1.7*10**308}, 32767, 127)" + ) + tdSql.execute( + f"insert into t2 values (now()-11m, {1-2**31}, {-3.4*10**38}, {1-2**63}, {-1.7*10**308}, -32767, -127)" + ) + tdSql.execute( + f"insert into t2 values (now()-12m, null , {-3.4*10**38}, null , {-1.7*10**308}, null , null)" + ) + + tdLog.printNoPrefix("==========step3:query timestamp type") + + tdSql.query("select * from t1 where ts between now()-1m and now()+10m") + tdSql.checkRows(10) + tdSql.query("select * from t1 where ts between '2021-01-01 00:00:00.000' and '2121-01-01 00:00:00.000'") + tdSql.checkRows(11) + tdSql.query("select * from t1 where ts between '1969-01-01 00:00:00.000' and '1969-12-31 23:59:59.999'") + tdSql.checkRows(0) + tdSql.query("select * from t1 where ts between -2793600 and 31507199") + tdSql.checkRows(0) + tdSql.query("select * from t1 where ts between 1609430400000 and 4765104000000") + tdSql.checkRows(11) + + tdLog.printNoPrefix("==========step4:query int type") + + tdSql.query("select * from t1 where c1 between 32767 and 32776") + tdSql.checkRows(10) + tdSql.query("select * from t1 where c1 between 32766.9 and 32776.1") + tdSql.checkRows(10) + tdSql.query("select * from t1 where c1 between 32776 and 32767") + tdSql.checkRows(0) + tdSql.error("select * from t1 where c1 between 'a' and 'e'") + # tdSql.query("select * from t1 where c1 between 0x64 and 0x69") + # tdSql.checkRows(6) + tdSql.error("select * from t1 where c1 not between 100 and 106") + tdSql.query(f"select * from t1 where c1 between {2**31-2} and {2**31+1}") + tdSql.checkRows(1) + tdSql.error(f"select * from t2 where c1 between null and {1-2**31}") + # tdSql.checkRows(3) + tdSql.query(f"select * from t2 where c1 between {-2**31} and {1-2**31}") + tdSql.checkRows(1) + + tdLog.printNoPrefix("==========step5:query float type") + + tdSql.query("select * from t1 where c2 between 20.0 and 21.0") + tdSql.checkRows(10) + tdSql.query(f"select * from t1 where c2 between {-3.4*10**38-1} and {3.4*10**38+1}") + tdSql.checkRows(11) + tdSql.query("select * from t1 where c2 between 21.0 and 20.0") + tdSql.checkRows(0) + tdSql.error("select * from t1 where c2 between 'DC3' and 'SYN'") + tdSql.error("select * from t1 where c2 not between 0.1 and 0.2") + # tdSql.query(f"select * from t1 where c2 between {pow(10,38)*3.4} and {pow(10,38)*3.4+1}") + # tdSql.checkRows(1) + tdSql.query(f"select * from t2 where c2 between {-3.4*10**38-1} and {-3.4*10**38}") + tdSql.checkRows(2) + tdSql.error(f"select * from t2 where c2 between null and {-3.4*10**38}") + # tdSql.checkRows(3) + + tdLog.printNoPrefix("==========step6:query bigint type") + + tdSql.query(f"select * from t1 where c3 between {2**31} and {2**31+10}") + tdSql.checkRows(10) + tdSql.error(f"select * from t1 where c3 between {-2**63} and {2**63}") + # tdSql.checkRows(11) + tdSql.query(f"select * from t1 where c3 between {2**31+10} and {2**31}") + tdSql.checkRows(0) + tdSql.error("select * from t1 where c3 between 'a' and 'z'") + tdSql.error("select * from t1 where c3 not between 1 and 2") + tdSql.query(f"select * from t1 where c3 between {2**63-2} and {2**63-1}") + tdSql.checkRows(1) + tdSql.error(f"select * from t2 where c3 between {-2**63} and {1-2**63}") + # tdSql.checkRows(3) + tdSql.error(f"select * from t2 where c3 between null and {1-2**63}") + # tdSql.checkRows(2) + + tdLog.printNoPrefix("==========step7:query double type") + + tdSql.query(f"select * from t1 where c4 between {3.4*10**38} and {3.4*10**38+10}") + tdSql.checkRows(10) + tdSql.query(f"select * from t1 where c4 between {1.7*10**308+1} and {1.7*10**308+2}") + # 因为精度原因，在超出bigint边界后，数值不能进行准确的判断 + # tdSql.checkRows(0) + tdSql.query(f"select * from t1 where c4 between {3.4*10**38+10} and {3.4*10**38}") + # tdSql.checkRows(0) + tdSql.error("select * from t1 where c4 between 'a' and 'z'") + tdSql.error("select * from t1 where c4 not between 1 and 2") + tdSql.query(f"select * from t1 where c4 between {1.7*10**308} and {1.7*10**308+1}") + tdSql.checkRows(1) + tdSql.query(f"select * from t2 where c4 between {-1.7*10**308-1} and {-1.7*10**308}") + # tdSql.checkRows(3) + tdSql.error(f"select * from t2 where c4 between null and {-1.7*10**308}") + # tdSql.checkRows(3) + + tdLog.printNoPrefix("==========step8:query smallint type") + + tdSql.query("select * from t1 where c5 between 127 and 136") + tdSql.checkRows(10) + tdSql.query("select * from t1 where c5 between 126.9 and 135.9") + tdSql.checkRows(9) + tdSql.query("select * from t1 where c5 between 136 and 127") + tdSql.checkRows(0) + tdSql.error("select * from t1 where c5 between '~' and 'ˆ'") + tdSql.error("select * from t1 where c5 not between 1 and 2") + tdSql.query("select * from t1 where c5 between 32767 and 32768") + tdSql.checkRows(1) + tdSql.query("select * from t2 where c5 between -32768 and -32767") + tdSql.checkRows(1) + tdSql.error("select * from t2 where c5 between null and -32767") + # tdSql.checkRows(1) + + tdLog.printNoPrefix("==========step9:query tinyint type") + + tdSql.query("select * from t1 where c6 between 0 and 9") + tdSql.checkRows(10) + tdSql.query("select * from t1 where c6 between -1.1 and 8.9") + tdSql.checkRows(9) + tdSql.query("select * from t1 where c6 between 9 and 0") + tdSql.checkRows(0) + tdSql.error("select * from t1 where c6 between 'NUL' and 'HT'") + tdSql.error("select * from t1 where c6 not between 1 and 2") + tdSql.query("select * from t1 where c6 between 127 and 128") + tdSql.checkRows(1) + tdSql.query("select * from t2 where c6 between -128 and -127") + tdSql.checkRows(1) + tdSql.error("select * from t2 where c6 between null and -127") + # tdSql.checkRows(3) + + tdLog.printNoPrefix("==========step10:invalid query type") + + tdSql.query("select * from supt where location between 'beijing' and 'shanghai'") + tdSql.checkRows(23) + # 非0值均解析为1，因此"between 负值 and o"解析为"between 1 and 0" + tdSql.query("select * from supt where isused between 0 and 1") + tdSql.checkRows(23) + tdSql.query("select * from supt where isused between -1 and 0") + tdSql.checkRows(0) + tdSql.error("select * from supt where isused between false and true") + tdSql.query("select * from supt where family between '拖拉机' and '自行车'") + tdSql.checkRows(23) + + tdLog.printNoPrefix("==========step11:query HEX/OCT/BIN type") + + tdSql.error("select * from t1 where c6 between 0x7f and 0x80") # check filter HEX + tdSql.error("select * from t1 where c6 between 0b1 and 0b11111") # check filter BIN + tdSql.error("select * from t1 where c6 between 0b1 and 0x80") + tdSql.error("select * from t1 where c6=0b1") + tdSql.error("select * from t1 where c6=0x1") + # 八进制数据会按照十进制数据进行判定 + tdSql.query("select * from t1 where c6 between 01 and 0200") # check filter OCT + tdSql.checkRows(10) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 9f5ddb36d0dbf415a5f5677843918d0557d709cf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Apr 2022 15:59:31 +0800 Subject: [PATCH 17/22] refactor(cluster): make the comment and code of udf to variable length --- include/common/tmsg.h | 55 +++--- include/util/tdef.h | 41 +++-- source/common/src/tmsg.c | 98 ++++++++--- source/dnode/mnode/impl/src/mndFunc.c | 195 +++++++++++---------- source/dnode/mnode/impl/test/func/func.cpp | 190 ++++++++------------ 5 files changed, 302 insertions(+), 277 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index cab8fea374..968db0c2ad 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -70,11 +70,7 @@ typedef uint16_t tmsg_t; #define TSDB_IE_TYPE_DNODE_EXT 6 #define TSDB_IE_TYPE_DNODE_STATE 7 -enum { - CONN_TYPE__QUERY = 1, - CONN_TYPE__TMQ, - CONN_TYPE__MAX -}; +enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__MAX }; enum { HEARTBEAT_KEY_DBINFO = 1, @@ -342,13 +338,13 @@ int32_t tSerializeSConnectReq(void* buf, int32_t bufLen, SConnectReq* pReq); int32_t tDeserializeSConnectReq(void* buf, int32_t bufLen, SConnectReq* pReq); typedef struct { - int32_t acctId; - int64_t clusterId; + int32_t acctId; + int64_t clusterId; uint32_t connId; - int8_t superUser; - int8_t connType; - SEpSet epSet; - char sVersion[128]; + int8_t superUser; + int8_t connType; + SEpSet epSet; + char sVersion[128]; } SConnectRsp; int32_t tSerializeSConnectRsp(void* buf, int32_t bufLen, SConnectRsp* pRsp); @@ -663,14 +659,13 @@ typedef struct { int32_t outputLen; int32_t bufSize; int64_t signature; - int32_t commentSize; - int32_t codeSize; - char pComment[TSDB_FUNC_COMMENT_LEN]; - char pCode[TSDB_FUNC_CODE_LEN]; + char* pComment; + char* pCode; } SCreateFuncReq; int32_t tSerializeSCreateFuncReq(void* buf, int32_t bufLen, SCreateFuncReq* pReq); int32_t tDeserializeSCreateFuncReq(void* buf, int32_t bufLen, SCreateFuncReq* pReq); +void tFreeSCreateFuncReq(SCreateFuncReq* pReq); typedef struct { char name[TSDB_FUNC_NAME_LEN]; @@ -687,6 +682,7 @@ typedef struct { int32_t tSerializeSRetrieveFuncReq(void* buf, int32_t bufLen, SRetrieveFuncReq* pReq); int32_t tDeserializeSRetrieveFuncReq(void* buf, int32_t bufLen, SRetrieveFuncReq* pReq); +void tFreeSRetrieveFuncReq(SRetrieveFuncReq* pReq); typedef struct { char name[TSDB_FUNC_NAME_LEN]; @@ -698,8 +694,8 @@ typedef struct { int64_t signature; int32_t commentSize; int32_t codeSize; - char pComment[TSDB_FUNC_COMMENT_LEN]; - char pCode[TSDB_FUNC_CODE_LEN]; + char* pComment; + char* pCode; } SFuncInfo; typedef struct { @@ -709,6 +705,7 @@ typedef struct { int32_t tSerializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp); int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp); +void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp* pRsp); typedef struct { int32_t statusInterval; @@ -1213,12 +1210,12 @@ typedef struct { #define STREAM_TRIGGER_WINDOW_CLOSE 2 typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - char outputSTbName[TSDB_TABLE_FNAME_LEN]; - int8_t igExists; - char* sql; - char* ast; - int8_t triggerType; + char name[TSDB_TOPIC_FNAME_LEN]; + char outputSTbName[TSDB_TABLE_FNAME_LEN]; + int8_t igExists; + char* sql; + char* ast; + int8_t triggerType; int64_t watermark; } SCMCreateStreamReq; @@ -1671,14 +1668,14 @@ typedef struct { int32_t pid; char fqdn[TSDB_FQDN_LEN]; int32_t subPlanNum; - SArray* subDesc; // SArray + SArray* subDesc; // SArray } SQueryDesc; typedef struct { - uint32_t connId; - int32_t pid; - char app[TSDB_APP_NAME_LEN]; - SArray* queryDesc; // SArray + uint32_t connId; + int32_t pid; + char app[TSDB_APP_NAME_LEN]; + SArray* queryDesc; // SArray } SQueryHbReqBasic; typedef struct { @@ -1742,7 +1739,7 @@ static FORCE_INLINE void tFreeClientHbReq(void* pReq) { } taosMemoryFreeClear(req->query); } - + if (req->info) { tFreeReqKvHash(req->info); taosHashCleanup(req->info); diff --git a/include/util/tdef.h b/include/util/tdef.h index 5fc30540ee..5b863f5920 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -128,17 +128,17 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_INS_TABLE_QUERIES "queries" #define TSDB_INS_TABLE_VNODES "vnodes" -#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" -#define TSDB_PERFS_TABLE_CONNECTIONS "connections" -#define TSDB_PERFS_TABLE_QUERIES "queries" -#define TSDB_PERFS_TABLE_TOPICS "topics" -#define TSDB_PERFS_TABLE_CONSUMERS "consumers" -#define TSDB_PERFS_TABLE_SUBSCRIBES "subscribes" +#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" +#define TSDB_PERFS_TABLE_CONNECTIONS "connections" +#define TSDB_PERFS_TABLE_QUERIES "queries" +#define TSDB_PERFS_TABLE_TOPICS "topics" +#define TSDB_PERFS_TABLE_CONSUMERS "consumers" +#define TSDB_PERFS_TABLE_SUBSCRIBES "subscribes" #define TSDB_INDEX_TYPE_SMA "SMA" #define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT" -#define TSDB_INS_USER_STABLES_DBNAME_COLID 2 +#define TSDB_INS_USER_STABLES_DBNAME_COLID 2 #define TSDB_TICK_PER_SECOND(precision) \ ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI ? 1e3L \ @@ -234,8 +234,8 @@ typedef enum ELogicConditionType { #define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_FUNC_NAME_LEN 65 -#define TSDB_FUNC_COMMENT_LEN 4096 -#define TSDB_FUNC_CODE_LEN (65535 - 512) +#define TSDB_FUNC_COMMENT_LEN 1024 * 1024 +#define TSDB_FUNC_CODE_LEN 10 * 1024 * 1024 #define TSDB_FUNC_BUF_SIZE 512 #define TSDB_FUNC_TYPE_SCALAR 1 #define TSDB_FUNC_TYPE_AGGREGATE 2 @@ -339,12 +339,12 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TOTAL_BLOCKS 10000 #define TSDB_DEFAULT_TOTAL_BLOCKS 6 -#define TSDB_MIN_DAYS_PER_FILE 60 // unit minute +#define TSDB_MIN_DAYS_PER_FILE 60 // unit minute #define TSDB_MAX_DAYS_PER_FILE (3650 * 1440) #define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440) #define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute -#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. +#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. #define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years #define TSDB_MIN_MIN_ROW_FBLOCK 10 @@ -419,11 +419,11 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_EXPLAIN_VERBOSE false -#define TSDB_MIN_EXPLAIN_RATIO 0 -#define TSDB_MAX_EXPLAIN_RATIO 1 -#define TSDB_DEFAULT_EXPLAIN_RATIO 0.001 +#define TSDB_MIN_EXPLAIN_RATIO 0 +#define TSDB_MAX_EXPLAIN_RATIO 1 +#define TSDB_DEFAULT_EXPLAIN_RATIO 0.001 -#define TSDB_EXPLAIN_RESULT_ROW_SIZE 1024 +#define TSDB_EXPLAIN_RESULT_ROW_SIZE 1024 #define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN" #define TSDB_MAX_JOIN_TABLE_NUM 10 @@ -509,13 +509,13 @@ enum { SND_WORKER_TYPE__UNIQUE, }; -#define MNODE_HANDLE -1 -#define QNODE_HANDLE 1 +#define MNODE_HANDLE -1 +#define QNODE_HANDLE 1 #define DEFAULT_HANDLE 0 -#define TSDB_CONFIG_OPTION_LEN 16 -#define TSDB_CONIIG_VALUE_LEN 48 -#define TSDB_CONFIG_NUMBER 8 +#define TSDB_CONFIG_OPTION_LEN 16 +#define TSDB_CONIIG_VALUE_LEN 48 +#define TSDB_CONFIG_NUMBER 8 #define QUERY_ID_SIZE 20 #define QUERY_OBJ_ID_SIZE 18 @@ -524,7 +524,6 @@ enum { #define MAX_NUM_STR_SIZE 40 - #ifdef __cplusplus } #endif diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 3adb0102ce..a21fc81bfa 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -142,10 +142,10 @@ static int32_t tSerializeSClientHbReq(SCoder *pEncoder, const SClientHbReq *pReq if (tEncodeU32(pEncoder, pReq->query->connId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->query->pid) < 0) return -1; if (tEncodeCStr(pEncoder, pReq->query->app) < 0) return -1; - + int32_t num = taosArrayGetSize(pReq->query->queryDesc); if (tEncodeI32(pEncoder, num) < 0) return -1; - + for (int32_t i = 0; i < num; ++i) { SQueryDesc *desc = taosArrayGet(pReq->query->queryDesc, i); if (tEncodeCStr(pEncoder, desc->sql) < 0) return -1; @@ -169,7 +169,7 @@ static int32_t tSerializeSClientHbReq(SCoder *pEncoder, const SClientHbReq *pReq if (tEncodeI32(pEncoder, queryNum) < 0) return -1; } } - + int32_t kvNum = taosHashGetSize(pReq->info); if (tEncodeI32(pEncoder, kvNum) < 0) return -1; void *pIter = taosHashIterate(pReq->info, NULL); @@ -200,7 +200,7 @@ static int32_t tDeserializeSClientHbReq(SCoder *pDecoder, SClientHbReq *pReq) { if (num > 0) { pReq->query->queryDesc = taosArrayInit(num, sizeof(SQueryDesc)); if (NULL == pReq->query->queryDesc) return -1; - + for (int32_t i = 0; i < num; ++i) { SQueryDesc desc = {0}; if (tDecodeCStrTo(pDecoder, desc.sql) < 0) return -1; @@ -217,7 +217,7 @@ static int32_t tDeserializeSClientHbReq(SCoder *pDecoder, SClientHbReq *pReq) { if (snum > 0) { desc.subDesc = taosArrayInit(snum, sizeof(SQuerySubDesc)); if (NULL == desc.subDesc) return -1; - + for (int32_t m = 0; m < snum; ++m) { SQuerySubDesc sDesc = {0}; if (tDecodeI64(pDecoder, &sDesc.tid) < 0) return -1; @@ -254,7 +254,7 @@ static int32_t tSerializeSClientHbRsp(SCoder *pEncoder, const SClientHbRsp *pRsp int32_t queryNum = 0; if (pRsp->query) { queryNum = 1; - if (tEncodeI32(pEncoder, queryNum) < 0) return -1; + if (tEncodeI32(pEncoder, queryNum) < 0) return -1; if (tEncodeU32(pEncoder, pRsp->query->connId) < 0) return -1; if (tEncodeU64(pEncoder, pRsp->query->killRid) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->query->totalDnodes) < 0) return -1; @@ -262,9 +262,9 @@ static int32_t tSerializeSClientHbRsp(SCoder *pEncoder, const SClientHbRsp *pRsp if (tEncodeI8(pEncoder, pRsp->query->killConnection) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pRsp->query->epSet) < 0) return -1; } else { - if (tEncodeI32(pEncoder, queryNum) < 0) return -1; + if (tEncodeI32(pEncoder, queryNum) < 0) return -1; } - + int32_t kvNum = taosArrayGetSize(pRsp->info); if (tEncodeI32(pEncoder, kvNum) < 0) return -1; for (int32_t i = 0; i < kvNum; i++) { @@ -1492,10 +1492,25 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq if (tEncodeI32(&encoder, pReq->outputLen) < 0) return -1; if (tEncodeI32(&encoder, pReq->bufSize) < 0) return -1; if (tEncodeI64(&encoder, pReq->signature) < 0) return -1; - if (tEncodeI32(&encoder, pReq->commentSize) < 0) return -1; - if (tEncodeI32(&encoder, pReq->codeSize) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->pComment) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->pCode) < 0) return -1; + + int32_t codeSize = 0; + if (pReq->pCode != NULL) { + codeSize = strlen(pReq->pCode) + 1; + } + if (tEncodeI32(&encoder, codeSize) < 0) return -1; + if (pReq->pCode != NULL) { + if (tEncodeCStr(&encoder, pReq->pCode) < 0) return -1; + } + + int32_t commentSize = 0; + if (pReq->pComment != NULL) { + commentSize = strlen(pReq->pComment) + 1; + } + if (tEncodeI32(&encoder, commentSize) < 0) return -1; + if (pReq->pComment != NULL) { + if (tEncodeCStr(&encoder, pReq->pComment) < 0) return -1; + } + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1516,16 +1531,40 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR if (tDecodeI32(&decoder, &pReq->outputLen) < 0) return -1; if (tDecodeI32(&decoder, &pReq->bufSize) < 0) return -1; if (tDecodeI64(&decoder, &pReq->signature) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->commentSize) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->codeSize) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->pComment) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->pCode) < 0) return -1; + + int32_t codeSize = 0; + if (tDecodeI32(&decoder, &codeSize) < 0) return -1; + if (codeSize > 0) { + pReq->pCode = taosMemoryCalloc(1, codeSize); + if (pReq->pCode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + if (tDecodeCStrTo(&decoder, pReq->pCode) < 0) return -1; + } + + int32_t commentSize = 0; + if (tDecodeI32(&decoder, &commentSize) < 0) return -1; + if (commentSize > 0) { + pReq->pComment = taosMemoryCalloc(1, commentSize); + if (pReq->pComment == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + if (tDecodeCStrTo(&decoder, pReq->pComment) < 0) return -1; + } + tEndDecode(&decoder); tCoderClear(&decoder); return 0; } +void tFreeSCreateFuncReq(SCreateFuncReq *pReq) { + taosMemoryFree(pReq->pCode); + taosMemoryFree(pReq->pComment); +} + int32_t tSerializeSDropFuncReq(void *buf, int32_t bufLen, SDropFuncReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -1594,6 +1633,8 @@ int32_t tDeserializeSRetrieveFuncReq(void *buf, int32_t bufLen, SRetrieveFuncReq return 0; } +void tFreeSRetrieveFuncReq(SRetrieveFuncReq *pReq) { taosArrayDestroy(pReq->pFuncNames); } + int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp *pRsp) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -1612,10 +1653,10 @@ int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp * if (tEncodeI32(&encoder, pInfo->outputLen) < 0) return -1; if (tEncodeI32(&encoder, pInfo->bufSize) < 0) return -1; if (tEncodeI64(&encoder, pInfo->signature) < 0) return -1; - if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1; if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1; - if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1; + if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1; if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1; + if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1; } tEndEncode(&encoder); @@ -1644,10 +1685,17 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp if (tDecodeI32(&decoder, &fInfo.outputLen) < 0) return -1; if (tDecodeI32(&decoder, &fInfo.bufSize) < 0) return -1; if (tDecodeI64(&decoder, &fInfo.signature) < 0) return -1; - if (tDecodeI32(&decoder, &fInfo.commentSize) < 0) return -1; if (tDecodeI32(&decoder, &fInfo.codeSize) < 0) return -1; - if (tDecodeCStrTo(&decoder, fInfo.pComment) < 0) return -1; + if (tDecodeI32(&decoder, &fInfo.commentSize) < 0) return -1; + fInfo.pCode = taosMemoryCalloc(1, fInfo.codeSize); + fInfo.pComment = taosMemoryCalloc(1, fInfo.commentSize); + if (fInfo.pCode == NULL || fInfo.pComment == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + if (tDecodeCStrTo(&decoder, fInfo.pCode) < 0) return -1; + if (tDecodeCStrTo(&decoder, fInfo.pComment) < 0) return -1; taosArrayPush(pRsp->pFuncInfos, &fInfo); } tEndDecode(&decoder); @@ -1656,6 +1704,16 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp return 0; } +void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp *pRsp) { + int32_t size = taosArrayGetSize(pRsp->pFuncInfos); + for (int32_t i = 0; i < size; ++i) { + SFuncInfo *pInfo = taosArrayGet(pRsp->pFuncInfos, i); + taosMemoryFree(pInfo->pCode); + taosMemoryFree(pInfo->pComment); + } + taosArrayDestroy(pRsp->pFuncInfos); +} + int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index a18afbea73..26c7b4d200 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -34,7 +34,7 @@ static int32_t mndDropFunc(SMnode *pMnode, SNodeMsg *pReq, SFuncObj *pFunc); static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq); static int32_t mndProcessDropFuncReq(SNodeMsg *pReq); static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq); -static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter); int32_t mndInitFunc(SMnode *pMnode) { @@ -63,28 +63,28 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) { int32_t size = pFunc->commentSize + pFunc->codeSize + sizeof(SFuncObj) + SDB_FUNC_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_FUNC, SDB_FUNC_VER, size); - if (pRaw == NULL) goto FUNC_ENCODE_OVER; + if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pFunc->name, TSDB_FUNC_NAME_LEN, FUNC_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pFunc->createdTime, FUNC_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pFunc->funcType, FUNC_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pFunc->scriptType, FUNC_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pFunc->align, FUNC_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pFunc->outputType, FUNC_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pFunc->outputLen, FUNC_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pFunc->bufSize, FUNC_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pFunc->signature, FUNC_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pFunc->commentSize, FUNC_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pFunc->codeSize, FUNC_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, FUNC_ENCODE_OVER) - SDB_SET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, FUNC_ENCODE_OVER) - SDB_SET_DATALEN(pRaw, dataPos, FUNC_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, pFunc->name, TSDB_FUNC_NAME_LEN, _OVER) + SDB_SET_INT64(pRaw, dataPos, pFunc->createdTime, _OVER) + SDB_SET_INT8(pRaw, dataPos, pFunc->funcType, _OVER) + SDB_SET_INT8(pRaw, dataPos, pFunc->scriptType, _OVER) + SDB_SET_INT8(pRaw, dataPos, pFunc->align, _OVER) + SDB_SET_INT8(pRaw, dataPos, pFunc->outputType, _OVER) + SDB_SET_INT32(pRaw, dataPos, pFunc->outputLen, _OVER) + SDB_SET_INT32(pRaw, dataPos, pFunc->bufSize, _OVER) + SDB_SET_INT64(pRaw, dataPos, pFunc->signature, _OVER) + SDB_SET_INT32(pRaw, dataPos, pFunc->commentSize, _OVER) + SDB_SET_INT32(pRaw, dataPos, pFunc->codeSize, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, _OVER) + SDB_SET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER); terrno = 0; -FUNC_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("func:%s, failed to encode to raw:%p since %s", pFunc->name, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -99,45 +99,45 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto FUNC_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sver != SDB_FUNC_VER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto FUNC_DECODE_OVER; + goto _OVER; } SSdbRow *pRow = sdbAllocRow(sizeof(SFuncObj)); - if (pRow == NULL) goto FUNC_DECODE_OVER; + if (pRow == NULL) goto _OVER; SFuncObj *pFunc = sdbGetRowObj(pRow); - if (pFunc == NULL) goto FUNC_DECODE_OVER; + if (pFunc == NULL) goto _OVER; int32_t dataPos = 0; - SDB_GET_BINARY(pRaw, dataPos, pFunc->name, TSDB_FUNC_NAME_LEN, FUNC_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pFunc->createdTime, FUNC_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pFunc->funcType, FUNC_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pFunc->scriptType, FUNC_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pFunc->align, FUNC_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pFunc->outputType, FUNC_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pFunc->outputLen, FUNC_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pFunc->bufSize, FUNC_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pFunc->signature, FUNC_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pFunc->commentSize, FUNC_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pFunc->codeSize, FUNC_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, pFunc->name, TSDB_FUNC_NAME_LEN, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pFunc->createdTime, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pFunc->funcType, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pFunc->scriptType, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pFunc->align, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pFunc->outputType, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pFunc->outputLen, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pFunc->bufSize, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pFunc->signature, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pFunc->commentSize, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pFunc->codeSize, _OVER) pFunc->pComment = taosMemoryCalloc(1, pFunc->commentSize); pFunc->pCode = taosMemoryCalloc(1, pFunc->codeSize); if (pFunc->pComment == NULL || pFunc->pCode == NULL) { - goto FUNC_DECODE_OVER; + goto _OVER; } - SDB_GET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, FUNC_DECODE_OVER) - SDB_GET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, FUNC_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, _OVER) + SDB_GET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, _OVER) terrno = 0; -FUNC_DECODE_OVER: +_OVER: if (terrno != 0) { mError("func:%s, failed to decode from raw:%p since %s", pFunc->name, pRaw, terrstr()); taosMemoryFreeClear(pRow); @@ -192,40 +192,40 @@ static int32_t mndCreateFunc(SMnode *pMnode, SNodeMsg *pReq, SCreateFuncReq *pCr func.outputLen = pCreate->outputLen; func.bufSize = pCreate->bufSize; func.signature = pCreate->signature; - func.commentSize = pCreate->commentSize; - func.codeSize = pCreate->codeSize; + func.commentSize = strlen(pCreate->pComment) + 1; + func.codeSize = strlen(pCreate->pCode) + 1; func.pComment = taosMemoryMalloc(func.commentSize); func.pCode = taosMemoryMalloc(func.codeSize); if (func.pCode == NULL || func.pCode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto CREATE_FUNC_OVER; + goto _OVER; } - memcpy(func.pComment, pCreate->pComment, pCreate->commentSize); + memcpy(func.pComment, pCreate->pComment, func.commentSize); memcpy(func.pCode, pCreate->pCode, func.codeSize); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_FUNC, &pReq->rpcMsg); - if (pTrans == NULL) goto CREATE_FUNC_OVER; + if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create func:%s", pTrans->id, pCreate->name); SSdbRaw *pRedoRaw = mndFuncActionEncode(&func); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto CREATE_FUNC_OVER; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) goto CREATE_FUNC_OVER; + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) goto _OVER; SSdbRaw *pUndoRaw = mndFuncActionEncode(&func); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto CREATE_FUNC_OVER; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) goto CREATE_FUNC_OVER; + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) goto _OVER; SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto CREATE_FUNC_OVER; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto CREATE_FUNC_OVER; + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto CREATE_FUNC_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; -CREATE_FUNC_OVER: +_OVER: taosMemoryFree(func.pCode); taosMemoryFree(func.pComment); mndTransDrop(pTrans); @@ -235,27 +235,27 @@ CREATE_FUNC_OVER: static int32_t mndDropFunc(SMnode *pMnode, SNodeMsg *pReq, SFuncObj *pFunc) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_FUNC, &pReq->rpcMsg); - if (pTrans == NULL) goto DROP_FUNC_OVER; + if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto DROP_FUNC_OVER; + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto DROP_FUNC_OVER; + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto DROP_FUNC_OVER; + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); - if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_FUNC_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; -DROP_FUNC_OVER: +_OVER: mndTransDrop(pTrans); return code; } @@ -269,7 +269,7 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) { if (tDeserializeSCreateFuncReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto CREATE_FUNC_OVER; + goto _OVER; } mDebug("func:%s, start to create", createReq.name); @@ -279,60 +279,66 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) { if (createReq.igExists) { mDebug("func:%s, already exist, ignore exist is set", createReq.name); code = 0; - goto CREATE_FUNC_OVER; + goto _OVER; } else { terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; - goto CREATE_FUNC_OVER; + goto _OVER; } } else if (terrno == TSDB_CODE_MND_FUNC_ALREADY_EXIST) { - goto CREATE_FUNC_OVER; + goto _OVER; } if (createReq.name[0] == 0) { terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; - goto CREATE_FUNC_OVER; + goto _OVER; } - if (createReq.commentSize <= 0 || createReq.commentSize > TSDB_FUNC_COMMENT_LEN) { + if (createReq.pComment == NULL) { terrno = TSDB_CODE_MND_INVALID_FUNC_COMMENT; - goto CREATE_FUNC_OVER; + goto _OVER; } - if (createReq.codeSize <= 0 || createReq.codeSize > TSDB_FUNC_CODE_LEN) { + if (createReq.pComment[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_FUNC_COMMENT; + goto _OVER; + } + + if (createReq.pCode == NULL) { terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; - goto CREATE_FUNC_OVER; + goto _OVER; } if (createReq.pCode[0] == 0) { terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; - goto CREATE_FUNC_OVER; + goto _OVER; } if (createReq.bufSize <= 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; - goto CREATE_FUNC_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto CREATE_FUNC_OVER; + goto _OVER; } if (mndCheckFuncAuth(pUser)) { - goto CREATE_FUNC_OVER; + goto _OVER; } code = mndCreateFunc(pMnode, pReq, &createReq); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -CREATE_FUNC_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("func:%s, failed to create since %s", createReq.name, terrstr()); } mndReleaseFunc(pMnode, pFunc); mndReleaseUser(pMnode, pUser); + tFreeSCreateFuncReq(&createReq); return code; } @@ -346,14 +352,14 @@ static int32_t mndProcessDropFuncReq(SNodeMsg *pReq) { if (tDeserializeSDropFuncReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto DROP_FUNC_OVER; + goto _OVER; } mDebug("func:%s, start to drop", dropReq.name); if (dropReq.name[0] == 0) { terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; - goto DROP_FUNC_OVER; + goto _OVER; } pFunc = mndAcquireFunc(pMnode, dropReq.name); @@ -361,27 +367,27 @@ static int32_t mndProcessDropFuncReq(SNodeMsg *pReq) { if (dropReq.igNotExists) { mDebug("func:%s, not exist, ignore not exist is set", dropReq.name); code = 0; - goto DROP_FUNC_OVER; + goto _OVER; } else { terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; - goto DROP_FUNC_OVER; + goto _OVER; } } pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto DROP_FUNC_OVER; + goto _OVER; } if (mndCheckFuncAuth(pUser)) { - goto DROP_FUNC_OVER; + goto _OVER; } code = mndDropFunc(pMnode, pReq, pFunc); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -DROP_FUNC_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("func:%s, failed to drop since %s", dropReq.name, terrstr()); } @@ -434,6 +440,12 @@ static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq) { funcInfo.signature = pFunc->signature; funcInfo.commentSize = pFunc->commentSize; funcInfo.codeSize = pFunc->codeSize; + funcInfo.pCode = taosMemoryCalloc(1, sizeof(funcInfo.codeSize)); + funcInfo.pComment = taosMemoryCalloc(1, sizeof(funcInfo.commentSize)); + if (funcInfo.pCode == NULL || funcInfo.pComment == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto RETRIEVE_FUNC_OVER; + } memcpy(funcInfo.pComment, pFunc->pComment, pFunc->commentSize); memcpy(funcInfo.pCode, pFunc->pCode, pFunc->codeSize); taosArrayPush(retrieveRsp.pFuncInfos, &funcInfo); @@ -455,8 +467,8 @@ static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq) { code = 0; RETRIEVE_FUNC_OVER: - taosArrayDestroy(retrieveReq.pFuncNames); - taosArrayDestroy(retrieveRsp.pFuncInfos); + tFreeSRetrieveFuncReq(&retrieveReq); + tFreeSRetrieveFuncRsp(&retrieveRsp); return code; } @@ -479,7 +491,7 @@ static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t le return tDataTypes[type].name; } -static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -496,34 +508,35 @@ static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pB char b1[tListLen(pFunc->name) + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(b1, pFunc->name, pShow->bytes[cols]); - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) b1, false); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)b1, false); - char* b2 = taosMemoryCalloc(1, pShow->bytes[cols]); + char *b2 = taosMemoryCalloc(1, pShow->bytes[cols]); STR_WITH_MAXSIZE_TO_VARSTR(b2, pFunc->pComment, pShow->bytes[cols]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) b2, false); + colDataAppend(pColInfo, numOfRows, (const char *)b2, false); int32_t isAgg = (pFunc->funcType == TSDB_FUNC_TYPE_AGGREGATE) ? 1 : 0; pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) &isAgg, false); + colDataAppend(pColInfo, numOfRows, (const char *)&isAgg, false); char b3[TSDB_TYPE_STR_MAX_LEN] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(b3, mnodeGenTypeStr(buf, TSDB_TYPE_STR_MAX_LEN, pFunc->outputType, pFunc->outputLen), pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(b3, mnodeGenTypeStr(buf, TSDB_TYPE_STR_MAX_LEN, pFunc->outputType, pFunc->outputLen), + pShow->bytes[cols]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) b3, false); + colDataAppend(pColInfo, numOfRows, (const char *)b3, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) &pFunc->createdTime, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pFunc->createdTime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) &pFunc->codeSize, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pFunc->codeSize, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) &pFunc->bufSize, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pFunc->bufSize, false); numOfRows++; sdbRelease(pSdb, pFunc); diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 6b9c410738..22b0387d8b 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -21,10 +21,25 @@ class MndTestFunc : public ::testing::Test { public: void SetUp() override {} void TearDown() override {} + + void SetCode(SCreateFuncReq* pReq, const char* pCode); + void SetComment(SCreateFuncReq* pReq, const char* pComment); }; Testbase MndTestFunc::test; +void MndTestFunc::SetCode(SCreateFuncReq* pReq, const char* pCode) { + int32_t len = strlen(pCode); + pReq->pCode = (char*)taosMemoryCalloc(1, len + 1); + strcpy(pReq->pCode, pCode); +} + +void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) { + int32_t len = strlen(pComment); + pReq->pComment = (char*)taosMemoryCalloc(1, len + 1); + strcpy(pReq->pComment, pComment); +} + TEST_F(MndTestFunc, 01_Show_Func) { test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", ""); EXPECT_EQ(test.GetShowRows(), 0); @@ -38,6 +53,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); tSerializeSCreateFuncReq(pReq, contLen, &createReq); + tFreeSCreateFuncReq(&createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -47,10 +63,12 @@ TEST_F(MndTestFunc, 02_Create_Func) { { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); + SetCode(&createReq, "code1"); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); tSerializeSCreateFuncReq(pReq, contLen, &createReq); + tFreeSCreateFuncReq(&createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -60,11 +78,28 @@ TEST_F(MndTestFunc, 02_Create_Func) { { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - createReq.commentSize = TSDB_FUNC_COMMENT_LEN + 1; + SetComment(&createReq, "comment1"); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); tSerializeSCreateFuncReq(pReq, contLen, &createReq); + tFreeSCreateFuncReq(&createReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); + } + + { + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "f1"); + SetCode(&createReq, "code1"); + SetComment(&createReq, ""); + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); + tFreeSCreateFuncReq(&createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -74,11 +109,13 @@ TEST_F(MndTestFunc, 02_Create_Func) { { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - createReq.commentSize = TSDB_FUNC_COMMENT_LEN; + SetCode(&createReq, ""); + SetComment(&createReq, "comment1"); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); tSerializeSCreateFuncReq(pReq, contLen, &createReq); + tFreeSCreateFuncReq(&createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -88,43 +125,13 @@ TEST_F(MndTestFunc, 02_Create_Func) { { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - createReq.commentSize = TSDB_FUNC_COMMENT_LEN; - createReq.codeSize = TSDB_FUNC_CODE_LEN + 1; - - int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateFuncReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); - } - - { - SCreateFuncReq createReq = {0}; - strcpy(createReq.name, "f1"); - createReq.commentSize = TSDB_FUNC_COMMENT_LEN; - createReq.codeSize = TSDB_FUNC_CODE_LEN; - - int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateFuncReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); - } - - { - SCreateFuncReq createReq = {0}; - strcpy(createReq.name, "f1"); - createReq.commentSize = TSDB_FUNC_COMMENT_LEN; - createReq.codeSize = TSDB_FUNC_CODE_LEN; - createReq.pCode[0] = 'a'; + SetCode(&createReq, "code1"); + SetComment(&createReq, "comment1"); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); tSerializeSCreateFuncReq(pReq, contLen, &createReq); + tFreeSCreateFuncReq(&createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -134,14 +141,14 @@ TEST_F(MndTestFunc, 02_Create_Func) { { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - createReq.commentSize = TSDB_FUNC_COMMENT_LEN; - createReq.codeSize = TSDB_FUNC_CODE_LEN; - createReq.pCode[0] = 'a'; + SetCode(&createReq, "code1"); + SetComment(&createReq, "comment1"); createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1; int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); tSerializeSCreateFuncReq(pReq, contLen, &createReq); + tFreeSCreateFuncReq(&createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -151,9 +158,8 @@ TEST_F(MndTestFunc, 02_Create_Func) { for (int32_t i = 0; i < 3; ++i) { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - createReq.commentSize = TSDB_FUNC_COMMENT_LEN; - createReq.codeSize = TSDB_FUNC_CODE_LEN; - createReq.pCode[0] = 'a'; + SetCode(&createReq, "code1"); + SetComment(&createReq, "comment1"); createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1; createReq.igExists = 0; if (i == 2) createReq.igExists = 1; @@ -163,16 +169,11 @@ TEST_F(MndTestFunc, 02_Create_Func) { createReq.outputLen = 12; createReq.bufSize = 4; createReq.signature = 5; - for (int32_t i = 0; i < createReq.commentSize - 1; ++i) { - createReq.pComment[i] = 'm'; - } - for (int32_t i = 0; i < createReq.codeSize - 1; ++i) { - createReq.pCode[i] = 'd'; - } int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); tSerializeSCreateFuncReq(pReq, contLen, &createReq); + tFreeSCreateFuncReq(&createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -197,7 +198,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); void* pReq = rpcMallocCont(contLen); tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); - taosArrayDestroy(retrieveReq.pFuncNames); + tFreeSRetrieveFuncReq(&retrieveReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -217,20 +218,10 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { EXPECT_EQ(pFuncInfo->outputLen, 12); EXPECT_EQ(pFuncInfo->bufSize, 4); EXPECT_EQ(pFuncInfo->signature, 5); - EXPECT_EQ(pFuncInfo->commentSize, TSDB_FUNC_COMMENT_LEN); - EXPECT_EQ(pFuncInfo->codeSize, TSDB_FUNC_CODE_LEN); + EXPECT_STREQ("comment1", pFuncInfo->pComment); + EXPECT_STREQ("code1", pFuncInfo->pCode); - char comments[TSDB_FUNC_COMMENT_LEN] = {0}; - for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) { - comments[i] = 'm'; - } - char codes[TSDB_FUNC_CODE_LEN] = {0}; - for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) { - codes[i] = 'd'; - } - EXPECT_STREQ(comments, pFuncInfo->pComment); - EXPECT_STREQ(codes, pFuncInfo->pCode); - taosArrayDestroy(retrieveRsp.pFuncInfos); + tFreeSRetrieveFuncRsp(&retrieveRsp); } { @@ -241,7 +232,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); void* pReq = rpcMallocCont(contLen); tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); - taosArrayDestroy(retrieveReq.pFuncNames); + tFreeSRetrieveFuncReq(&retrieveReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -259,7 +250,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); void* pReq = rpcMallocCont(contLen); tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); - taosArrayDestroy(retrieveReq.pFuncNames); + tFreeSRetrieveFuncReq(&retrieveReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -275,7 +266,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); void* pReq = rpcMallocCont(contLen); tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); - taosArrayDestroy(retrieveReq.pFuncNames); + tFreeSRetrieveFuncReq(&retrieveReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -285,8 +276,6 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f2"); - createReq.commentSize = 1024; - createReq.codeSize = 9527; createReq.igExists = 1; createReq.funcType = 2; createReq.scriptType = 3; @@ -294,16 +283,13 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { createReq.outputLen = 24; createReq.bufSize = 6; createReq.signature = 18; - for (int32_t i = 0; i < createReq.commentSize - 1; ++i) { - createReq.pComment[i] = 'p'; - } - for (int32_t i = 0; i < createReq.codeSize - 1; ++i) { - createReq.pCode[i] = 'q'; - } + SetCode(&createReq, "code2"); + SetComment(&createReq, "comment2"); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); tSerializeSCreateFuncReq(pReq, contLen, &createReq); + tFreeSCreateFuncReq(&createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -322,7 +308,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); void* pReq = rpcMallocCont(contLen); tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); - taosArrayDestroy(retrieveReq.pFuncNames); + tFreeSRetrieveFuncReq(&retrieveReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -342,21 +328,13 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { EXPECT_EQ(pFuncInfo->outputLen, 24); EXPECT_EQ(pFuncInfo->bufSize, 6); EXPECT_EQ(pFuncInfo->signature, 18); - EXPECT_EQ(pFuncInfo->commentSize, 1024); - EXPECT_EQ(pFuncInfo->codeSize, 9527); + EXPECT_EQ(pFuncInfo->commentSize, strlen("comment2") + 1); + EXPECT_EQ(pFuncInfo->codeSize, strlen("code2") + 1); - char comments[TSDB_FUNC_COMMENT_LEN] = {0}; - for (int32_t i = 0; i < 1024 - 1; ++i) { - comments[i] = 'p'; - } - char codes[TSDB_FUNC_CODE_LEN] = {0}; - for (int32_t i = 0; i < 9527 - 1; ++i) { - codes[i] = 'q'; - } + EXPECT_STREQ("comment2", pFuncInfo->pComment); + EXPECT_STREQ("code2", pFuncInfo->pCode); - EXPECT_STREQ(comments, pFuncInfo->pComment); - EXPECT_STREQ(codes, pFuncInfo->pCode); - taosArrayDestroy(retrieveRsp.pFuncInfos); + tFreeSRetrieveFuncRsp(&retrieveRsp); } { @@ -369,7 +347,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); void* pReq = rpcMallocCont(contLen); tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); - taosArrayDestroy(retrieveReq.pFuncNames); + tFreeSRetrieveFuncReq(&retrieveReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -389,20 +367,10 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { EXPECT_EQ(pFuncInfo->outputLen, 24); EXPECT_EQ(pFuncInfo->bufSize, 6); EXPECT_EQ(pFuncInfo->signature, 18); - EXPECT_EQ(pFuncInfo->commentSize, 1024); - EXPECT_EQ(pFuncInfo->codeSize, 9527); - - char comments[TSDB_FUNC_COMMENT_LEN] = {0}; - for (int32_t i = 0; i < 1024 - 1; ++i) { - comments[i] = 'p'; - } - char codes[TSDB_FUNC_CODE_LEN] = {0}; - for (int32_t i = 0; i < 9527 - 1; ++i) { - codes[i] = 'q'; - } - - EXPECT_STREQ(comments, pFuncInfo->pComment); - EXPECT_STREQ(codes, pFuncInfo->pCode); + EXPECT_EQ(pFuncInfo->commentSize, strlen("comment2") + 1); + EXPECT_EQ(pFuncInfo->codeSize, strlen("code2") + 1); + EXPECT_STREQ("comment2", pFuncInfo->pComment); + EXPECT_STREQ("code2", pFuncInfo->pCode); } { @@ -414,21 +382,11 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { EXPECT_EQ(pFuncInfo->outputLen, 12); EXPECT_EQ(pFuncInfo->bufSize, 4); EXPECT_EQ(pFuncInfo->signature, 5); - EXPECT_EQ(pFuncInfo->commentSize, TSDB_FUNC_COMMENT_LEN); - EXPECT_EQ(pFuncInfo->codeSize, TSDB_FUNC_CODE_LEN); - - char comments[TSDB_FUNC_COMMENT_LEN] = {0}; - for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) { - comments[i] = 'm'; - } - char codes[TSDB_FUNC_CODE_LEN] = {0}; - for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) { - codes[i] = 'd'; - } - EXPECT_STREQ(comments, pFuncInfo->pComment); - EXPECT_STREQ(codes, pFuncInfo->pCode); + EXPECT_STREQ("comment1", pFuncInfo->pComment); + EXPECT_STREQ("code1", pFuncInfo->pCode); } - taosArrayDestroy(retrieveRsp.pFuncInfos); + + tFreeSRetrieveFuncRsp(&retrieveRsp); } { @@ -441,7 +399,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); void* pReq = rpcMallocCont(contLen); tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); - taosArrayDestroy(retrieveReq.pFuncNames); + tFreeSRetrieveFuncReq(&retrieveReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); From feb5e712bf0cc8ae10fdca153e6450886aee89d0 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 18 Apr 2022 16:57:50 +0800 Subject: [PATCH 18/22] [test: add test case for session] --- tests/script/tsim/query/session.sim | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/tests/script/tsim/query/session.sim b/tests/script/tsim/query/session.sim index 6f4010c3f3..d5e77c7b28 100644 --- a/tests/script/tsim/query/session.sim +++ b/tests/script/tsim/query/session.sim @@ -26,14 +26,6 @@ sql connect $vgroups = 4 $dbNamme = d0 -print ====> create database d1 precision 'us' -sql create database d1 precision 'us' -sql use d1 -sql create table dev_001 (ts timestamp ,i timestamp ,j int) -sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) -sql create table secondts(ts timestamp,t2 timestamp,i int) -sql insert into secondts values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) - print ====> create database $dbNamme vgroups $vgroups sql create database $dbNamme vgroups $vgroups sql show databases @@ -296,11 +288,14 @@ sql_error select count(*) from dev_001 session(i,1y) sql_error select count(*) from dev_001 session(ts,1d) where ts <'2020-05-20 0:0:0' # vnode does not return the precision of the table -#print ====> create database d1 precision 'us' -#sql create database d1 precision 'us' -#sql use d1 -#sql create table dev_001 (ts timestamp ,i timestamp ,j int) -#sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) +print ====> create database d1 precision 'us' +sql create database d1 precision 'us' +sql use d1 +sql create table dev_001 (ts timestamp ,i timestamp ,j int) +sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) +sql create table secondts(ts timestamp,t2 timestamp,i int) +sql insert into secondts values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) + #print ====> select count(*) from dev_001 session(ts,1u) #sql select _wstartts, count(*) from dev_001 session(ts,1u) #print rows: $rows @@ -312,14 +307,12 @@ sql_error select count(*) from dev_001 session(ts,1d) where ts <'2020-05-20 0:0: # print expect 2, actual: $rows # return -1 #endi -# #if $data01 != 1 then # return -1 #endi -#sql_error select count(*) from dev_001 session(i,1s) -sql create table secondts(ts timestamp,t2 timestamp,i int) -#sql_error select count(*) from secondts session(t2,2s) +sql_error select count(*) from dev_001 session(i,1s) +sql_error select count(*) from secondts session(t2,2s) if $loop_test == 0 then print =============== stop and restart taosd From 52f4699a52e2da8ea50e942dca4a1900fa8479f3 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 18 Apr 2022 17:34:10 +0800 Subject: [PATCH 19/22] fix(os): getline memory free error. --- source/os/src/osFile.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 4b55d4912c..c2c4af18e5 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -772,6 +772,9 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { if (pFile == NULL) { return -1; } + if (*ptrBuf != NULL) { + taosMemoryFreeClear(*ptrBuf); + } assert(pFile->fp != NULL); size_t len = 0; From 6f2dbd4169bb6d24dc62a075372c1b8dd3259256 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Apr 2022 18:30:41 +0800 Subject: [PATCH 20/22] fix(cluster): crash while create rollup sma --- source/dnode/mnode/impl/inc/mndDef.h | 21 +-- source/dnode/mnode/impl/src/mndDb.c | 4 + source/dnode/mnode/impl/src/mndStb.c | 244 +++++++++++++-------------- 3 files changed, 137 insertions(+), 132 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 5b96416fd7..4a6455c0ed 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -275,16 +275,17 @@ typedef struct { } SDbCfg; typedef struct { - char name[TSDB_DB_FNAME_LEN]; - char acct[TSDB_USER_LEN]; - char createUser[TSDB_USER_LEN]; - int64_t createdTime; - int64_t updateTime; - int64_t uid; - int32_t cfgVersion; - int32_t vgVersion; - int8_t hashMethod; // default is 1 - SDbCfg cfg; + char name[TSDB_DB_FNAME_LEN]; + char acct[TSDB_USER_LEN]; + char createUser[TSDB_USER_LEN]; + int64_t createdTime; + int64_t updateTime; + int64_t uid; + int32_t cfgVersion; + int32_t vgVersion; + int8_t hashMethod; // default is 1 + SDbCfg cfg; + SRWLatch lock; } SDbObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9889743a82..755e2828de 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -219,10 +219,14 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { mTrace("db:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); + taosWLockLatch(&pOld->lock); + SArray *pOldRetensions = pOld->cfg.pRetensions; pOld->updateTime = pNew->updateTime; pOld->cfgVersion = pNew->cfgVersion; pOld->vgVersion = pNew->vgVersion; memcpy(&pOld->cfg, &pNew->cfg, sizeof(SDbCfg)); + pNew->cfg.pRetensions = pOldRetensions; + taosWUnLockLatch(&pOld->lock); return 0; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 0ce73005e8..777141c5d2 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -74,59 +74,59 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags + pStb->numOfSmas) * sizeof(SSchema) + TSDB_STB_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size); - if (pRaw == NULL) goto STB_ENCODE_OVER; + if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_FNAME_LEN, STB_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pStb->db, TSDB_DB_FNAME_LEN, STB_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pStb->createdTime, STB_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pStb->updateTime, STB_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pStb->uid, STB_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pStb->dbUid, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pStb->version, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, (int32_t)(pStb->xFilesFactor * 10000), STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pStb->aggregationMethod, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pStb->delay, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pStb->ttl, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pStb->numOfSmas, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pStb->commentLen, STB_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pStb->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_INT64(pRaw, dataPos, pStb->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pStb->updateTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pStb->uid, _OVER) + SDB_SET_INT64(pRaw, dataPos, pStb->dbUid, _OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->version, _OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, _OVER) + SDB_SET_INT32(pRaw, dataPos, (int32_t)(pStb->xFilesFactor * 10000), _OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->aggregationMethod, _OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->delay, _OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->ttl, _OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, _OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, _OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->numOfSmas, _OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->commentLen, _OVER) for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; - SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) + SDB_SET_INT8(pRaw, dataPos, pSchema->type, _OVER) + SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) } for (int32_t i = 0; i < pStb->numOfTags; ++i) { SSchema *pSchema = &pStb->pTags[i]; - SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) + SDB_SET_INT8(pRaw, dataPos, pSchema->type, _OVER) + SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) } for (int32_t i = 0; i < pStb->numOfSmas; ++i) { SSchema *pSchema = &pStb->pSmas[i]; - SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) + SDB_SET_INT8(pRaw, dataPos, pSchema->type, _OVER) + SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) } if (pStb->commentLen > 0) { - SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, STB_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, _OVER) } - SDB_SET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, STB_ENCODE_OVER) - SDB_SET_DATALEN(pRaw, dataPos, STB_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; -STB_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("stb:%s, failed to encode to raw:%p since %s", pStb->name, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -141,80 +141,80 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto STB_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sver != TSDB_STB_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto STB_DECODE_OVER; + goto _OVER; } SSdbRow *pRow = sdbAllocRow(sizeof(SStbObj)); - if (pRow == NULL) goto STB_DECODE_OVER; + if (pRow == NULL) goto _OVER; SStbObj *pStb = sdbGetRowObj(pRow); - if (pStb == NULL) goto STB_DECODE_OVER; + if (pStb == NULL) goto _OVER; int32_t dataPos = 0; - SDB_GET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_FNAME_LEN, STB_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pStb->db, TSDB_DB_FNAME_LEN, STB_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pStb->createdTime, STB_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pStb->updateTime, STB_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pStb->uid, STB_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pStb->dbUid, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pStb->version, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pStb->nextColId, STB_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pStb->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pStb->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pStb->updateTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pStb->uid, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pStb->dbUid, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->version, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->nextColId, _OVER) int32_t xFilesFactor = 0; - SDB_GET_INT32(pRaw, dataPos, &xFilesFactor, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &xFilesFactor, _OVER) pStb->xFilesFactor = xFilesFactor / 10000.0f; - SDB_GET_INT32(pRaw, dataPos, &pStb->aggregationMethod, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pStb->delay, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pStb->numOfSmas, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->aggregationMethod, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->delay, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->numOfSmas, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, _OVER) pStb->pColumns = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema)); pStb->pTags = taosMemoryCalloc(pStb->numOfTags, sizeof(SSchema)); pStb->pSmas = taosMemoryCalloc(pStb->numOfSmas, sizeof(SSchema)); if (pStb->pColumns == NULL || pStb->pTags == NULL || pStb->pSmas == NULL) { - goto STB_DECODE_OVER; + goto _OVER; } for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; - SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) - SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) + SDB_GET_INT8(pRaw, dataPos, &pSchema->type, _OVER) + SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) } for (int32_t i = 0; i < pStb->numOfTags; ++i) { SSchema *pSchema = &pStb->pTags[i]; - SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) - SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) + SDB_GET_INT8(pRaw, dataPos, &pSchema->type, _OVER) + SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) } for (int32_t i = 0; i < pStb->numOfSmas; ++i) { SSchema *pSchema = &pStb->pSmas[i]; - SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) - SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) + SDB_GET_INT8(pRaw, dataPos, &pSchema->type, _OVER) + SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) } if (pStb->commentLen > 0) { pStb->comment = taosMemoryCalloc(pStb->commentLen, 1); - if (pStb->comment == NULL) goto STB_DECODE_OVER; - SDB_GET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, STB_DECODE_OVER) + if (pStb->comment == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, _OVER) } - SDB_GET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, STB_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, _OVER) terrno = 0; -STB_DECODE_OVER: +_OVER: if (terrno != 0) { mError("stb:%s, failed to decode from raw:%p since %s", pStb->name, pRaw, terrstr()); taosMemoryFreeClear(pStb->pColumns); @@ -724,21 +724,21 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &pReq->rpcMsg); - if (pTrans == NULL) goto CREATE_STB_OVER; + if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); mndTransSetDbInfo(pTrans, pDb); - if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto CREATE_STB_OVER; - if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto CREATE_STB_OVER; - if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto CREATE_STB_OVER; - if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto CREATE_STB_OVER; - if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto CREATE_STB_OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto CREATE_STB_OVER; + if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; + if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; + if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; + if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; + if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; -CREATE_STB_OVER: +_OVER: mndTransDrop(pTrans); return code; } @@ -754,13 +754,13 @@ static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { if (tDeserializeSMCreateStbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto CREATE_STB_OVER; + goto _OVER; } mDebug("stb:%s, start to create", createReq.name); if (mndCheckCreateStbReq(&createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto CREATE_STB_OVER; + goto _OVER; } pStb = mndAcquireStb(pMnode, createReq.name); @@ -768,40 +768,40 @@ static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { if (createReq.igExists) { mDebug("stb:%s, already exist, ignore exist is set", createReq.name); code = 0; - goto CREATE_STB_OVER; + goto _OVER; } else { terrno = TSDB_CODE_MND_STB_ALREADY_EXIST; - goto CREATE_STB_OVER; + goto _OVER; } } else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) { - goto CREATE_STB_OVER; + goto _OVER; } pTopicStb = mndAcquireStb(pMnode, createReq.name); if (pTopicStb != NULL) { terrno = TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC; - goto CREATE_STB_OVER; + goto _OVER; } pDb = mndAcquireDbByStb(pMnode, createReq.name); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - goto CREATE_STB_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto CREATE_STB_OVER; + goto _OVER; } if (mndCheckWriteAuth(pUser, pDb) != 0) { - goto CREATE_STB_OVER; + goto _OVER; } code = mndCreateStb(pMnode, pReq, &createReq, pDb); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -CREATE_STB_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("stb:%s, failed to create since %s", createReq.name, terrstr()); } @@ -1211,23 +1211,23 @@ static int32_t mndAlterStb(SMnode *pMnode, SNodeMsg *pReq, const SMAltertbReq *p break; } - if (code != 0) goto ALTER_STB_OVER; + if (code != 0) goto _OVER; code = -1; pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_ALTER_STB, &pReq->rpcMsg); - if (pTrans == NULL) goto ALTER_STB_OVER; + if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to alter stb:%s", pTrans->id, pAlter->name); mndTransSetDbInfo(pTrans, pDb); - if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto ALTER_STB_OVER; - if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto ALTER_STB_OVER; - if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto ALTER_STB_OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto ALTER_STB_OVER; + if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; + if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; + if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; -ALTER_STB_OVER: +_OVER: mndTransDrop(pTrans); taosMemoryFreeClear(stbObj.pTags); taosMemoryFreeClear(stbObj.pColumns); @@ -1244,37 +1244,37 @@ static int32_t mndProcessMAlterStbReq(SNodeMsg *pReq) { if (tDeserializeSMAlterStbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto ALTER_STB_OVER; + goto _OVER; } mDebug("stb:%s, start to alter", alterReq.name); - if (mndCheckAlterStbReq(&alterReq) != 0) goto ALTER_STB_OVER; + if (mndCheckAlterStbReq(&alterReq) != 0) goto _OVER; pDb = mndAcquireDbByStb(pMnode, alterReq.name); if (pDb == NULL) { terrno = TSDB_CODE_MND_INVALID_DB; - goto ALTER_STB_OVER; + goto _OVER; } pStb = mndAcquireStb(pMnode, alterReq.name); if (pStb == NULL) { terrno = TSDB_CODE_MND_STB_NOT_EXIST; - goto ALTER_STB_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto ALTER_STB_OVER; + goto _OVER; } if (mndCheckWriteAuth(pUser, pDb) != 0) { - goto ALTER_STB_OVER; + goto _OVER; } code = mndAlterStb(pMnode, pReq, &alterReq, pDb, pStb); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -ALTER_STB_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("stb:%s, failed to alter since %s", alterReq.name, terrstr()); } @@ -1353,19 +1353,19 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * static int32_t mndDropStb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_STB, &pReq->rpcMsg); - if (pTrans == NULL) goto DROP_STB_OVER; + if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); mndTransSetDbInfo(pTrans, pDb); - if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; - if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; - if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto DROP_STB_OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_STB_OVER; + if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER; + if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; + if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; -DROP_STB_OVER: +_OVER: mndTransDrop(pTrans); return code; } @@ -1380,7 +1380,7 @@ static int32_t mndProcessMDropStbReq(SNodeMsg *pReq) { if (tDeserializeSMDropStbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto DROP_STB_OVER; + goto _OVER; } mDebug("stb:%s, start to drop", dropReq.name); @@ -1390,32 +1390,32 @@ static int32_t mndProcessMDropStbReq(SNodeMsg *pReq) { if (dropReq.igNotExists) { mDebug("stb:%s, not exist, ignore not exist is set", dropReq.name); code = 0; - goto DROP_STB_OVER; + goto _OVER; } else { terrno = TSDB_CODE_MND_STB_NOT_EXIST; - goto DROP_STB_OVER; + goto _OVER; } } pDb = mndAcquireDbByStb(pMnode, dropReq.name); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - goto DROP_STB_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto DROP_STB_OVER; + goto _OVER; } if (mndCheckWriteAuth(pUser, pDb) != 0) { - goto DROP_STB_OVER; + goto _OVER; } code = mndDropStb(pMnode, pReq, pDb, pStb); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -DROP_STB_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("stb:%s, failed to drop since %s", dropReq.name, terrstr()); } @@ -1509,36 +1509,36 @@ static int32_t mndProcessTableMetaReq(SNodeMsg *pReq) { if (tDeserializeSTableInfoReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &infoReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto RETRIEVE_META_OVER; + goto _OVER; } if (0 == strcmp(infoReq.dbFName, TSDB_INFORMATION_SCHEMA_DB)) { mDebug("information_schema table:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); if (mndBuildInsTableSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { - goto RETRIEVE_META_OVER; + goto _OVER; } } else if (0 == strcmp(infoReq.dbFName, TSDB_PERFORMANCE_SCHEMA_DB)) { mDebug("performance_schema table:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); if (mndBuildPerfsTableSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { - goto RETRIEVE_META_OVER; + goto _OVER; } } else { mDebug("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { - goto RETRIEVE_META_OVER; + goto _OVER; } } int32_t rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); if (rspLen < 0) { terrno = TSDB_CODE_INVALID_MSG; - goto RETRIEVE_META_OVER; + goto _OVER; } void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto RETRIEVE_META_OVER; + goto _OVER; } tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); @@ -1548,7 +1548,7 @@ static int32_t mndProcessTableMetaReq(SNodeMsg *pReq) { mDebug("stb:%s.%s, meta is retrieved", infoReq.dbFName, infoReq.tbName); -RETRIEVE_META_OVER: +_OVER: if (code != 0) { mError("stb:%s.%s, failed to retrieve meta since %s", infoReq.dbFName, infoReq.tbName, terrstr()); } From f0ab7dbc96b2d53462a40e5eaafd0016c366e33d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Apr 2022 20:29:10 +0800 Subject: [PATCH 21/22] fix: memory leak in tconfig.c in client --- source/client/src/clientMain.c | 1 + tools/shell/src/shellEngine.c | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 2e6cd4ce17..903018d5c3 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -71,6 +71,7 @@ void taos_cleanup(void) { schedulerDestroy(); tscInfo("all local resources released"); + taosCleanupCfg(); taosCloseLog(); } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index ac2010efa3..548d2169b1 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -91,11 +91,6 @@ TAOS *shellInit(SShellArguments *_args) { _args->user = TSDB_DEFAULT_USER; } - SConfig *pCfg = cfgInit(); - if (NULL == pCfg) return NULL; - - if (0 != taosAddClientLogCfg(pCfg)) return NULL; - // Connect to the database. TAOS *con = NULL; if (_args->auth == NULL) { From 784563667682ee353e6363b53ae21241f683a506 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Apr 2022 20:55:01 +0800 Subject: [PATCH 22/22] test: update data.sim to repeat valgrind error --- tests/script/tmp/data.sim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/tmp/data.sim b/tests/script/tmp/data.sim index faac5b2828..92fc9dccc9 100644 --- a/tests/script/tmp/data.sim +++ b/tests/script/tmp/data.sim @@ -3,6 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect +return sql create database db sql create table db.tb (ts timestamp, i int) sql insert into db.tb values(now, 1)