From 387983f6f4be62b93544431a5485561814a30f75 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 7 Dec 2022 11:44:39 +0800 Subject: [PATCH 01/18] fix(query): fix percentile crash when input has more than 10 million records --- source/libs/function/src/tpercentile.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index e5727f1472..56f55bde32 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -367,16 +367,16 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { pSlot->info.data = NULL; } - SArray *pPageIdList = (SArray *)taosHashGet(pBucket->groupPagesMap, &groupId, sizeof(groupId)); + SArray **pPageIdList = taosHashGet(pBucket->groupPagesMap, &groupId, sizeof(groupId)); if (pPageIdList == NULL) { SArray *pList = taosArrayInit(4, sizeof(int32_t)); taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pList, POINTER_BYTES); - pPageIdList = pList; + pPageIdList = &pList; } pSlot->info.data = getNewBufPage(pBucket->pBuffer, &pageId); pSlot->info.pageId = pageId; - taosArrayPush(pPageIdList, &pageId); + taosArrayPush(*pPageIdList, &pageId); } memcpy(pSlot->info.data->data + pSlot->info.data->num * pBucket->bytes, d, pBucket->bytes); From 15b8b289725b2ba38a9e4280441d73decf59984d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 7 Dec 2022 17:52:49 +0800 Subject: [PATCH 02/18] fix(query): SMA ensure support info pColAgg capacity over column numfor this table --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 9 ++++++++- source/libs/executor/src/scanoperator.c | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index c4040644b1..7065364eaf 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -174,7 +174,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableL void tsdbReaderClose(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader); void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow); -int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockSMA, bool *allHave); +int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock* pBlock, bool *allHave); SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a4581f5472..00e6a5c541 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -4112,8 +4112,9 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ } } -int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg ***pBlockSMA, bool* allHave) { +int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pBlock, bool* allHave) { int32_t code = 0; + SColumnDataAgg ***pBlockSMA = &pBlock->pBlockAgg; *allHave = false; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { @@ -4161,6 +4162,12 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg ***pBlockS int32_t i = 0, j = 0; size_t size = taosArrayGetSize(pSup->pColAgg); + // ensure capacity + if(pBlock->pDataBlock) { + size_t colsNum = taosArrayGetSize(pBlock->pDataBlock); + taosArrayEnsureCap(pSup->pColAgg, colsNum); + } + SSDataBlock* pResBlock = pReader->pResBlock; if (pResBlock->pBlockAgg == NULL) { size_t num = taosArrayGetSize(pResBlock->pDataBlock); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0f35d3778d..eae2ff3c72 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -224,7 +224,7 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { bool allColumnsHaveAgg = true; - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pBlock->pBlockAgg, &allColumnsHaveAgg); + int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } From cc285d49006cc10ed5ab7fad6aaf2951cc7fac93 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 7 Dec 2022 18:00:32 +0800 Subject: [PATCH 03/18] fix(query): SMA ensure support info pColAgg capacity over column numfor this table --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 7065364eaf..99260ffefd 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -174,7 +174,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableL void tsdbReaderClose(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader); void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow); -int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock* pBlock, bool *allHave); +int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock* pDataBlock, bool *allHave); SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 00e6a5c541..3ec4f63c30 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -4112,9 +4112,9 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ } } -int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pBlock, bool* allHave) { +int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave) { int32_t code = 0; - SColumnDataAgg ***pBlockSMA = &pBlock->pBlockAgg; + SColumnDataAgg ***pBlockSMA = &pDataBlock->pBlockAgg; *allHave = false; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { @@ -4163,8 +4163,8 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pBlock, bool size_t size = taosArrayGetSize(pSup->pColAgg); // ensure capacity - if(pBlock->pDataBlock) { - size_t colsNum = taosArrayGetSize(pBlock->pDataBlock); + if(pDataBlock->pDataBlock) { + size_t colsNum = taosArrayGetSize(pDataBlock->pDataBlock); taosArrayEnsureCap(pSup->pColAgg, colsNum); } From 66f4ee9a41315ecdcb44e522f5a68cc28ef84c90 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 7 Dec 2022 21:57:49 +0800 Subject: [PATCH 04/18] enh: alter db replica return quickly, operation executed asynchronously --- source/dnode/mnode/impl/src/mndDb.c | 8 +++++++- tests/script/tsim/db/alter_replica_13.sim | 19 +++++++++++++++++++ tests/script/tsim/db/alter_replica_31.sim | 20 ++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 5fa86dffe2..43155124c1 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -825,7 +825,13 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); code = mndAlterDb(pMnode, pReq, pDb, &dbObj); - if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + + if (dbObj.cfg.replications != pDb->cfg.replications) { + // return quickly, operation executed asynchronously + mInfo("db:%s, alter db replica from %d to %d", pDb->name, pDb->cfg.replications, dbObj.cfg.replications); + } else { + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + } _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { diff --git a/tests/script/tsim/db/alter_replica_13.sim b/tests/script/tsim/db/alter_replica_13.sim index 4eafb86198..d75acb50ad 100644 --- a/tests/script/tsim/db/alter_replica_13.sim +++ b/tests/script/tsim/db/alter_replica_13.sim @@ -116,6 +116,25 @@ endi print ============= step4: alter database sql alter database db replica 3 +$wt = 0 +stepwt1: + $wt = $wt + 1 + sleep 1000 + if $wt == 200 then + print ====> dnode not ready! + return -1 + endi +sql show transactions +if $rows != 0 then + print wait 1 seconds to alter + goto stepwt1 +endi + +sql show db.vgroups +print ---> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data27 $data28 $data29 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data26 $data37 $data38 $data39 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data36 $data47 $data48 $data49 $leaderIndex = 0 diff --git a/tests/script/tsim/db/alter_replica_31.sim b/tests/script/tsim/db/alter_replica_31.sim index 47e1fda79f..b689c9de10 100644 --- a/tests/script/tsim/db/alter_replica_31.sim +++ b/tests/script/tsim/db/alter_replica_31.sim @@ -148,6 +148,26 @@ endi print ============= step3: alter database sql alter database db replica 1 +wt = 0 + stepwt1: + $wt = $wt + 1 + sleep 1000 + if $wt == 200 then + print ====> dnode not ready! + return -1 + endi +sql show transactions +if $rows != 0 then + print wait 1 seconds to alter + goto stepwt1 +endi + +sql show db.vgroups +print ---> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data27 $data28 $data29 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data26 $data37 $data38 $data39 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data36 $data47 $data48 $data49 + $hasleader = 0 $x = 0 From 1dd1e497f95f18179b5ac2135481f016ef4f84ea Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 7 Dec 2022 23:57:51 +0800 Subject: [PATCH 05/18] test: update case --- tests/script/tsim/db/alter_replica_31.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/tsim/db/alter_replica_31.sim b/tests/script/tsim/db/alter_replica_31.sim index b689c9de10..74e8007791 100644 --- a/tests/script/tsim/db/alter_replica_31.sim +++ b/tests/script/tsim/db/alter_replica_31.sim @@ -148,7 +148,7 @@ endi print ============= step3: alter database sql alter database db replica 1 -wt = 0 +$wt = 0 stepwt1: $wt = $wt + 1 sleep 1000 From 3a3bf89cb819f17069d4e6543c9301b7b06a8c52 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 8 Dec 2022 01:39:01 +0800 Subject: [PATCH 06/18] fix: keep option of database with ns precision --- source/libs/parser/src/parTranslater.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 85a4e70124..2afca5e35c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3879,12 +3879,20 @@ static int32_t checkDbKeepOption(STranslateContext* pCxt, SDatabaseOptions* pOpt pOptions->keep[2] = getBigintFromValueNode((SValueNode*)nodesListGetNode(pOptions->pKeep, 2)); } + int64_t tsdbMaxKeep = TSDB_MAX_KEEP; + if (pOptions->precision == TSDB_TIME_PRECISION_NANO) { + int64_t now = taosGetTimestampSec(); + if (now < 0) now = 0; + tsdbMaxKeep = now / 60 + 292 * 365 * 1440; + tsdbMaxKeep = TMIN(tsdbMaxKeep, TSDB_MAX_KEEP); + } + if (pOptions->keep[0] < TSDB_MIN_KEEP || pOptions->keep[1] < TSDB_MIN_KEEP || pOptions->keep[2] < TSDB_MIN_KEEP || - pOptions->keep[0] > TSDB_MAX_KEEP || pOptions->keep[1] > TSDB_MAX_KEEP || pOptions->keep[2] > TSDB_MAX_KEEP) { + pOptions->keep[0] > tsdbMaxKeep || pOptions->keep[1] > tsdbMaxKeep || pOptions->keep[2] > tsdbMaxKeep) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, "Invalid option keep: %" PRId64 ", %" PRId64 ", %" PRId64 " valid range: [%dm, %dm]", pOptions->keep[0], pOptions->keep[1], pOptions->keep[2], TSDB_MIN_KEEP, - TSDB_MAX_KEEP); + tsdbMaxKeep); } if (!((pOptions->keep[0] <= pOptions->keep[1]) && (pOptions->keep[1] <= pOptions->keep[2]))) { @@ -4036,7 +4044,10 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName TSDB_MAX_MINROWS_FBLOCK); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbKeepOption(pCxt, pOptions); + code = checkDbPrecisionOption(pCxt, pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbKeepOption(pCxt, pOptions); // use precision } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "pages", pOptions->pages, TSDB_MIN_PAGES_PER_VNODE, TSDB_MAX_PAGES_PER_VNODE); @@ -4049,9 +4060,6 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbRangeOption(pCxt, "tsdbPagesize", pOptions->tsdbPageSize, TSDB_MIN_TSDB_PAGESIZE, TSDB_MAX_TSDB_PAGESIZE); } - if (TSDB_CODE_SUCCESS == code) { - code = checkDbPrecisionOption(pCxt, pOptions); - } if (TSDB_CODE_SUCCESS == code) { code = checkDbEnumOption(pCxt, "replications", pOptions->replica, TSDB_MIN_DB_REPLICA, TSDB_MAX_DB_REPLICA); } From 2d75447bb214e6badccd34521c523e441e3c518d Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 8 Dec 2022 02:11:39 +0800 Subject: [PATCH 07/18] fix: update diskId when migrate DFileSet --- source/dnode/vnode/src/tsdb/tsdbFS.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 72d9c4f69e..1942634d89 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -962,6 +962,8 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { } } + pDFileSet->diskId = pSet->diskId; + goto _exit; } } From 1c04c97362ecc34226f5e3dc9a9a575bfa0dbc99 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 8 Dec 2022 09:11:14 +0800 Subject: [PATCH 08/18] refact: rename taosAssert --- include/util/tlog.h | 4 ++-- source/util/src/tlog.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/util/tlog.h b/include/util/tlog.h index b6a389b6d9..e6ef7f388f 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -83,8 +83,8 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons #endif ; -bool taosAssertLog(bool condition, const char *file, int32_t line, const char *format, ...); -#define ASSERTS(condition, ...) taosAssertLog(condition, __FILE__, __LINE__, __VA_ARGS__) +bool taosAssert(bool condition, const char *file, int32_t line, const char *format, ...); +#define ASSERTS(condition, ...) taosAssert(condition, __FILE__, __LINE__, __VA_ARGS__) #define ASSERT(condition) ASSERTS(condition, "assert info not provided") // clang-format off diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index cea08e46fe..3825b91c6d 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -780,7 +780,7 @@ cmp_end: return ret; } -bool taosAssertLog(bool condition, const char *file, int32_t line, const char *format, ...) { +bool taosAssert(bool condition, const char *file, int32_t line, const char *format, ...) { if (condition) return false; const char *flags = "UTL FATAL "; From dd821a28ee80c888446a5d0f3baa85fa9a10d77d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 8 Dec 2022 09:41:42 +0800 Subject: [PATCH 09/18] enh: collect and record the signal that the dnode was killed in the log file --- source/dnode/mgmt/exe/dmMain.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 6963803df0..175bbc6a39 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -17,6 +17,7 @@ #include "dmMgmt.h" #include "mnode.h" #include "tconfig.h" +#include "tglobal.h" // clang-format off #define DM_APOLLO_URL "The apollo string to use when configuring the server, such as: -a 'jsonFile:./tests/cfg.json', cfg.json text can be '{\"fqdn\":\"td1\"}'." @@ -45,9 +46,30 @@ static struct { SArray *pArgs; // SConfigPair } global = {0}; -static void dmStopDnode(int signum, void *info, void *ctx) { dmStop(); } +static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetAllDebugFlag(143, true); } +static void dmSetAssert(int32_t signum, void *sigInfo, void *context) { tsAssert = 1; } + +static void dmStopDnode(int signum, void *sigInfo, void *context) { + // taosIgnSignal(SIGUSR1); + // taosIgnSignal(SIGUSR2); + taosIgnSignal(SIGTERM); + taosIgnSignal(SIGHUP); + taosIgnSignal(SIGINT); + taosIgnSignal(SIGABRT); + taosIgnSignal(SIGBREAK); + + dInfo("shut down signal is %d", signum); +#ifndef WINDOWS + dInfo("sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, + taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid)); +#endif + + dmStop(); +} static void dmSetSignalHandle() { + taosSetSignal(SIGUSR1, dmSetDebugFlag); + taosSetSignal(SIGUSR2, dmSetAssert); taosSetSignal(SIGTERM, dmStopDnode); taosSetSignal(SIGHUP, dmStopDnode); taosSetSignal(SIGINT, dmStopDnode); From 332ecd1d927179a88c6e0a8b105232077c0d5b88 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 8 Dec 2022 10:07:55 +0800 Subject: [PATCH 10/18] fix: mem leak --- source/client/src/clientTmq.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index ade0c95227..766a94caf1 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -875,6 +875,7 @@ void tmqFreeImpl(void* handle) { tmq_t* tmq = (tmq_t*)handle; // TODO stop timer + tmqClearUnhandleMsg(tmq); if (tmq->mqueue) taosCloseQueue(tmq->mqueue); if (tmq->delayedTask) taosCloseQueue(tmq->delayedTask); if (tmq->qall) taosFreeQall(tmq->qall); @@ -884,8 +885,7 @@ void tmqFreeImpl(void* handle) { int32_t sz = taosArrayGetSize(tmq->clientTopics); for (int32_t i = 0; i < sz; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema); - int32_t vgSz = taosArrayGetSize(pTopic->vgs); + taosMemoryFreeClear(pTopic->schema.pSchema); taosArrayDestroy(pTopic->vgs); } taosArrayDestroy(tmq->clientTopics); @@ -1304,7 +1304,6 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { for (int32_t i = 0; i < sz; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema); - int32_t vgSz = taosArrayGetSize(pTopic->vgs); taosArrayDestroy(pTopic->vgs); } taosArrayDestroy(tmq->clientTopics); @@ -1410,7 +1409,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { return -1; } void* pReq = taosMemoryCalloc(1, tlen); - if (tlen < 0) { + if (pReq == NULL) { tscError("failed to malloc askEpReq msg, size:%d", tlen); return -1; } @@ -1738,7 +1737,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { taosFreeQitem(pollRspWrapper); return pRsp; } else { - tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d\n", + tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", pollRspWrapper->taosxRsp.head.epoch, consumerEpoch); taosFreeQitem(pollRspWrapper); } From 1d34b7edf8d22d07f9e50762ca936ee599dc975b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 8 Dec 2022 10:50:24 +0800 Subject: [PATCH 11/18] enh: print dnode startup parameters --- include/os/osDir.h | 1 + include/os/osSystem.h | 8 ++++++-- source/dnode/mgmt/exe/dmMain.c | 15 +++++++++++++++ source/os/src/osDir.c | 8 ++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/include/os/osDir.h b/include/os/osDir.h index 40012f8141..73871602c5 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -62,6 +62,7 @@ int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen); bool taosIsDir(const char *dirname); char *taosDirName(char *dirname); char *taosDirEntryBaseName(char *dirname); +void taosGetCwd(char *buf, int32_t len); TdDirPtr taosOpenDir(const char *dirname); TdDirEntryPtr taosReadDir(TdDirPtr pDir); diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 31ec513fca..ed5213ce34 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -62,8 +62,12 @@ void taosResetTerminalMode(); taosMemoryFree(strings); \ } #else -#define taosPrintTrace(flags, level, dflag) \ - {} +#define taosPrintTrace(flags, level, dflag) \ + { \ + taosPrintLog(flags, level, dflag, \ + "backtrace not implemented on windows, so detailed stack information cannot be printed"); \ + \ \ + } #endif #ifdef __cplusplus diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 6963803df0..b4e2379736 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -105,6 +105,19 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { return 0; } +static void dmPrintArgs(int32_t argc, char const *argv[]) { + char path[1024] = {0}; + taosGetCwd(path, sizeof(path)); + + char args[1024] = {0}; + int32_t arglen = snprintf(args, sizeof(args), "%s", argv[0]); + for (int32_t i = 1; i < argc; ++i) { + arglen = arglen + snprintf(args + arglen, sizeof(args) - arglen, " %s", argv[i]); + } + + dInfo("startup path:%s args:%s", path, args); +} + static void dmGenerateGrant() { mndGenerateMachineCode(); } static void dmPrintVersion() { @@ -194,6 +207,8 @@ int mainWindows(int argc, char **argv) { return -1; } + dmPrintArgs(argc, argv); + if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) { dError("failed to start since read config error"); taosCloseLog(); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index d219873b5a..9175067568 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -497,3 +497,11 @@ int32_t taosCloseDir(TdDirPtr *ppDir) { return 0; #endif } + +void taosGetCwd(char *buf, int32_t len) { +#if !defined(WINDOWS) + (void)getcwd(buf, len - 1); +#else + strncpy(buf, "not implemented on windows", len -1); +#endif +} From 1a066d19f7390b15f37cc7d60f31902fa39304ab Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 8 Dec 2022 10:55:08 +0800 Subject: [PATCH 12/18] fix: dismiss coverity issues --- source/dnode/vnode/src/meta/metaTable.c | 4 ++++ source/dnode/vnode/src/tsdb/tsdbCache.c | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 0b00f036de..632675ee1d 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -1353,6 +1353,10 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { goto end; } + if (stbEntry.stbEntry.schemaTag.pSchema == NULL) { + goto end; + } + pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; STagVal tagVal = {.cid = pTagColumn->colId}; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 7309ea3407..37fae4f709 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -952,7 +952,10 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs SArray *pDelIdxArray = taosArrayInit(32, sizeof(SDelIdx)); code = tsdbReadDelIdx(pDelFReader, pDelIdxArray); - if (code) goto _err; + if (code) { + tsdbDelFReaderClose(&pDelFReader); + goto _err; + } SDelIdx *delIdx = taosArraySearch(pDelIdxArray, &(SDelIdx){.suid = suid, .uid = uid}, tCmprDelIdx, TD_EQ); From ad69067a012a2336d591bdc6abcb84129e711c9d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 8 Dec 2022 10:57:21 +0800 Subject: [PATCH 13/18] fix asan stack-scope-after-use errror --- source/libs/function/src/tpercentile.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 56f55bde32..cf8e239190 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -367,16 +367,18 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { pSlot->info.data = NULL; } - SArray **pPageIdList = taosHashGet(pBucket->groupPagesMap, &groupId, sizeof(groupId)); - if (pPageIdList == NULL) { - SArray *pList = taosArrayInit(4, sizeof(int32_t)); - taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pList, POINTER_BYTES); - pPageIdList = &pList; + SArray *pPageIdList; + void *p = taosHashGet(pBucket->groupPagesMap, &groupId, sizeof(groupId)); + if (p == NULL) { + pPageIdList = taosArrayInit(4, sizeof(int32_t)); + taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pPageIdList, POINTER_BYTES); + } else { + pPageIdList = *(SArray **)p; } pSlot->info.data = getNewBufPage(pBucket->pBuffer, &pageId); pSlot->info.pageId = pageId; - taosArrayPush(*pPageIdList, &pageId); + taosArrayPush(pPageIdList, &pageId); } memcpy(pSlot->info.data->data + pSlot->info.data->num * pBucket->bytes, d, pBucket->bytes); From 942152947b3e489f298eb61fc54dbf0e69864a4f Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 8 Dec 2022 11:36:34 +0800 Subject: [PATCH 14/18] fix:stream update data --- source/libs/executor/src/timewindowoperator.c | 47 ++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index d9a011a892..6f1ab8ca65 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -863,19 +863,20 @@ static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) { int32_t compareWinRes(void* pKey, void* data, int32_t index) { SArray* res = (SArray*)data; - SWinKey* pos = taosArrayGet(res, index); - SResKeyPos* pData = (SResKeyPos*)pKey; - if (*(int64_t*)pData->key == pos->ts) { - if (pData->groupId > pos->groupId) { - return 1; - } else if (pData->groupId < pos->groupId) { - return -1; - } - return 0; - } else if (*(int64_t*)pData->key > pos->ts) { + SWinKey* pDataPos = taosArrayGet(res, index); + SResKeyPos* pRKey = (SResKeyPos*)pKey; + if (pRKey->groupId > pDataPos->groupId) { return 1; + } else if (pRKey->groupId < pDataPos->groupId) { + return -1; } - return -1; + + if (*(int64_t*)pRKey->key > pDataPos->ts) { + return 1; + } else if (*(int64_t*)pRKey->key < pDataPos->ts){ + return -1; + } + return 0; } static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) { @@ -1400,19 +1401,21 @@ static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { int32_t compareWinKey(void* pKey, void* data, int32_t index) { SArray* res = (SArray*)data; - SWinKey* pos = taosArrayGet(res, index); - SWinKey* pData = (SWinKey*)pKey; - if (pData->ts == pos->ts) { - if (pData->groupId > pos->groupId) { - return 1; - } else if (pData->groupId < pos->groupId) { - return -1; - } - return 0; - } else if (pData->ts > pos->ts) { + SWinKey* pDataPos = taosArrayGet(res, index); + SWinKey* pWKey = (SWinKey*)pKey; + + if (pWKey->groupId > pDataPos->groupId) { return 1; + } else if (pWKey->groupId < pDataPos->groupId) { + return -1; } - return -1; + + if (pWKey->ts > pDataPos->ts) { + return 1; + } else if (pWKey->ts < pDataPos->ts) { + return -1; + } + return 0; } static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval, From 4a6f41f7f1612fa02be3dfaf0d45baae0fb4f309 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 8 Dec 2022 11:56:29 +0800 Subject: [PATCH 15/18] fix: keep option of database with precision ns --- include/util/tdef.h | 1 + source/libs/parser/src/parTranslater.c | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 58c8c1b51a..0ebc78bedb 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -309,6 +309,7 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_DURATION_PER_FILE (10 * 1440) #define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute #define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. +#define TSDB_MAX_KEEP_NS (29200 * 1440) // data in db to be reserved. #define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years #define TSDB_MIN_MINROWS_FBLOCK 10 #define TSDB_MAX_MINROWS_FBLOCK 1000 diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2afca5e35c..10e45901e5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3881,10 +3881,7 @@ static int32_t checkDbKeepOption(STranslateContext* pCxt, SDatabaseOptions* pOpt int64_t tsdbMaxKeep = TSDB_MAX_KEEP; if (pOptions->precision == TSDB_TIME_PRECISION_NANO) { - int64_t now = taosGetTimestampSec(); - if (now < 0) now = 0; - tsdbMaxKeep = now / 60 + 292 * 365 * 1440; - tsdbMaxKeep = TMIN(tsdbMaxKeep, TSDB_MAX_KEEP); + tsdbMaxKeep = TSDB_MAX_KEEP_NS; } if (pOptions->keep[0] < TSDB_MIN_KEEP || pOptions->keep[1] < TSDB_MIN_KEEP || pOptions->keep[2] < TSDB_MIN_KEEP || From 399dace80f2942d5fa1d01d4da2e16a002e4768a Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 8 Dec 2022 12:00:04 +0800 Subject: [PATCH 16/18] chore: code format optimization --- source/dnode/vnode/src/tsdb/tsdbFS.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 1942634d89..7dc839773f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -963,7 +963,6 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { } pDFileSet->diskId = pSet->diskId; - goto _exit; } } From a3c1d3e1bac3519931d0236e9506e146480a5510 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 8 Dec 2022 12:26:10 +0800 Subject: [PATCH 17/18] fix: compile error in windows --- include/os/osSystem.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/os/osSystem.h b/include/os/osSystem.h index ed5213ce34..8f1f5c58d5 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -66,8 +66,7 @@ void taosResetTerminalMode(); { \ taosPrintLog(flags, level, dflag, \ "backtrace not implemented on windows, so detailed stack information cannot be printed"); \ - \ \ - } + } #endif #ifdef __cplusplus From 45528081d53825353188a08ac09d99646683c675 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 8 Dec 2022 12:30:40 +0800 Subject: [PATCH 18/18] chore: keep option of ns database --- include/util/tdef.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 0ebc78bedb..e0089014aa 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -307,9 +307,9 @@ typedef enum ELogicConditionType { #define TSDB_MIN_DURATION_PER_FILE 60 // unit minute #define TSDB_MAX_DURATION_PER_FILE (3650 * 1440) #define TSDB_DEFAULT_DURATION_PER_FILE (10 * 1440) -#define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute -#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. -#define TSDB_MAX_KEEP_NS (29200 * 1440) // data in db to be reserved. +#define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute +#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. +#define TSDB_MAX_KEEP_NS (365 * 292 * 1440) // data in db to be reserved. #define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years #define TSDB_MIN_MINROWS_FBLOCK 10 #define TSDB_MAX_MINROWS_FBLOCK 1000