From d9042f5ed38bdd9529709d685f555d49b43b558e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 25 May 2023 17:24:11 +0800 Subject: [PATCH 01/10] fix:memory leak --- source/client/inc/clientSml.h | 2 +- source/client/src/clientSml.c | 69 ++++++++++++++++------------- source/client/src/clientSmlJson.c | 15 +++++-- source/client/src/clientSmlLine.c | 8 +++- source/client/src/clientSmlTelnet.c | 5 ++- tests/parallel_test/cases.task | 4 +- 6 files changed, 64 insertions(+), 39 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index f9cbffa748..c9eb951014 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -251,7 +251,7 @@ int32_t smlClearForRerun(SSmlHandle *info); int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg); uint8_t smlGetTimestampLen(int64_t num); void clearColValArray(SArray* pCols); -void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag); +void smlDestroyTableInfo(void *para); void freeSSmlKv(void* data); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 92e4d0f8da..5748c06e8b 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -230,6 +230,16 @@ void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tin } } +static void smlDestroySTableMeta(void *para) { + SSmlSTableMeta *meta = *(SSmlSTableMeta**)para; + taosHashCleanup(meta->tagHash); + taosHashCleanup(meta->colHash); + taosArrayDestroy(meta->tags); + taosArrayDestroy(meta->cols); + taosMemoryFree(meta->tableMeta); + taosMemoryFree(meta); +} + SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); if (!meta) { @@ -264,7 +274,7 @@ SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { return meta; cleanup: - taosMemoryFree(meta); + smlDestroySTableMeta(meta); return NULL; } @@ -1093,15 +1103,6 @@ static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { } } -static void smlDestroySTableMeta(SSmlSTableMeta *meta) { - taosHashCleanup(meta->tagHash); - taosHashCleanup(meta->colHash); - taosArrayDestroy(meta->tags); - taosArrayDestroy(meta->cols); - taosMemoryFree(meta->tableMeta); - taosMemoryFree(meta); -} - static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg) { for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); @@ -1141,7 +1142,8 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols return TSDB_CODE_SUCCESS; } -void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { +void smlDestroyTableInfo(void *para) { + SSmlTableInfo *tag = *(SSmlTableInfo**)para; for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); taosHashCleanup(kvHash); @@ -1178,18 +1180,18 @@ void smlDestroyInfo(SSmlHandle *info) { qDestroyQuery(info->pQuery); // destroy info->childTables - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); - while (oneTable) { - smlDestroyTableInfo(info, *oneTable); - oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); - } +// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); +// while (oneTable) { +// smlDestroyTableInfo(oneTable); +// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); +// } // destroy info->superTables - SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); - while (oneSTable) { - smlDestroySTableMeta(*oneSTable); - oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); - } +// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); +// while (oneSTable) { +// smlDestroySTableMeta(*oneSTable); +// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); +// } // destroy info->pVgHash taosHashCleanup(info->pVgHash); @@ -1248,6 +1250,8 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->tableUids = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + taosHashSetFreeFp(info->superTables, smlDestroySTableMeta); + taosHashSetFreeFp(info->childTables, smlDestroyTableInfo); info->id = smlGenId(); info->pQuery = smlInitHandle(); @@ -1354,6 +1358,9 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); + if(meta == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); terrno = 0; smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); @@ -1473,18 +1480,18 @@ static void smlPrintStatisticInfo(SSmlHandle *info) { int32_t smlClearForRerun(SSmlHandle *info) { info->reRun = false; // clear info->childTables - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); - while (oneTable) { - smlDestroyTableInfo(info, *oneTable); - oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); - } +// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); +// while (oneTable) { +// smlDestroyTableInfo(info, *oneTable); +// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); +// } // clear info->superTables - SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); - while (oneSTable) { - smlDestroySTableMeta(*oneSTable); - oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); - } +// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); +// while (oneSTable) { +// smlDestroySTableMeta(*oneSTable); +// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); +// } taosHashClear(info->childTables); taosHashClear(info->superTables); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 7ccf930964..ad5b199c4e 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -695,6 +695,9 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo return TSDB_CODE_SUCCESS; } sMeta = smlBuildSTableMeta(info->dataFormat); + if(sMeta == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ @@ -784,7 +787,7 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); if (tinfo->tableDataCtx == NULL) { smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(info, tinfo); + smlDestroyTableInfo(&tinfo); return TSDB_CODE_SML_INVALID_DATA; } } @@ -1048,12 +1051,18 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { return TSDB_CODE_TSC_INVALID_JSON; } - info->lineNum = payloadNum; - info->dataFormat = true; + if (unlikely(info->lines != NULL)) { + for (int i = 0; i < info->lineNum; i++) { + taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); + if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag); + } taosMemoryFree(info->lines); info->lines = NULL; } + info->lineNum = payloadNum; + info->dataFormat = true; + ret = smlClearForRerun(info); if (ret != TSDB_CODE_SUCCESS) { return ret; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index e79093398e..4bf4ae091f 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -168,6 +168,9 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SUCCESS; } sMeta = smlBuildSTableMeta(info->dataFormat); + if(sMeta == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); for (int i = pTableMeta->tableInfo.numOfColumns; @@ -326,7 +329,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin info->currSTableMeta->uid = tinfo->uid; tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); if (tinfo->tableDataCtx == NULL) { - smlDestroyTableInfo(info, tinfo); + smlDestroyTableInfo(&tinfo); smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); return TSDB_CODE_SML_INVALID_DATA; } @@ -372,6 +375,9 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin return TSDB_CODE_SUCCESS; } *tmp = smlBuildSTableMeta(info->dataFormat); + if(*tmp == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } (*tmp)->tableMeta = pTableMeta; taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES); diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index 42b8001e59..6dd8842797 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -91,6 +91,9 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS return TSDB_CODE_SUCCESS; } sMeta = smlBuildSTableMeta(info->dataFormat); + if(sMeta == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ @@ -212,7 +215,7 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); if (tinfo->tableDataCtx == NULL) { smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(info, tinfo); + smlDestroyTableInfo(&tinfo); return TSDB_CODE_SML_INVALID_DATA; } } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ab4663e5b3..b09d8f68ec 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -571,7 +571,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_stable.py @@ -1089,7 +1089,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 4 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 4 From 8ee1288611ce3a922e296a6c1519df7a95aca43d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 25 May 2023 17:47:55 +0800 Subject: [PATCH 02/10] fix(cache): remove deleted data --- source/dnode/vnode/src/tsdb/tsdbCache.c | 62 ++++++++++++---------- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 6 +-- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index d33f04c145..830f44b411 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -937,14 +937,14 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE } // build keys & multi get from rocks - int num_keys = pTSchema->numOfCols; - char **keys_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); - size_t *keys_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); + int num_keys = pTSchema->numOfCols; + char **keys_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); + size_t *keys_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); + const size_t klen = ROCKS_KEY_LEN; for (int i = 0; i < num_keys; ++i) { int16_t cid = pTSchema->columns[i].colId; - size_t klen = ROCKS_KEY_LEN; - char *keys = taosMemoryCalloc(2, sizeof(SLastKey)); + char *keys = taosMemoryCalloc(2, sizeof(SLastKey)); ((SLastKey *)keys)[0] = (SLastKey){.ltype = 1, .uid = uid, .cid = cid}; ((SLastKey *)keys)[1] = (SLastKey){.ltype = 0, .uid = uid, .cid = cid}; @@ -960,39 +960,35 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE rocksMayWrite(pTsdb, true, false, false); rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys * 2, (const char *const *)keys_list, keys_list_sizes, values_list, values_list_sizes, errs); - for (int i = 0; i < num_keys; ++i) { - taosMemoryFree(keys_list[i]); - } for (int i = 0; i < num_keys * 2; ++i) { - rocksdb_free(errs[i]); + if (errs[i]) { + rocksdb_free(errs[i]); + } } - taosMemoryFree(keys_list); - taosMemoryFree(keys_list_sizes); taosMemoryFree(errs); rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; for (int i = 0; i < num_keys; ++i) { SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i]); if (NULL != pLastCol && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { - SLastKey *key = &(SLastKey){.ltype = 1, .uid = uid, .cid = pLastCol->colVal.cid}; - size_t klen = ROCKS_KEY_LEN; - - rocksdb_writebatch_delete(wb, (char *)key, klen); - taosLRUCacheErase(pTsdb->lruCache, key, klen); + rocksdb_writebatch_delete(wb, keys_list[i], klen); } + taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen); pLastCol = tsdbCacheDeserialize(values_list[i + num_keys]); if (NULL != pLastCol && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { - SLastKey *key = &(SLastKey){.ltype = 0, .uid = uid, .cid = pLastCol->colVal.cid}; - size_t klen = ROCKS_KEY_LEN; - - rocksdb_writebatch_delete(wb, (char *)key, klen); - taosLRUCacheErase(pTsdb->lruCache, key, klen); + rocksdb_writebatch_delete(wb, keys_list[num_keys + i], klen); } + taosLRUCacheErase(pTsdb->lruCache, keys_list[num_keys + i], klen); rocksdb_free(values_list[i]); rocksdb_free(values_list[i + num_keys]); } + for (int i = 0; i < num_keys; ++i) { + taosMemoryFree(keys_list[i]); + } + taosMemoryFree(keys_list); + taosMemoryFree(keys_list_sizes); taosMemoryFree(values_list); taosMemoryFree(values_list_sizes); @@ -1871,10 +1867,14 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie if (isLast && (pColData->flag & HAS_VALUE)) { skipBlock = false; break; - } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { + } /*else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { skipBlock = false; break; - } + }*/ + } + + if (!isLast) { + skipBlock = false; } if (skipBlock) { @@ -1908,6 +1908,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie if (checkRemainingRow) { bool skipBlock = true; int inputColIndex = 0; + if (aCols[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { + ++inputColIndex; + } for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) { SColData *pColData = &state->pBlockData->aColData[colIndex]; int16_t cid = pColData->cid; @@ -1916,15 +1919,19 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie if (isLast && (pColData->flag & HAS_VALUE)) { skipBlock = false; break; - } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { + } /*else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { skipBlock = false; break; - } + }*/ ++inputColIndex; } } + if (!isLast) { + skipBlock = false; + } + if (skipBlock) { if (--state->iBlock < 0) { tsdbDataFReaderClose(state->pDataFReader); @@ -2144,7 +2151,8 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { if (key->ts > pItemBack->ts) { return false; } else if (key->ts >= pItemFront->ts && key->ts <= pItemBack->ts) { - if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { + // if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { + if (key->version <= pItemFront->version || key->version <= pItemBack->version) { return true; } else { return false; @@ -2959,7 +2967,7 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, do { TSDBROW *pRow = NULL; - nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); + nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); if (!pRow) { break; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 80967a906f..6d223e00c5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -190,9 +190,9 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid tsdbCacheDeleteLast(pTsdb->lruCache, pTbData->uid, eKey); } */ - if (eKey >= pTbData->maxKey && sKey <= pTbData->maxKey) { - tsdbCacheDel(pTsdb, suid, uid, sKey, eKey); - } + // if (eKey >= pTbData->maxKey && sKey <= pTbData->maxKey) { + tsdbCacheDel(pTsdb, suid, uid, sKey, eKey); + //} tsdbTrace("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 " at version %" PRId64, From 40fa0e06201153504d06d1bc0ff4aef896d3cdc9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 25 May 2023 18:07:36 +0800 Subject: [PATCH 03/10] fix:memory leak --- source/client/src/clientSml.c | 3 ++- source/client/src/clientSmlJson.c | 1 + source/client/src/clientSmlLine.c | 2 ++ source/client/src/clientSmlTelnet.c | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 5748c06e8b..85cba9cda6 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -236,7 +236,7 @@ static void smlDestroySTableMeta(void *para) { taosHashCleanup(meta->colHash); taosArrayDestroy(meta->tags); taosArrayDestroy(meta->cols); - taosMemoryFree(meta->tableMeta); + taosMemoryFreeClear(meta->tableMeta); taosMemoryFree(meta); } @@ -1045,6 +1045,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } } + taosMemoryFreeClear(sTableData->tableMeta); sTableData->tableMeta = pTableMeta; uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid, pTableMeta->sversion, pTableMeta->tversion) tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index ad5b199c4e..0f59505f8c 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -696,6 +696,7 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo } sMeta = smlBuildSTableMeta(info->dataFormat); if(sMeta == NULL){ + taosMemoryFreeClear(pTableMeta); return TSDB_CODE_OUT_OF_MEMORY; } sMeta->tableMeta = pTableMeta; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 4bf4ae091f..40e014458e 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -169,6 +169,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } sMeta = smlBuildSTableMeta(info->dataFormat); if(sMeta == NULL){ + taosMemoryFreeClear(pTableMeta); return TSDB_CODE_OUT_OF_MEMORY; } sMeta->tableMeta = pTableMeta; @@ -376,6 +377,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } *tmp = smlBuildSTableMeta(info->dataFormat); if(*tmp == NULL){ + taosMemoryFreeClear(pTableMeta); return TSDB_CODE_OUT_OF_MEMORY; } (*tmp)->tableMeta = pTableMeta; diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index 6dd8842797..b3f45a3107 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -92,6 +92,7 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS } sMeta = smlBuildSTableMeta(info->dataFormat); if(sMeta == NULL){ + taosMemoryFreeClear(pTableMeta); return TSDB_CODE_OUT_OF_MEMORY; } sMeta->tableMeta = pTableMeta; From 75eb724c37297c6010b013914b09ea0e5c31065c Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Thu, 25 May 2023 17:28:50 +0800 Subject: [PATCH 04/10] fix tsdb read error version --- source/dnode/vnode/src/tq/tq.c | 1 + source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0a3173b3cb..25df80479d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1083,6 +1083,7 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t qDebug("s-task:%s set the start wal offset to be:%"PRId64, pTask->id.idStr, sversion); walReaderSeekVer(pTask->exec.pWalReader, sversion); + pTask->chkInfo.currentVer = sversion; if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b4b090249e..2fb86db8b8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1848,7 +1848,7 @@ static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBloc SDataBlockToLoadInfo info = {0}; getBlockToLoadInfo(&info, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader, pReader); bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf || - info.overlapWithDelInfo || info.overlapWithLastBlock); + info.overlapWithDelInfo || info.overlapWithLastBlock || info.partiallyRequired); return isCleanFileBlock; } From 208ba2ef4d507aec918d7001b399b9b68f5255ff Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 26 May 2023 10:45:37 +0800 Subject: [PATCH 05/10] fix:memory leak --- source/client/src/clientImpl.c | 1 + source/libs/parser/src/parTranslater.c | 11 ++++------- source/libs/parser/src/parUtil.c | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 82177302c7..5963e419e1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1757,6 +1757,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int return TSDB_CODE_TSC_INTERNAL_ERROR; } + taosMemoryFreeClear(pResultInfo->convertJson); pResultInfo->convertJson = taosMemoryCalloc(1, dataLen); if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; char* p1 = pResultInfo->convertJson; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 864513f15d..39dab556f3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8008,6 +8008,9 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { isJson = true; code = buildJsonTagVal(pCxt, pTagSchema, pVal, pTagArray, ppTag); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode((SNode*)pVal); + } taosArrayPush(tagName, pTagSchema->name); } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL && !pVal->isNull) { char* tmpVal = nodesGetValueFromNode(pVal); @@ -8328,13 +8331,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS SArray* pTagVals = taosArrayInit(1, sizeof(STagVal)); int32_t code = TSDB_CODE_SUCCESS; STag* pTag = NULL; - do { - code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf); - if (TSDB_CODE_SUCCESS != code) { - break; - } - } while (0); - + code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf); taosArrayDestroy(pTagVals); if (code != TSDB_CODE_SUCCESS) { return code; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 5597bd3df8..8fa92a9d2e 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -416,7 +416,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi end: taosHashCleanup(keyHash); if (retCode == TSDB_CODE_SUCCESS) { - tTagNew(pTagVals, 1, true, ppTag); + retCode = tTagNew(pTagVals, 1, true, ppTag); } for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { STagVal* p = (STagVal*)taosArrayGet(pTagVals, i); From bb481c52b6c03c8b0c8f27877a7b93401e5c2eb3 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 26 May 2023 11:26:55 +0800 Subject: [PATCH 06/10] cache/skyline: fix skyline false condition --- source/dnode/vnode/src/tsdb/tsdbCache.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 830f44b411..8757661216 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -825,7 +825,7 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h); SLastCol lastCol = *pLastCol; - // reallocVarData(&lastCol.colVal); + reallocVarData(&lastCol.colVal); taosArrayPush(pLastArray, &lastCol); if (h) { @@ -2151,11 +2151,15 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { if (key->ts > pItemBack->ts) { return false; } else if (key->ts >= pItemFront->ts && key->ts <= pItemBack->ts) { - // if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { - if (key->version <= pItemFront->version || key->version <= pItemBack->version) { + if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { + // if (key->version <= pItemFront->version || key->version <= pItemBack->version) { return true; } else { - return false; + if (*iSkyline > 1) { + --*iSkyline; + } else { + return false; + } } } else { if (*iSkyline > 1) { From fcf2b58aad3010b10dbb9f24465ce7d6faf61eff Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 26 May 2023 12:48:37 +0800 Subject: [PATCH 07/10] check offline when force --- include/util/taoserror.h | 1 + source/dnode/mnode/impl/src/mndDnode.c | 11 +++++++++-- source/util/src/terror.c | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 636a5e63a7..9e5229870e 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -409,6 +409,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MNODE_ALREADY_IS_VOTER TAOS_DEF_ERROR_CODE(0, 0x0413) // internal #define TSDB_CODE_MNODE_ONLY_TWO_MNODE TAOS_DEF_ERROR_CODE(0, 0x0414) // internal #define TSDB_CODE_MNODE_NO_NEED_RESTORE TAOS_DEF_ERROR_CODE(0, 0x0415) // internal +#define TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0416) // vnode // #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index cee3b3c61d..99694ae4b8 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -986,8 +986,15 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } int32_t numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); - if ((numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) && !force) { - if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) { + if (numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) { + bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs()); + if (isonline && force) { + terrno = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE; + mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), + numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); + goto _OVER; + } + if (!isonline && !force) { terrno = TSDB_CODE_DNODE_OFFLINE; mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 31727f7535..e28e67f83a 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -325,6 +325,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NOT_CATCH_UP, "Mnode didn't catch th TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_ALREADY_IS_VOTER, "Mnode already is a leader") TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_ONLY_TWO_MNODE, "Only two mnodes exist") TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NO_NEED_RESTORE, "No need to restore on this dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE, "Please use this command when the dnode is offline") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed") From 6e29d7879f9b9dec98f9d7faef4ac78aa0fafdeb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 26 May 2023 14:13:01 +0800 Subject: [PATCH 08/10] cache/var-data: realloc var data --- source/dnode/vnode/src/tsdb/tsdbCache.c | 3 ++- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 24 ++++++++++----------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 8757661216..aea7c45bd6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -703,6 +703,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr *pTmpLastCol = *pLastCol; pLastCol = pTmpLastCol; + reallocVarData(&pLastCol->colVal); size_t charge = sizeof(*pLastCol); if (IS_VAR_DATA_TYPE(pLastCol->colVal.type)) { charge += pLastCol->colVal.value.nData; @@ -853,8 +854,8 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache SLastCol lastCol = *pLastCol; reallocVarData(&lastCol.colVal); - taosArraySet(pLastArray, idxKey->idx, &lastCol); + if (h) { taosLRUCacheRelease(pCache, h, false); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 0e0b7a2ffa..5df1ea0672 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -315,14 +315,14 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 tsdbCacheGetBatch(pr->pTsdb, pKeyInfo->uid, pRow, pr, ltype); // tsdbCacheGet(pr->pTsdb, pKeyInfo->uid, pRow, pr, ltype); if (TARRAY_SIZE(pRow) <= 0) { - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); continue; } SLastCol* pColVal = taosArrayGet(pRow, 0); if (COL_VAL_IS_NONE(&pColVal->colVal)) { - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); continue; } @@ -381,8 +381,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } } - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); } if (hasRes) { @@ -394,20 +394,20 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); if (TARRAY_SIZE(pRow) <= 0) { - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); continue; } SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, 0); if (COL_VAL_IS_NONE(&pColVal->colVal)) { - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); continue; } saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); - // taosArrayClearEx(pRow, freeItem); - taosArrayClear(pRow); + taosArrayClearEx(pRow, freeItem); + // taosArrayClear(pRow); taosArrayPush(pTableUidList, &uid); From 5ebe75599e4a7f853726acd14661599c3f56d74e Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Fri, 26 May 2023 10:09:05 +0800 Subject: [PATCH 09/10] fix pause issue --- source/libs/stream/src/streamExec.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 637d661343..95b97e080a 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -327,7 +327,11 @@ int32_t streamExecForAll(SStreamTask* pTask) { while (1) { if (streamTaskShouldPause(&pTask->status)) { - return 0; + if (batchSize > 1) { + break; + } else { + return 0; + } } SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); From 34b9b322cdcbd17dcb83a4e9a4d343257d64945b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 26 May 2023 15:01:57 +0800 Subject: [PATCH 10/10] cache/var-data: realloc for cache reading --- source/dnode/vnode/src/tsdb/tsdbCache.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index aea7c45bd6..63629f8623 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -790,7 +790,9 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA code = -1; } - taosArraySet(pLastArray, idxKey->idx, pLastCol); + SLastCol lastCol = *pLastCol; + reallocVarData(&lastCol.colVal); + taosArraySet(pLastArray, idxKey->idx, &lastCol); taosArrayRemove(remainCols, j); taosMemoryFree(values_list[i]);