From 413a57f4cbf1863fdd0fe5d651624cf80e7a3c47 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 13 Aug 2024 18:55:48 +0800 Subject: [PATCH 1/9] fix(query): retry when column dropped in latest schema --- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 4 +++- source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index 2665cc1aaf..e6720466ce 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -463,7 +463,9 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe if (cid < blockCol.cid) { const STColumn *tcol = tTSchemaSearchColumn(pTSchema, cid); - ASSERT(tcol); + if (tcol == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER, &lino, _exit); + } SBlockCol none = { .cid = cid, .type = tcol->type, diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index 6e24311017..3d30ea0d93 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -309,7 +309,9 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk * if (cid < blockCol.cid) { const STColumn *tcol = tTSchemaSearchColumn(pTSchema, cid); - ASSERT(tcol); + if (tcol == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER, &lino, _exit); + } SBlockCol none = { .cid = cid, .type = tcol->type, From fe07324d72bf289d27764e9d664e988817f9f2bc Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 14 Aug 2024 12:42:46 +0800 Subject: [PATCH 2/9] fix(tsdb): return error code when loading data --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index e55ede560e..e9761d8c87 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -1004,6 +1004,10 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF pSttDataInfo->numOfRows += numOfRows; } } else { + if(terrno != TSDB_CODE_SUCCESS) { + code = terrno; + goto _end; + } if (!pMTree->ignoreEarlierTs) { pMTree->ignoreEarlierTs = pIter->ignoreEarlierTs; } From cec7643ea825920081f491e66be52f4ae3145e5d Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 14 Aug 2024 13:57:33 +0800 Subject: [PATCH 3/9] enh: code optimization --- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 4 +--- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 5 +---- source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 4 +--- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index e6720466ce..a9c7ea3961 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -463,9 +463,7 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe if (cid < blockCol.cid) { const STColumn *tcol = tTSchemaSearchColumn(pTSchema, cid); - if (tcol == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER, &lino, _exit); - } + TSDB_CHECK_NULL(tcol, code,lino,_exit,TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER); SBlockCol none = { .cid = cid, .type = tcol->type, diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index e9761d8c87..9d99c51587 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -1004,10 +1004,7 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF pSttDataInfo->numOfRows += numOfRows; } } else { - if(terrno != TSDB_CODE_SUCCESS) { - code = terrno; - goto _end; - } + TAOS_CHECK_GOTO(terrno, NULL, _end); if (!pMTree->ignoreEarlierTs) { pMTree->ignoreEarlierTs = pIter->ignoreEarlierTs; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index 3d30ea0d93..d3990c645c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -309,9 +309,7 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk * if (cid < blockCol.cid) { const STColumn *tcol = tTSchemaSearchColumn(pTSchema, cid); - if (tcol == NULL) { - TAOS_CHECK_GOTO(TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER, &lino, _exit); - } + TSDB_CHECK_NULL(tcol, code,lino,_exit,TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER); SBlockCol none = { .cid = cid, .type = tcol->type, From f6ad63ac14516b7a904f9746744bc8980457f830 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 14 Aug 2024 13:59:33 +0800 Subject: [PATCH 4/9] enh: code optimization --- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 2 +- source/dnode/vnode/src/tsdb/tsdbSttFileRW.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index a9c7ea3961..7e7ea59a5b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -463,7 +463,7 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe if (cid < blockCol.cid) { const STColumn *tcol = tTSchemaSearchColumn(pTSchema, cid); - TSDB_CHECK_NULL(tcol, code,lino,_exit,TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER); + TSDB_CHECK_NULL(tcol, code, lino, _exit, TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER); SBlockCol none = { .cid = cid, .type = tcol->type, diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index d3990c645c..e3d7f9d45f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -309,7 +309,7 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk * if (cid < blockCol.cid) { const STColumn *tcol = tTSchemaSearchColumn(pTSchema, cid); - TSDB_CHECK_NULL(tcol, code,lino,_exit,TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER); + TSDB_CHECK_NULL(tcol, code, lino, _exit, TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER); SBlockCol none = { .cid = cid, .type = tcol->type, From 6c917646b2aa30182772e0ddd6bae4376ffb7995 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 14 Aug 2024 16:15:37 +0800 Subject: [PATCH 5/9] fix(tsdb/cache): return error code --- source/dnode/vnode/src/tsdb/tsdbCache.c | 44 +++++++++++-------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 1216f0da81..46a498409b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1368,7 +1368,7 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SArray *remainCols, SCacheRowsReader *pr, int8_t ltype) { - int32_t code = 0; + int32_t code = 0, lino = 0; rocksdb_writebatch_t *wb = NULL; SArray *pTmpColArray = NULL; @@ -1413,9 +1413,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr if (NULL == lastTmpIndexArray) { lastTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t)); if (!lastTmpIndexArray) { - taosArrayDestroy(lastrowTmpIndexArray); - - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } } (void)taosArrayPush(lastTmpIndexArray, &(i)); @@ -1426,9 +1424,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr if (NULL == lastrowTmpIndexArray) { lastrowTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t)); if (!lastrowTmpIndexArray) { - taosArrayDestroy(lastTmpIndexArray); - - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } } (void)taosArrayPush(lastrowTmpIndexArray, &(i)); @@ -1440,13 +1436,11 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr pTmpColArray = taosArrayInit(lastIndex + lastrowIndex, sizeof(SLastCol)); if (!pTmpColArray) { - taosArrayDestroy(lastrowTmpIndexArray); - taosArrayDestroy(lastTmpIndexArray); - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } if (lastTmpIndexArray != NULL) { - (void)mergeLastCid(uid, pTsdb, &lastTmpColArray, pr, lastColIds, lastIndex, lastSlotIds); + TAOS_CHECK_EXIT(mergeLastCid(uid, pTsdb, &lastTmpColArray, pr, lastColIds, lastIndex, lastSlotIds)); for (int i = 0; i < taosArrayGetSize(lastTmpColArray); i++) { (void)taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastTmpIndexArray, i), taosArrayGet(lastTmpColArray, i)); @@ -1454,7 +1448,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr } if (lastrowTmpIndexArray != NULL) { - (void)mergeLastRowCid(uid, pTsdb, &lastrowTmpColArray, pr, lastrowColIds, lastrowIndex, lastrowSlotIds); + TAOS_CHECK_EXIT(mergeLastRowCid(uid, pTsdb, &lastrowTmpColArray, pr, lastrowColIds, lastrowIndex, lastrowSlotIds)); for (int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) { (void)taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastrowTmpIndexArray, i), taosArrayGet(lastrowTmpColArray, i)); @@ -1475,7 +1469,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr .colVal = COL_VAL_NONE(idxKey->key.cid, pr->pSchema->columns[slotIds[i]].type)}; if (!pLastCol) { pLastCol = &noneCol; - TAOS_CHECK_RETURN(reallocVarData(&pLastCol->colVal)); + TAOS_CHECK_EXIT(reallocVarData(&pLastCol->colVal)); } taosArraySet(pLastArray, idxKey->idx, pLastCol); @@ -1490,12 +1484,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); if (!pTmpLastCol) { - taosMemoryFree(slotIds); - taosMemoryFree(lastColIds); - taosMemoryFree(lastSlotIds); - taosMemoryFree(lastrowColIds); - taosMemoryFree(lastrowSlotIds); - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } *pTmpLastCol = *pLastCol; pLastCol = pTmpLastCol; @@ -1504,12 +1493,12 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr for (int8_t i = 0; i < pLastCol->rowKey.numOfPKs; i++) { SValue *pValue = &pLastCol->rowKey.pks[i]; if (IS_VAR_DATA_TYPE(pValue->type)) { - TAOS_CHECK_RETURN(reallocVarDataVal(pValue)); + TAOS_CHECK_EXIT(reallocVarDataVal(pValue)); charge += pValue->nData; } } if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) { - TAOS_CHECK_RETURN(reallocVarData(&pLastCol->colVal)); + TAOS_CHECK_EXIT(reallocVarData(&pLastCol->colVal)); charge += pLastCol->colVal.value.nData; } @@ -1538,6 +1527,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr rocksMayWrite(pTsdb, false, true, false); } +_exit: taosArrayDestroy(lastrowTmpIndexArray); taosArrayDestroy(lastrowTmpColArray); taosArrayDestroy(lastTmpIndexArray); @@ -3015,11 +3005,13 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC // inverse iterator CacheNextRowIter iter = {0}; - (void)nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->info.suid, pr->pLDataIterArray, pr->pReadSnap, pr->lastTs, pr); + code = + nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->info.suid, pr->pLDataIterArray, pr->pReadSnap, pr->lastTs, pr); + TAOS_CHECK_GOTO(code, &lino, _err); do { TSDBROW *pRow = NULL; - (void)nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); + code = nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); if (!pRow) { break; @@ -3199,11 +3191,13 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, // inverse iterator CacheNextRowIter iter = {0}; - (void)nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->info.suid, pr->pLDataIterArray, pr->pReadSnap, pr->lastTs, pr); + code = + nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->info.suid, pr->pLDataIterArray, pr->pReadSnap, pr->lastTs, pr); + TAOS_CHECK_GOTO(code, &lino, _err); do { TSDBROW *pRow = NULL; - (void)nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); + code = nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); if (!pRow) { break; From ab45ab1459801a068b8d339d777c3656ccd343d2 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 14 Aug 2024 16:40:12 +0800 Subject: [PATCH 6/9] fix(tsdb/read): memory leak --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 6783eb2cbd..31101165fd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2440,20 +2440,23 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info); if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(info.pKeyRangeList); pReader->code = code; return false; } code = initMemDataIterator(pScanInfo, pReader); if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(info.pKeyRangeList); pReader->code = code; return false; } code = initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost); if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(info.pKeyRangeList); pReader->code = code; - return code; + return false; } if (conf.rspRows) { From 92704f8b7add1cdc5ce20c9ef0e92e2ece11d96b Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 14 Aug 2024 22:06:18 +0800 Subject: [PATCH 7/9] fix: check error code --- source/dnode/vnode/src/tsdb/tsdbCache.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 46a498409b..0b020830e1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -2891,11 +2891,13 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI if (!pIter->pSkyline) { pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); + TSDB_CHECK_NULL(pIter->pSkyline, code, lino, _err, TSDB_CODE_OUT_OF_MEMORY); uint64_t uid = pIter->idx.uid; STableLoadInfo *pInfo = getTableLoadInfo(pIter->pr, uid); if (pInfo->pTombData == NULL) { pInfo->pTombData = taosArrayInit(4, sizeof(SDelData)); + TSDB_CHECK_NULL(pInfo->pTombData, code, lino, _err, TSDB_CODE_OUT_OF_MEMORY); } (void)taosArrayAddAll(pInfo->pTombData, pIter->pMemDelData); @@ -2903,6 +2905,7 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI size_t delSize = TARRAY_SIZE(pInfo->pTombData); if (delSize > 0) { code = tsdbBuildDeleteSkyline(pInfo->pTombData, 0, (int32_t)(delSize - 1), pIter->pSkyline); + TAOS_CHECK_GOTO(code, &lino, _err); } pIter->iSkyline = taosArrayGetSize(pIter->pSkyline) - 1; } From 1059650e573a9a8795574449f04449868edd9dd5 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 14 Aug 2024 22:20:03 +0800 Subject: [PATCH 8/9] enh: code optimization --- source/dnode/vnode/src/tsdb/tsdbCache.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index cc5759a151..aa92597211 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1440,11 +1440,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr int lastrowIndex = 0; if (!slotIds || !lastColIds || !lastSlotIds || !lastrowColIds || !lastrowSlotIds) { - taosMemoryFree(slotIds); - taosMemoryFree(lastColIds); - taosMemoryFree(lastSlotIds); - taosMemoryFree(lastrowColIds); - TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_MEMORY); } for (int i = 0; i < num_keys; ++i) { From cc8bff453a1e0fa14a0b8aad6c26a0de67305262 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 15 Aug 2024 10:45:16 +0800 Subject: [PATCH 9/9] fix(insert): return error when parsing csv file --- source/libs/parser/src/parInsertSql.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 70bd43559c..cb94cd42f7 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2238,6 +2238,8 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt if (pStmt->insertType != TSDB_QUERY_TYPE_FILE_INSERT) { return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is exclusive", NULL); } + } else { + return buildInvalidOperationMsg(&pCxt->msg, tstrerror(code)); } // just record pTableCxt whose data come from file