From 8cd92624ae3d49d486b8258b056c365925dc228e Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 14 Oct 2024 09:10:06 +0000 Subject: [PATCH 01/11] fix/TD-32547-memleak --- source/dnode/mgmt/exe/dmMain.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index ddef0537f8..ba162bd84f 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" #include "mnode.h" +#include "osFile.h" #include "tconfig.h" #include "tglobal.h" #include "version.h" @@ -415,6 +416,9 @@ int mainWindows(int argc, char **argv) { return code; } int ret = dmUpdateEncryptKey(global.encryptKey, toLogFile); + if (taosCloseFile(&pFile) != 0) { + encryptError("failed to close file:%p", pFile); + } taosCloseLog(); taosCleanupArgs(); return ret; From c1557c423f72a74e8c9e3ecc3674beb90f6a785d Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 15 Oct 2024 01:12:52 +0000 Subject: [PATCH 02/11] fix/TD-32555-reset-actionpos-when-rollbak --- source/dnode/mnode/impl/src/mndTrans.c | 44 +++++++++++++++----------- source/util/src/terror.c | 2 +- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index fcf7905cd1..657601f5ae 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1252,8 +1252,9 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) { pAction->errCode = pRsp->code; pTrans->lastErrorNo = pRsp->code; - mInfo("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, - mndTransStr(pAction->stage), action, pRsp->code, pAction->acceptableCode, pAction->retryCode); + mInfo("trans:%d, %s:%d response is received, received code:0x%x(%s), accept:0x%x(%s) retry:0x%x(%s)", transId, + mndTransStr(pAction->stage), action, pRsp->code, tstrerror(pRsp->code), pAction->acceptableCode, + tstrerror(pAction->acceptableCode), pAction->retryCode, tstrerror(pAction->retryCode)); } else { mInfo("trans:%d, invalid action, index:%d, code:0x%x", transId, action, pRsp->code); } @@ -1469,8 +1470,8 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { - mError("trans:%d, failed to execute redoActions since:%s, code:0x%x, topHalf:%d", pTrans->id, terrstr(), terrno, - topHalf); + mError("trans:%d, failed to execute redoActions since:%s, code:0x%x, topHalf(TransContext):%d", pTrans->id, + terrstr(), terrno, topHalf); } return code; } @@ -1478,7 +1479,8 @@ static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans, bool t static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { - mError("trans:%d, failed to execute undoActions since %s. topHalf:%d", pTrans->id, terrstr(), topHalf); + mError("trans:%d, failed to execute undoActions since %s. topHalf(TransContext):%d", pTrans->id, terrstr(), + topHalf); } return code; } @@ -1486,7 +1488,8 @@ static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans, bool t static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->commitActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS && code != TSDB_CODE_MND_TRANS_CTX_SWITCH) { - mError("trans:%d, failed to execute commitActions since %s. topHalf:%d", pTrans->id, terrstr(), topHalf); + mError("trans:%d, failed to execute commitActions since %s. topHalf(TransContext):%d", pTrans->id, terrstr(), + topHalf); } return code; } @@ -1500,11 +1503,15 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr return code; } - mInfo("trans:%d, execute %d actions serial, current action:%d", pTrans->id, numOfActions, pTrans->actionPos); + mInfo("trans:%d, execute %d actions serial, begin at action:%d, stage:%s", pTrans->id, numOfActions, + pTrans->actionPos, mndTransStr(pTrans->stage)); for (int32_t action = pTrans->actionPos; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pActions, action); + mInfo("trans:%d, current action:%d, stage:%s, actionType(0:log,1:msg):%d", pTrans->id, pTrans->actionPos, + mndTransStr(pAction->stage), pAction->actionType); + code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf); if (code == 0) { if (pAction->msgSent) { @@ -1536,8 +1543,8 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr if (mndCannotExecuteTransAction(pMnode, topHalf)) { pTrans->lastErrorNo = code; pTrans->code = code; - mInfo("trans:%d, %s:%d, topHalf:%d, not execute next action, code:%s", pTrans->id, mndTransStr(pAction->stage), - action, topHalf, tstrerror(code)); + mInfo("trans:%d, %s:%d, topHalf(TransContext):%d, not execute next action, code:%s", pTrans->id, + mndTransStr(pAction->stage), action, topHalf, tstrerror(code)); break; } @@ -1561,7 +1568,8 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr break; } else if (code == pAction->retryCode || code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_SYN_NOT_LEADER) { - mInfo("trans:%d, %s:%d receive code:0x%x and retry", pTrans->id, mndTransStr(pAction->stage), pAction->id, code); + mInfo("trans:%d, %s:%d receive code:0x%x(%s) and retry", pTrans->id, mndTransStr(pAction->stage), pAction->id, + code, tstrerror(code)); pTrans->lastErrorNo = code; taosMsleep(300); action--; @@ -1570,8 +1578,8 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr terrno = code; pTrans->lastErrorNo = code; pTrans->code = code; - mInfo("trans:%d, %s:%d receive code:0x%x and wait another schedule, failedTimes:%d", pTrans->id, - mndTransStr(pAction->stage), pAction->id, code, pTrans->failedTimes); + mInfo("trans:%d, %s:%d receive code:0x%x(%s) and wait another schedule, failedTimes:%d", pTrans->id, + mndTransStr(pAction->stage), pAction->id, code, tstrerror(code), pTrans->failedTimes); break; } } @@ -1647,8 +1655,8 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool } else { continueExec = false; } - mInfo("trans:%d, cannot execute redo action stage, topHalf:%d, continueExec:%d, code:%s", pTrans->id, topHalf, - continueExec, tstrerror(code)); + mInfo("trans:%d, cannot execute redo action stage, topHalf(TransContext):%d, continueExec:%d, code:%s", pTrans->id, + topHalf, continueExec, tstrerror(code)); return continueExec; } @@ -1680,7 +1688,9 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool } pTrans->stage = TRN_STAGE_ROLLBACK; - mError("trans:%d, stage from redoAction to rollback since %s", pTrans->id, terrstr()); + pTrans->actionPos = 0; + mError("trans:%d, stage from redoAction to rollback since %s, and set actionPos to %d", pTrans->id, terrstr(), + pTrans->actionPos); continueExec = true; } else { mError("trans:%d, stage keep on redoAction since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); @@ -1773,8 +1783,6 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool to if (code == 0) { pTrans->stage = TRN_STAGE_UNDO_ACTION; - pTrans->actionPos = 0; - mInfo("trans:%d, stage from rollback to undoAction, actionPos:%d", pTrans->id, pTrans->actionPos); continueExec = true; } else { pTrans->failedTimes++; @@ -1829,7 +1837,7 @@ void mndTransExecuteImp(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = true; while (continueExec) { - mInfo("trans:%d, continue to execute, stage:%s createTime:%" PRId64 " topHalf:%d", pTrans->id, + mInfo("trans:%d, continue to execute, stage:%s createTime:%" PRId64 " topHalf(TransContext):%d", pTrans->id, mndTransStr(pTrans->stage), pTrans->createdTime, topHalf); pTrans->lastExecTime = taosGetTimestampMs(); switch (pTrans->stage) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 1c94a5f2e4..35f0bc73eb 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -314,7 +314,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CLOG_IS_NULL, "Transaction commitlog TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL, "Unable to establish connection While execute transaction and will continue in the background") TAOS_DEFINE_ERROR(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED, "Last Transaction not finished") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_SYNC_TIMEOUT, "Sync timeout While execute transaction and will continue in the background") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CTX_SWITCH, "Transaction context switch") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CTX_SWITCH, "Wrong transaction execution context") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT_COMPACT, "Transaction not completed due to conflict with compact") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error") From c372959cef3d6da7077dff3d3514f0c08a2736dc Mon Sep 17 00:00:00 2001 From: Jinqing Kuang Date: Tue, 15 Oct 2024 11:32:26 +0800 Subject: [PATCH 03/11] fix(query)[TD-30667]. Fix avx/avx2 check and compilation on ARM platform --- cmake/cmake.define | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index fd8f42de2f..9fae397363 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -160,12 +160,21 @@ ELSE () CHECK_C_COMPILER_FLAG("-msse4.2" COMPILER_SUPPORT_SSE42) ENDIF() - CHECK_C_COMPILER_FLAG("-mfma" COMPILER_SUPPORT_FMA) - CHECK_C_COMPILER_FLAG("-mavx" COMPILER_SUPPORT_AVX) - CHECK_C_COMPILER_FLAG("-mavx2" COMPILER_SUPPORT_AVX2) - CHECK_C_COMPILER_FLAG("-mavx512f" COMPILER_SUPPORT_AVX512F) - CHECK_C_COMPILER_FLAG("-mavx512vbmi" COMPILER_SUPPORT_AVX512BMI) - CHECK_C_COMPILER_FLAG("-mavx512vl" COMPILER_SUPPORT_AVX512VL) + IF (TD_ARM_64 OR TD_ARM_32) + SET(COMPILER_SUPPORT_FMA false) + SET(COMPILER_SUPPORT_AVX false) + SET(COMPILER_SUPPORT_AVX2 false) + SET(COMPILER_SUPPORT_AVX512F false) + SET(COMPILER_SUPPORT_AVX512BMI false) + SET(COMPILER_SUPPORT_AVX512VL false) + ELSE() + CHECK_C_COMPILER_FLAG("-mfma" COMPILER_SUPPORT_FMA) + CHECK_C_COMPILER_FLAG("-mavx" COMPILER_SUPPORT_AVX) + CHECK_C_COMPILER_FLAG("-mavx2" COMPILER_SUPPORT_AVX2) + CHECK_C_COMPILER_FLAG("-mavx512f" COMPILER_SUPPORT_AVX512F) + CHECK_C_COMPILER_FLAG("-mavx512vbmi" COMPILER_SUPPORT_AVX512BMI) + CHECK_C_COMPILER_FLAG("-mavx512vl" COMPILER_SUPPORT_AVX512VL) + ENDIF() IF (COMPILER_SUPPORT_SSE42) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse4.2") @@ -182,12 +191,13 @@ ELSE () IF (COMPILER_SUPPORT_AVX) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx") + MESSAGE(STATUS "AVX instructions is ACTIVATED") ENDIF() IF (COMPILER_SUPPORT_AVX2) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2") + MESSAGE(STATUS "AVX2 instructions is ACTIVATED") ENDIF() - MESSAGE(STATUS "SIMD instructions (AVX/AVX2) is ACTIVATED") IF ("${SIMD_AVX512_SUPPORT}" MATCHES "true") IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI) From a00dfb52e10760423ae2f0b87fd9f57feb2f3d25 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 15 Oct 2024 14:00:15 +0800 Subject: [PATCH 04/11] fix:[TS-5528] insert error in sml --- source/client/inc/clientSml.h | 1 + source/client/src/clientSml.c | 8 ++++++-- source/client/src/clientSmlLine.c | 4 ++++ tests/system-test/2-query/sml.py | 4 ++++ utils/test/c/sml_test.c | 32 +++++++++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 2 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 209c376f30..a6aca2fddf 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -199,6 +199,7 @@ typedef struct { SArray *preLineTagKV; SArray *maxTagKVs; SArray *maxColKVs; + SArray *escapedStringList; SSmlLineInfo preLine; STableMeta *currSTableMeta; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index d5cca55701..80f583bbee 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -479,6 +479,7 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) } clearColValArraySml(info->currTableDataCtx->pValues); + taosArrayClearP(info->escapedStringList, taosMemoryFree); if (unlikely(ret != TSDB_CODE_SUCCESS)) { smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); return ret; @@ -1608,6 +1609,7 @@ void smlDestroyInfo(SSmlHandle *info) { taosArrayDestroy(info->valueJsonArray); taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); + taosArrayDestroyP(info->escapedStringList, taosMemoryFree); if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { @@ -1667,8 +1669,9 @@ int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle) { info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); - - if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || info->preLineTagKV == NULL) { + info->escapedStringList = taosArrayInit(8, POINTER_BYTES); + if (info->tagJsonArray == NULL || info->valueJsonArray == NULL || + info->preLineTagKV == NULL || info->escapedStringList == NULL) { uError("SML:0x%" PRIx64 " failed to allocate memory", info->id); code = terrno; goto FAILED; @@ -1949,6 +1952,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { } } + taosArrayClearP(info->escapedStringList, taosMemoryFree); (void)memset(&info->preLine, 0, sizeof(SSmlLineInfo)); info->currSTableMeta = NULL; info->currTableDataCtx = NULL; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index e620ca9b0c..fabda47ae1 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -451,6 +451,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (info->dataFormat) { bool isAligned = isSmlColAligned(info, cnt, &kv); + if (kv.type == TSDB_DATA_TYPE_BINARY && valueEscaped) { + taosArrayPush(info->escapedStringList, &kv.value); + kv.value = NULL; + } freeSSmlKv(&kv); if(!isAligned){ return TSDB_CODE_SUCCESS; diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index e28f3b1edd..4a8760cd67 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -105,6 +105,10 @@ class TDTestCase: tdSql.query(f"select * from ts3724.`stb2.`") tdSql.checkRows(1) + tdSql.query(f"select * from ts5528.device_log_yuelan_cs1") + tdSql.checkRows(2) + tdSql.checkData(0, 1, '{"deviceId":"星宇公司-861701069493741","headers":{"_uid":"4e3599eacd62834995c77b38ad95f88d","creatorId":"1199596756811550720","deviceNmae":"861701069493741","productId":"yuelan","productName":"悦蓝cat1穿戴设备"},"messageType":"REPORT_PROPERTY","properties":{"lat":35.265527067449185,"lng":118.49713144245987,"location":"118.49713144245987,35.265527067449185"},"timestamp":1728719963230}') + tdSql.checkData(1, 1, '{"deviceId":"星宇公司-861701069065507","headers":{"_uid":"9045d6b78b4ffaf1e2d244e912ebbff8","creatorId":"1199596756811550720","deviceNmae":"861701069065507","productId":"yuelan","productName":"悦蓝cat1穿戴设备"},"messageType":"REPORT_PROPERTY","properties":{"lat":36.788241914043425,"lng":119.15042325460891,"location":"119.15042325460891,36.788241914043425"},"timestamp":1728719964105}') # tdSql.query(f"select * from td24559.stb order by _ts") # tdSql.checkRows(4) # tdSql.checkData(0, 2, "POINT (4.343000 89.342000)") diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index a3830d2518..4f4f09bd85 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -2098,12 +2098,44 @@ int sml_td29373_Test() { return code; } +int sml_ts5528_test(){ + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "drop database if exists ts5528"); + taos_free_result(pRes); + + pRes = taos_query(taos, "create database if not exists ts5528"); + taos_free_result(pRes); + + // check column name duplication + char *sql[] = { + "device_log_yuelan_cs1,deviceId=861701069493741 content=\"{\\\"deviceId\\\":\\\"星宇公司-861701069493741\\\",\\\"headers\\\":{\\\"_uid\\\":\\\"4e3599eacd62834995c77b38ad95f88d\\\",\\\"creatorId\\\":\\\"1199596756811550720\\\",\\\"deviceNmae\\\":\\\"861701069493741\\\",\\\"productId\\\":\\\"yuelan\\\",\\\"productName\\\":\\\"悦蓝cat1穿戴设备\\\"},\\\"messageType\\\":\\\"REPORT_PROPERTY\\\",\\\"properties\\\":{\\\"lat\\\":35.265527067449185,\\\"lng\\\":118.49713144245987,\\\"location\\\":\\\"118.49713144245987,35.265527067449185\\\"},\\\"timestamp\\\":1728719963230}\",createTime=1728719963230i64,id=\"4e3599eacd62834995c77b38ad95f88d\",messageId=\"\",timestamp=1728719963230i64,type=\"reportProperty\" 1728719963230", + "device_log_yuelan_cs1,deviceId=861701069065507 content=\"{\\\"deviceId\\\":\\\"星宇公司-861701069065507\\\",\\\"headers\\\":{\\\"_uid\\\":\\\"9045d6b78b4ffaf1e2d244e912ebbff8\\\",\\\"creatorId\\\":\\\"1199596756811550720\\\",\\\"deviceNmae\\\":\\\"861701069065507\\\",\\\"productId\\\":\\\"yuelan\\\",\\\"productName\\\":\\\"悦蓝cat1穿戴设备\\\"},\\\"messageType\\\":\\\"REPORT_PROPERTY\\\",\\\"properties\\\":{\\\"lat\\\":36.788241914043425,\\\"lng\\\":119.15042325460891,\\\"location\\\":\\\"119.15042325460891,36.788241914043425\\\"},\\\"timestamp\\\":1728719964105}\",createTime=1728719964105i64,id=\"9045d6b78b4ffaf1e2d244e912ebbff8\",messageId=\"\",timestamp=1728719964105i64,type=\"reportProperty\" 1728719964105", + }; + pRes = taos_query(taos, "use ts5528"); + taos_free_result(pRes); + + for( int i = 0; i < 2; i++){ + int32_t totalRows = 0; + pRes = taos_schemaless_insert_raw(taos, sql[i], strlen(sql[i]), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + taos_free_result(pRes); + int code = taos_errno(pRes); + if (code != 0) { + return code; + } + } + printf("%s result success\n", __FUNCTION__); + return 0; +} int main(int argc, char *argv[]) { if (argc == 2) { taos_options(TSDB_OPTION_CONFIGDIR, argv[1]); } int ret = 0; + ret = sml_ts5528_test(); + ASSERT(!ret); ret = sml_td29691_Test(); ASSERT(ret); ret = sml_td29373_Test(); From b40db3fab138bce2270818b3d3553617be206560 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Tue, 15 Oct 2024 14:13:24 +0800 Subject: [PATCH 05/11] Update 17-mongodb.md --- docs/zh/06-advanced/05-data-in/17-mongodb.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/05-data-in/17-mongodb.md b/docs/zh/06-advanced/05-data-in/17-mongodb.md index 2434e2e122..5311bc43c6 100644 --- a/docs/zh/06-advanced/05-data-in/17-mongodb.md +++ b/docs/zh/06-advanced/05-data-in/17-mongodb.md @@ -13,7 +13,7 @@ MongoDB 是一个介于关系型数据库与非关系型数据库之间的产品 ### 1. 新增数据源 -在数据写入页面中点击左上角的 **+新增数据源** 按钮进入新增数据源页面，如下图所示： +在数据写入页面中点击右上角的 **+新增数据源** 按钮进入新增数据源页面，如下图所示： ![Common-zh00-EnterDataSourcePage.png](./pic/Common-zh00-EnterDataSourcePage.png "进入新增数据源页面") From 986f92d31766482b6aab5784d94980e71760236f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 15 Oct 2024 14:30:36 +0800 Subject: [PATCH 06/11] fix:[TS-5528] insert error in sml --- utils/test/c/sml_test.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 4f4f09bd85..1d8d82ccb9 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -2119,12 +2119,14 @@ int sml_ts5528_test(){ int32_t totalRows = 0; pRes = taos_schemaless_insert_raw(taos, sql[i], strlen(sql[i]), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); - taos_free_result(pRes); int code = taos_errno(pRes); + taos_free_result(pRes); if (code != 0) { + taos_close(taos); return code; } } + taos_close(taos); printf("%s result success\n", __FUNCTION__); return 0; } From f048de5b44c27e1b22c5992dd66fa643913126b4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 15 Oct 2024 14:35:58 +0800 Subject: [PATCH 07/11] Revert "stmt2/interlace: separate interlace batch adding into exec2" This reverts commit bcfd74e697a3972acd1d001e482d9def2a01ee74. --- source/client/src/clientStmt2.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index b78e0d0f56..0837154fce 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1200,6 +1200,22 @@ static int stmtAddBatch2(TAOS_STMT2* stmt) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_ADD_BATCH)); + if (pStmt->sql.stbInterlaceMode) { + int64_t startUs2 = taosGetTimestampUs(); + pStmt->stat.addBatchUs += startUs2 - startUs; + + pStmt->sql.siInfo.tableColsReady = false; + + SStmtQNode* param = NULL; + STMT_ERR_RET(stmtAllocQNodeFromBuf(&pStmt->sql.siInfo.tbBuf, (void**)¶m)); + param->restoreTbCols = true; + param->next = NULL; + + stmtEnqueue(pStmt, param); + + return TSDB_CODE_SUCCESS; + } + STMT_ERR_RET(stmtCacheBlock(pStmt)); return TSDB_CODE_SUCCESS; @@ -1611,22 +1627,6 @@ int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); - if (pStmt->sql.stbInterlaceMode) { - int64_t startUs2 = taosGetTimestampUs(); - pStmt->stat.addBatchUs += startUs2 - startUs; - - pStmt->sql.siInfo.tableColsReady = false; - - SStmtQNode* param = NULL; - STMT_ERR_RET(stmtAllocQNodeFromBuf(&pStmt->sql.siInfo.tbBuf, (void**)¶m)); - param->restoreTbCols = true; - param->next = NULL; - - stmtEnqueue(pStmt, param); - - return TSDB_CODE_SUCCESS; - } - if (STMT_TYPE_QUERY != pStmt->sql.type) { if (pStmt->sql.stbInterlaceMode) { int64_t startTs = taosGetTimestampUs(); From 7348eddc092f9e7f98c7ff449fe0cc443947e534 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 15 Oct 2024 14:51:07 +0800 Subject: [PATCH 08/11] stmt2/add batch: move interlace add batch into exec2 --- source/client/src/clientStmt2.c | 8 +++++-- source/libs/parser/src/parUtil.c | 40 ++++++++++++++++++-------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index 0837154fce..2f046b61d6 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1419,10 +1419,10 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) { if (pStmt->sql.stbInterlaceMode) { STMT_ERR_RET(stmtAppendTablePostHandle(pStmt, param)); + } else { + STMT_ERR_RET(stmtAddBatch2(pStmt)); } - STMT_ERR_RET(stmtAddBatch2(pStmt)); - pStmt->stat.bindDataUs4 += taosGetTimestampUs() - startUs4; return TSDB_CODE_SUCCESS; @@ -1625,6 +1625,10 @@ int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) { return pStmt->errCode; } + if (pStmt->sql.stbInterlaceMode) { + STMT_ERR_RET(stmtAddBatch2(pStmt)); + } + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); if (STMT_TYPE_QUERY != pStmt->sql.type) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 98676160cb..9c2977d289 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -247,7 +247,10 @@ int32_t generateSyntaxErrMsgExt(SMsgBuf* pBuf, int32_t errCode, const char* pFor } int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) { - strncpy(pBuf->buf, msg, pBuf->len); + if (pBuf->buf) { + strncpy(pBuf->buf, msg, pBuf->len); + } + return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -259,7 +262,6 @@ int32_t buildInvalidOperationMsgExt(SMsgBuf* pBuf, const char* pFormat, ...) { return TSDB_CODE_TSC_INVALID_OPERATION; } - int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr) { if (pBuf == NULL) return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; const char* msgFormat1 = "syntax error near \'%s\'"; @@ -328,7 +330,7 @@ STableMeta* tableMetaDup(const STableMeta* pTableMeta) { STableMeta* p = taosMemoryMalloc(size + schemaExtSize); if (NULL == p) return NULL; - memcpy(p, pTableMeta, schemaExtSize+size); + memcpy(p, pTableMeta, schemaExtSize + size); if (hasSchemaExt) { p->schemaExt = (SSchemaExt*)(((char*)p) + size); } else { @@ -453,7 +455,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi // strcpy(val.colName, colName); val.pKey = jsonKey; retCode = taosHashPut(keyHash, jsonKey, keyLen, &keyLen, - CHAR_BYTES); // add key to hash to remove dumplicate, value is useless + CHAR_BYTES); // add key to hash to remove dumplicate, value is useless if (TSDB_CODE_SUCCESS != retCode) { goto end; } @@ -649,7 +651,7 @@ static int32_t buildTableReq(SHashObj* pTablesHash, SArray** pTables) { char* pKey = taosHashGetKey(p, &len); char fullName[TSDB_TABLE_FNAME_LEN] = {0}; strncpy(fullName, pKey, len); - SName name = {0}; + SName name = {0}; int32_t code = tNameFromString(&name, fullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); if (TSDB_CODE_SUCCESS == code) { if (NULL == taosArrayPush(*pTables, &name)) { @@ -816,9 +818,10 @@ int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalog return code; } -int32_t createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint, SNode** ppSelect) { +int32_t createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint, + SNode** ppSelect) { SSelectStmt* select = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_SELECT_STMT, (SNode**)&select); + int32_t code = nodesMakeNode(QUERY_NODE_SELECT_STMT, (SNode**)&select); if (NULL == select) { return code; } @@ -861,7 +864,7 @@ int32_t getMetaDataFromHash(const char* pKey, int32_t len, SHashObj* pHash, void static int32_t putTableDataToCache(const SArray* pTableReq, const SArray* pTableData, SHashObj** pTable) { int32_t ntables = taosArrayGetSize(pTableReq); for (int32_t i = 0; i < ntables; ++i) { - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(taosArrayGet(pTableReq, i), fullName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -892,7 +895,7 @@ static int32_t putDbTableDataToCache(const SArray* pDbReq, const SArray* pTableD STablesReq* pReq = taosArrayGet(pDbReq, i); int32_t ntables = taosArrayGetSize(pReq->pTables); for (int32_t j = 0; j < ntables; ++j) { - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(taosArrayGet(pReq->pTables, j), fullName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1088,7 +1091,7 @@ int32_t buildTableMetaFromViewMeta(STableMeta** pMeta, SViewMeta* pViewMeta) { } int32_t getViewMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) { - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pName, fullName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1139,7 +1142,7 @@ int32_t reserveTableVgroupInCacheExt(const SName* pName, SParseMetaCache* pMetaC } int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup) { - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pName, fullName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1299,7 +1302,7 @@ int32_t reserveTableCfgInCache(int32_t acctId, const char* pDb, const char* pTab return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableCfg); } -int32_t reserveTableTSMAInfoInCache(int32_t acctId, const char *pDb, const char *pTable, SParseMetaCache *pMetaCache) { +int32_t reserveTableTSMAInfoInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) { return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableTSMAs); } @@ -1308,9 +1311,10 @@ int32_t reserveTSMAInfoInCache(int32_t acctId, const char* pDb, const char* pTsm } int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes) { - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pName, fullName); - if (TSDB_CODE_SUCCESS != code) return code;; + if (TSDB_CODE_SUCCESS != code) return code; + ; SArray* pSmaIndexes = NULL; code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pTableIndex, (void**)&pSmaIndexes); if (TSDB_CODE_SUCCESS == code && NULL != pSmaIndexes) { @@ -1323,7 +1327,7 @@ int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, } int32_t getTableTsmasFromCache(SParseMetaCache* pMetaCache, const SName* pTbName, SArray** pTsmas) { - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pTbName, tbFName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1337,7 +1341,7 @@ int32_t getTableTsmasFromCache(SParseMetaCache* pMetaCache, const SName* pTbName } int32_t getTsmaFromCache(SParseMetaCache* pMetaCache, const SName* pTsmaName, STableTSMAInfo** pTsma) { - char tsmaFName[TSDB_TABLE_FNAME_LEN]; + char tsmaFName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pTsmaName, tsmaFName); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1349,7 +1353,7 @@ int32_t getTsmaFromCache(SParseMetaCache* pMetaCache, const SName* pTsmaName, ST return TSDB_CODE_PAR_INTERNAL_ERROR; } *pTsma = taosArrayGetP(pTsmaRsp->pTsmas, 0); - } else if (code == TSDB_CODE_PAR_INTERNAL_ERROR){ + } else if (code == TSDB_CODE_PAR_INTERNAL_ERROR) { code = TSDB_CODE_MND_SMA_NOT_EXIST; } return code; @@ -1410,7 +1414,7 @@ err: } int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput) { - char fullName[TSDB_TABLE_FNAME_LEN]; + char fullName[TSDB_TABLE_FNAME_LEN]; int32_t code = tNameExtractFullName(pName, fullName); if (TSDB_CODE_SUCCESS != code) { return code; From 490466747184737a44688eec38db16fd0b004020 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 15 Oct 2024 15:19:37 +0800 Subject: [PATCH 09/11] fix:[TS-5528] insert error in sml --- source/client/src/clientSml.c | 5 ++--- source/client/src/clientSmlLine.c | 7 ++++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 80f583bbee..46aaf29b6b 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -262,7 +262,7 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml return TSDB_CODE_SUCCESS; } -bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { +void isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { // cnt begin 0, add ts so + 2 if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { goto END; @@ -288,12 +288,11 @@ bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { maxKV->length = kv->length; info->needModifySchema = true; } - return true; + return; END: info->dataFormat = false; info->reRun = true; - return false; } bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index fabda47ae1..c1f3431698 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -452,7 +452,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (info->dataFormat) { bool isAligned = isSmlColAligned(info, cnt, &kv); if (kv.type == TSDB_DATA_TYPE_BINARY && valueEscaped) { - taosArrayPush(info->escapedStringList, &kv.value); + if (taosArrayPush(info->escapedStringList, &kv.value) == NULL){ + freeSSmlKv(&kv); + return terrno; + } kv.value = NULL; } freeSSmlKv(&kv); @@ -463,10 +466,12 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); if (currElement->colArray == NULL) { + freeSSmlKv(&kv); return terrno; } } if (taosArrayPush(currElement->colArray, &kv) == NULL){ // reserve for timestamp + freeSSmlKv(&kv); return terrno; } } From c9f835e60d742e5608b902f11e0e910c4ccd0dd2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 15 Oct 2024 15:25:28 +0800 Subject: [PATCH 10/11] fix:[TS-5528] insert error in sml --- source/client/src/clientSml.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 46aaf29b6b..80f583bbee 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -262,7 +262,7 @@ int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSml return TSDB_CODE_SUCCESS; } -void isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { +bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { // cnt begin 0, add ts so + 2 if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { goto END; @@ -288,11 +288,12 @@ void isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { maxKV->length = kv->length; info->needModifySchema = true; } - return; + return true; END: info->dataFormat = false; info->reRun = true; + return false; } bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { From 056fbe5eb3ad9e92000c8d4e8c903fd4721a1e3f Mon Sep 17 00:00:00 2001 From: Jinqing Kuang Date: Tue, 15 Oct 2024 15:30:04 +0800 Subject: [PATCH 11/11] fix(query)[TD-30667]. Fix compilation with AVX instructions on older GCC versions --- include/util/tcompression.h | 2 +- source/util/src/tdecompress.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index d32d20b727..1f09b750cb 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -156,9 +156,9 @@ int32_t getWordLength(char type); int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type); int32_t tsDecompressFloatImpAvx2(const char *input, int32_t nelements, char *output); int32_t tsDecompressDoubleImpAvx2(const char *input, int32_t nelements, char *output); -void tsDecompressTimestampAvx2(const char *input, int32_t nelements, char *output, bool bigEndian); #endif #ifdef __AVX512VL__ +void tsDecompressTimestampAvx2(const char *input, int32_t nelements, char *output, bool bigEndian); void tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, bool bigEndian); #endif diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c index 60a1f1c938..81223d7311 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompress.c @@ -449,7 +449,9 @@ int32_t tsDecompressDoubleImpAvx2(const char *input, const int32_t nelements, ch } return (int32_t)(out - output); } +#endif +#if __AVX512VL__ // decode two timestamps in one loop. void tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, bool bigEndian) { int64_t *ostream = (int64_t *)output; @@ -588,9 +590,7 @@ void tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, } return; } -#endif -#if __AVX512VL__ void tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, bool UNUSED_PARAM(bigEndian)) { int64_t *ostream = (int64_t *)output;