From f5c959da6f6c894ca0fee53150f90ce3a09e561a Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 17 Oct 2024 03:34:57 +0000 Subject: [PATCH 01/59] feat/TS-5484-audit-delete --- include/common/tglobal.h | 1 + include/common/tmsg.h | 11 +++++ include/common/tmsgdef.h | 1 + include/libs/audit/audit.h | 1 - include/util/tdef.h | 1 + source/common/src/tglobal.c | 5 ++ source/common/src/tmsg.c | 54 +++++++++++++++++++++ source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 1 + source/dnode/mnode/impl/src/mndDnode.c | 17 +++++++ 9 files changed, 91 insertions(+), 1 deletion(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index cf918c6e0d..3a7f307cfa 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -140,6 +140,7 @@ extern bool tsMonitorForceV2; // audit extern bool tsEnableAudit; extern bool tsEnableAuditCreateTable; +extern bool tsEnableAuditDelete; extern int32_t tsAuditInterval; // telem diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1a10f02c96..0008bc0e9d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1795,6 +1795,17 @@ int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); int32_t tDeserializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); void tFreeSStatisReq(SStatisReq* pReq); +typedef struct { + char db[TSDB_DB_FNAME_LEN]; + char table[TSDB_TABLE_NAME_LEN]; + char operation[AUDIT_OPERATION_LEN]; + int32_t sqlLen; + char* pSql; +} SAuditReq; +int32_t tSerializeSAuditReq(void* buf, int32_t bufLen, SAuditReq* pReq); +int32_t tDeserializeSAuditReq(void* buf, int32_t bufLen, SAuditReq* pReq); +void tFreeSAuditReq(SAuditReq* pReq); + typedef struct { int32_t dnodeId; int64_t clusterId; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 40464dc29a..4a2206bd2b 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -254,6 +254,7 @@ TD_DEF_MSG_TYPE(TDMT_MND_STREAM_DROP_ORPHANTASKS, "stream-drop-orphan-tasks", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_TASK_RESET, "stream-reset-tasks", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_UPDATE_DNODE_INFO, "update-dnode-info", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_AUDIT, "audit", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8 diff --git a/include/libs/audit/audit.h b/include/libs/audit/audit.h index 2e786ab2b3..f5710256e9 100644 --- a/include/libs/audit/audit.h +++ b/include/libs/audit/audit.h @@ -29,7 +29,6 @@ extern "C" { #endif #define AUDIT_DETAIL_MAX 65472 -#define AUDIT_OPERATION_LEN 20 typedef struct { const char *server; diff --git a/include/util/tdef.h b/include/util/tdef.h index a2bc77d819..970570683d 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -604,6 +604,7 @@ enum { RAND_ERR_MEMORY = 1, RAND_ERR_FILE = 2, RAND_ERR_NETWORK = 4 }; #define MONITOR_TAG_VALUE_LEN 300 #define MONITOR_METRIC_NAME_LEN 100 +#define AUDIT_OPERATION_LEN 20 #ifdef __cplusplus } #endif diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3c05294264..b85fea632e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -117,6 +117,7 @@ bool tsMonitorForceV2 = true; // audit bool tsEnableAudit = true; bool tsEnableAuditCreateTable = true; +bool tsEnableAuditDelete = true; int32_t tsAuditInterval = 5000; // telem @@ -763,6 +764,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableAuditDelete", tsEnableAuditDelete, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE)); @@ -1448,6 +1450,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "auditCreateTable"); tsEnableAuditCreateTable = pItem->bval; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "enableAuditDelete"); + tsEnableAuditDelete = pItem->bval; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "auditInterval"); tsAuditInterval = pItem->i32; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 4c4b78278e..10c84d9bf8 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1802,6 +1802,60 @@ _exit: void tFreeSDropUserReq(SDropUserReq *pReq) { FREESQL(); } +int32_t tSerializeSAuditReq(void *buf, int32_t bufLen, SAuditReq *pReq) { + SEncoder encoder = {0}; + int32_t code = 0; + int32_t lino; + int32_t tlen; + tEncoderInit(&encoder, buf, bufLen); + + TAOS_CHECK_EXIT(tStartEncode(&encoder)); + + TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->operation)); + TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->db)); + TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->table)); + TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->sqlLen)); + TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->pSql)); + + tEndEncode(&encoder); + +_exit: + if (code) { + tlen = code; + } else { + tlen = encoder.pos; + } + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSAuditReq(void *buf, int32_t bufLen, SAuditReq *pReq) { + SDecoder decoder = {0}; + int32_t code = 0; + int32_t lino; + tDecoderInit(&decoder, buf, bufLen); + + TAOS_CHECK_EXIT(tStartDecode(&decoder)); + + TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->operation)); + TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->db)); + TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->table)); + TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->sqlLen)); + if (pReq->sqlLen > 0) { + pReq->pSql = taosMemoryMalloc(pReq->sqlLen + 1); + if (pReq->pSql == NULL) { + TAOS_CHECK_EXIT(terrno); + } + TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->pSql)); + } + tEndDecode(&decoder); +_exit: + tDecoderClear(&decoder); + return code; +} + +void tFreeSAuditReq(SAuditReq *pReq) { taosMemoryFreeClear(pReq->pSql); } + SIpWhiteList *cloneIpWhiteList(SIpWhiteList *pIpWhiteList) { if (pIpWhiteList == NULL) return NULL; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 7204cde8f7..a182307c49 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -208,6 +208,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_VIEW_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STATIS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_AUDIT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_COMPACT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_CLUSTER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 04041646eb..8eedcaa906 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -86,6 +86,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq); static int32_t mndProcessNotifyReq(SRpcMsg *pReq); static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq); static int32_t mndProcessStatisReq(SRpcMsg *pReq); +static int32_t mndProcessAuditReq(SRpcMsg *pReq); static int32_t mndProcessUpdateDnodeInfoReq(SRpcMsg *pReq); static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pRsp); static int32_t mndProcessCreateEncryptKeyRsp(SRpcMsg *pRsp); @@ -125,6 +126,7 @@ int32_t mndInitDnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq); mndSetMsgHandle(pMnode, TDMT_MND_RESTORE_DNODE, mndProcessRestoreDnodeReq); mndSetMsgHandle(pMnode, TDMT_MND_STATIS, mndProcessStatisReq); + mndSetMsgHandle(pMnode, TDMT_MND_AUDIT, mndProcessAuditReq); mndSetMsgHandle(pMnode, TDMT_MND_CREATE_ENCRYPT_KEY, mndProcessCreateEncryptKeyReq); mndSetMsgHandle(pMnode, TDMT_DND_CREATE_ENCRYPT_KEY_RSP, mndProcessCreateEncryptKeyRsp); mndSetMsgHandle(pMnode, TDMT_MND_UPDATE_DNODE_INFO, mndProcessUpdateDnodeInfoReq); @@ -604,6 +606,21 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { return 0; } +static int32_t mndProcessAuditReq(SRpcMsg *pReq) { + if (tsEnableAudit && tsEnableAuditDelete) { + SMnode *pMnode = pReq->info.node; + SAuditReq auditReq = {0}; + + TAOS_CHECK_RETURN(tDeserializeSAuditReq(pReq->pCont, pReq->contLen, &auditReq)); + + auditAddRecord(pReq, pMnode->clusterId, auditReq.operation, auditReq.db, auditReq.table, auditReq.pSql, + auditReq.sqlLen); + + tFreeSAuditReq(&auditReq); + } + return 0; +} + static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) { int32_t code = 0, lino = 0; SDnodeInfoReq infoReq = {0}; From 41acc1ea5ab7f68250f0b1f960b2dc4ad4d53e79 Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 18 Oct 2024 03:43:39 +0000 Subject: [PATCH 02/59] feat/TS-5484-audit-delete-client-pull-config --- include/common/tmsg.h | 2 ++ source/dnode/mnode/impl/src/mndProfile.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0008bc0e9d..74907992e3 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1009,6 +1009,7 @@ typedef struct { char sDetailVer[128]; int64_t whiteListVer; SMonitorParas monitorParas; + int8_t enableAuditDelete; } SConnectRsp; int32_t tSerializeSConnectRsp(void* buf, int32_t bufLen, SConnectRsp* pRsp); @@ -3369,6 +3370,7 @@ typedef struct { int32_t svrTimestamp; SArray* rsps; // SArray SMonitorParas monitorParas; + int8_t enableAuditDelete; } SClientHbBatchRsp; static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) { return taosIntHash_64(key, keyLen); } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index a1ffee9b06..d0047cace1 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -305,6 +305,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { connectRsp.monitorParas.tsSlowLogMaxLen = tsSlowLogMaxLen; connectRsp.monitorParas.tsSlowLogThreshold = tsSlowLogThreshold; connectRsp.monitorParas.tsSlowLogThresholdTest = tsSlowLogThresholdTest; + connectRsp.enableAuditDelete = tsEnableAuditDelete; tstrncpy(connectRsp.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN); connectRsp.whiteListVer = pUser->ipWhiteListVer; @@ -709,6 +710,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { tstrncpy(batchRsp.monitorParas.tsSlowLogExceptDb, tsSlowLogExceptDb, TSDB_DB_NAME_LEN); batchRsp.monitorParas.tsSlowLogMaxLen = tsSlowLogMaxLen; batchRsp.monitorParas.tsSlowLogScope = tsSlowLogScope; + batchRsp.enableAuditDelete = tsEnableAuditDelete; int32_t sz = taosArrayGetSize(batchReq.reqs); for (int i = 0; i < sz; i++) { From 0a9d1282b28cf9fbff94e621527ba0b1d53110e5 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Sat, 12 Oct 2024 18:02:30 +0800 Subject: [PATCH 03/59] setResultDataPtr --- source/client/inc/clientInt.h | 3 +-- source/client/src/clientImpl.c | 27 +++++++++++++-------------- source/client/src/clientTmq.c | 3 +-- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 8d45e8b4a8..2ce466dee1 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -295,8 +295,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); void doSetOneRowPtr(SReqResultInfo* pResultInfo); void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4); -int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows, - bool convertUcs4); +int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4); int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); void doFreeReqResultInfo(SReqResultInfo* pResInfo); int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 9131d29f30..08dbe7af6f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2341,14 +2341,13 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int return TSDB_CODE_SUCCESS; } -int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows, - bool convertUcs4) { - if (numOfCols <= 0 || pFields == NULL || pResultInfo == NULL) { +int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { + if (pResultInfo->numOfCols <= 0 || pResultInfo->fields == NULL || pResultInfo == NULL) { tscError("setResultDataPtr paras error"); return TSDB_CODE_TSC_INTERNAL_ERROR; } - if (numOfRows == 0) { + if (pResultInfo->numOfRows == 0) { return TSDB_CODE_SUCCESS; } @@ -2356,7 +2355,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 if (code != TSDB_CODE_SUCCESS) { return code; } - code = doConvertJson(pResultInfo, numOfCols, numOfRows); + code = doConvertJson(pResultInfo, pResultInfo->numOfCols, pResultInfo->numOfRows); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2376,9 +2375,9 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 int32_t cols = *(int32_t*)p; p += sizeof(int32_t); - if (rows != numOfRows || cols != numOfCols) { - tscError("setResultDataPtr paras error:rows;%d numOfRows:%d cols:%d numOfCols:%d", rows, numOfRows, cols, - numOfCols); + if (rows != pResultInfo->numOfRows || cols != pResultInfo->numOfCols) { + tscError("setResultDataPtr paras error:rows;%d numOfRows:%" PRId64 " cols:%d numOfCols:%d", rows, pResultInfo->numOfRows, cols, + pResultInfo->numOfCols); return TSDB_CODE_TSC_INTERNAL_ERROR; } @@ -2389,7 +2388,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 p += sizeof(uint64_t); // check fields - for (int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { int8_t type = *(int8_t*)p; p += sizeof(int8_t); @@ -2398,10 +2397,10 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 } int32_t* colLength = (int32_t*)p; - p += sizeof(int32_t) * numOfCols; + p += sizeof(int32_t) * pResultInfo->numOfCols; char* pStart = p; - for (int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { if (blockVersion == BLOCK_VERSION_1) { colLength[i] = htonl(colLength[i]); } @@ -2412,7 +2411,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { pResultInfo->pCol[i].offset = (int32_t*)pStart; - pStart += numOfRows * sizeof(int32_t); + pStart += pResultInfo->numOfRows * sizeof(int32_t); } else { pResultInfo->pCol[i].nullbitmap = pStart; pStart += BitmapLen(pResultInfo->numOfRows); @@ -2429,7 +2428,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 p += sizeof(bool); if (convertUcs4) { - code = doConvertUCS4(pResultInfo, numOfRows, numOfCols, colLength); + code = doConvertUCS4(pResultInfo, pResultInfo->numOfRows, pResultInfo->numOfCols, colLength); } return code; @@ -2542,7 +2541,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR pResultInfo->totalRows += pResultInfo->numOfRows; int32_t code = - setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows, convertUcs4); + setResultDataPtr(pResultInfo, convertUcs4); return code; } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index fd6ca831d1..d8a67d62e2 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -2868,8 +2868,7 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes pRspObj->resInfo.precision = precision; pRspObj->resInfo.totalRows += pRspObj->resInfo.numOfRows; - int32_t code = setResultDataPtr(&pRspObj->resInfo, pRspObj->resInfo.fields, pRspObj->resInfo.numOfCols, - pRspObj->resInfo.numOfRows, convertUcs4); + int32_t code = setResultDataPtr(&pRspObj->resInfo, convertUcs4); if (code != 0) { return code; } From 161eb54b56d4b37c51086c93149f6d0d7d269b04 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 17 Oct 2024 15:47:26 +0800 Subject: [PATCH 04/59] enh: setResultDataPtr safe check --- source/client/src/clientImpl.c | 42 ++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 08dbe7af6f..713c07e60f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2079,12 +2079,12 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { return TSDB_CODE_SUCCESS; } -static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) { +static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) { int32_t idx = -1; iconv_t conv = taosAcquireConv(&idx, C2M); if (conv == (iconv_t)-1) return TSDB_CODE_TSC_INTERNAL_ERROR; - for (int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { int32_t type = pResultInfo->fields[i].type; int32_t bytes = pResultInfo->fields[i].bytes; @@ -2098,7 +2098,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int pResultInfo->convertBuf[i] = p; SResultColumn* pCol = &pResultInfo->pCol[i]; - for (int32_t j = 0; j < numOfRows; ++j) { + for (int32_t j = 0; j < pResultInfo->numOfRows; ++j) { if (pCol->offset[j] != -1) { char* pStart = pCol->offset[j] + pCol->pData; @@ -2131,10 +2131,13 @@ int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { numOfCols * (sizeof(int8_t) + sizeof(int32_t)); } -static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { +static int32_t estimateJsonLen(SReqResultInfo* pResultInfo) { char* p = (char*)pResultInfo->pData; int32_t blockVersion = *(int32_t*)p; + int32_t numOfRows = pResultInfo->numOfRows; + int32_t numOfCols = pResultInfo->numOfCols; + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column // length | int32_t cols = *(int32_t*)(p + sizeof(int32_t) * 3); @@ -2196,7 +2199,9 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i return len; } -static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { +static int32_t doConvertJson(SReqResultInfo* pResultInfo) { + int32_t numOfRows = pResultInfo->numOfRows; + int32_t numOfCols = pResultInfo->numOfCols; bool needConvert = false; for (int32_t i = 0; i < numOfCols; ++i) { if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) { @@ -2213,7 +2218,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int char* p = (char*)pResultInfo->pData; int32_t blockVersion = *(int32_t*)p; - int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows); + int32_t dataLen = estimateJsonLen(pResultInfo); if (dataLen <= 0) { return TSDB_CODE_TSC_INTERNAL_ERROR; } @@ -2342,7 +2347,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int } int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { - if (pResultInfo->numOfCols <= 0 || pResultInfo->fields == NULL || pResultInfo == NULL) { + if (pResultInfo == NULL || pResultInfo->numOfCols <= 0 || pResultInfo->fields == NULL) { tscError("setResultDataPtr paras error"); return TSDB_CODE_TSC_INTERNAL_ERROR; } @@ -2351,11 +2356,16 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { return TSDB_CODE_SUCCESS; } + if (pResultInfo->pData == NULL) { + tscError("estimateJsonLen error: pData is NULL"); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } + int32_t code = doPrepareResPtr(pResultInfo); if (code != TSDB_CODE_SUCCESS) { return code; } - code = doConvertJson(pResultInfo, pResultInfo->numOfCols, pResultInfo->numOfRows); + code = doConvertJson(pResultInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2401,6 +2411,10 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { char* pStart = p; for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { + if ((pStart - pResultInfo->pData) >= dataLen) { + tscError("setResultDataPtr invalid offset over dataLen %d", dataLen); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } if (blockVersion == BLOCK_VERSION_1) { colLength[i] = htonl(colLength[i]); } @@ -2424,11 +2438,17 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { pStart += colLength[i]; } + p = pStart; // bool blankFill = *(bool*)p; p += sizeof(bool); + int32_t offset = p - pResultInfo->pData; + if (offset > dataLen) { + tscError("invalid offset %d, dataLen %d", offset, dataLen); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } if (convertUcs4) { - code = doConvertUCS4(pResultInfo, pResultInfo->numOfRows, pResultInfo->numOfCols, colLength); + code = doConvertUCS4(pResultInfo, colLength); } return code; @@ -2535,6 +2555,10 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR return TSDB_CODE_TSC_INTERNAL_ERROR; } } + } else { + pResultInfo->pData = NULL; + pResultInfo->payloadLen = 0; + return TSDB_CODE_TSC_INTERNAL_ERROR; } // TODO handle the compressed case From bee96ce5e253ce0046bcb8a82cd8997e1c6b152e Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 17 Oct 2024 16:40:28 +0800 Subject: [PATCH 05/59] fix: payloadLen may be zero --- source/client/src/clientImpl.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 713c07e60f..7c14359f64 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2555,10 +2555,6 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR return TSDB_CODE_TSC_INTERNAL_ERROR; } } - } else { - pResultInfo->pData = NULL; - pResultInfo->payloadLen = 0; - return TSDB_CODE_TSC_INTERNAL_ERROR; } // TODO handle the compressed case From 04a9d67965b4bc9a7a7968b68244e1e67fe09271 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 17 Oct 2024 17:39:11 +0800 Subject: [PATCH 06/59] enh: type check --- include/common/ttypes.h | 1 + source/client/src/clientImpl.c | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 3934553b1c..0f0f30a1c3 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -284,6 +284,7 @@ typedef struct { #define IS_VALID_UINT64(_t) ((_t) >= 0 && (_t) <= UINT64_MAX) #define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) #define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) +#define IS_INVALID_TYPE(_t) ((_t) < TSDB_DATA_TYPE_NULL || (_t) >= TSDB_DATA_TYPE_MAX) #define IS_CONVERT_AS_SIGNED(_t) \ (IS_SIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7c14359f64..4f268aa96e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2346,6 +2346,16 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo) { return TSDB_CODE_SUCCESS; } +int32_t checkResultInfo(SReqResultInfo* pResultInfo) { + if (pResultInfo->totalRows < pResultInfo->numOfRows) { + tscError("checkResultInfo error: totalRows:%" PRId64 " < numOfRows:%" PRId64, pResultInfo->totalRows, + pResultInfo->numOfRows); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { if (pResultInfo == NULL || pResultInfo->numOfCols <= 0 || pResultInfo->fields == NULL) { tscError("setResultDataPtr paras error"); @@ -2422,7 +2432,10 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { tscError("invalid colLength %d, dataLen %d", colLength[i], dataLen); return TSDB_CODE_TSC_INTERNAL_ERROR; } - + if (IS_INVALID_TYPE(pResultInfo->fields[i].type)) { + tscError("invalid type %d", pResultInfo->fields[i].type); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { pResultInfo->pCol[i].offset = (int32_t*)pStart; pStart += pResultInfo->numOfRows * sizeof(int32_t); @@ -2450,6 +2463,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { if (convertUcs4) { code = doConvertUCS4(pResultInfo, colLength); } + code = checkResultInfo(pResultInfo); return code; } From d8218b3872531b718c95a1191310f597e2dda92f Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 17 Oct 2024 19:05:23 +0800 Subject: [PATCH 07/59] fix: resultDataLen with json col --- source/client/src/clientImpl.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 4f268aa96e..d8c01ee17c 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2196,6 +2196,10 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo) { } pStart += colLen; } + + // Ensure the complete structure of the block, including the blankfill field, + // even though it is not used on the client side. + len += sizeof(bool); return len; } @@ -2341,6 +2345,11 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo) { pStart1 += colLen1; } + // Ensure the complete structure of the block, including the blankfill field, + // even though it is not used on the client side. + // (void)memcpy(pStart1, pStart, sizeof(bool)); + totalLen += sizeof(bool); + *(int32_t*)(pResultInfo->convertJson + 4) = totalLen; pResultInfo->pData = pResultInfo->convertJson; return TSDB_CODE_SUCCESS; @@ -2367,7 +2376,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { } if (pResultInfo->pData == NULL) { - tscError("estimateJsonLen error: pData is NULL"); + tscError("setResultDataPtr error: pData is NULL"); return TSDB_CODE_TSC_INTERNAL_ERROR; } From d01a3481bca8a3e8bbcfc6da5ec08570a719314e Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 18 Oct 2024 16:16:57 +0800 Subject: [PATCH 08/59] block buf len check --- include/common/tdatablock.h | 2 +- source/client/src/clientImpl.c | 14 ++++++++++++-- source/client/src/clientMsgHandler.c | 10 ++++++---- source/common/src/tdatablock.c | 13 ++++++++++++- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 6 +++--- source/dnode/mnode/impl/src/mndShow.c | 6 +++--- source/dnode/vnode/src/tq/tqScan.c | 5 +++-- source/libs/command/src/command.c | 5 +++-- source/libs/command/src/explain.c | 5 +++-- source/libs/executor/src/dataDispatcher.c | 13 +++++++------ source/libs/stream/src/streamDispatch.c | 10 ++++++---- 11 files changed, 59 insertions(+), 30 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 99cdb53103..c6f0b4d517 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -266,7 +266,7 @@ SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId) int32_t bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index, SColumnInfoData** pColInfoData); int32_t blockGetEncodeSize(const SSDataBlock* pBlock); -int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols); +int32_t blockEncode(const SSDataBlock* pBlock, char* data, size_t dataLen, int32_t numOfCols); int32_t blockDecode(SSDataBlock* pBlock, const char* pData, const char** pEndPos); // for debug diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d8c01ee17c..69b6fe312c 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2355,12 +2355,22 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo) { return TSDB_CODE_SUCCESS; } -int32_t checkResultInfo(SReqResultInfo* pResultInfo) { +int32_t resultInfoSafeCheck(SReqResultInfo* pResultInfo) { if (pResultInfo->totalRows < pResultInfo->numOfRows) { tscError("checkResultInfo error: totalRows:%" PRId64 " < numOfRows:%" PRId64, pResultInfo->totalRows, pResultInfo->numOfRows); return TSDB_CODE_TSC_INTERNAL_ERROR; } + for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { + if (pResultInfo->fields[i].bytes <= 0) { + tscError("checkResultInfo error: bytes:%d <= 0", pResultInfo->fields[i].bytes); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } + if(!IS_VAR_DATA_TYPE(pResultInfo->fields[i].type) && TYPE_BYTES[pResultInfo->fields[i].type] != pResultInfo->fields[i].bytes) { + tscError("checkResultInfo error: type:%d bytes:%d != %d", pResultInfo->fields[i].type, pResultInfo->fields[i].bytes, TYPE_BYTES[pResultInfo->fields[i].type]); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } + } return TSDB_CODE_SUCCESS; } @@ -2472,7 +2482,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { if (convertUcs4) { code = doConvertUCS4(pResultInfo, colLength); } - code = checkResultInfo(pResultInfo); + code = resultInfoSafeCheck(pResultInfo); return code; } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index aef3cef1c5..f8a0f384de 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -588,7 +588,8 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { return code; } - size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; + size_t dataEncodeSize = blockGetEncodeSize(pBlock); + size_t rspSize = sizeof(SRetrieveTableRsp) + dataEncodeSize + PAYLOAD_PREFIX_LEN; *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { code = terrno; @@ -603,7 +604,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); - int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, SHOW_VARIABLES_RESULT_COLS); + int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeSize, SHOW_VARIABLES_RESULT_COLS); if(len < 0) { uError("buildShowVariablesRsp error, len:%d", len); code = terrno; @@ -741,7 +742,8 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr return code; } - size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; + size_t dataEncodeSize = blockGetEncodeSize(pBlock); + size_t rspSize = sizeof(SRetrieveTableRsp) + dataEncodeSize + PAYLOAD_PREFIX_LEN; *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { code = terrno; @@ -757,7 +759,7 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS); - int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, COMPACT_DB_RESULT_COLS); + int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeSize, COMPACT_DB_RESULT_COLS); if(len < 0) { uError("buildRetriveTableRspForCompactDb error, len:%d", len); code = terrno; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 60d1f34cf7..2da83a6b74 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -3041,7 +3041,7 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha } // return length of encoded data, return -1 if failed -int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { +int32_t blockEncode(const SSDataBlock* pBlock, char* data, size_t dataBuflen, int32_t numOfCols) { blockDataCheck(pBlock, false); int32_t dataLen = 0; @@ -3106,9 +3106,11 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { size_t metaSize = 0; if (IS_VAR_DATA_TYPE(pColRes->info.type)) { metaSize = numOfRows * sizeof(int32_t); + if(dataLen + metaSize > dataBuflen) goto _exit; memcpy(data, pColRes->varmeta.offset, metaSize); } else { metaSize = BitmapLen(numOfRows); + if(dataLen + metaSize > dataBuflen) goto _exit; memcpy(data, pColRes->nullbitmap, metaSize); } @@ -3127,12 +3129,14 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { } colSizes[col] += colSize; dataLen += colSize; + if(dataLen > dataBuflen) goto _exit; (void) memmove(data, pColData, colSize); data += colSize; } } else { colSizes[col] = colDataGetLength(pColRes, numOfRows); dataLen += colSizes[col]; + if(dataLen > dataBuflen) goto _exit; if (pColRes->pData != NULL) { (void) memmove(data, pColRes->pData, colSizes[col]); } @@ -3156,7 +3160,14 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { *actualLen = dataLen; *groupId = pBlock->info.id.groupId; + if (dataLen > dataBuflen) goto _exit; + return dataLen; + +_exit: + uError("blockEncode dataLen:%d, dataBuflen:%" PRIx64, dataLen, dataBuflen); + terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + return -1; } int32_t blockDecode(SSDataBlock* pBlock, const char* pData, const char** pEndPos) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 1446faab77..0f3f42acb9 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -548,8 +548,8 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols + - blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(numOfCols); + size_t dataEncodeSize = blockGetEncodeSize(pBlock); + size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols + dataEncodeSize; SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { @@ -574,7 +574,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { pStart += sizeof(SSysTableSchema); } - int32_t len = blockEncode(pBlock, pStart, numOfCols); + int32_t len = blockEncode(pBlock, pStart, dataEncodeSize, numOfCols); if (len < 0) { dError("failed to retrieve data since %s", tstrerror(code)); blockDataDestroy(pBlock); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 264fea3476..01dc329d04 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -333,8 +333,8 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { mDebug("show:0x%" PRIx64 ", stop retrieve data, rowsRead:%d numOfRows:%d", pShow->id, rowsRead, pShow->numOfRows); } - size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns + - blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)); + size_t dataEncodeSize = blockGetEncodeSize(pBlock); + size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns + dataEncodeSize; SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { @@ -361,7 +361,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { pStart += sizeof(SSysTableSchema); } - int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); + int32_t len = blockEncode(pBlock, pStart, dataEncodeSize, pShow->pMeta->numOfColumns); if(len < 0){ mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, tstrerror(code)); code = terrno; diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index dbc1b16cf5..5632297a2b 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -16,7 +16,8 @@ #include "tq.h" int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) { - int32_t dataStrLen = sizeof(SRetrieveTableRspForTmq) + blockGetEncodeSize(pBlock); + size_t dataEncodeSize = blockGetEncodeSize(pBlock); + int32_t dataStrLen = sizeof(SRetrieveTableRspForTmq) + dataEncodeSize; void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) { return terrno; @@ -28,7 +29,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t pRetrieve->compressed = 0; pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, dataEncodeSize, numOfCols); if(actualLen < 0){ taosMemoryFree(buf); return terrno; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index b2417a8597..db3e553c7a 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -35,7 +35,8 @@ extern SConfig* tsCfg; static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRetrieveTableRsp** pRsp) { - size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; + size_t dataEncodeSize = blockGetEncodeSize(pBlock); + size_t rspSize = sizeof(SRetrieveTableRsp) + dataEncodeSize + PAYLOAD_PREFIX_LEN; *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { return terrno; @@ -49,7 +50,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); - int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, numOfCols); + int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeSize, numOfCols); if(len < 0) { taosMemoryFree(*pRsp); return terrno; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index b82bba250f..4f4f67caf7 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1966,7 +1966,8 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { pBlock->info.rows = rowNum; - int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; + size_t dataEncodeSize = blockGetEncodeSize(pBlock); + int32_t rspSize = sizeof(SRetrieveTableRsp) + dataEncodeSize + PAYLOAD_PREFIX_LEN; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); if (NULL == rsp) { @@ -1977,7 +1978,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htobe64((int64_t)rowNum); - int32_t len = blockEncode(pBlock, rsp->data + PAYLOAD_PREFIX_LEN, taosArrayGetSize(pBlock->pDataBlock)); + int32_t len = blockEncode(pBlock, rsp->data + PAYLOAD_PREFIX_LEN, dataEncodeSize, taosArrayGetSize(pBlock->pDataBlock)); if(len < 0) { qError("qExplainGetRspFromCtx: blockEncode failed"); QRY_ERR_JRET(terrno); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 8acd569358..8ab1558172 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -84,17 +84,18 @@ static int32_t toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pBuf->useSize = sizeof(SDataCacheEntry); { + size_t dataEncodeSize = pBuf->allocSize + 8; if ((pBuf->allocSize > tsCompressMsgSize) && (tsCompressMsgSize > 0) && pHandle->pManager->cfg.compress) { if (pHandle->pCompressBuf == NULL) { // allocate additional 8 bytes to avoid invalid write if compress failed to reduce the size - pHandle->pCompressBuf = taosMemoryMalloc(pBuf->allocSize + 8); + pHandle->pCompressBuf = taosMemoryMalloc(dataEncodeSize); if (NULL == pHandle->pCompressBuf) { QRY_RET(terrno); } - pHandle->bufSize = pBuf->allocSize + 8; + pHandle->bufSize = dataEncodeSize; } else { - if (pHandle->bufSize < pBuf->allocSize + 8) { - pHandle->bufSize = pBuf->allocSize + 8; + if (pHandle->bufSize < dataEncodeSize) { + pHandle->bufSize = dataEncodeSize; void* p = taosMemoryRealloc(pHandle->pCompressBuf, pHandle->bufSize); if (p != NULL) { pHandle->pCompressBuf = p; @@ -105,7 +106,7 @@ static int32_t toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* } } - int32_t dataLen = blockEncode(pInput->pData, pHandle->pCompressBuf, numOfCols); + int32_t dataLen = blockEncode(pInput->pData, pHandle->pCompressBuf, dataEncodeSize, numOfCols); if(dataLen < 0) { qError("failed to encode data block, code: %d", dataLen); return terrno; @@ -123,7 +124,7 @@ static int32_t toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* TAOS_MEMCPY(pEntry->data, pHandle->pCompressBuf, dataLen); } } else { - pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); + pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, dataEncodeSize, numOfCols); if(pEntry->dataLen < 0) { qError("failed to encode data block, code: %d", pEntry->dataLen); return terrno; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 62d60ff664..2b91f0f806 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -145,7 +145,8 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBlock, SStreamRetrieveReq* req) { SRetrieveTableRsp* pRetrieve = NULL; - int32_t len = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; + size_t dataEncodeSize = blockGetEncodeSize(pBlock); + int32_t len = sizeof(SRetrieveTableRsp) + dataEncodeSize + PAYLOAD_PREFIX_LEN; pRetrieve = taosMemoryCalloc(1, len); if (pRetrieve == NULL) return terrno; @@ -162,7 +163,7 @@ static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBl pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN, numOfCols); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN, dataEncodeSize, numOfCols); if (actualLen < 0) { taosMemoryFree(pRetrieve); return terrno; @@ -1203,7 +1204,8 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { } int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) { - int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; + size_t dataEncodeSize = blockGetEncodeSize(pBlock); + int32_t dataStrLen = sizeof(SRetrieveTableRsp) + dataEncodeSize + PAYLOAD_PREFIX_LEN; void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) { return terrno; @@ -1225,7 +1227,7 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); pRetrieve->numOfCols = htonl(numOfCols); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN, numOfCols); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN, dataEncodeSize, numOfCols); if (actualLen < 0) { taosMemoryFree(buf); return terrno; From f35d2847b91388ace4baa16d3aacfd44167f6115 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Sun, 20 Oct 2024 22:17:16 +0800 Subject: [PATCH 09/59] enh: blockencode --- source/client/src/clientMsgHandler.c | 12 ++++++------ source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 6 +++--- source/dnode/mnode/impl/src/mndShow.c | 6 +++--- source/dnode/vnode/src/tq/tqScan.c | 6 +++--- source/libs/command/src/command.c | 6 +++--- source/libs/command/src/explain.c | 6 +++--- source/libs/executor/src/dataDispatcher.c | 14 +++++++------- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index f8a0f384de..099a224f85 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -588,8 +588,8 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { return code; } - size_t dataEncodeSize = blockGetEncodeSize(pBlock); - size_t rspSize = sizeof(SRetrieveTableRsp) + dataEncodeSize + PAYLOAD_PREFIX_LEN; + size_t dataEncodeBufSize = blockGetEncodeSize(pBlock); + size_t rspSize = sizeof(SRetrieveTableRsp) + dataEncodeBufSize + PAYLOAD_PREFIX_LEN; *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { code = terrno; @@ -604,7 +604,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); - int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeSize, SHOW_VARIABLES_RESULT_COLS); + int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeBufSize, SHOW_VARIABLES_RESULT_COLS); if(len < 0) { uError("buildShowVariablesRsp error, len:%d", len); code = terrno; @@ -742,8 +742,8 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr return code; } - size_t dataEncodeSize = blockGetEncodeSize(pBlock); - size_t rspSize = sizeof(SRetrieveTableRsp) + dataEncodeSize + PAYLOAD_PREFIX_LEN; + size_t dataEncodeBufSize = blockGetEncodeSize(pBlock); + size_t rspSize = sizeof(SRetrieveTableRsp) + dataEncodeBufSize + PAYLOAD_PREFIX_LEN; *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { code = terrno; @@ -759,7 +759,7 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS); - int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeSize, COMPACT_DB_RESULT_COLS); + int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeBufSize, COMPACT_DB_RESULT_COLS); if(len < 0) { uError("buildRetriveTableRspForCompactDb error, len:%d", len); code = terrno; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 0f3f42acb9..d6b792ca74 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -548,8 +548,8 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - size_t dataEncodeSize = blockGetEncodeSize(pBlock); - size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols + dataEncodeSize; + size_t dataEncodeBufSize = blockGetEncodeSize(pBlock); + size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols + dataEncodeBufSize; SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { @@ -574,7 +574,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { pStart += sizeof(SSysTableSchema); } - int32_t len = blockEncode(pBlock, pStart, dataEncodeSize, numOfCols); + int32_t len = blockEncode(pBlock, pStart, dataEncodeBufSize, numOfCols); if (len < 0) { dError("failed to retrieve data since %s", tstrerror(code)); blockDataDestroy(pBlock); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 01dc329d04..29f6c32dbe 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -333,8 +333,8 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { mDebug("show:0x%" PRIx64 ", stop retrieve data, rowsRead:%d numOfRows:%d", pShow->id, rowsRead, pShow->numOfRows); } - size_t dataEncodeSize = blockGetEncodeSize(pBlock); - size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns + dataEncodeSize; + size_t dataEncodeBufSize = blockGetEncodeSize(pBlock); + size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns + dataEncodeBufSize; SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { @@ -361,7 +361,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { pStart += sizeof(SSysTableSchema); } - int32_t len = blockEncode(pBlock, pStart, dataEncodeSize, pShow->pMeta->numOfColumns); + int32_t len = blockEncode(pBlock, pStart, dataEncodeBufSize, pShow->pMeta->numOfColumns); if(len < 0){ mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, tstrerror(code)); code = terrno; diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 5632297a2b..14cf41f1f9 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -16,8 +16,8 @@ #include "tq.h" int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) { - size_t dataEncodeSize = blockGetEncodeSize(pBlock); - int32_t dataStrLen = sizeof(SRetrieveTableRspForTmq) + dataEncodeSize; + size_t dataEncodeBufSize = blockGetEncodeSize(pBlock); + int32_t dataStrLen = sizeof(SRetrieveTableRspForTmq) + dataEncodeBufSize; void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) { return terrno; @@ -29,7 +29,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t pRetrieve->compressed = 0; pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, dataEncodeSize, numOfCols); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, dataEncodeBufSize, numOfCols); if(actualLen < 0){ taosMemoryFree(buf); return terrno; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index db3e553c7a..471a079e4c 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -35,8 +35,8 @@ extern SConfig* tsCfg; static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRetrieveTableRsp** pRsp) { - size_t dataEncodeSize = blockGetEncodeSize(pBlock); - size_t rspSize = sizeof(SRetrieveTableRsp) + dataEncodeSize + PAYLOAD_PREFIX_LEN; + size_t dataEncodeBufSize = blockGetEncodeSize(pBlock); + size_t rspSize = sizeof(SRetrieveTableRsp) + dataEncodeBufSize + PAYLOAD_PREFIX_LEN; *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { return terrno; @@ -50,7 +50,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); - int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeSize, numOfCols); + int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeBufSize, numOfCols); if(len < 0) { taosMemoryFree(*pRsp); return terrno; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 4f4f67caf7..42c214fac7 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1966,8 +1966,8 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { pBlock->info.rows = rowNum; - size_t dataEncodeSize = blockGetEncodeSize(pBlock); - int32_t rspSize = sizeof(SRetrieveTableRsp) + dataEncodeSize + PAYLOAD_PREFIX_LEN; + size_t dataEncodeBufSize = blockGetEncodeSize(pBlock); + int32_t rspSize = sizeof(SRetrieveTableRsp) + dataEncodeBufSize + PAYLOAD_PREFIX_LEN; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); if (NULL == rsp) { @@ -1978,7 +1978,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htobe64((int64_t)rowNum); - int32_t len = blockEncode(pBlock, rsp->data + PAYLOAD_PREFIX_LEN, dataEncodeSize, taosArrayGetSize(pBlock->pDataBlock)); + int32_t len = blockEncode(pBlock, rsp->data + PAYLOAD_PREFIX_LEN, dataEncodeBufSize, taosArrayGetSize(pBlock->pDataBlock)); if(len < 0) { qError("qExplainGetRspFromCtx: blockEncode failed"); QRY_ERR_JRET(terrno); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 8ab1558172..b9fbf06ef6 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -84,18 +84,18 @@ static int32_t toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pBuf->useSize = sizeof(SDataCacheEntry); { - size_t dataEncodeSize = pBuf->allocSize + 8; + size_t dataEncodeBufSize = pBuf->allocSize + 8; if ((pBuf->allocSize > tsCompressMsgSize) && (tsCompressMsgSize > 0) && pHandle->pManager->cfg.compress) { if (pHandle->pCompressBuf == NULL) { // allocate additional 8 bytes to avoid invalid write if compress failed to reduce the size - pHandle->pCompressBuf = taosMemoryMalloc(dataEncodeSize); + pHandle->pCompressBuf = taosMemoryMalloc(dataEncodeBufSize); if (NULL == pHandle->pCompressBuf) { QRY_RET(terrno); } - pHandle->bufSize = dataEncodeSize; + pHandle->bufSize = dataEncodeBufSize; } else { - if (pHandle->bufSize < dataEncodeSize) { - pHandle->bufSize = dataEncodeSize; + if (pHandle->bufSize < dataEncodeBufSize) { + pHandle->bufSize = dataEncodeBufSize; void* p = taosMemoryRealloc(pHandle->pCompressBuf, pHandle->bufSize); if (p != NULL) { pHandle->pCompressBuf = p; @@ -106,7 +106,7 @@ static int32_t toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* } } - int32_t dataLen = blockEncode(pInput->pData, pHandle->pCompressBuf, dataEncodeSize, numOfCols); + int32_t dataLen = blockEncode(pInput->pData, pHandle->pCompressBuf, dataEncodeBufSize, numOfCols); if(dataLen < 0) { qError("failed to encode data block, code: %d", dataLen); return terrno; @@ -124,7 +124,7 @@ static int32_t toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* TAOS_MEMCPY(pEntry->data, pHandle->pCompressBuf, dataLen); } } else { - pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, dataEncodeSize, numOfCols); + pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, pBuf->allocSize, numOfCols); if(pEntry->dataLen < 0) { qError("failed to encode data block, code: %d", pEntry->dataLen); return terrno; From 70c79abddca56cbe91a6f4b42bc7d83f09d22c6d Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 21 Oct 2024 09:27:46 +0800 Subject: [PATCH 10/59] fix: concat null failed --- source/client/src/clientImpl.c | 2 +- source/common/src/tdatablock.c | 2 +- source/libs/executor/src/dataDispatcher.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 69b6fe312c..323a5aed3f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2362,7 +2362,7 @@ int32_t resultInfoSafeCheck(SReqResultInfo* pResultInfo) { return TSDB_CODE_TSC_INTERNAL_ERROR; } for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { - if (pResultInfo->fields[i].bytes <= 0) { + if (pResultInfo->fields[i].bytes < 0) { tscError("checkResultInfo error: bytes:%d <= 0", pResultInfo->fields[i].bytes); return TSDB_CODE_TSC_INTERNAL_ERROR; } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 2da83a6b74..d41b433468 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -3165,7 +3165,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, size_t dataBuflen, in return dataLen; _exit: - uError("blockEncode dataLen:%d, dataBuflen:%" PRIx64, dataLen, dataBuflen); + uError("blockEncode dataLen:%d, dataBuflen:%" PRIu64, dataLen, dataBuflen); terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; return -1; } diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index b9fbf06ef6..d358cedfb7 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -84,10 +84,10 @@ static int32_t toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pBuf->useSize = sizeof(SDataCacheEntry); { + // allocate additional 8 bytes to avoid invalid write if compress failed to reduce the size size_t dataEncodeBufSize = pBuf->allocSize + 8; if ((pBuf->allocSize > tsCompressMsgSize) && (tsCompressMsgSize > 0) && pHandle->pManager->cfg.compress) { if (pHandle->pCompressBuf == NULL) { - // allocate additional 8 bytes to avoid invalid write if compress failed to reduce the size pHandle->pCompressBuf = taosMemoryMalloc(dataEncodeBufSize); if (NULL == pHandle->pCompressBuf) { QRY_RET(terrno); From 7094a8ddcfd49be3074d6001bd96d0e7fa41c9db Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 22 Oct 2024 14:15:48 +0800 Subject: [PATCH 11/59] fix: bytes of repeat and timezone --- include/libs/scalar/scalar.h | 1 + source/client/src/clientImpl.c | 21 ------ source/libs/executor/src/dataDispatcher.c | 78 +++++++++++++++++++++++ source/libs/function/src/builtins.c | 12 ++-- source/libs/scalar/src/sclfunc.c | 4 ++ 5 files changed, 91 insertions(+), 25 deletions(-) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index fd936dd087..4b89a6a439 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -105,6 +105,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t timeZoneStrLen(); int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t weekdayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t dayofweekFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 323a5aed3f..a3e327ee51 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2355,26 +2355,6 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo) { return TSDB_CODE_SUCCESS; } -int32_t resultInfoSafeCheck(SReqResultInfo* pResultInfo) { - if (pResultInfo->totalRows < pResultInfo->numOfRows) { - tscError("checkResultInfo error: totalRows:%" PRId64 " < numOfRows:%" PRId64, pResultInfo->totalRows, - pResultInfo->numOfRows); - return TSDB_CODE_TSC_INTERNAL_ERROR; - } - for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { - if (pResultInfo->fields[i].bytes < 0) { - tscError("checkResultInfo error: bytes:%d <= 0", pResultInfo->fields[i].bytes); - return TSDB_CODE_TSC_INTERNAL_ERROR; - } - if(!IS_VAR_DATA_TYPE(pResultInfo->fields[i].type) && TYPE_BYTES[pResultInfo->fields[i].type] != pResultInfo->fields[i].bytes) { - tscError("checkResultInfo error: type:%d bytes:%d != %d", pResultInfo->fields[i].type, pResultInfo->fields[i].bytes, TYPE_BYTES[pResultInfo->fields[i].type]); - return TSDB_CODE_TSC_INTERNAL_ERROR; - } - } - - return TSDB_CODE_SUCCESS; -} - int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { if (pResultInfo == NULL || pResultInfo->numOfCols <= 0 || pResultInfo->fields == NULL) { tscError("setResultDataPtr paras error"); @@ -2482,7 +2462,6 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) { if (convertUcs4) { code = doConvertUCS4(pResultInfo, colLength); } - code = resultInfoSafeCheck(pResultInfo); return code; } diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index d358cedfb7..983309b7c2 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -54,6 +54,78 @@ typedef struct SDataDispatchHandle { TdThreadMutex mutex; } SDataDispatchHandle; +static int32_t inputSafetyCheck(SDataDispatchHandle* pHandle, const SInputData* pInput) { + if (pInput == NULL || pInput->pData == NULL || pInput->pData->info.rows <= 0) { + qError("invalid input data"); + return TSDB_CODE_QRY_INVALID_INPUT; + } + SDataBlockDescNode* pSchema = pHandle->pSchema; + if (pSchema == NULL || pSchema->outputRowSize > pInput->pData->info.rowSize) { + qError("invalid schema"); + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SNode* pNode; + int32_t numOfCols = 0; + int32_t realOutputRowSize = 0; + FOREACH(pNode, pHandle->pSchema->pSlots) { + SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode; + if (pSlotDesc->output) { + realOutputRowSize += pSlotDesc->dataType.bytes; + ++numOfCols; + } else { + break; + } + } + if (realOutputRowSize != pSchema->outputRowSize) { + qError("invalid schema, realOutputRowSize:%d, outputRowSize:%d", realOutputRowSize, pSchema->outputRowSize); + return TSDB_CODE_QRY_INVALID_INPUT; + } + + if (numOfCols > taosArrayGetSize(pInput->pData->pDataBlock)) { + qError("invalid column number, schema:%d, input:%" PRIu64, numOfCols, taosArrayGetSize(pInput->pData->pDataBlock)); + return TSDB_CODE_QRY_INVALID_INPUT; + } + + int32_t colNum = 0; + FOREACH(pNode, pHandle->pSchema->pSlots) { + SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode; + if (pSlotDesc->output) { + SColumnInfoData* pColInfoData = taosArrayGet(pInput->pData->pDataBlock, colNum); + if (pColInfoData == NULL) { + return -1; + } + if (pColInfoData->info.bytes < 0) { + qError("invalid column bytes, schema:%d, input:%d", pSlotDesc->dataType.bytes, pColInfoData->info.bytes); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } + if (!IS_VAR_DATA_TYPE(pColInfoData->info.type) && + TYPE_BYTES[pColInfoData->info.type] != pColInfoData->info.bytes) { + qError("invalid column bytes, schema:%d, input:%d", TYPE_BYTES[pColInfoData->info.type], + pColInfoData->info.bytes); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } + if (pColInfoData->info.type != pSlotDesc->dataType.type) { + qError("invalid column type, schema:%d, input:%d", pSlotDesc->dataType.type, pColInfoData->info.type); + return TSDB_CODE_QRY_INVALID_INPUT; + } + if (pColInfoData->info.bytes != pSlotDesc->dataType.bytes) { + qError("invalid column bytes, schema:%d, input:%d", pSlotDesc->dataType.bytes, pColInfoData->info.bytes); + return TSDB_CODE_QRY_INVALID_INPUT; + } + + if (IS_INVALID_TYPE(pColInfoData->info.type)) { + qError("invalid column type, type:%d", pColInfoData->info.type); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } + ++colNum; + } + } + + + return TSDB_CODE_SUCCESS; +} + // clang-format off // data format: // +----------------+------------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ @@ -67,6 +139,12 @@ static int32_t toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* int32_t numOfCols = 0; SNode* pNode; + int32_t code = inputSafetyCheck(pHandle, pInput); + if (code) { + qError("failed to check input data, code:%d", code); + return code; + } + FOREACH(pNode, pHandle->pSchema->pSlots) { SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode; if (pSlotDesc->output) { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 1fd99125a0..1d3a43c498 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -695,7 +695,8 @@ static int32_t translateIsFilledPseudoColumn(SFunctionNode* pFunc, char* pErrBuf } static int32_t translateTimezone(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - pFunc->node.resType = (SDataType){.bytes = TD_TIMEZONE_LEN, .type = TSDB_DATA_TYPE_BINARY}; + int32_t bytesLen = timeZoneStrLen(); + pFunc->node.resType = (SDataType){.bytes = bytesLen, .type = TSDB_DATA_TYPE_BINARY}; return TSDB_CODE_SUCCESS; } @@ -2465,9 +2466,12 @@ static int32_t translateRepeat(SFunctionNode* pFunc, char* pErrBuf, int32_t len) uint8_t type = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type; int32_t orgLen = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->bytes; - int32_t count = TMAX((int32_t)((SValueNode*)nodesListGetNode(pFunc->pParameterList, 1))->datum.i, 1); - - int32_t resLen = orgLen * count; + int32_t resLen; + if (nodeType(nodesListGetNode(pFunc->pParameterList, 1)) == QUERY_NODE_VALUE) { + resLen = orgLen * TMAX((int32_t)((SValueNode*)nodesListGetNode(pFunc->pParameterList, 1))->datum.i, 1); + } else { + resLen = TSDB_MAX_BINARY_LEN; + } pFunc->node.resType = (SDataType){.bytes = resLen, .type = type}; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 6f6362a8f7..9aa67c441b 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2662,6 +2662,10 @@ int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut return TSDB_CODE_SUCCESS; } +int32_t timeZoneStrLen() { + return sizeof(VarDataLenT) + strlen(tsTimezoneStr); +} + int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { char output[TD_TIMEZONE_LEN + VARSTR_HEADER_SIZE] = {0}; (void)memcpy(varDataVal(output), tsTimezoneStr, TD_TIMEZONE_LEN); From f57e3f3c8511222ce74a03d574b233f27cd9d342 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 22 Oct 2024 16:22:27 +0800 Subject: [PATCH 12/59] enh: safetyCheck --- include/common/tglobal.h | 1 + source/common/src/tdatablock.c | 2 +- source/common/src/tglobal.c | 11 +++++++++-- source/libs/executor/src/dataDispatcher.c | 3 +++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 41fb692e42..a13693d38c 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -153,6 +153,7 @@ extern bool tsEnableCrashReport; extern char *tsTelemUri; extern char *tsClientCrashReportUri; extern char *tsSvrCrashReportUri; +extern bool tsEnableSafetyCheck; // query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d41b433468..23efe48209 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -3165,7 +3165,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, size_t dataBuflen, in return dataLen; _exit: - uError("blockEncode dataLen:%d, dataBuflen:%" PRIu64, dataLen, dataBuflen); + uError("blockEncode dataLen:%d, dataBuflen:%zu", dataLen, dataBuflen); terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; return -1; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7cff5de008..06791c5ec3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -139,6 +139,7 @@ bool tsEnableCrashReport = true; #endif char *tsClientCrashReportUri = "/ccrashreport"; char *tsSvrCrashReportUri = "/dcrashreport"; +bool tsEnableSafetyCheck = true; // schemaless bool tsSmlDot2Underline = true; @@ -607,6 +608,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { TAOS_CHECK_RETURN( cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableSafetyCheck", tsEnableSafetyCheck, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, TSDB_MAX_RPC_THREADS); @@ -1299,6 +1301,9 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "tsmaDataDeleteMark"); tsmaDataDeleteMark = pItem->i32; + + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "enableSafetyCheck"); + tsEnableSafetyCheck = pItem->bval; TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -2039,7 +2044,8 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"s3UploadDelaySec", &tsS3UploadDelaySec}, {"supportVnodes", &tsNumOfSupportVnodes}, {"experimental", &tsExperimental}, - {"maxTsmaNum", &tsMaxTsmaNum}}; + {"maxTsmaNum", &tsMaxTsmaNum}, + {"enableSafetyCheck", &tsEnableSafetyCheck}}; if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) { code = taosCfgSetOption(options, tListLen(options), pItem, false); @@ -2295,7 +2301,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { {"experimental", &tsExperimental}, {"multiResultFunctionStarReturnTags", &tsMultiResultFunctionStarReturnTags}, {"maxTsmaCalcDelay", &tsMaxTsmaCalcDelay}, - {"tsmaDataDeleteMark", &tsmaDataDeleteMark}}; + {"tsmaDataDeleteMark", &tsmaDataDeleteMark}, + {"enableSafetyCheck", &tsEnableSafetyCheck}}; if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) { code = taosCfgSetOption(options, tListLen(options), pItem, false); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 983309b7c2..5c3bc88b5b 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -55,6 +55,9 @@ typedef struct SDataDispatchHandle { } SDataDispatchHandle; static int32_t inputSafetyCheck(SDataDispatchHandle* pHandle, const SInputData* pInput) { + if(!tsEnableSafetyCheck) { + return TSDB_CODE_SUCCESS; + } if (pInput == NULL || pInput->pData == NULL || pInput->pData->info.rows <= 0) { qError("invalid input data"); return TSDB_CODE_QRY_INVALID_INPUT; From 2ae51f311847a00450e22f4b4eab2a475e4b657f Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Wed, 23 Oct 2024 06:20:51 +0800 Subject: [PATCH 13/59] enh: safetyCheck config --- source/common/src/tglobal.c | 2 +- source/libs/executor/src/dataDispatcher.c | 2 +- tests/pytest/util/dnodes.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 06791c5ec3..87cf1d7587 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -139,7 +139,7 @@ bool tsEnableCrashReport = true; #endif char *tsClientCrashReportUri = "/ccrashreport"; char *tsSvrCrashReportUri = "/dcrashreport"; -bool tsEnableSafetyCheck = true; +bool tsEnableSafetyCheck = false; // schemaless bool tsSmlDot2Underline = true; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 5c3bc88b5b..b7422d849c 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -86,7 +86,7 @@ static int32_t inputSafetyCheck(SDataDispatchHandle* pHandle, const SInputData* } if (numOfCols > taosArrayGetSize(pInput->pData->pDataBlock)) { - qError("invalid column number, schema:%d, input:%" PRIu64, numOfCols, taosArrayGetSize(pInput->pData->pDataBlock)); + qError("invalid column number, schema:%d, input:%zu", numOfCols, taosArrayGetSize(pInput->pData->pDataBlock)); return TSDB_CODE_QRY_INVALID_INPUT; } diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index bb6f8ff030..7a836c181b 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -48,6 +48,7 @@ class TDSimClient: "telemetryReporting": "0", "tqDebugflag": "135", "stDebugflag":"135", + "enableSafetyCheck":"1" } def getLogDir(self): @@ -149,7 +150,8 @@ class TDDnode: "statusInterval": "1", "enableQueryHb": "1", "supportVnodes": "1024", - "telemetryReporting": "0" + "telemetryReporting": "0", + "enableSafetyCheck":"1" } def init(self, path, remoteIP = ""): From 00d374358a42a721cb7f4c8c84f7992ee39476c1 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Wed, 23 Oct 2024 11:38:46 +0800 Subject: [PATCH 14/59] fix: rowSize check --- source/libs/executor/src/dataDispatcher.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index b7422d849c..732b2566a9 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -63,7 +63,7 @@ static int32_t inputSafetyCheck(SDataDispatchHandle* pHandle, const SInputData* return TSDB_CODE_QRY_INVALID_INPUT; } SDataBlockDescNode* pSchema = pHandle->pSchema; - if (pSchema == NULL || pSchema->outputRowSize > pInput->pData->info.rowSize) { + if (pSchema == NULL || pSchema->totalRowSize != pInput->pData->info.rowSize) { qError("invalid schema"); return TSDB_CODE_QRY_INVALID_INPUT; } @@ -77,6 +77,7 @@ static int32_t inputSafetyCheck(SDataDispatchHandle* pHandle, const SInputData* realOutputRowSize += pSlotDesc->dataType.bytes; ++numOfCols; } else { + // Slots must be sorted, and slots with 'output' set to true must come first break; } } From ea6c90e023af33ea8e6a6823ca0d195fdb928bfa Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Wed, 23 Oct 2024 13:52:53 +0800 Subject: [PATCH 15/59] fix: repeat.csv --- tests/army/query/function/ans/repeat.csv | 68 ++++++++++++------------ 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/tests/army/query/function/ans/repeat.csv b/tests/army/query/function/ans/repeat.csv index d8f8b3050f..c303f164a9 100644 --- a/tests/army/query/function/ans/repeat.csv +++ b/tests/army/query/function/ans/repeat.csv @@ -108,13 +108,13 @@ taos> select repeat(nch1, id) from ts_4893.meters where id > 0 order by ts limit novelnovelnovelnovelnovel | taos> select repeat(var1, id) from ts_4893.meters where id > 0 order by ts limit 5 - repeat(var1, id) | -=================== - person | - novelnovel | - plateplateplate | - 一二三四五六... | - updateupdateu... | + repeat(var1, id) | +================================= + person | + novelnovel | + plateplateplate | + 一二三四五六七八九十一二三... | + updateupdateupdateupdateupdate | taos> select repeat('nch1', id) from ts_4893.meters where id > 0 order by ts limit 5 repeat('nch1', id) | @@ -229,32 +229,32 @@ taos> select repeat(var1, 3) from ts_4893.meters order by ts limit 10 plateplateplate | taos> select repeat(name, groupid) from ts_4893.d0 order by ts limit 10 - repeat(name, groupid) | -======================== - lili | - x | - lili | - x | - lili | - taos | - haha | - taos | - taos | - haha | + repeat(name, groupid) | +================================= + lili | + x | + lili | + x | + lili | + taos | + haha | + taos | + taos | + haha | taos> select repeat(name, groupid) from ts_4893.meters order by ts limit 10 - repeat(name, groupid) | -======================== - lili | - x | - lili | - x | - lili | - taos | - haha | - taos | - taos | - haha | + repeat(name, groupid) | +================================= + lili | + x | + lili | + x | + lili | + taos | + haha | + taos | + taos | + haha | taos> select repeat(nch1, groupid) from ts_4893.d0 order by ts limit 10 repeat(nch1, groupid) | @@ -355,9 +355,9 @@ taos> select repeat('你好', 2) 你好你好 | taos> select repeat('abc', length('abc')) - repeat('abc', length('abc')) | -=============================== - abcabcabc | + repeat('abc', length('abc')) | +================================= + abcabcabc | taos> select repeat(concat('A', 'B', 'C'), 3) repeat(concat('A', 'B', 'C'), 3) | From e8835c2caa0004dc5db18983ce074a389a90cd34 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Wed, 23 Oct 2024 15:37:49 +0800 Subject: [PATCH 16/59] blockDataCheck --- include/common/tdatablock.h | 2 +- include/common/tglobal.h | 7 +++- source/common/src/tdatablock.c | 39 ++++++++++++------- source/common/src/tglobal.c | 16 ++++---- source/libs/executor/src/dataDispatcher.c | 2 +- .../libs/executor/src/dynqueryctrloperator.c | 7 +++- source/libs/executor/src/executor.c | 13 +++++-- source/libs/executor/src/executorInt.c | 9 +++-- source/libs/executor/src/groupcacheoperator.c | 2 +- source/libs/executor/src/mergeoperator.c | 10 +++-- source/libs/executor/src/operator.c | 20 +++++++--- source/libs/executor/src/sortoperator.c | 11 ++++-- tests/pytest/util/dnodes.py | 4 +- 13 files changed, 93 insertions(+), 49 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index c6f0b4d517..3b24ef9490 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -233,7 +233,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo); * @brief find how many rows already in order start from first row */ int32_t blockDataGetSortedRows(SSDataBlock* pDataBlock, SArray* pOrderInfo); -void blockDataCheck(const SSDataBlock* pDataBlock, bool forceChk); +int32_t blockDataCheck(const SSDataBlock* pDataBlock); int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); diff --git a/include/common/tglobal.h b/include/common/tglobal.h index a13693d38c..5a52abe9ee 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -153,7 +153,12 @@ extern bool tsEnableCrashReport; extern char *tsTelemUri; extern char *tsClientCrashReportUri; extern char *tsSvrCrashReportUri; -extern bool tsEnableSafetyCheck; +extern int8_t tsSafetyCheckLevel; +enum { + TSDB_SAFETY_CHECK_LEVELL_NEVER = 0, + TSDB_SAFETY_CHECK_LEVELL_NORMAL = 1, + TSDB_SAFETY_CHECK_LEVELL_BYROW = 2, +}; // query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 23efe48209..57df4a6006 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -18,6 +18,7 @@ #include "tcompare.h" #include "tlog.h" #include "tname.h" +#include "tglobal.h" #define MALLOC_ALIGN_BYTES 32 @@ -3042,7 +3043,11 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha // return length of encoded data, return -1 if failed int32_t blockEncode(const SSDataBlock* pBlock, char* data, size_t dataBuflen, int32_t numOfCols) { - blockDataCheck(pBlock, false); + int32_t code = blockDataCheck(pBlock); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return -1; + } int32_t dataLen = 0; @@ -3297,9 +3302,13 @@ int32_t blockDecode(SSDataBlock* pBlock, const char* pData, const char** pEndPos *pEndPos = pStart; - blockDataCheck(pBlock, false); + code = blockDataCheck(pBlock); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return code; + } - return code; + return TSDB_CODE_SUCCESS; } int32_t trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList) { @@ -3509,20 +3518,19 @@ int32_t blockDataGetSortedRows(SSDataBlock* pDataBlock, SArray* pOrderInfo) { return nextRowIdx; } -void blockDataCheck(const SSDataBlock* pDataBlock, bool forceChk) { - return; - - if (NULL == pDataBlock || pDataBlock->info.rows == 0) { - return; +#define BLOCK_DATA_CHECK_TRESSA(o) \ + if (!(o)) { \ + uError("blockDataCheck failed! line:%d", __LINE__); \ + return TSDB_CODE_INTERNAL_ERROR; \ + } +int32_t blockDataCheck(const SSDataBlock* pDataBlock) { + if (tsSafetyCheckLevel == TSDB_SAFETY_CHECK_LEVELL_NEVER || NULL == pDataBlock || pDataBlock->info.rows == 0) { + return TSDB_CODE_SUCCESS; } -#define BLOCK_DATA_CHECK_TRESSA(o) ; -//#define BLOCK_DATA_CHECK_TRESSA(o) A S S E R T(o) - BLOCK_DATA_CHECK_TRESSA(pDataBlock->info.rows > 0); - - if (!pDataBlock->info.dataLoad && !forceChk) { - return; + if (!pDataBlock->info.dataLoad) { + return TSDB_CODE_SUCCESS; } bool isVarType = false; @@ -3544,6 +3552,7 @@ void blockDataCheck(const SSDataBlock* pDataBlock, bool forceChk) { nextPos = 0; for (int64_t r = 0; r < checkRows; ++r) { + if (tsSafetyCheckLevel <= TSDB_SAFETY_CHECK_LEVELL_NORMAL) break; if (!colDataIsNull_s(pCol, r)) { BLOCK_DATA_CHECK_TRESSA(pCol->pData); BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.length <= pCol->varmeta.allocLen); @@ -3578,7 +3587,7 @@ void blockDataCheck(const SSDataBlock* pDataBlock, bool forceChk) { } } - return; + return TSDB_CODE_SUCCESS; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 87cf1d7587..6b222b15ab 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -137,9 +137,9 @@ bool tsEnableCrashReport = false; #else bool tsEnableCrashReport = true; #endif -char *tsClientCrashReportUri = "/ccrashreport"; -char *tsSvrCrashReportUri = "/dcrashreport"; -bool tsEnableSafetyCheck = false; +char *tsClientCrashReportUri = "/ccrashreport"; +char *tsSvrCrashReportUri = "/dcrashreport"; +int8_t tsSafetyCheckLevel = TSDB_SAFETY_CHECK_LEVELL_NEVER; // schemaless bool tsSmlDot2Underline = true; @@ -608,7 +608,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { TAOS_CHECK_RETURN( cfgAddInt64(pCfg, "randErrorDivisor", tsRandErrDivisor, 1, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "randErrorScope", tsRandErrScope, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableSafetyCheck", tsEnableSafetyCheck, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "safetyCheckLevel", tsSafetyCheckLevel, 0, 5, CFG_SCOPE_BOTH, CFG_DYN_BOTH)); tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, TSDB_MAX_RPC_THREADS); @@ -1302,8 +1302,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "tsmaDataDeleteMark"); tsmaDataDeleteMark = pItem->i32; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "enableSafetyCheck"); - tsEnableSafetyCheck = pItem->bval; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "safetyCheckLevel"); + tsSafetyCheckLevel = pItem->i32; TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -2045,7 +2045,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { {"supportVnodes", &tsNumOfSupportVnodes}, {"experimental", &tsExperimental}, {"maxTsmaNum", &tsMaxTsmaNum}, - {"enableSafetyCheck", &tsEnableSafetyCheck}}; + {"safetyCheckLevel", &tsSafetyCheckLevel}}; if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) { code = taosCfgSetOption(options, tListLen(options), pItem, false); @@ -2302,7 +2302,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { {"multiResultFunctionStarReturnTags", &tsMultiResultFunctionStarReturnTags}, {"maxTsmaCalcDelay", &tsMaxTsmaCalcDelay}, {"tsmaDataDeleteMark", &tsmaDataDeleteMark}, - {"enableSafetyCheck", &tsEnableSafetyCheck}}; + {"safetyCheckLevel", &tsSafetyCheckLevel}}; if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) { code = taosCfgSetOption(options, tListLen(options), pItem, false); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 732b2566a9..236d6a4d3e 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -55,7 +55,7 @@ typedef struct SDataDispatchHandle { } SDataDispatchHandle; static int32_t inputSafetyCheck(SDataDispatchHandle* pHandle, const SInputData* pInput) { - if(!tsEnableSafetyCheck) { + if(tsSafetyCheckLevel == TSDB_SAFETY_CHECK_LEVELL_NEVER) { return TSDB_CODE_SUCCESS; } if (pInput == NULL || pInput->pData == NULL || pInput->pData->info.rows <= 0) { diff --git a/source/libs/executor/src/dynqueryctrloperator.c b/source/libs/executor/src/dynqueryctrloperator.c index eb49057d89..62f199387e 100644 --- a/source/libs/executor/src/dynqueryctrloperator.c +++ b/source/libs/executor/src/dynqueryctrloperator.c @@ -528,7 +528,12 @@ static void seqJoinLaunchNewRetrieveImpl(SOperatorInfo* pOperator, SSDataBlock** qDebug("%s dynamic post task begin", GET_TASKID(pOperator->pTaskInfo)); code = pOperator->pDownstream[1]->fpSet.getNextExtFn(pOperator->pDownstream[1], pParam, ppRes); if (*ppRes && (code == 0)) { - blockDataCheck(*ppRes, false); + code = blockDataCheck(*ppRes); + if (code) { + qError("Invalid block data, blockDataCheck failed, error:%s", tstrerror(code)); + pOperator->pTaskInfo->code = code; + T_LONG_JMP(pOperator->pTaskInfo->env, pOperator->pTaskInfo->code); + } pPost->isStarted = true; pStbJoin->execInfo.postBlkNum++; pStbJoin->execInfo.postBlkRows += (*ppRes)->info.rows; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 27dd687f40..632daf0aa0 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -699,12 +699,12 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo if (pTaskInfo->pOpParam && !pTaskInfo->paramSet) { pTaskInfo->paramSet = true; code = pTaskInfo->pRoot->fpSet.getNextExtFn(pTaskInfo->pRoot, pTaskInfo->pOpParam, &pRes); - blockDataCheck(pRes, false); } else { code = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot, &pRes); - blockDataCheck(pRes, false); } + QUERY_CHECK_CODE(code, lino, _end); + code = blockDataCheck(pRes); QUERY_CHECK_CODE(code, lino, _end); if (pRes == NULL) { @@ -749,7 +749,8 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo } code = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot, &pRes); - blockDataCheck(pRes, false); + QUERY_CHECK_CODE(code, lino, _end); + code = blockDataCheck(pRes); QUERY_CHECK_CODE(code, lino, _end); } @@ -848,7 +849,11 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { qError("%s failed at line %d, code:%s %s", __func__, __LINE__, tstrerror(code), GET_TASKID(pTaskInfo)); } - blockDataCheck(*pRes, false); + code = blockDataCheck(*pRes); + if (code) { + pTaskInfo->code = code; + qError("%s failed at line %d, code:%s %s", __func__, __LINE__, tstrerror(code), GET_TASKID(pTaskInfo)); + } uint64_t el = (taosGetTimestampUs() - st); diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 64a07c4653..b39cf4014d 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -616,11 +616,12 @@ int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* p } } } - code = TSDB_CODE_SUCCESS; - + code = blockDataCheck(pBlock); + QUERY_CHECK_CODE(code, lino, _err); _err: - blockDataCheck(pBlock, true); - + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } colDataDestroy(p); taosMemoryFree(p); return code; diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index 10b372319b..d47ab366b6 100644 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -765,7 +765,7 @@ static FORCE_INLINE int32_t getBlkFromDownstreamOperator(struct SOperatorInfo* p } } - blockDataCheck(pBlock, false); + code = blockDataCheck(pBlock); *ppRes = pBlock; return code; diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c index 7fd6b91e52..4a288a277b 100644 --- a/source/libs/executor/src/mergeoperator.c +++ b/source/libs/executor/src/mergeoperator.c @@ -65,11 +65,14 @@ static int32_t sortMergeloadNextDataBlock(void* param, SSDataBlock** ppBlock); int32_t sortMergeloadNextDataBlock(void* param, SSDataBlock** ppBlock) { SOperatorInfo* pOperator = (SOperatorInfo*)param; - int32_t code = pOperator->fpSet.getNextFn(pOperator, ppBlock); - blockDataCheck(*ppBlock, false); + int32_t code = pOperator->fpSet.getNextFn(pOperator, ppBlock); if (code) { qError("failed to get next data block from upstream, %s code:%s", __func__, tstrerror(code)); } + blockDataCheck(*ppBlock); + if (code) { + qError("failed to check data block got from upstream, %s code:%s", __func__, tstrerror(code)); + } return code; } @@ -526,7 +529,8 @@ int32_t doMultiwayMerge(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { if ((*pResBlock) != NULL) { pOperator->resultInfo.totalRows += (*pResBlock)->info.rows; - blockDataCheck(*pResBlock, false); + code = blockDataCheck(*pResBlock); + QUERY_CHECK_CODE(code, lino, _end); } else { setOperatorCompleted(pOperator); } diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 7914f9f320..f88b544b0f 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -870,15 +870,25 @@ int32_t setOperatorParams(struct SOperatorInfo* pOperator, SOperatorParam* pInpu SSDataBlock* getNextBlockFromDownstream(struct SOperatorInfo* pOperator, int32_t idx) { SSDataBlock* p = NULL; - int32_t code = getNextBlockFromDownstreamImpl(pOperator, idx, true, &p); - blockDataCheck(p, false); - return (code == 0)? p:NULL; + int32_t code = getNextBlockFromDownstreamImpl(pOperator, idx, true, &p); + if (code == TSDB_CODE_SUCCESS) { + code = blockDataCheck(p); + if (code != TSDB_CODE_SUCCESS) { + qError("blockDataCheck failed, code:%s", tstrerror(code)); + } + } + return (code == 0) ? p : NULL; } SSDataBlock* getNextBlockFromDownstreamRemain(struct SOperatorInfo* pOperator, int32_t idx) { SSDataBlock* p = NULL; - int32_t code = getNextBlockFromDownstreamImpl(pOperator, idx, false, &p); - blockDataCheck(p, false); + int32_t code = getNextBlockFromDownstreamImpl(pOperator, idx, false, &p); + if (code == TSDB_CODE_SUCCESS) { + code = blockDataCheck(p); + if (code != TSDB_CODE_SUCCESS) { + qError("blockDataCheck failed, code:%s", tstrerror(code)); + } + } return (code == 0)? p:NULL; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 1c241dffec..a6ca20c5ee 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -334,10 +334,14 @@ static int32_t getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t loadNextDataBlock(void* param, SSDataBlock** ppBlock) { SOperatorInfo* pOperator = (SOperatorInfo*)param; - int32_t code = pOperator->fpSet.getNextFn(pOperator, ppBlock); - blockDataCheck(*ppBlock, false); + int32_t code = pOperator->fpSet.getNextFn(pOperator, ppBlock); if (code) { qError("failed to get next data block from upstream, %s code:%s", __func__, tstrerror(code)); + } else { + code = blockDataCheck(*ppBlock); + if (code) { + qError("failed to check block data, %s code:%s", __func__, tstrerror(code)); + } } return code; } @@ -630,7 +634,8 @@ int32_t fetchNextGroupSortDataBlock(void* param, SSDataBlock** ppBlock) { QUERY_CHECK_CODE(code, lino, _end); if (block != NULL) { - blockDataCheck(block, false); + code = blockDataCheck(block); + QUERY_CHECK_CODE(code, lino, _end); if (block->info.id.groupId == grpSortOpInfo->currGroupId) { grpSortOpInfo->childOpStatus = CHILD_OP_SAME_GROUP; *ppBlock = block; diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 7a836c181b..3832530218 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -48,7 +48,7 @@ class TDSimClient: "telemetryReporting": "0", "tqDebugflag": "135", "stDebugflag":"135", - "enableSafetyCheck":"1" + "safetyCheckLevel":"2" } def getLogDir(self): @@ -151,7 +151,7 @@ class TDDnode: "enableQueryHb": "1", "supportVnodes": "1024", "telemetryReporting": "0", - "enableSafetyCheck":"1" + "safetyCheckLevel":"2" } def init(self, path, remoteIP = ""): From 1e8565eca839a2476133f22b9f9adb86a07e5bc4 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Wed, 23 Oct 2024 15:51:46 +0800 Subject: [PATCH 17/59] fix: check return value --- source/libs/executor/src/mergeoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c index 4a288a277b..0dfe89e10e 100644 --- a/source/libs/executor/src/mergeoperator.c +++ b/source/libs/executor/src/mergeoperator.c @@ -69,7 +69,7 @@ int32_t sortMergeloadNextDataBlock(void* param, SSDataBlock** ppBlock) { if (code) { qError("failed to get next data block from upstream, %s code:%s", __func__, tstrerror(code)); } - blockDataCheck(*ppBlock); + code = blockDataCheck(*ppBlock); if (code) { qError("failed to check data block got from upstream, %s code:%s", __func__, tstrerror(code)); } From a7bbb7863b87e92e81b79925f8308622169ec071 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Wed, 23 Oct 2024 17:21:22 +0800 Subject: [PATCH 18/59] fix: varType length check --- source/common/src/tdatablock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 57df4a6006..cb5b746844 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -3570,7 +3570,7 @@ int32_t blockDataCheck(const SSDataBlock* pDataBlock) { colLen = varDataTLen(pCol->pData + pCol->varmeta.offset[r]); BLOCK_DATA_CHECK_TRESSA(colLen >= VARSTR_HEADER_SIZE); - BLOCK_DATA_CHECK_TRESSA(colLen <= pCol->info.bytes); + BLOCK_DATA_CHECK_TRESSA(colLen <= pCol->varmeta.length); if (pCol->reassigned) { BLOCK_DATA_CHECK_TRESSA((pCol->varmeta.offset[r] + colLen) <= pCol->varmeta.length); From a29f6145b0560a84ce4a02180a39573dc6a873b7 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 24 Oct 2024 06:40:37 +0800 Subject: [PATCH 19/59] check skip:pcol.pdata --- source/common/src/tdatablock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index cb5b746844..ee551d54db 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -3554,7 +3554,7 @@ int32_t blockDataCheck(const SSDataBlock* pDataBlock) { for (int64_t r = 0; r < checkRows; ++r) { if (tsSafetyCheckLevel <= TSDB_SAFETY_CHECK_LEVELL_NORMAL) break; if (!colDataIsNull_s(pCol, r)) { - BLOCK_DATA_CHECK_TRESSA(pCol->pData); + // BLOCK_DATA_CHECK_TRESSA(pCol->pData); BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.length <= pCol->varmeta.allocLen); if (isVarType) { From f91226b1d6357aca99b3b0976e5115629bf3be67 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 24 Oct 2024 06:53:47 +0800 Subject: [PATCH 20/59] skip: pCol.pData Null --- source/common/src/tdatablock.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index ee551d54db..b47abeeb65 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -3543,6 +3543,7 @@ int32_t blockDataCheck(const SSDataBlock* pDataBlock) { SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pDataBlock->pDataBlock, i); isVarType = IS_VAR_DATA_TYPE(pCol->info.type); checkRows = pDataBlock->info.rows; + if(pCol->pData == NULL) continue; if (isVarType) { BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset); From 3a505157f372fad76b57716a27d0f7562281f14e Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 24 Oct 2024 10:33:18 +0800 Subject: [PATCH 21/59] fix: type overflow --- source/common/src/tdatablock.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b47abeeb65..b8e9579c6b 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -3582,7 +3582,15 @@ int32_t blockDataCheck(const SSDataBlock* pDataBlock) { typeValue = *(char*)(pCol->pData + pCol->varmeta.offset[r] + colLen - 1); } else { - GET_TYPED_DATA(typeValue, int64_t, pCol->info.type, colDataGetNumData(pCol, r)); + if (TSDB_DATA_TYPE_FLOAT == pCol->info.type) { + float v = 0; + GET_TYPED_DATA(v, float, pCol->info.type, colDataGetNumData(pCol, r)); + } else if (TSDB_DATA_TYPE_DOUBLE == pCol->info.type) { + double v = 0; + GET_TYPED_DATA(v, double, pCol->info.type, colDataGetNumData(pCol, r)); + } else { + GET_TYPED_DATA(typeValue, int64_t, pCol->info.type, colDataGetNumData(pCol, r)); + } } } } From 5d20fde69209fda9e89ec1b7019cbaf4d7727559 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 24 Oct 2024 17:44:13 +0800 Subject: [PATCH 22/59] safetycheck, use reserve state --- include/common/tmsg.h | 1 + source/common/src/tdatablock.c | 10 +++++----- source/libs/executor/src/executil.c | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 01808d4f2f..14e3f9b0eb 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1215,6 +1215,7 @@ typedef struct { int32_t bytes; int8_t type; uint8_t pk; + bool reserve; } SColumnInfo; typedef struct STimeWindow { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b8e9579c6b..9240e18700 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -3543,7 +3543,7 @@ int32_t blockDataCheck(const SSDataBlock* pDataBlock) { SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pDataBlock->pDataBlock, i); isVarType = IS_VAR_DATA_TYPE(pCol->info.type); checkRows = pDataBlock->info.rows; - if(pCol->pData == NULL) continue; + if (pCol->info.reserve == true) continue; if (isVarType) { BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset); @@ -3555,12 +3555,12 @@ int32_t blockDataCheck(const SSDataBlock* pDataBlock) { for (int64_t r = 0; r < checkRows; ++r) { if (tsSafetyCheckLevel <= TSDB_SAFETY_CHECK_LEVELL_NORMAL) break; if (!colDataIsNull_s(pCol, r)) { - // BLOCK_DATA_CHECK_TRESSA(pCol->pData); + BLOCK_DATA_CHECK_TRESSA(pCol->pData); BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.length <= pCol->varmeta.allocLen); - + if (isVarType) { BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.allocLen > 0); - BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset[r] < pCol->varmeta.length); + BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset[r] <= pCol->varmeta.length); if (pCol->reassigned) { BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset[r] >= 0); } else if (0 == r) { @@ -3571,7 +3571,7 @@ int32_t blockDataCheck(const SSDataBlock* pDataBlock) { colLen = varDataTLen(pCol->pData + pCol->varmeta.offset[r]); BLOCK_DATA_CHECK_TRESSA(colLen >= VARSTR_HEADER_SIZE); - BLOCK_DATA_CHECK_TRESSA(colLen <= pCol->varmeta.length); + BLOCK_DATA_CHECK_TRESSA(colLen <= pCol->info.bytes); if (pCol->reassigned) { BLOCK_DATA_CHECK_TRESSA((pCol->varmeta.offset[r] + colLen) <= pCol->varmeta.length); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index b15cc2ab45..1a43d42348 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -390,6 +390,7 @@ SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode) { createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId); idata.info.scale = pDescNode->dataType.scale; idata.info.precision = pDescNode->dataType.precision; + idata.info.reserve = pDescNode->reserve; code = blockDataAppendColInfo(pBlock, &idata); if (code != TSDB_CODE_SUCCESS) { From b50a89804f4ce4400297bf4a3268c28e8643eed8 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 25 Oct 2024 10:16:28 +0800 Subject: [PATCH 23/59] fix: reserve check --- source/common/src/tdatablock.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 9240e18700..d59c9252f3 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -3541,9 +3541,10 @@ int32_t blockDataCheck(const SSDataBlock* pDataBlock) { int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < colNum; ++i) { SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pDataBlock->pDataBlock, i); + BLOCK_DATA_CHECK_TRESSA(pCol != NULL); isVarType = IS_VAR_DATA_TYPE(pCol->info.type); checkRows = pDataBlock->info.rows; - if (pCol->info.reserve == true) continue; + if (pCol->info.reserve == false) continue; if (isVarType) { BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset); From 0a8b3714f9496276f6af5e67b1935c8ba2ed7b40 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 25 Oct 2024 18:45:29 +0800 Subject: [PATCH 24/59] fix: reserve --- source/libs/planner/src/planPhysiCreater.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 738ccf3224..a44fbbdbee 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -233,7 +233,7 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD int32_t len = 0; code = getSlotKey(pNode, NULL, &name, &len, 16); if (TSDB_CODE_SUCCESS == code) { - code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pNode, slotId, true, false)); + code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pNode, slotId, true, true)); } code = putSlotToHash(name, len, pDataBlockDesc->dataBlockId, slotId, pNode, pHash); if (TSDB_CODE_SUCCESS == code) { @@ -367,7 +367,7 @@ static int32_t addDataBlockSlotsForProject(SPhysiPlanContext* pCxt, const char* } static int32_t pushdownDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { - return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, NULL, true, true); + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, NULL, true, false); } typedef struct SSetSlotIdCxt { From 323088dc7113d1c5e2e3a80f14e682e02ba73697 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Sun, 27 Oct 2024 22:41:34 +0800 Subject: [PATCH 25/59] fix: create partialfunction --- include/libs/function/functionMgt.h | 1 + source/libs/function/src/builtins.c | 2 +- source/libs/function/src/functionMgt.c | 36 +++++++++++++++++++------- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index e5bacf85b2..0b6928fa50 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -290,6 +290,7 @@ bool fmIsElapsedFunc(int32_t funcId); void getLastCacheDataType(SDataType* pType, int32_t pkBytes); int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc); +int32_t createFunctionWithSrcFunc(const char* pName, const SFunctionNode* pSrcFunc, SNodeList* pParameterList, SFunctionNode** pFunc); int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMidFunc, SFunctionNode** pMergeFunc); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 7c659bdef5..bd72498508 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1648,7 +1648,7 @@ static int32_t translateOutVarchar(SFunctionNode* pFunc, char* pErrBuf, int32_t bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE; break; case FUNCTION_TYPE_TIMEZONE: - bytes = TD_TIMEZONE_LEN; + bytes = timeZoneStrLen(); break; case FUNCTION_TYPE_IRATE_PARTIAL: bytes = getIrateInfoSize((pFunc->hasPk) ? pFunc->pkBytes : 0) + VARSTR_HEADER_SIZE; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index aaa66441ee..a406b23c59 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -412,6 +412,27 @@ int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNo return code; } +int32_t createFunctionWithSrcFunc(const char* pName, const SFunctionNode* pSrcFunc, SNodeList* pParameterList, SFunctionNode** ppFunc) { + int32_t code = nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)ppFunc); + if (NULL == *ppFunc) { + return code; + } + + (*ppFunc)->hasPk = pSrcFunc->hasPk; + (*ppFunc)->pkBytes = pSrcFunc->pkBytes; + + (void)snprintf((*ppFunc)->functionName, sizeof((*ppFunc)->functionName), "%s", pName); + (*ppFunc)->pParameterList = pParameterList; + code = getFuncInfo((*ppFunc)); + if (TSDB_CODE_SUCCESS != code) { + (*ppFunc)->pParameterList = NULL; + nodesDestroyNode((SNode*)*ppFunc); + *ppFunc = NULL; + return code; + } + return code; +} + static int32_t createColumnByFunc(const SFunctionNode* pFunc, SColumnNode** ppCol) { int32_t code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)ppCol); if (NULL == *ppCol) { @@ -438,7 +459,8 @@ static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNod if (NULL == pParameterList) { return code; } - code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pPartialFunc, pParameterList,pPartialFunc ); + code = + createFunctionWithSrcFunc(funcMgtBuiltins[pSrcFunc->funcId].pPartialFunc, pSrcFunc, pParameterList, pPartialFunc); if (TSDB_CODE_SUCCESS != code) { nodesDestroyList(pParameterList); return code; @@ -452,8 +474,6 @@ static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNod return TSDB_CODE_FAILED; } tstrncpy((*pPartialFunc)->node.aliasName, name, TSDB_COL_NAME_LEN); - (*pPartialFunc)->hasPk = pSrcFunc->hasPk; - (*pPartialFunc)->pkBytes = pSrcFunc->pkBytes; return TSDB_CODE_SUCCESS; } @@ -479,9 +499,9 @@ static int32_t createMidFunction(const SFunctionNode* pSrcFunc, const SFunctionN int32_t code = createMergeFuncPara(pSrcFunc, pPartialFunc, &pParameterList); if (TSDB_CODE_SUCCESS == code) { if(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc != NULL){ - code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc, pParameterList, &pFunc); + code = createFunctionWithSrcFunc(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc, pSrcFunc, pParameterList, &pFunc); }else{ - code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList, &pFunc); + code = createFunctionWithSrcFunc(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pSrcFunc, pParameterList, &pFunc); } } if (TSDB_CODE_SUCCESS == code) { @@ -493,8 +513,6 @@ static int32_t createMidFunction(const SFunctionNode* pSrcFunc, const SFunctionN } else { nodesDestroyList(pParameterList); } - (*pMidFunc)->hasPk = pPartialFunc->hasPk; - (*pMidFunc)->pkBytes = pPartialFunc->pkBytes; return code; } @@ -505,7 +523,7 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio int32_t code = createMergeFuncPara(pSrcFunc, pPartialFunc, &pParameterList); if (TSDB_CODE_SUCCESS == code) { - code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList, &pFunc); + code = createFunctionWithSrcFunc(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pSrcFunc, pParameterList, &pFunc); } if (TSDB_CODE_SUCCESS == code) { pFunc->hasOriginalFunc = true; @@ -522,8 +540,6 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio } else { nodesDestroyList(pParameterList); } - (*pMergeFunc)->hasPk = pPartialFunc->hasPk; - (*pMergeFunc)->pkBytes = pPartialFunc->pkBytes; return code; } From b02ad5c8e28ede23bbf29f7ab2bed07713657802 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Sun, 27 Oct 2024 23:35:21 +0800 Subject: [PATCH 26/59] fix to_char length --- source/libs/scalar/src/sclfunc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 9aa67c441b..da4a39a551 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2413,7 +2413,7 @@ int32_t toCharFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOu char *ts = colDataGetData(pInput[0].columnData, i); char *formatData = colDataGetData(pInput[1].columnData, pInput[1].numOfRows > 1 ? i : 0); - len = TMIN(TS_FORMAT_MAX_LEN - 1, varDataLen(formatData)); + len = TMIN(TS_FORMAT_MAX_LEN - VARSTR_HEADER_SIZE, varDataLen(formatData)); if (pInput[1].numOfRows > 1 || i == 0) { (void)strncpy(format, varDataVal(formatData), len); format[len] = '\0'; From fb34549ae76d58bd930fc5fca13ce9059fd5b644 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 28 Oct 2024 11:50:25 +0800 Subject: [PATCH 27/59] fix: json tag length check --- source/common/src/tdatablock.c | 13 ++++++++++--- source/libs/function/src/builtins.c | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d59c9252f3..5ac5ffdd9b 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -3569,11 +3569,18 @@ int32_t blockDataCheck(const SSDataBlock* pDataBlock) { } else { BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset[r] == nextPos); } - - colLen = varDataTLen(pCol->pData + pCol->varmeta.offset[r]); + + char* pColData = pCol->pData + pCol->varmeta.offset[r]; + int32_t colSize = 0; + if (pCol->info.type == TSDB_DATA_TYPE_JSON) { + colLen = getJsonValueLen(pColData); + } else { + colLen = varDataTLen(pColData); + } + BLOCK_DATA_CHECK_TRESSA(colLen >= VARSTR_HEADER_SIZE); BLOCK_DATA_CHECK_TRESSA(colLen <= pCol->info.bytes); - + if (pCol->reassigned) { BLOCK_DATA_CHECK_TRESSA((pCol->varmeta.offset[r] + colLen) <= pCol->varmeta.length); } else { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index bd72498508..e66ddf5197 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1606,7 +1606,7 @@ static int32_t translateOutVarchar(SFunctionNode* pFunc, char* pErrBuf, int32_t break; case FUNCTION_TYPE_BLOCK_DIST: case FUNCTION_TYPE_BLOCK_DIST_INFO: - bytes = 128; + bytes = sizeof(STableBlockDistInfo); break; case FUNCTION_TYPE_TO_CHAR: bytes = 4096; From 85989212c64e8cdeb16ef865a177a569bbb0be61 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 28 Oct 2024 19:38:56 +0800 Subject: [PATCH 28/59] fix: set scolumnInfo nodata flag where table scan notload --- include/common/tmsg.h | 2 +- source/common/src/tdatablock.c | 8 ++++++-- source/libs/executor/src/executil.c | 2 +- source/libs/executor/src/scanoperator.c | 13 +++++++++++++ source/libs/planner/src/planPhysiCreater.c | 8 ++++---- 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 14e3f9b0eb..d6aad934e0 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1215,7 +1215,7 @@ typedef struct { int32_t bytes; int8_t type; uint8_t pk; - bool reserve; + bool noData; } SColumnInfo; typedef struct STimeWindow { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5ac5ffdd9b..4724a3d241 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -3544,7 +3544,7 @@ int32_t blockDataCheck(const SSDataBlock* pDataBlock) { BLOCK_DATA_CHECK_TRESSA(pCol != NULL); isVarType = IS_VAR_DATA_TYPE(pCol->info.type); checkRows = pDataBlock->info.rows; - if (pCol->info.reserve == false) continue; + if (pCol->info.noData == true) continue; if (isVarType) { BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset); @@ -3578,7 +3578,11 @@ int32_t blockDataCheck(const SSDataBlock* pDataBlock) { colLen = varDataTLen(pColData); } - BLOCK_DATA_CHECK_TRESSA(colLen >= VARSTR_HEADER_SIZE); + if (pCol->info.type == TSDB_DATA_TYPE_JSON) { + BLOCK_DATA_CHECK_TRESSA(colLen >= CHAR_BYTES); + } else { + BLOCK_DATA_CHECK_TRESSA(colLen >= VARSTR_HEADER_SIZE); + } BLOCK_DATA_CHECK_TRESSA(colLen <= pCol->info.bytes); if (pCol->reassigned) { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1a43d42348..8cd573e971 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -390,7 +390,7 @@ SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode) { createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId); idata.info.scale = pDescNode->dataType.scale; idata.info.precision = pDescNode->dataType.precision; - idata.info.reserve = pDescNode->reserve; + idata.info.noData = pDescNode->reserve; code = blockDataAppendColInfo(pBlock, &idata); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index bae9926f63..f936e95005 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1461,6 +1461,18 @@ static void destroyTableScanOperatorInfo(void* param) { taosMemoryFreeClear(param); } +static void resetClolumnReserve(SSDataBlock* pBlock, int32_t dataRequireFlag) { + if (pBlock && dataRequireFlag == FUNC_DATA_REQUIRED_NOT_LOAD) { + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); + if (pCol) { + pCol->info.noData = true; + } + } + } +} + int32_t createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) { @@ -1511,6 +1523,7 @@ int32_t createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHa pInfo->base.readerAPI = pTaskInfo->storageAPI.tsdReader; initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); + resetClolumnReserve(pInfo->pResBlock, pInfo->base.dataBlockLoadFlag); QUERY_CHECK_NULL(pInfo->pResBlock, code, lino, _error, terrno); code = prepareDataBlockBuf(pInfo->pResBlock, &pInfo->base.matchInfo); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index a48504b79d..a67071516d 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -233,7 +233,7 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD int32_t len = 0; code = getSlotKey(pNode, NULL, &name, &len, 16); if (TSDB_CODE_SUCCESS == code) { - code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pNode, slotId, true, true)); + code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pNode, slotId, true, false)); } code = putSlotToHash(name, len, pDataBlockDesc->dataBlockId, slotId, pNode, pHash); if (TSDB_CODE_SUCCESS == code) { @@ -341,7 +341,7 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, } static int32_t addDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { - return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, NULL, false, false); + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, NULL, false, true); } static int32_t addDataBlockSlot(SPhysiPlanContext* pCxt, SNode** pNode, SDataBlockDescNode* pDataBlockDesc) { @@ -363,11 +363,11 @@ static int32_t addDataBlockSlot(SPhysiPlanContext* pCxt, SNode** pNode, SDataBlo static int32_t addDataBlockSlotsForProject(SPhysiPlanContext* pCxt, const char* pStmtName, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { - return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, false, false); + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, false, true); } static int32_t pushdownDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { - return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, NULL, true, false); + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, NULL, true, true); } typedef struct SSetSlotIdCxt { From 1463bd3efdcfc2fa3d5333ad44fb327c45b17804 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 29 Oct 2024 00:03:40 +0800 Subject: [PATCH 29/59] fix: show streams --- source/common/src/systable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index eef38bf18e..211aa20645 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -165,8 +165,8 @@ static const SSysDbTableSchema userStbsSchema[] = { static const SSysDbTableSchema streamSchema[] = { {.name = "stream_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, - {.name = "stream_id", .bytes = 16 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "history_id", .bytes = 16 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "stream_id", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "history_id", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "source_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, From 36e115d2cd0799c5e577981bfde1e313e9420361 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 29 Oct 2024 15:49:16 +0800 Subject: [PATCH 30/59] fix: colDataSetVal repeatedly sets the same row --- source/common/src/systable.c | 6 +++--- source/common/src/tdatablock.c | 12 ++++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 211aa20645..cb11e4a24e 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -190,9 +190,9 @@ static const SSysDbTableSchema streamTaskSchema[] = { {.name = "stage", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, {.name = "in_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "process_total", .bytes = 14, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "process_throughput", .bytes = 14, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "out_total", .bytes = 14, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "out_throughput", .bytes = 14, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "process_throughput", .bytes = 15, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "out_total", .bytes = 15, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "out_throughput", .bytes = 15, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, // {.name = "dispatch_throughput", .bytes = 12, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, // {.name = "dispatch_total", .bytes = 12, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, // {.name = "out_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4724a3d241..3187d2315a 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -109,6 +109,14 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const dataLen = varDataTLen(pData); } + if(rowIndex == 0 && pColumnInfoData->varmeta.length > 0) { + return 0; + } + if (pColumnInfoData->varmeta.offset[rowIndex] > 0 && + pColumnInfoData->varmeta.offset[rowIndex] < pColumnInfoData->varmeta.length) { + return 0; + } + SVarColAttr* pAttr = &pColumnInfoData->varmeta; if (pAttr->allocLen < pAttr->length + dataLen) { uint32_t newSize = pAttr->allocLen; @@ -3552,7 +3560,7 @@ int32_t blockDataCheck(const SSDataBlock* pDataBlock) { BLOCK_DATA_CHECK_TRESSA(pCol->nullbitmap); } - nextPos = 0; + nextPos = -1; for (int64_t r = 0; r < checkRows; ++r) { if (tsSafetyCheckLevel <= TSDB_SAFETY_CHECK_LEVELL_NORMAL) break; if (!colDataIsNull_s(pCol, r)) { @@ -3564,7 +3572,7 @@ int32_t blockDataCheck(const SSDataBlock* pDataBlock) { BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset[r] <= pCol->varmeta.length); if (pCol->reassigned) { BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset[r] >= 0); - } else if (0 == r) { + } else if (0 == r || nextPos == -1) { nextPos = pCol->varmeta.offset[r]; } else { BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset[r] == nextPos); From 694e7577703a5d5e23c1d2f3ddcef09dc48c86a6 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 29 Oct 2024 19:50:21 +0800 Subject: [PATCH 31/59] fix: coldata set val --- source/common/src/tdatablock.c | 35 +++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 3187d2315a..c185ac9d86 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -87,6 +87,16 @@ int32_t getJsonValueLen(const char* data) { return dataLen; } +static int32_t getDataLen(int32_t type, const char* pData) { + int32_t dataLen = 0; + if (type == TSDB_DATA_TYPE_JSON) { + dataLen = getJsonValueLen(pData); + } else { + dataLen = varDataTLen(pData); + } + return dataLen; +} + int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull) { if (isNull || pData == NULL) { // There is a placehold for each NULL value of binary or nchar type. @@ -102,19 +112,18 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const int32_t type = pColumnInfoData->info.type; if (IS_VAR_DATA_TYPE(type)) { - int32_t dataLen = 0; - if (type == TSDB_DATA_TYPE_JSON) { - dataLen = getJsonValueLen(pData); - } else { - dataLen = varDataTLen(pData); - } - - if(rowIndex == 0 && pColumnInfoData->varmeta.length > 0) { - return 0; - } - if (pColumnInfoData->varmeta.offset[rowIndex] > 0 && - pColumnInfoData->varmeta.offset[rowIndex] < pColumnInfoData->varmeta.length) { - return 0; + int32_t dataLen = getDataLen(type, pData); + if (pColumnInfoData->varmeta.offset[rowIndex] > 0) { + if (rowIndex == 0) { + pColumnInfoData->varmeta.length = 0; + } else { + int32_t start = pColumnInfoData->varmeta.offset[rowIndex - 1]; + int32_t lastDataLen = getDataLen(type, pColumnInfoData->pData + start); + if (start + lastDataLen < pColumnInfoData->varmeta.length) { + uInfo("column data is reassigned, row:%d, offset:%d, length:%d", rowIndex, start, lastDataLen); + pColumnInfoData->varmeta.length = start + lastDataLen; + } + } } SVarColAttr* pAttr = &pColumnInfoData->varmeta; From 62fbb62b9ce036ab47c148f5a162df8c0e482567 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 29 Oct 2024 23:06:57 +0800 Subject: [PATCH 32/59] fix: merge conflict --- source/libs/command/src/command.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 2f44da6c4f..6272ac7049 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -52,8 +52,6 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeBufSize, numOfCols); if(len < 0) { - int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, dataEncodeBufSize, numOfCols); - if (len < 0) { taosMemoryFree(*pRsp); return terrno; } From 0b5f2ec57b8d8bcf8c8632527eccabc185f76285 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 30 Oct 2024 11:11:29 +0800 Subject: [PATCH 33/59] test(blob): testing & fixes for blob --- tests/army/storage/blob/s3Basic.json | 2 +- tests/army/storage/blob/s3Basic1.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/army/storage/blob/s3Basic.json b/tests/army/storage/blob/s3Basic.json index ee341b2096..2b911a989f 100644 --- a/tests/army/storage/blob/s3Basic.json +++ b/tests/army/storage/blob/s3Basic.json @@ -20,7 +20,7 @@ "replica": 1, "duration":"10d", "s3_keeplocal":"30d", - "s3_chunksize":"131072", + "s3_chunkpages":"131072", "tsdb_pagesize":"1", "s3_compact":"1", "wal_retention_size":"1", diff --git a/tests/army/storage/blob/s3Basic1.json b/tests/army/storage/blob/s3Basic1.json index 02be308443..087f89edec 100644 --- a/tests/army/storage/blob/s3Basic1.json +++ b/tests/army/storage/blob/s3Basic1.json @@ -20,7 +20,7 @@ "replica": 1, "duration":"10d", "s3_keeplocal":"30d", - "s3_chunksize":"131072", + "s3_chunkpages":"131072", "tsdb_pagesize":"1", "s3_compact":"1", "wal_retention_size":"1", From 6d4d79a4d3c5724fbcfcc19d392c697196bac7a1 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Wed, 30 Oct 2024 14:04:12 +0800 Subject: [PATCH 34/59] fix: geometry ST_AsText --- source/libs/function/src/builtins.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index e66ddf5197..d91060fb6e 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1533,14 +1533,16 @@ static int32_t translateToJson(SFunctionNode* pFunc, char* pErrBuf, int32_t len) static int32_t translateOutGeom(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { FUNC_ERR_RET(validateParam(pFunc, pErrBuf, len)); - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_GEOMETRY].bytes, .type = TSDB_DATA_TYPE_GEOMETRY}; + SDataType dt = *getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0)); + pFunc->node.resType = (SDataType){.bytes = dt.bytes, .type = TSDB_DATA_TYPE_GEOMETRY}; return TSDB_CODE_SUCCESS; } static int32_t translateInGeomOutStr(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { FUNC_ERR_RET(validateParam(pFunc, pErrBuf, len)); - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_VARCHAR].bytes, .type = TSDB_DATA_TYPE_VARCHAR}; + SDataType dt = *getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0)); + pFunc->node.resType = (SDataType){.bytes = dt.bytes, .type = TSDB_DATA_TYPE_VARCHAR}; return TSDB_CODE_SUCCESS; } From e9ef7a85744d7a84c6008fe4ea7cb192a03297ac Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 30 Oct 2024 14:29:26 +0800 Subject: [PATCH 35/59] rename s3_chunksize to s3_chunkpages --- tests/army/storage/blob/ablob.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/army/storage/blob/ablob.py b/tests/army/storage/blob/ablob.py index fae492a3df..f6e783d6f3 100644 --- a/tests/army/storage/blob/ablob.py +++ b/tests/army/storage/blob/ablob.py @@ -152,13 +152,13 @@ class TDTestCase(TBase): if keepLocal is not None: kw1 = f"s3_keeplocal {keepLocal}" if chunkSize is not None: - kw2 = f"s3_chunksize {chunkSize}" + kw2 = f"s3_chunkpages {chunkSize}" if compact is not None: kw3 = f"s3_compact {compact}" sql = f" create database db1 vgroups 1 duration 1h {kw1} {kw2} {kw3}" tdSql.execute(sql, show=True) - #sql = f"select name,s3_keeplocal,s3_chunksize,s3_compact from information_schema.ins_databases where name='db1';" + #sql = f"select name,s3_keeplocal,s3_chunkpages,s3_compact from information_schema.ins_databases where name='db1';" sql = f"select * from information_schema.ins_databases where name='db1';" tdSql.query(sql) # 29 30 31 -> chunksize keeplocal compact @@ -178,9 +178,9 @@ class TDTestCase(TBase): f"create database db2 s3_keeplocal -1", f"create database db2 s3_keeplocal 0", f"create database db2 s3_keeplocal 365001", - f"create database db2 s3_chunksize -1", - f"create database db2 s3_chunksize 0", - f"create database db2 s3_chunksize 900000000", + f"create database db2 s3_chunkpages -1", + f"create database db2 s3_chunkpages 0", + f"create database db2 s3_chunkpages 900000000", f"create database db2 s3_compact -1", f"create database db2 s3_compact 100", f"create database db2 duration 1d s3_keeplocal 1d" From 73be7ddbc68ef247ff293af7506113d027fe402c Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Wed, 30 Oct 2024 19:38:08 +0800 Subject: [PATCH 36/59] fix: set tablefield of stream block to NULL --- source/dnode/vnode/src/tq/tqUtil.c | 3 +++ source/libs/executor/src/scanoperator.c | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index b4866b8c65..1edc3d42ab 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -626,6 +626,9 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void* tmp = taosArrayGet(pDelBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); TSDB_CHECK_NULL(tmp, code, line, END, terrno) colDataSetNULL(tmp, i); + tmp = taosArrayGet(pDelBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); + TSDB_CHECK_NULL(tmp, code, line, END, terrno) + colDataSetNULL(tmp, i); } if (type == 0) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index f936e95005..918be0465f 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2115,6 +2115,7 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + SColumnInfoData* pDestTableNameInxCol = taosArrayGet(pDestBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { uint64_t groupId = pSrcGp[i]; if (groupId == 0) { @@ -2152,7 +2153,7 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr QUERY_CHECK_CODE(code, lino, _end); colDataSetNULL(pDestCalStartTsCol, i); - colDataSetNULL(pDestCalEndTsCol, i); + colDataSetNULL(pDestTableNameInxCol, i); pDestBlock->info.rows++; } @@ -2205,6 +2206,7 @@ static int32_t generateCountScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcB SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + SColumnInfoData* pDestTableNameInxCol = taosArrayGet(pDestBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { uint64_t groupId = pSrcGp[i]; if (groupId == 0) { @@ -2233,6 +2235,8 @@ static int32_t generateCountScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcB code = colDataSetVal(pDestCalEndTsCol, i, (const char*)&range.win.ekey, false); QUERY_CHECK_CODE(code, lino, _end); + colDataSetNULL(pDestTableNameInxCol, i); + pDestBlock->info.rows++; } @@ -2287,6 +2291,7 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS SColumnInfoData* pGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + SColumnInfoData* pDestTableNameInxCol = taosArrayGet(pDestBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); for (int32_t i = 0; i < pSrcBlock->info.rows;) { uint64_t srcUid = srcUidData[i]; uint64_t groupId = srcGp[i]; @@ -2319,6 +2324,8 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS code = colDataSetVal(pGpCol, pDestBlock->info.rows, (const char*)(&groupId), false); QUERY_CHECK_CODE(code, lino, _end); + colDataSetNULL(pDestTableNameInxCol, pDestBlock->info.rows); + pDestBlock->info.rows++; } @@ -3074,6 +3081,7 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S colDataSetNULL(taosArrayGet(pDst->pDataBlock, GROUPID_COLUMN_INDEX), j); colDataSetNULL(taosArrayGet(pDst->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX), j); colDataSetNULL(taosArrayGet(pDst->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX), j); + colDataSetNULL(taosArrayGet(pDst->pDataBlock, TABLE_NAME_COLUMN_INDEX), j); j++; } } @@ -3670,6 +3678,9 @@ FETCH_NEXT_BLOCK: // printDataBlock(pInfo->pCheckpointRes, "stream scan ck", GET_TASKID(pTaskInfo)); (*ppRes) = pInfo->pCheckpointRes; return code; + } else { + qError("stream scan error, invalid block type %d, %s", pInfo->blockType, id); + code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; } _end: From ad83a9a6509244e1c04ec53de5a4eb2d2d5f9e45 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 31 Oct 2024 11:16:39 +0800 Subject: [PATCH 37/59] fix: reset offset when trimdatablock --- source/common/src/tdatablock.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c185ac9d86..49dd9948f2 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -114,16 +114,7 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const if (IS_VAR_DATA_TYPE(type)) { int32_t dataLen = getDataLen(type, pData); if (pColumnInfoData->varmeta.offset[rowIndex] > 0) { - if (rowIndex == 0) { - pColumnInfoData->varmeta.length = 0; - } else { - int32_t start = pColumnInfoData->varmeta.offset[rowIndex - 1]; - int32_t lastDataLen = getDataLen(type, pColumnInfoData->pData + start); - if (start + lastDataLen < pColumnInfoData->varmeta.length) { - uInfo("column data is reassigned, row:%d, offset:%d, length:%d", rowIndex, start, lastDataLen); - pColumnInfoData->varmeta.length = start + lastDataLen; - } - } + pColumnInfoData->varmeta.length = pColumnInfoData->varmeta.offset[rowIndex]; } SVarColAttr* pAttr = &pColumnInfoData->varmeta; @@ -3391,6 +3382,7 @@ int32_t trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolL } memcpy(p2, p1, len); + pDst->varmeta.offset[numOfRows] = -1; code = colDataSetVal(pDst, numOfRows, p2, false); taosMemoryFree(p2); if (code) { From 974591f2658ce8b53cf360db386ce038df24c208 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 31 Oct 2024 14:23:28 +0800 Subject: [PATCH 38/59] fix: colDataSetOrCover --- include/common/tdatablock.h | 1 + source/common/src/tdatablock.c | 19 +++++++++++++++---- source/libs/executor/src/executorInt.c | 2 +- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 3b24ef9490..6578999db4 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -190,6 +190,7 @@ static FORCE_INLINE void colDataSetDouble(SColumnInfoData* pColumnInfoData, uint int32_t getJsonValueLen(const char* data); int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); +int32_t colDataSetValOrCover(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData); int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, bool trimValue); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 49dd9948f2..94ebcc7033 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -97,8 +97,8 @@ static int32_t getDataLen(int32_t type, const char* pData) { return dataLen; } -int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull) { - if (isNull || pData == NULL) { +static int32_t colDataSetValHelp(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull) { + if (isNull || pData == NULL) { // There is a placehold for each NULL value of binary or nchar type. if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { pColumnInfoData->varmeta.offset[rowIndex] = -1; // it is a null value of VAR type. @@ -143,7 +143,7 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const uint32_t len = pColumnInfoData->varmeta.length; pColumnInfoData->varmeta.offset[rowIndex] = len; - (void) memmove(pColumnInfoData->pData + len, pData, dataLen); + (void)memmove(pColumnInfoData->pData + len, pData, dataLen); pColumnInfoData->varmeta.length += dataLen; } else { memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex, pData, pColumnInfoData->info.bytes); @@ -153,6 +153,18 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const return 0; } +int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull) { + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + pColumnInfoData->varmeta.offset[rowIndex] = -1; + } + + return colDataSetValHelp(pColumnInfoData, rowIndex, pData, isNull); +} + +int32_t colDataSetValOrCover(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull) { + return colDataSetValHelp(pColumnInfoData, rowIndex, pData, isNull); +} + int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData) { int32_t type = pColumnInfoData->info.type; @@ -3382,7 +3394,6 @@ int32_t trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolL } memcpy(p2, p1, len); - pDst->varmeta.offset[numOfRows] = -1; code = colDataSetVal(pDst, numOfRows, p2, false); taosMemoryFree(p2); if (code) { diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index b39cf4014d..1b823bf69d 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -702,7 +702,7 @@ int32_t copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResu QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { - code = colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + code = colDataSetValOrCover(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); QUERY_CHECK_CODE(code, lino, _end); } } From b663adf946bf6d247c5ebd9ee0e363a7ecc2a144 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Thu, 31 Oct 2024 17:33:08 +0800 Subject: [PATCH 39/59] feat: delete report --- include/libs/monitor/clientMonitor.h | 3 ++ source/client/src/clientImpl.c | 2 + source/client/src/clientMonitor.c | 71 ++++++++++++++++++++++++++-- 3 files changed, 73 insertions(+), 3 deletions(-) diff --git a/include/libs/monitor/clientMonitor.h b/include/libs/monitor/clientMonitor.h index 0085173ecd..b09a1ac11c 100644 --- a/include/libs/monitor/clientMonitor.h +++ b/include/libs/monitor/clientMonitor.h @@ -24,6 +24,7 @@ extern "C" { #include "thash.h" #include "query.h" #include "tqueue.h" +#include "clientInt.h" typedef enum { SQL_RESULT_SUCCESS = 0, @@ -81,6 +82,8 @@ void monitorCreateClientCounter(int64_t clusterId, const char* name, void monitorCounterInc(int64_t clusterId, const char* counterName, const char** label_values); const char* monitorResultStr(SQL_RESULT_CODE code); int32_t monitorPutData2MonitorQueue(MonitorSlowLogData data); + +void clientOperateReport(SRequestObj* pRequest); #ifdef __cplusplus } #endif diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 2c67cafdf5..13ca874c47 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2945,6 +2945,8 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t s tscDebug("taos_query end with sql:%s", sql); + clientOperateReport(pRequest); + return pRequest; } diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 595c871953..0624d8d94d 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -2,8 +2,6 @@ #include "cJSON.h" #include "clientInt.h" #include "clientLog.h" -#include "os.h" -#include "tglobal.h" #include "tmisce.h" #include "tqueue.h" #include "ttime.h" @@ -932,4 +930,71 @@ int32_t monitorPutData2MonitorQueue(MonitorSlowLogData data) { taosFreeQitem(slowLogData); } return 0; -} \ No newline at end of file +} + +int32_t reportCB(void* param, SDataBuf* pMsg, int32_t code) { + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); + tscDebug("[del report]delete reportCB code:%d", code); + return 0; +} + +void clientOperateReport(SRequestObj* pRequest) { + if (pRequest == NULL || pRequest->pQuery == NULL) { + tscError("[del report]invalid request"); + return; + } + + if (QUERY_NODE_DELETE_STMT == nodeType(pRequest->pQuery->pRoot)) { + SDeleteStmt* pStmt = (SDeleteStmt*)pRequest->pQuery->pRoot; + STscObj* pTscObj = pRequest->pTscObj; + + if(nodeType(pStmt->pFromTable) != QUERY_NODE_REAL_TABLE) { + tscError("[del report]invalid from table node type:%d", nodeType(pStmt->pFromTable)); + return; + } + SRealTableNode* pTable = (SRealTableNode*)pStmt->pFromTable; + SAuditReq req; + req.pSql = pRequest->sqlstr; + req.sqlLen = pRequest->sqlLen; + tsnprintf(req.table, TSDB_TABLE_NAME_LEN, "%s", pTable->table.tableName); + tsnprintf(req.db, TSDB_DB_FNAME_LEN, "%s", pTable->table.dbName); + tsnprintf(req.operation, AUDIT_OPERATION_LEN, "delete"); + int32_t tlen = tSerializeSAuditReq(NULL, 0, &req); + void* pReq = taosMemoryCalloc(1, tlen); + if (pReq == NULL) { + tscError("[del report]failed to allocate memory for req"); + return; + } + + if (tSerializeSAuditReq(pReq, tlen, &req) < 0) { + tscError("[del report]failed to serialize req"); + taosMemoryFree(pReq); + return; + } + + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + tscError("[del report]failed to allocate memory for sendInfo"); + taosMemoryFree(pReq); + return; + } + + sendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = tlen, .handle = NULL}; + + sendInfo->requestId = generateRequestId(); + sendInfo->requestObjRefId = 0; + sendInfo->param = NULL; + sendInfo->fp = reportCB; + sendInfo->msgType = TDMT_MND_AUDIT; + + SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + + int32_t code = asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); + if (code != 0) { + tscError("[del report]failed to send msg to server, code:%d", code); + taosMemoryFree(sendInfo); + } + tscDebug("[del report]delete data, sql:%s", req.pSql); + } +} From 6010e52a2389a77897a0bf474066973baf0f40ac Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 31 Oct 2024 17:42:06 +0800 Subject: [PATCH 40/59] blob/param: new cases for default value checking --- tests/army/storage/blob/ablob.py | 39 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/tests/army/storage/blob/ablob.py b/tests/army/storage/blob/ablob.py index f6e783d6f3..d3e00f3424 100644 --- a/tests/army/storage/blob/ablob.py +++ b/tests/army/storage/blob/ablob.py @@ -31,8 +31,6 @@ from frame.eos import * class TDTestCase(TBase): - index = eutil.cpuRand(20) + 1 - bucketName = f"ci-bucket{index}" updatecfgDict = { "supportVnodes":"1000", 's3EndPoint': 'https://.blob.core.windows.net', @@ -44,7 +42,6 @@ class TDTestCase(TBase): 's3MigrateEnabled': '1' } - tdLog.info(f"assign bucketName is {bucketName}\n") maxFileSize = (128 + 10) * 1014 * 1024 # add 10M buffer def insertData(self): @@ -172,6 +169,23 @@ class TDTestCase(TBase): sql = "drop database db1" tdSql.execute(sql) + def checkDefault(self, keepLocal, chunkSize, compact): + sql = f" create database db1 vgroups 1" + tdSql.execute(sql, show=True) + #sql = f"select name,s3_keeplocal,s3_chunkpages,s3_compact from information_schema.ins_databases where name='db1';" + sql = f"select * from information_schema.ins_databases where name='db1';" + tdSql.query(sql) + # 29 30 31 -> chunksize keeplocal compact + if chunkSize is not None: + tdSql.checkData(0, 29, chunkSize) + if keepLocal is not None: + keepLocalm = keepLocal * 24 * 60 + tdSql.checkData(0, 30, f"{keepLocalm}m") + if compact is not None: + tdSql.checkData(0, 31, compact) + sql = "drop database db1" + tdSql.execute(sql) + def checkExcept(self): # errors sqls = [ @@ -226,16 +240,7 @@ class TDTestCase(TBase): # except self.checkExcept() - - # - def preDb(self, vgroups): - cnt = int(time.time())%2 + 1 - for i in range(cnt): - vg = eutil.cpuRand(9) + 1 - sql = f"create database predb vgroups {vg}" - tdSql.execute(sql, show=True) - sql = "drop database predb" - tdSql.execute(sql, show=True) + self.checkDefault(365, 131072, 1) # history def insertHistory(self): @@ -287,9 +292,6 @@ class TDTestCase(TBase): if eos.isArm64Cpu(): tdLog.success(f"{__file__} arm64 ignore executed") else: - - self.preDb(10) - # insert data self.insertData() @@ -311,7 +313,6 @@ class TDTestCase(TBase): # check insert correct again self.checkInsertCorrect() - # check stream correct and drop stream #self.checkStreamCorrect() @@ -321,7 +322,7 @@ class TDTestCase(TBase): # insert history disorder data self.insertHistory() - # checkBasic + # check db params self.checkBasic() #self.checkInsertCorrect() @@ -335,10 +336,8 @@ class TDTestCase(TBase): # drop database and free s3 file self.dropDb() - tdLog.success(f"{__file__} successfully executed") - tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) From 8c3e4ce63daac2a6140d05f56a97a16a197423a0 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 1 Nov 2024 09:48:11 +0800 Subject: [PATCH 41/59] enh: set default safetyCheckLevel to normal --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 1aef08bc7e..b57db2a84a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -139,7 +139,7 @@ bool tsEnableCrashReport = true; #endif char *tsClientCrashReportUri = "/ccrashreport"; char *tsSvrCrashReportUri = "/dcrashreport"; -int8_t tsSafetyCheckLevel = TSDB_SAFETY_CHECK_LEVELL_NEVER; +int8_t tsSafetyCheckLevel = TSDB_SAFETY_CHECK_LEVELL_NORMAL; // schemaless bool tsSmlDot2Underline = true; From 08267fec5ceb1a7149b81912f6669220c52119f0 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 1 Nov 2024 11:40:53 +0800 Subject: [PATCH 42/59] feat: delete report --- source/client/src/clientImpl.c | 3 +- source/client/src/clientMonitor.c | 120 +++++++++++++++++------------- 2 files changed, 71 insertions(+), 52 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 13ca874c47..04c536a2d0 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2836,6 +2836,7 @@ void syncQueryFn(void* param, void* res, int32_t code) { if (pParam->pRequest) { pParam->pRequest->code = code; + clientOperateReport(pParam->pRequest); } if (TSDB_CODE_SUCCESS != tsem_post(&pParam->sem)) { @@ -2945,8 +2946,6 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t s tscDebug("taos_query end with sql:%s", sql); - clientOperateReport(pRequest); - return pRequest; } diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 0624d8d94d..3469543e4d 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -939,6 +939,75 @@ int32_t reportCB(void* param, SDataBuf* pMsg, int32_t code) { return 0; } +int32_t senAuditInfo(STscObj* pTscObj, void* pReq, int32_t len) { + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + tscError("[del report]failed to allocate memory for sendInfo"); + return terrno; + } + + sendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = len, .handle = NULL}; + + sendInfo->requestId = generateRequestId(); + sendInfo->requestObjRefId = 0; + sendInfo->param = NULL; + sendInfo->fp = reportCB; + sendInfo->msgType = TDMT_MND_AUDIT; + + SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + + int32_t code = asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); + if (code != 0) { + tscError("[del report]failed to send msg to server, code:%d", code); + taosMemoryFree(sendInfo); + return code; + } + return TSDB_CODE_SUCCESS; +} + +static void reportDeleteSql(SRequestObj* pRequest) { + SDeleteStmt* pStmt = (SDeleteStmt*)pRequest->pQuery->pRoot; + STscObj* pTscObj = pRequest->pTscObj; + + if (pRequest->code != TSDB_CODE_SUCCESS) { + tscDebug("[del report]delete request result code:%d", pRequest->code); + return; + } + + if (nodeType(pStmt->pFromTable) != QUERY_NODE_REAL_TABLE) { + tscError("[del report]invalid from table node type:%d", nodeType(pStmt->pFromTable)); + return; + } + + SRealTableNode* pTable = (SRealTableNode*)pStmt->pFromTable; + SAuditReq req; + req.pSql = pRequest->sqlstr; + req.sqlLen = pRequest->sqlLen; + tsnprintf(req.table, TSDB_TABLE_NAME_LEN, "%s", pTable->table.tableName); + tsnprintf(req.db, TSDB_DB_FNAME_LEN, "%s", pTable->table.dbName); + tsnprintf(req.operation, AUDIT_OPERATION_LEN, "delete"); + int32_t tlen = tSerializeSAuditReq(NULL, 0, &req); + void* pReq = taosMemoryCalloc(1, tlen); + if (pReq == NULL) { + tscError("[del report]failed to allocate memory for req"); + return; + } + + if (tSerializeSAuditReq(pReq, tlen, &req) < 0) { + tscError("[del report]failed to serialize req"); + taosMemoryFree(pReq); + return; + } + + int32_t code = senAuditInfo(pRequest->pTscObj, pReq, tlen); + if (code != 0) { + tscError("[del report]failed to send audit info, code:%d", code); + taosMemoryFree(pReq); + return; + } + tscDebug("[del report]delete data, sql:%s", req.pSql); +} + void clientOperateReport(SRequestObj* pRequest) { if (pRequest == NULL || pRequest->pQuery == NULL) { tscError("[del report]invalid request"); @@ -946,55 +1015,6 @@ void clientOperateReport(SRequestObj* pRequest) { } if (QUERY_NODE_DELETE_STMT == nodeType(pRequest->pQuery->pRoot)) { - SDeleteStmt* pStmt = (SDeleteStmt*)pRequest->pQuery->pRoot; - STscObj* pTscObj = pRequest->pTscObj; - - if(nodeType(pStmt->pFromTable) != QUERY_NODE_REAL_TABLE) { - tscError("[del report]invalid from table node type:%d", nodeType(pStmt->pFromTable)); - return; - } - SRealTableNode* pTable = (SRealTableNode*)pStmt->pFromTable; - SAuditReq req; - req.pSql = pRequest->sqlstr; - req.sqlLen = pRequest->sqlLen; - tsnprintf(req.table, TSDB_TABLE_NAME_LEN, "%s", pTable->table.tableName); - tsnprintf(req.db, TSDB_DB_FNAME_LEN, "%s", pTable->table.dbName); - tsnprintf(req.operation, AUDIT_OPERATION_LEN, "delete"); - int32_t tlen = tSerializeSAuditReq(NULL, 0, &req); - void* pReq = taosMemoryCalloc(1, tlen); - if (pReq == NULL) { - tscError("[del report]failed to allocate memory for req"); - return; - } - - if (tSerializeSAuditReq(pReq, tlen, &req) < 0) { - tscError("[del report]failed to serialize req"); - taosMemoryFree(pReq); - return; - } - - SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (sendInfo == NULL) { - tscError("[del report]failed to allocate memory for sendInfo"); - taosMemoryFree(pReq); - return; - } - - sendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = tlen, .handle = NULL}; - - sendInfo->requestId = generateRequestId(); - sendInfo->requestObjRefId = 0; - sendInfo->param = NULL; - sendInfo->fp = reportCB; - sendInfo->msgType = TDMT_MND_AUDIT; - - SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - - int32_t code = asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); - if (code != 0) { - tscError("[del report]failed to send msg to server, code:%d", code); - taosMemoryFree(sendInfo); - } - tscDebug("[del report]delete data, sql:%s", req.pSql); + reportDeleteSql(pRequest); } } From 6f68677b7ea8dee9cf3e818ec407d05c59a244b4 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Fri, 1 Nov 2024 13:40:00 +0800 Subject: [PATCH 43/59] enh: delete report switch --- source/client/src/clientMonitor.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 3469543e4d..44ed262880 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -17,6 +17,7 @@ STaosQueue* monitorQueue; SHashObj* monitorSlowLogHash; char tmpSlowLogPath[PATH_MAX] = {0}; TdThread monitorThread; +extern bool tsEnableAuditDelete; static int32_t getSlowLogTmpDir(char* tmpPath, int32_t size) { int ret = tsnprintf(tmpPath, size, "%s/tdengine_slow_log/", tsTempDir); @@ -1014,7 +1015,7 @@ void clientOperateReport(SRequestObj* pRequest) { return; } - if (QUERY_NODE_DELETE_STMT == nodeType(pRequest->pQuery->pRoot)) { + if (tsEnableAuditDelete && QUERY_NODE_DELETE_STMT == nodeType(pRequest->pQuery->pRoot)) { reportDeleteSql(pRequest); } } From 39a0916b803444155742c4a764acaaf5dba6c675 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 1 Nov 2024 15:38:33 +0800 Subject: [PATCH 44/59] fix: return value --- source/client/src/clientMonitor.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 44ed262880..3b19e070b8 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -984,9 +984,9 @@ static void reportDeleteSql(SRequestObj* pRequest) { SAuditReq req; req.pSql = pRequest->sqlstr; req.sqlLen = pRequest->sqlLen; - tsnprintf(req.table, TSDB_TABLE_NAME_LEN, "%s", pTable->table.tableName); - tsnprintf(req.db, TSDB_DB_FNAME_LEN, "%s", pTable->table.dbName); - tsnprintf(req.operation, AUDIT_OPERATION_LEN, "delete"); + TAOS_UNUSED(tsnprintf(req.table, TSDB_TABLE_NAME_LEN, "%s", pTable->table.tableName)); + TAOS_UNUSED(tsnprintf(req.db, TSDB_DB_FNAME_LEN, "%s", pTable->table.dbName)); + TAOS_UNUSED(tsnprintf(req.operation, AUDIT_OPERATION_LEN, "delete")); int32_t tlen = tSerializeSAuditReq(NULL, 0, &req); void* pReq = taosMemoryCalloc(1, tlen); if (pReq == NULL) { From 7041e6474493a184a2abb5c56a6ce5719adcd44d Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Sun, 3 Nov 2024 23:00:31 +0800 Subject: [PATCH 45/59] add colDataSetVal func desc --- include/common/tdatablock.h | 4 +++ source/libs/executor/src/dataDispatcher.c | 39 +++++++++++++++++++---- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 6578999db4..1103b89ccb 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -189,7 +189,11 @@ static FORCE_INLINE void colDataSetDouble(SColumnInfoData* pColumnInfoData, uint int32_t getJsonValueLen(const char* data); +// For the VAR_DATA_TYPE type, new data is inserted strictly according to the position of SVarColAttr.length. +// If the same row is inserted repeatedly, data holes will result. int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); +// For the VAR_DATA_TYPE type, if a row already has data before inserting it (judged by offset != -1), +// it will be inserted at the original position and the old data will be overwritten. int32_t colDataSetValOrCover(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData); int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 236d6a4d3e..48f4ed3ed1 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -70,21 +70,15 @@ static int32_t inputSafetyCheck(SDataDispatchHandle* pHandle, const SInputData* SNode* pNode; int32_t numOfCols = 0; - int32_t realOutputRowSize = 0; FOREACH(pNode, pHandle->pSchema->pSlots) { SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode; if (pSlotDesc->output) { - realOutputRowSize += pSlotDesc->dataType.bytes; ++numOfCols; } else { // Slots must be sorted, and slots with 'output' set to true must come first break; } } - if (realOutputRowSize != pSchema->outputRowSize) { - qError("invalid schema, realOutputRowSize:%d, outputRowSize:%d", realOutputRowSize, pSchema->outputRowSize); - return TSDB_CODE_QRY_INVALID_INPUT; - } if (numOfCols > taosArrayGetSize(pInput->pData->pDataBlock)) { qError("invalid column number, schema:%d, input:%zu", numOfCols, taosArrayGetSize(pInput->pData->pDataBlock)); @@ -397,8 +391,41 @@ static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) { return TSDB_CODE_SUCCESS; } +static int32_t blockDescNodeCheck(SDataBlockDescNode* pInputDataBlockDesc) { + if(tsSafetyCheckLevel == TSDB_SAFETY_CHECK_LEVELL_NEVER) { + return TSDB_CODE_SUCCESS; + } + + if (pInputDataBlockDesc == NULL) { + qError("invalid schema"); + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SNode* pNode; + int32_t realOutputRowSize = 0; + FOREACH(pNode, pInputDataBlockDesc->pSlots) { + SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode; + if (pSlotDesc->output) { + realOutputRowSize += pSlotDesc->dataType.bytes; + } else { + // Slots must be sorted, and slots with 'output' set to true must come first + break; + } + } + if (realOutputRowSize != pInputDataBlockDesc->outputRowSize) { + qError("invalid schema, realOutputRowSize:%d, outputRowSize:%d", realOutputRowSize, pInputDataBlockDesc->outputRowSize); + return TSDB_CODE_QRY_INVALID_INPUT; + } + return TSDB_CODE_SUCCESS; +} + int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) { int32_t code; + code = blockDescNodeCheck(pDataSink->pInputDataBlockDesc); + if (code) { + qError("failed to check input data block desc, code:%d", code); + return code; + } SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle)); if (NULL == dispatcher) { From 0f535e06aaec3b3f97ed8e7113a3f46e475cf562 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 4 Nov 2024 11:12:18 +0800 Subject: [PATCH 46/59] feat/TS-5484-audit-delete-fix-review --- source/common/src/tmsg.c | 13 +++++++++++++ source/dnode/mnode/impl/src/mndDnode.c | 3 +++ 2 files changed, 16 insertions(+) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8b55f2c4e9..458badc764 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -567,6 +567,7 @@ int32_t tSerializeSClientHbBatchRsp(void *buf, int32_t bufLen, const SClientHbBa TAOS_CHECK_EXIT(tSerializeSClientHbRsp(&encoder, pRsp)); } TAOS_CHECK_EXIT(tSerializeSMonitorParas(&encoder, &pBatchRsp->monitorParas)); + TAOS_CHECK_EXIT(tEncodeI8(&encoder, pBatchRsp->enableAuditDelete)); tEndEncode(&encoder); _exit: @@ -609,6 +610,12 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR TAOS_CHECK_EXIT(tDeserializeSMonitorParas(&decoder, &pBatchRsp->monitorParas)); } + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pBatchRsp->enableAuditDelete)); + } else { + pBatchRsp->enableAuditDelete = 0; + } + tEndDecode(&decoder); _exit: @@ -6348,6 +6355,7 @@ int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->authVer)); TAOS_CHECK_EXIT(tEncodeI64(&encoder, pRsp->whiteListVer)); TAOS_CHECK_EXIT(tSerializeSMonitorParas(&encoder, &pRsp->monitorParas)); + TAOS_CHECK_EXIT(tEncodeI8(&encoder, pRsp->enableAuditDelete)); tEndEncode(&encoder); _exit: @@ -6399,6 +6407,11 @@ int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (!tDecodeIsEnd(&decoder)) { TAOS_CHECK_EXIT(tDeserializeSMonitorParas(&decoder, &pRsp->monitorParas)); } + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pRsp->enableAuditDelete)); + } else { + pRsp->enableAuditDelete = 0; + } tEndDecode(&decoder); _exit: diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 08e9886df4..24ae8382f9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -607,12 +607,15 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { } static int32_t mndProcessAuditReq(SRpcMsg *pReq) { + mTrace("process audit req:%p", pReq); if (tsEnableAudit && tsEnableAuditDelete) { SMnode *pMnode = pReq->info.node; SAuditReq auditReq = {0}; TAOS_CHECK_RETURN(tDeserializeSAuditReq(pReq->pCont, pReq->contLen, &auditReq)); + mDebug("received audit req:%s, %s, %s, %s", auditReq.operation, auditReq.db, auditReq.table, auditReq.pSql); + auditAddRecord(pReq, pMnode->clusterId, auditReq.operation, auditReq.db, auditReq.table, auditReq.pSql, auditReq.sqlLen); From 296b2a7b8e9f586ed509f5f54b1b16d291274ac6 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Mon, 4 Nov 2024 12:56:12 +0800 Subject: [PATCH 47/59] cfg: enableAuditDelete --- source/client/inc/clientInt.h | 6 +++++- source/client/src/clientEnv.c | 20 ++++++++++---------- source/client/src/clientHb.c | 3 ++- source/client/src/clientMonitor.c | 20 +++++++++++++++----- source/client/src/clientMsgHandler.c | 3 ++- source/client/src/clientSml.c | 2 +- 6 files changed, 35 insertions(+), 19 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 114bc00125..7aefdf5435 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -108,6 +108,10 @@ typedef struct SQueryExecMetric { int64_t execCostUs; } SQueryExecMetric; +typedef struct { + SMonitorParas monitorParas; + int8_t enableAuditDelete; +} SAppInstServerCFG; struct SAppInstInfo { int64_t numOfConns; SCorEpSet mgmtEp; @@ -121,7 +125,7 @@ struct SAppInstInfo { void* pTransporter; SAppHbMgr* pAppHbMgr; char* instKey; - SMonitorParas monitorParas; + SAppInstServerCFG serverCfg; }; typedef struct SAppInfo { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index f892575f0a..d759abe293 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -166,11 +166,11 @@ static int32_t generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "type", cJSON_CreateNumber(reqType))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject( json, "rows_num", cJSON_CreateNumber(pRequest->body.resInfo.numOfRows + pRequest->body.resInfo.totalRows))); - if (pRequest->sqlstr != NULL && strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen) { - char tmp = pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen]; - pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = '\0'; + if (pRequest->sqlstr != NULL && strlen(pRequest->sqlstr) > pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogMaxLen) { + char tmp = pRequest->sqlstr[pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogMaxLen]; + pRequest->sqlstr[pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogMaxLen] = '\0'; ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr))); - pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = tmp; + pRequest->sqlstr[pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogMaxLen] = tmp; } else { ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr))); } @@ -284,7 +284,7 @@ static void deregisterRequest(SRequestObj *pRequest) { } } - if (pTscObj->pAppInfo->monitorParas.tsEnableMonitor) { + if (pTscObj->pAppInfo->serverCfg.monitorParas.tsEnableMonitor) { if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) { sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT); } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { @@ -294,15 +294,15 @@ static void deregisterRequest(SRequestObj *pRequest) { } } - if ((duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThreshold * 1000000UL || - duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThresholdTest * 1000000UL) && - checkSlowLogExceptDb(pRequest, pTscObj->pAppInfo->monitorParas.tsSlowLogExceptDb)) { + if ((duration >= pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogThreshold * 1000000UL || + duration >= pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogThresholdTest * 1000000UL) && + checkSlowLogExceptDb(pRequest, pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogExceptDb)) { (void)atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); - if (pTscObj->pAppInfo->monitorParas.tsSlowLogScope & reqType) { + if (pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogScope & reqType) { taosPrintSlowLog("PID:%d, Conn:%u,QID:0x%" PRIx64 ", Start:%" PRId64 " us, Duration:%" PRId64 "us, SQL:%s", taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration, pRequest->sqlstr); - if (pTscObj->pAppInfo->monitorParas.tsEnableMonitor) { + if (pTscObj->pAppInfo->serverCfg.monitorParas.tsEnableMonitor) { slowQueryLog(pTscObj->id, pRequest->killed, pRequest->code, duration); if (TSDB_CODE_SUCCESS != generateWriteSlowLog(pTscObj, pRequest, reqType, duration)) { tscError("failed to generate write slow log"); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 75716d0bd2..07be4bb596 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -605,7 +605,8 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { return code; } - pInst->monitorParas = pRsp.monitorParas; + pInst->serverCfg.monitorParas = pRsp.monitorParas; + pInst->serverCfg.enableAuditDelete = pRsp.enableAuditDelete; tscDebug("[monitor] paras from hb, clusterId:%" PRIx64 " monitorParas threshold:%d scope:%d", pInst->clusterId, pRsp.monitorParas.tsSlowLogThreshold, pRsp.monitorParas.tsSlowLogScope); diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 3b19e070b8..901f4ceef8 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -215,7 +215,7 @@ static void reportSendProcess(void* param, void* tmrId) { SEpSet ep = getEpSet_s(&pInst->mgmtEp); generateClusterReport(pMonitor->registry, pInst->pTransporter, &ep); bool reset = - taosTmrReset(reportSendProcess, pInst->monitorParas.tsMonitorInterval * 1000, param, monitorTimer, &tmrId); + taosTmrReset(reportSendProcess, pInst->serverCfg.monitorParas.tsMonitorInterval * 1000, param, monitorTimer, &tmrId); tscDebug("reset timer, pMonitor:%p, %d", pMonitor, reset); taosRUnLockLatch(&monitorLock); } @@ -288,7 +288,7 @@ void monitorCreateClient(int64_t clusterId) { goto fail; } pMonitor->timer = - taosTmrStart(reportSendProcess, pInst->monitorParas.tsMonitorInterval * 1000, (void*)pMonitor, monitorTimer); + taosTmrStart(reportSendProcess, pInst->serverCfg.monitorParas.tsMonitorInterval * 1000, (void*)pMonitor, monitorTimer); if (pMonitor->timer == NULL) { tscError("failed to start timer"); goto fail; @@ -659,7 +659,7 @@ static void monitorSendAllSlowLog() { taosHashCancelIterate(monitorSlowLogHash, pIter); return; } - if (t - pClient->lastCheckTime > pInst->monitorParas.tsMonitorInterval * 1000) { + if (t - pClient->lastCheckTime > pInst->serverCfg.monitorParas.tsMonitorInterval * 1000) { pClient->lastCheckTime = t; } else { continue; @@ -685,7 +685,7 @@ static void monitorSendAllSlowLog() { static void monitorSendAllSlowLogFromTempDir(int64_t clusterId) { SAppInstInfo* pInst = getAppInstByClusterId((int64_t)clusterId); - if (pInst == NULL || !pInst->monitorParas.tsEnableMonitor) { + if (pInst == NULL || !pInst->serverCfg.monitorParas.tsEnableMonitor) { tscInfo("[monitor] monitor is disabled, skip send slow log"); return; } @@ -970,6 +970,16 @@ static void reportDeleteSql(SRequestObj* pRequest) { SDeleteStmt* pStmt = (SDeleteStmt*)pRequest->pQuery->pRoot; STscObj* pTscObj = pRequest->pTscObj; + if (pTscObj == NULL || pTscObj->pAppInfo == NULL) { + tscError("[del report]invalid tsc obj"); + return; + } + + if(pTscObj->pAppInfo->serverCfg.enableAuditDelete == 0) { + tscDebug("[del report]audit delete is disabled"); + return; + } + if (pRequest->code != TSDB_CODE_SUCCESS) { tscDebug("[del report]delete request result code:%d", pRequest->code); return; @@ -1015,7 +1025,7 @@ void clientOperateReport(SRequestObj* pRequest) { return; } - if (tsEnableAuditDelete && QUERY_NODE_DELETE_STMT == nodeType(pRequest->pQuery->pRoot)) { + if (QUERY_NODE_DELETE_STMT == nodeType(pRequest->pQuery->pRoot)) { reportDeleteSql(pRequest); } } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index aef3cef1c5..34007c2be7 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -135,7 +135,8 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { // update the appInstInfo pTscObj->pAppInfo->clusterId = connectRsp.clusterId; - pTscObj->pAppInfo->monitorParas = connectRsp.monitorParas; + pTscObj->pAppInfo->serverCfg.monitorParas = connectRsp.monitorParas; + pTscObj->pAppInfo->serverCfg.enableAuditDelete = connectRsp.enableAuditDelete; tscDebug("[monitor] paras from connect rsp, clusterId:%" PRIx64 " monitorParas threshold:%d scope:%d", connectRsp.clusterId, connectRsp.monitorParas.tsSlowLogThreshold, connectRsp.monitorParas.tsSlowLogScope); lastClusterId = connectRsp.clusterId; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index e8221c8b8d..0f649f99ff 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1695,7 +1695,7 @@ END: } void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd) { - if (request->pTscObj->pAppInfo->monitorParas.tsSlowLogScope & SLOW_LOG_TYPE_INSERT) { + if (request->pTscObj->pAppInfo->serverCfg.monitorParas.tsSlowLogScope & SLOW_LOG_TYPE_INSERT) { int32_t len = 0; int32_t rlen = 0; char *p = NULL; From 71e27240cb3be095f5e6abf9f7e2d00722cfa7f3 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 4 Nov 2024 14:00:27 +0800 Subject: [PATCH 48/59] stream/rocks: fix meta file dir --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index b82d06b6c7..5923efbac9 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -5065,7 +5065,7 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { goto _ERROR; } memset(dstBuf, 0, cap); - nBytes = snprintf(dstDir, cap, "%s%s%s", dstDir, TD_DIRSEP, chkpMeta); + nBytes = snprintf(dstBuf, cap, "%s%s%s", dstDir, TD_DIRSEP, chkpMeta); if (nBytes <= 0 || nBytes >= cap) { code = TSDB_CODE_OUT_OF_RANGE; goto _ERROR; From 175a3f7d57b887ab414f4108577fec6e619ad017 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 4 Nov 2024 14:22:28 +0800 Subject: [PATCH 49/59] strea/checkpoint: fix meta file path --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 5923efbac9..b69e191059 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -5071,7 +5071,7 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname, SArray* list) { goto _ERROR; } - TdFilePtr pFile = taosOpenFile(dstDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + TdFilePtr pFile = taosOpenFile(dstBuf, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { code = terrno; stError("chkp failed to create meta file: %s, reason:%s", dstDir, tstrerror(code)); From 51e9f93ae68ebfbfd0f758ec16c9bce2131d0041 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 4 Nov 2024 18:44:50 +0800 Subject: [PATCH 50/59] requestid --- source/client/src/clientMonitor.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientMonitor.c b/source/client/src/clientMonitor.c index 901f4ceef8..5f7e11b6a3 100644 --- a/source/client/src/clientMonitor.c +++ b/source/client/src/clientMonitor.c @@ -940,7 +940,7 @@ int32_t reportCB(void* param, SDataBuf* pMsg, int32_t code) { return 0; } -int32_t senAuditInfo(STscObj* pTscObj, void* pReq, int32_t len) { +int32_t senAuditInfo(STscObj* pTscObj, void* pReq, int32_t len, uint64_t requestId) { SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { tscError("[del report]failed to allocate memory for sendInfo"); @@ -949,7 +949,7 @@ int32_t senAuditInfo(STscObj* pTscObj, void* pReq, int32_t len) { sendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = len, .handle = NULL}; - sendInfo->requestId = generateRequestId(); + sendInfo->requestId = requestId; sendInfo->requestObjRefId = 0; sendInfo->param = NULL; sendInfo->fp = reportCB; @@ -1010,7 +1010,7 @@ static void reportDeleteSql(SRequestObj* pRequest) { return; } - int32_t code = senAuditInfo(pRequest->pTscObj, pReq, tlen); + int32_t code = senAuditInfo(pRequest->pTscObj, pReq, tlen, pRequest->requestId); if (code != 0) { tscError("[del report]failed to send audit info, code:%d", code); taosMemoryFree(pReq); @@ -1020,7 +1020,7 @@ static void reportDeleteSql(SRequestObj* pRequest) { } void clientOperateReport(SRequestObj* pRequest) { - if (pRequest == NULL || pRequest->pQuery == NULL) { + if (pRequest == NULL || pRequest->pQuery == NULL || pRequest->pQuery->pRoot == NULL) { tscError("[del report]invalid request"); return; } From 109c1a799f5a58f16582f783fca7ee4573c70a1f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 4 Nov 2024 18:49:36 +0800 Subject: [PATCH 51/59] test/blob: perf json file for taos benchmark --- tests/army/storage/blob/perf.json | 67 +++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 tests/army/storage/blob/perf.json diff --git a/tests/army/storage/blob/perf.json b/tests/army/storage/blob/perf.json new file mode 100644 index 0000000000..002515873e --- /dev/null +++ b/tests/army/storage/blob/perf.json @@ -0,0 +1,67 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 4000, + "prepared_rand": 500, + "thread_count": 4, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 3, + "replica": 3, + "duration":"10d", + "s3_keeplocal":"30d", + "s3_chunkpages":"131072", + "tsdb_pagesize":"1", + "s3_compact":"1", + "wal_retention_size":"1", + "wal_retention_period":"1", + "flush_each_batch":"no", + "keep": "3650d" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 500, + "insert_rows": 200000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 100, + "start_timestamp": 1600000000000, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" ,"max": 1,"min": 1}, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 50}, + { "type": "nchar", "name": "nch", "len": 100} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} + From d572f9e38779047463ebb35bdf1d4f45b9541af8 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 4 Nov 2024 19:21:27 +0800 Subject: [PATCH 52/59] blob/doc: connect blob directly without flexify --- docs/zh/08-operation/12-multi.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/zh/08-operation/12-multi.md b/docs/zh/08-operation/12-multi.md index 5489226ce1..1e81a7ff1e 100644 --- a/docs/zh/08-operation/12-multi.md +++ b/docs/zh/08-operation/12-multi.md @@ -163,3 +163,15 @@ s3BucketName td-test - 认为全部 S3 服务均指向同一数据源，对各个 S3 服务操作完全等价 - 在某一 S3 服务上操作失败后会切换至其他服务，全部服务都失败后将返回最后产生的错误码 - 最大支持的 S3 服务配置数为 10 + +### 不依赖 Flexify 服务 + +用户界面同 S3，不同的地方在于下面三个参数的配置： + +| # | 参数 | 示例值 | 描述 | +| :--- | :----------- | :--------------------------------------- | :----------------------------------------------------------- | +| 1 | s3EndPoint | https://fd2d01c73.blob.core.windows.net | Blob URL | +| 2 | s3AccessKey | fd2d01c73:veUy/iRBeWaI2YAerl+AStw6PPqg== | 冒号分隔的用户 accountId:accountKey | +| 3 | s3BucketName | test-container | Container name | + +其中 fd2d01c73 是账户 ID；微软 Blob 存储服务只支持 Https 协议，不支持 Http。 From 90d1e014db7dbcbddfa0d6d303dbbede7f266638 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 5 Nov 2024 14:25:24 +0800 Subject: [PATCH 53/59] az/begin: remove duplicate begin & end of empty impl. --- source/libs/azure/src/az.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index 5f95624c94..9a1f95b142 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -22,6 +22,10 @@ #include "taoserror.h" #include "tglobal.h" +int32_t azBegin() { return TSDB_CODE_SUCCESS; } + +void azEnd() {} + #if defined(USE_S3) #include @@ -40,10 +44,6 @@ extern char tsS3BucketName[TSDB_FQDN_LEN]; extern int8_t tsS3Enabled; extern int8_t tsS3EpNum; -int32_t azBegin() { return TSDB_CODE_SUCCESS; } - -void azEnd() {} - static void checkPrint(const char *fmt, ...) { va_list arg_ptr; va_start(arg_ptr, fmt); @@ -524,10 +524,6 @@ int32_t azDeleteObjects(const char *object_name[], int nobject) { #else -int32_t azBegin() { return TSDB_CODE_SUCCESS; } - -void azEnd() {} - int32_t azCheckCfg() { return TSDB_CODE_SUCCESS; } int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { From 7316b339b6fa3b17a45120402ad12cb551b03f8d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 5 Nov 2024 14:34:20 +0800 Subject: [PATCH 54/59] az/get object by prefix: catch all cpp eceptions --- source/libs/azure/src/az.cpp | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index 9a1f95b142..1380b58bbd 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -32,7 +32,6 @@ void azEnd() {} #include #include "td_block_blob_client.hpp" -// Add appropriate using namespace directives using namespace Azure::Storage; using namespace Azure::Storage::Blobs; @@ -223,7 +222,6 @@ static int32_t azPutObjectFromFileOffsetImpl(const char *file, const char *objec uint8_t blobContent[] = "Hello Azure!"; // Create the block blob client // BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blobName); - // TDBlockBlobClient blobClient(containerClient.GetBlobClient(blobName)); TDBlockBlobClient blobClient(containerClient.GetBlobClient(object_name)); blobClient.UploadFrom(file, offset, size); @@ -467,7 +465,7 @@ int32_t azGetObjectToFile(const char *object_name, const char *fileName) { TAOS_RETURN(code); } -int32_t azGetObjectsByPrefix(const char *prefix, const char *path) { +static int32_t azGetObjectsByPrefixImpl(const char *prefix, const char *path) { const std::string delimiter = "/"; std::string accountName = tsS3AccessKeyId[0]; std::string accountKey = tsS3AccessKeySecret[0]; @@ -514,6 +512,23 @@ int32_t azGetObjectsByPrefix(const char *prefix, const char *path) { return 0; } +int32_t azGetObjectsByPrefix(const char *prefix, const char *path) { + int32_t code = 0; + + try { + code = azGetObjectsByPrefixImpl(prefix, path); + } catch (const std::exception &e) { + azError("%s: Reason Phrase: %s", __func__, e.what()); + + code = TAOS_SYSTEM_ERROR(EIO); + azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + + TAOS_RETURN(code); + } + + TAOS_RETURN(code); +} + int32_t azDeleteObjects(const char *object_name[], int nobject) { for (int i = 0; i < nobject; ++i) { azDeleteObjectsByPrefix(object_name[i]); From a51bd9a24ff5661615cdf13487de53d9fd6da697 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 5 Nov 2024 15:42:57 +0800 Subject: [PATCH 55/59] enh: cal output col counts --- source/libs/executor/src/dataDispatcher.c | 38 ++++++++++++++--------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 48f4ed3ed1..f255d0b95c 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -45,6 +45,7 @@ typedef struct SDataDispatchHandle { SDataBlockDescNode* pSchema; STaosQueue* pDataBlocks; SDataDispatchBuf nextOutput; + int32_t outPutColCounts; int32_t status; bool queryEnd; uint64_t useconds; @@ -68,23 +69,12 @@ static int32_t inputSafetyCheck(SDataDispatchHandle* pHandle, const SInputData* return TSDB_CODE_QRY_INVALID_INPUT; } - SNode* pNode; - int32_t numOfCols = 0; - FOREACH(pNode, pHandle->pSchema->pSlots) { - SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode; - if (pSlotDesc->output) { - ++numOfCols; - } else { - // Slots must be sorted, and slots with 'output' set to true must come first - break; - } - } - - if (numOfCols > taosArrayGetSize(pInput->pData->pDataBlock)) { - qError("invalid column number, schema:%d, input:%zu", numOfCols, taosArrayGetSize(pInput->pData->pDataBlock)); + if (pHandle->outPutColCounts > taosArrayGetSize(pInput->pData->pDataBlock)) { + qError("invalid column number, schema:%d, input:%zu", pHandle->outPutColCounts, taosArrayGetSize(pInput->pData->pDataBlock)); return TSDB_CODE_QRY_INVALID_INPUT; } + SNode* pNode; int32_t colNum = 0; FOREACH(pNode, pHandle->pSchema->pSlots) { SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode; @@ -419,6 +409,25 @@ static int32_t blockDescNodeCheck(SDataBlockDescNode* pInputDataBlockDesc) { return TSDB_CODE_SUCCESS; } +int32_t getOutputColCounts(SDataBlockDescNode* pInputDataBlockDesc) { + if (pInputDataBlockDesc == NULL) { + qError("invalid schema"); + return 0; + } + SNode* pNode; + int32_t numOfCols = 0; + FOREACH(pNode, pInputDataBlockDesc->pSlots) { + SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode; + if (pSlotDesc->output) { + ++numOfCols; + } else { + // Slots must be sorted, and slots with 'output' set to true must come first + break; + } + } + return numOfCols; +} + int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) { int32_t code; code = blockDescNodeCheck(pDataSink->pInputDataBlockDesc); @@ -443,6 +452,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD dispatcher->pManager = pManager; pManager = NULL; dispatcher->pSchema = pDataSink->pInputDataBlockDesc; + dispatcher->outPutColCounts = getOutputColCounts(dispatcher->pSchema); dispatcher->status = DS_BUF_EMPTY; dispatcher->queryEnd = false; code = taosOpenQueue(&dispatcher->pDataBlocks); From 688b159bec22791b93218fa03d412016e636b809 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 5 Nov 2024 17:09:52 +0800 Subject: [PATCH 56/59] fix:[TD-32166] change createRequest to buildRequest to avoid printing error because request not in pRequests hash --- source/client/src/clientEnv.c | 2 +- source/client/src/clientRawBlockWrite.c | 8 ++++---- source/client/src/clientSml.c | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 21f7c93036..b1eb4a683f 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -686,7 +686,7 @@ void doDestroyRequest(void *p) { int32_t code = taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self)); if (TSDB_CODE_SUCCESS != code) { - tscError("failed to remove request from hash, code:%s", tstrerror(code)); + tscWarn("failed to remove request from hash, code:%s", tstrerror(code)); } schedulerFreeJob(&pRequest->body.queryJob, 0); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index acba8117c6..213c18ac8a 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1590,7 +1590,7 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat SHashObj* pVgHash = NULL; SRequestObj* pRequest = NULL; - RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, reqid, &pRequest)); + RAW_RETURN_CHECK(buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, reqid)); uDebug(LOG_ID_TAG " write raw block with field, rows:%d, pData:%p, tbname:%s, fields:%p, numFields:%d", LOG_ID_VALUE, rows, pData, tbname, fields, numFields); @@ -1651,7 +1651,7 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha SHashObj* pVgHash = NULL; SRequestObj* pRequest = NULL; - RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, reqid, &pRequest)); + RAW_RETURN_CHECK(buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, reqid)); uDebug(LOG_ID_TAG " write raw block, rows:%d, pData:%p, tbname:%s", LOG_ID_VALUE, rows, pData, tbname); @@ -1721,7 +1721,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { STableMeta* pTableMeta = NULL; SRequestObj* pRequest = NULL; - RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest)); + RAW_RETURN_CHECK(buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0)); uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen); pRequest->syncQuery = true; @@ -1869,7 +1869,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) SHashObj* pCreateTbHash = NULL; SRequestObj* pRequest = NULL; - RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest)); + RAW_RETURN_CHECK(buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0)); uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen); pRequest->syncQuery = true; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index e8221c8b8d..90c7895d94 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1740,7 +1740,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, SSmlHandle *info = NULL; int cnt = 0; while (1) { - SML_CHECK_CODE(createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid, &request)); + SML_CHECK_CODE(buildRequest(*(int64_t*)taos, "", 0, NULL, false, &request, reqid)); SSmlMsgBuf msg = {request->msgBufLen, request->msgBuf}; request->code = smlBuildSmlInfo(taos, &info); SML_CHECK_CODE(request->code); From 16570822e8bce1de77b59c0d56318fd309faffac Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 5 Nov 2024 17:16:20 +0800 Subject: [PATCH 57/59] blob/config: check dnode s3 params --- tests/system-test/2-query/db.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py index d4e5f89aa3..ee6b517061 100644 --- a/tests/system-test/2-query/db.py +++ b/tests/system-test/2-query/db.py @@ -62,12 +62,30 @@ class TDTestCase: tdSql.query("show dnode 1 variables like '____debugFlag'") tdSql.checkRows(2) - tdSql.query("show dnode 1 variables like 's3MigrateEna%'") + tdSql.query("show dnode 1 variables like 's3MigrateEnab%'") tdSql.checkRows(1) tdSql.checkData(0, 0, 1) tdSql.checkData(0, 1, 's3MigrateEnabled') tdSql.checkData(0, 2, 0) + tdSql.query("show dnode 1 variables like 's3MigrateIntervalSec%'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 's3MigrateIntervalSec') + tdSql.checkData(0, 2, 3600) + + tdSql.query("show dnode 1 variables like 's3PageCacheSize%'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 's3PageCacheSize') + tdSql.checkData(0, 2, 4096) + + tdSql.query("show dnode 1 variables like 's3UploadDelaySec%'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 's3UploadDelaySec') + tdSql.checkData(0, 2, 60) + def threadTest(self, threadID): print(f"Thread {threadID} starting...") tdsqln = tdCom.newTdSql() From c54855cdd995da7f6f40936afd9f9e107c0a4d03 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 5 Nov 2024 17:18:20 +0800 Subject: [PATCH 58/59] fix:[TD-32166] change createRequest to buildRequest to avoid printing error because request not in pRequests hash --- source/client/src/clientRawBlockWrite.c | 84 ++++++++++++------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 229e137a21..1799f29eb4 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -163,7 +163,7 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche } RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tags", tags)); - end: +end: *pJson = json; } @@ -197,7 +197,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) { return code; } - end: +end: return code; } static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** pJson) { @@ -338,7 +338,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** break; } - end: +end: tFreeSMAltertbReq(&req); *pJson = json; } @@ -358,7 +358,7 @@ static void processCreateStb(SMqMetaRsp* metaRsp, cJSON** pJson) { } buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE, &req.colCmpr, pJson); - end: +end: uDebug("create stable return"); tDecoderClear(&coder); } @@ -378,7 +378,7 @@ static void processAlterStb(SMqMetaRsp* metaRsp, cJSON** pJson) { } buildAlterSTableJson(req.alterOriData, req.alterOriDataLen, pJson); - end: +end: uDebug("alter stable return"); tDecoderClear(&coder); } @@ -485,7 +485,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { RAW_FALSE_CHECK(cJSON_AddItemToArray(tags, tag)); } - end: +end: RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tags", tags)); taosArrayDestroy(pTagVals); } @@ -514,7 +514,7 @@ static void buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs, cJSO } RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "createList", createList)); - end: +end: *pJson = json; } @@ -542,7 +542,7 @@ static void processCreateTable(SMqMetaRsp* metaRsp, cJSON** pJson) { } } - end: +end: uDebug("create table return"); tDeleteSVCreateTbBatchReq(&req); tDecoderClear(&decoder); @@ -585,7 +585,7 @@ static void processAutoCreateTable(SMqDataRsp* rsp, char** string) { *string = cJSON_PrintUnformatted(pJson); cJSON_Delete(pJson); - end: +end: uDebug("auto created table return, sql json:%s", *string); for (int i = 0; decoder && pCreateReq && i < rsp->createTableNum; i++) { tDecoderClear(&decoder[i]); @@ -773,7 +773,7 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { break; } - end: +end: uDebug("alter table return"); tDecoderClear(&decoder); *pJson = json; @@ -808,7 +808,7 @@ static void processDropSTable(SMqMetaRsp* metaRsp, cJSON** pJson) { RAW_NULL_CHECK(tableName); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableName", tableName)); - end: +end: uDebug("processDropSTable return"); tDecoderClear(&decoder); *pJson = json; @@ -844,7 +844,7 @@ static void processDeleteTable(SMqMetaRsp* metaRsp, cJSON** pJson) { RAW_NULL_CHECK(sqlJson); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", sqlJson)); - end: +end: uDebug("processDeleteTable return"); tDecoderClear(&coder); *pJson = json; @@ -881,7 +881,7 @@ static void processDropTable(SMqMetaRsp* metaRsp, cJSON** pJson) { } RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tableNameList", tableNameList)); - end: +end: uDebug("processDropTable return"); tDecoderClear(&decoder); *pJson = json; @@ -989,7 +989,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { code = pRequest->code; - end: +end: uDebug(LOG_ID_TAG " create stable return, msg:%s", LOG_ID_VALUE, tstrerror(code)); destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); @@ -1023,9 +1023,9 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { SCatalog* pCatalog = NULL; RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog)); SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)}; SName pName = {0}; toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName); STableMeta* pTableMeta = NULL; @@ -1088,7 +1088,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { code = pRequest->code; - end: +end: uDebug(LOG_ID_TAG " drop stable return, msg:%s", LOG_ID_VALUE, tstrerror(code)); destroyRequest(pRequest); tDecoderClear(&coder); @@ -1142,9 +1142,9 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { taosHashSetFreeFp(pVgroupHashmap, destroyCreateTbReqBatch); SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName)); RAW_NULL_CHECK(pRequest->tableList); @@ -1269,7 +1269,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { code = pRequest->code; - end: +end: uDebug(LOG_ID_TAG " create table return, msg:%s", LOG_ID_VALUE, tstrerror(code)); tDeleteSVCreateTbBatchReq(&req); @@ -1328,9 +1328,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch); SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName)); RAW_NULL_CHECK(pRequest->tableList); // loop to create table @@ -1395,7 +1395,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { } code = pRequest->code; - end: +end: uDebug(LOG_ID_TAG " drop table return, msg:%s", LOG_ID_VALUE, tstrerror(code)); taosHashCleanup(pVgroupHashmap); destroyRequest(pRequest); @@ -1433,7 +1433,7 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { } taos_free_result(res); - end: +end: uDebug("connId:0x%" PRIx64 " delete data sql:%s, code:%s", *(int64_t*)taos, sql, tstrerror(code)); tDecoderClear(&coder); return code; @@ -1473,9 +1473,9 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { SCatalog* pCatalog = NULL; RAW_RETURN_CHECK(catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog)); SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; SVgroupInfo pInfo = {0}; SName pName = {0}; @@ -1543,7 +1543,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { code = handleAlterTbExecRes(pRes->res, pCatalog); } } - end: +end: uDebug(LOG_ID_TAG " alter table return, meta:%p, len:%d, msg:%s", LOG_ID_VALUE, meta, metaLen, tstrerror(code)); taosArrayDestroy(pArray); if (pVgData) taosMemoryFreeClear(pVgData->pData); @@ -1608,7 +1608,7 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; - end: +end: uDebug(LOG_ID_TAG " write raw block with field return, msg:%s", LOG_ID_VALUE, tstrerror(code)); taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); @@ -1668,7 +1668,7 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; - end: +end: uDebug(LOG_ID_TAG " write raw block return, msg:%s", LOG_ID_VALUE, tstrerror(code)); taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); @@ -1719,7 +1719,7 @@ static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) { } return 0; - end: +end: tDecoderClear(&decoderTmp); tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); return code; @@ -1826,7 +1826,7 @@ static int32_t getRawCache(SHashObj** pVgHash, SHashObj** pNameHash, SHashObj** } return 0; - end: +end: taosHashCleanup(*pMetaHash); taosHashCleanup(*pNameHash); taosHashCleanup(*pVgHash); @@ -1848,7 +1848,7 @@ static int32_t buildRawRequest(TAOS* taos, SRequestObj** pRequest, SCatalog** pC conn->requestObjRefId = (*pRequest)->self; conn->mgmtEps = getEpSet_s(&(*pRequest)->pTscObj->pAppInfo->mgmtEp); - end: +end: return code; } @@ -1925,7 +1925,7 @@ static int32_t processCacheMeta(SHashObj* pVgHash, SHashObj* pNameHash, SHashObj } *pMeta = pTableMeta; - end: +end: return code; } @@ -1993,7 +1993,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { break; } - end: +end: uDebug(LOG_ID_TAG " write raw data return, msg:%s", LOG_ID_VALUE, tstrerror(code)); tDeleteMqDataRsp(&rspObj.dataRsp); tDecoderClear(&decoder); @@ -2075,7 +2075,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) break; } - end: +end: uDebug(LOG_ID_TAG " write raw metadata return, msg:%s", LOG_ID_VALUE, tstrerror(code)); tDeleteSTaosxRsp(&rspObj.dataRsp); void* pIter = taosHashIterate(pCreateTbHash, NULL); @@ -2150,7 +2150,7 @@ static void processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp, char** string) { *string = fullStr; return; - end: +end: cJSON_Delete(pJson); tDeleteMqBatchMetaRsp(&rsp); } @@ -2233,7 +2233,7 @@ static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, SMqDataRsp* rspObj, raw->raw = buf; raw->raw_len = len; return code; - FAILED: +FAILED: tEncoderClear(&encoder); taosMemoryFree(buf); return code; @@ -2377,7 +2377,7 @@ static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen } } - end: +end: tDeleteMqBatchMetaRsp(&rsp); return code; } \ No newline at end of file From 1790f2929e5749c0a63d38a695f07dcb2106e701 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 6 Nov 2024 09:41:36 +0800 Subject: [PATCH 59/59] docs: data analysis --- .../06-advanced/06-data-analysis/01-arima.md | 22 +++++++-------- .../06-data-analysis/02-holtwinters.md | 16 +++++------ .../06-data-analysis/03-anomaly-detection.md | 10 +++---- .../zh/06-advanced/06-data-analysis/addins.md | 28 ++++++++++--------- docs/zh/06-advanced/06-data-analysis/index.md | 2 +- 5 files changed, 40 insertions(+), 38 deletions(-) diff --git a/docs/zh/06-advanced/06-data-analysis/01-arima.md b/docs/zh/06-advanced/06-data-analysis/01-arima.md index 56383b7e86..b9d63e924f 100644 --- a/docs/zh/06-advanced/06-data-analysis/01-arima.md +++ b/docs/zh/06-advanced/06-data-analysis/01-arima.md @@ -8,11 +8,11 @@ sidebar_label: "ARIMA" ## 功能概述 ARIMA 即自回归移动平均模型（Autoregressive Integrated Moving Average, ARIMA），也记作 ARIMA(p,d,q)，是统计模型中最常见的一种用来进行时间序列预测的模型。 -ARIMA 模型是一种自回归模型，只需要自变量即可预测后续的值。ARIMA 模型要求时序数据**平稳**，或经过差分处理后平稳，如果是不平稳的数据，**无法**获得正确的结果。 +ARIMA 模型是一种自回归模型，只需要自变量即可预测后续的值。ARIMA 模型要求时间序列**平稳**，或经过差分处理后平稳，如果是不平稳的数据，**无法**获得正确的结果。 >平稳的时间序列：其性质不随观测时间的变化而变化。具有趋势或季节性的时间序列不是平稳时间序列——趋势和季节性使得时间序列在不同时段呈现不同性质。 -以下参数可以动态输入控制预测过程中生成 合适的 ARIMA 的模型。 +以下参数可以动态输入，控制预测过程中生成合适的 ARIMA 模型。 - p= 自回归模型阶数 - d= 差分阶数 @@ -21,13 +21,13 @@ ARIMA 模型是一种自回归模型，只需要自变量即可预测后续的 ### 参数 分析平台中使用自动化的 ARIMA 模型进行计算，因此每次计算的时候会根据输入的数据自动拟合最合适的模型，然后根据该模型进行预测输出结果。 -|参数名称|说明|必填项| -|---|---|---| -|period|输入时间序列每个周期包含的数据点个数。如果不设置该参数或该参数设置为 0，将使用非季节性/周期性的 ARIMA 模型预测。|选填| -|start_p|自回归模型阶数的起始值，0 开始的整数，不推荐大于 10 |选填| -|max_p|自回归模型阶数的结束值，0 开始的整数，不推荐大于 10 |选填| -|start_q|移动平均模型阶数的起始值，0 开始的整数，不推荐大于 10 |选填| -|max_q|移动平均模型阶数的结束值，0 开始的整数，不推荐大于 10 |选填| +|参数|说明|必填项| +|---|---|-----| +|period|输入时间序列每个周期包含的数据点个数，如果不设置该参数或该参数设置为 0，将使用非季节性/周期性的 ARIMA 模型预测|选填| +|start_p|自回归模型阶数的起始值，0 开始的整数，不推荐大于 10|选填| +|max_p|自回归模型阶数的结束值，0 开始的整数，不推荐大于 10|选填| +|start_q|移动平均模型阶数的起始值，0 开始的整数，不推荐大于 10|选填| +|max_q|移动平均模型阶数的结束值，0 开始的整数，不推荐大于 10|选填| |d|差分阶数|选填| `start_p`、`max_p` `start_q` `max_q` 四个参数约束了模型在多大的范围内去搜寻合适的最优解。相同输入数据的条件下，参数范围越大，消耗的资源越多，系统响应的时间越长。 @@ -40,11 +40,11 @@ FORECAST(i32, "algo=arima,alpha=95,period=10,start_p=1,max_p=5,start_q=1,max_q=5 ```json5 { -"rows": fc_rows, // 预测结果的行数 +"rows": fc_rows, // 返回结果的行数 "period": period, // 返回结果的周期性，同输入 "alpha": alpha, // 返回结果的置信区间，同输入 "algo": "arima", // 返回结果使用的算法 -"mse":mse, // 拟合输入时序数据时候生成模型的最小均方误差(MSE) +"mse": mse, // 拟合输入时间序列时候生成模型的最小均方误差(MSE) "res": res // 列模式的结果 } ``` diff --git a/docs/zh/06-advanced/06-data-analysis/02-holtwinters.md b/docs/zh/06-advanced/06-data-analysis/02-holtwinters.md index 6fb4bffdc9..38662ca2b3 100644 --- a/docs/zh/06-advanced/06-data-analysis/02-holtwinters.md +++ b/docs/zh/06-advanced/06-data-analysis/02-holtwinters.md @@ -8,15 +8,15 @@ sidebar_label: "HoltWinters" ## 功能概述 HoltWinters 模型又称为多次指数平滑模型（EMA）。适用于含有线性趋势和周期波动的非平稳序列，利用指数平滑法让模型参数不断适应非平稳序列的变化，并对未来趋势进行**短期**预测。 HoltWinters 有两种不同的季节性组成部分，当季节变化在该时间序列中大致保持不变时，通常选择**加法模型**；而当季节变化与时间序列的水平成比例变化时，通常选择**乘法模型**。 -该模型对于返回数据也不提供计算的置信区间范围结果。在 95% 置信区间的上下界结果与预测结果相同。 +该模型对于返回数据不提供计算的置信区间范围结果，在 95% 置信区间的上下界结果与预测结果相同。 ### 参数 -分析平台中使用自动化的 ARIMA 模型进行计算，因此每次计算的时候会根据输入的数据自动拟合最合适的模型，然后根据该模型进行预测输出结果。 -|参数名称|说明|必填项| +分析平台中使用自动化的 HoltWinters 模型进行计算，因此每次计算的时候会根据输入的数据自动拟合最合适的模型，然后根据该模型进行预测输出结果。 +|参数|说明|必填项| |---|---|---| -|period|输入时间序列每个周期包含的数据点个数。如果不设置该参数或该参数设置为 0， 将使用一次（简单）指数平滑方式进行数据拟合，并据此进行未来数据的预测|选填| +|period|输入时间序列每个周期包含的数据点个数。如果不设置该参数或该参数设置为 0，将使用一次（简单）指数平滑方式进行数据拟合，并据此进行未来数据的预测|选填| |trend|趋势模型使用加法模型还是乘法模型|选填| |seasonal|季节性采用加法模型还是乘法模型|选填| @@ -30,11 +30,11 @@ FORECAST(i32, "algo=holtwinters,period=10,trend=mul,seasonal=mul") ```json5 { -"rows": rows, // 结果的行数 -"period": period, // 返回结果的周期性， 该结果与输入的周期性相同，如果没有周期性，该值为 0 +"rows": rows, // 返回结果的行数 +"period": period, // 返回结果的周期性，该结果与输入的周期性相同，如果没有周期性，该值为 0 "algo": 'holtwinters' // 返回结果使用的计算模型 -"mse":mse, // 最小均方误差（minmum square error） -"res": res // 具体的结果，按照列形式返回的结果。一般意义上包含了 两列 [timestamp][fc_results]。 +"mse": mse, // 最小均方误差（minmum square error） +"res": res // 具体的结果，按照列形式返回的结果。一般意义上包含了两列 [timestamp][fc_results]。 } ``` diff --git a/docs/zh/06-advanced/06-data-analysis/03-anomaly-detection.md b/docs/zh/06-advanced/06-data-analysis/03-anomaly-detection.md index d0da330ab3..bdfa455ae3 100644 --- a/docs/zh/06-advanced/06-data-analysis/03-anomaly-detection.md +++ b/docs/zh/06-advanced/06-data-analysis/03-anomaly-detection.md @@ -3,7 +3,7 @@ title: "Anomaly-detection" sidebar_label: "Anomaly-detection" --- -本节讲述 异常检测算法模型的使用方法。 +本节讲述异常检测算法模型的使用方法。 ## 概述 分析平台提供了 6 种异常检查模型，6 种异常检查模型分为 3 个类别，分别属于基于统计的异常检测模型、基于数据密度的检测模型、基于深度学习的异常检测模型。在不指定异常检测使用的方法的情况下，默认调用 iqr 的方法进行计算。 @@ -11,20 +11,20 @@ sidebar_label: "Anomaly-detection" ### 统计学异常检测方法 -- k-sigma[1]: 即 ***68–95–99.7 rule*** 。***k***值默认为 3， 即序列均值的 3 倍标准差范围为边界，超过边界的是异常值。KSigma 要求数据整体上服从正态分布，如果一个点偏离均值 K 倍标准差，则该点被视为异常点. +- k-sigma[1]: 即 ***68–95–99.7 rule*** 。***k***值默认为 3，即序列均值的 3 倍标准差范围为边界，超过边界的是异常值。KSigma 要求数据整体上服从正态分布，如果一个点偏离均值 K 倍标准差，则该点被视为异常点. -|参数名称|说明|是否必选|默认值| +|参数|说明|是否必选|默认值| |---|---|---|---| |k|标准差倍数|选填|3| -- IQR[2]：四分位距 (Interquartile range, IQR) 是一种衡量变异性的方法. 四分位数将一个按等级排序的数据集划分为四个相等的部分。即 Q1（第 1 个四分位数）、Q2（第 2 个四分位数）和 Q3（第 3 个四分位数）。IQR 定义为 Q3–Q1，位于 Q3+1.5 。无输入参数。 +- IQR[2]：四分位距 (Interquartile range, IQR) 是一种衡量变异性的方法. 四分位数将一个按等级排序的数据集划分为四个相等的部分。即 Q1（第 1 个四分位数）、Q2（第 2 个四分位数）和 Q3（第 3 个四分位数）。IQR 定义为 Q3–Q1，位于 Q3+1.5。无输入参数。 - Grubbs[3]: 又称为 Grubbs' test，即最大标准残差测试。Grubbs 通常用作检验最大值、最小值偏离均值的程度是否为异常，该单变量数据集遵循近似标准正态分布。非正态分布数据集不能使用该方法。无输入参数。 - SHESD[4]： 带有季节性的 ESD 检测算法。ESD 可以检测时间序列数据的多异常点。需要指定异常点比例的上界***k***，最差的情况是至多 49.9%。数据集的异常比例一般不超过 5% -|参数名称|说明|是否必选|默认值| +|参数|说明|是否必选|默认值| |---|---|---|---| |k|异常点在输入数据集中占比，范围是$`1\le K \le 49.9`$ |选填|5| diff --git a/docs/zh/06-advanced/06-data-analysis/addins.md b/docs/zh/06-advanced/06-data-analysis/addins.md index aeaf08d490..c0b8921718 100644 --- a/docs/zh/06-advanced/06-data-analysis/addins.md +++ b/docs/zh/06-advanced/06-data-analysis/addins.md @@ -3,7 +3,7 @@ title: "addins" sidebar_label: "addins" --- -本节说明如何将自己开发的新预测算法和异常检测算法整合到 TDengine 分析平台， 并能够通过 SQL 语句进行调用。 +本节说明如何将自己开发的预测算法和异常检测算法整合到 TDengine 分析平台，并能够通过 SQL 语句进行调用。 ## 目录结构 @@ -11,14 +11,14 @@ sidebar_label: "addins" |目录|说明| |---|---| -|taos|Python 源代码目录，其下包含了算法具体保存目录 algo，放置杂项目录 misc， 单元测试和集成测试目录 test。 algo 目录下 ad 放置异常检测算法代码， fc 放置预测算法代码| +|taos|Python 源代码目录，其下包含了算法具体保存目录 algo，放置杂项目录 misc，单元测试和集成测试目录 test。 algo 目录下 ad 放置异常检测算法代码，fc 放置预测算法代码| |script|是安装脚本和发布脚本放置目录| |model|放置针对数据集完成的训练模型| |cfg|配置文件目录| ## 约定与限制 -定义异常检测算法的 Python 代码文件 需放在 /taos/algo/ad 目录中，预测算法 Python 代码文件需要放在 /taos/algo/fc 目录中，以确保系统启动的时候能够正常加载对应目录下的 Python 文件。 +定义异常检测算法的 Python 代码文件需放在 /taos/algo/ad 目录中，预测算法 Python 代码文件需要放在 /taos/algo/fc 目录中，以确保系统启动的时候能够正常加载对应目录下的 Python 文件。 ### 类命名规范 @@ -27,33 +27,35 @@ sidebar_label: "addins" ### 类继承约定 -异常检测算法需要从 `AbstractAnomalyDetectionService` 继承，并实现其核心抽象方法 `execute`. -预测算法需要从 `AbstractForecastService` 继承，同样需要实现其核心抽象方法 `execute`。 +- 异常检测算法需要从 `AbstractAnomalyDetectionService` 继承，并实现其核心抽象方法 `execute` +- 预测算法需要从 `AbstractForecastService` 继承，同样需要实现其核心抽象方法 `execute` ### 类属性初始化 -每个算法实现的类需要静态初始化两个类属性，分别是 +每个算法实现的类需要静态初始化两个类属性，分别是： -`name`: 的触发调用关键词，全小写英文字母。 -`desc`：该算法的描述信息。 +- `name`：触发调用的关键词，全小写英文字母 +- `desc`：算法的描述信息 ### 核心方法输入与输出约定 `execute` 是算法处理的核心方法。调用该方法的时候，`self.list` 已经设置好输入数组。 + 异常检测输出结果 `execute` 的返回值是长度与 `self.list` 相同的数组，数组位置为 -1 的即为异常值点。例如：输入数组是 [2, 2, 2, 2, 100]， 如果 100 是异常点，那么返回值是 [1, 1, 1, 1, -1]。 + 预测输出结果 对于预测算法，`AbstractForecastService` 的对象属性说明如下： |属性名称|说明|默认值| |---|---|---| -|period|输入时序数据的周期性，多少个数据点表示一个完整的周期。如果没有周期性，那么设置为 0 即可| 0| -|start_ts|预测数据的开始时间| 0| +|period|输入时间序列的周期性，多少个数据点表示一个完整的周期。如果没有周期性，那么设置为 0 即可| 0| +|start_ts|预测结果的开始时间| 0| |time_step|预测结果的两个数据点之间时间间隔|0 | -|fc_rows|预测结果数量| 0 | -|return_conf|返回结果中是否包含执行区间范围，如果算法计算结果不包含置信区间，那么上界和下界与自身相同| 1| -|conf|执行区间分位数 0.05| +|fc_rows|预测结果的数量| 0 | +|return_conf|预测结果中是否包含置信区间范围，如果不包含置信区间，那么上界和下界与自身相同| 1| +|conf|置信区间分位数 0.05| 预测返回结果如下： diff --git a/docs/zh/06-advanced/06-data-analysis/index.md b/docs/zh/06-advanced/06-data-analysis/index.md index df04c91d28..2cbea1caba 100644 --- a/docs/zh/06-advanced/06-data-analysis/index.md +++ b/docs/zh/06-advanced/06-data-analysis/index.md @@ -156,7 +156,7 @@ SHOW ANODES; SHOW ANODES FULL; ``` -#### 强制刷新 TDengine 集群中分析算法缓存 +#### 强制刷新集群中的分析算法缓存 ```SQL UPDATE ANODE {node_id} UPDATE ALL ANODES