From 9dd306e1ef80ec21863d0ef7c4e06c2899a346e6 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 22 Nov 2023 13:56:16 +0800 Subject: [PATCH 001/102] enhance: duration notification --- include/libs/executor/storageapi.h | 14 ++++++++++++++ source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/tsdb/tsdbRead2.c | 12 ++++++++++++ source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 3 +++ source/dnode/vnode/src/vnd/vnodeInitApi.c | 2 ++ source/libs/executor/src/scanoperator.c | 9 +++++++-- 6 files changed, 39 insertions(+), 2 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index d5f1da957d..b45103f94a 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -165,6 +165,18 @@ typedef struct { // clang-format off /*-------------------------------------------------new api format---------------------------------------------------*/ +typedef enum { + TSD_READER_NOTIFY_DURATION +} ETsdReaderNotifyType; + +typedef union { + struct { + int32_t fileSetId; + } duration; +} STsdReaderNotifyInfo; + +typedef void (*TsdReaderNotifyCbFn)(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param); + typedef struct TsdReader { int32_t (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly, @@ -183,6 +195,8 @@ typedef struct TsdReader { int32_t (*tsdReaderGetDataBlockDistInfo)(); int64_t (*tsdReaderGetNumOfInMemRows)(); void (*tsdReaderNotifyClosing)(); + + void (*tsdSetSetNotifyCb)(void* pReader, TsdReaderNotifyCbFn notifyFn, void* param); } TsdReader; typedef struct SStoreCacheReader { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 6a0c991be4..db01f7d995 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -191,6 +191,7 @@ void *tsdbGetIvtIdx2(SMeta *pMeta); uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader); void tsdbReaderSetCloseFlag(STsdbReader *pReader); int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); +void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param); //====================================================================================================================== int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 6169014d9f..2412639722 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -241,6 +241,11 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo tsdbDebug("%p file found fid:%d for qrange:%" PRId64 "-%" PRId64 ", %s", pReader, fid, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); + if (pReader->notifyFn) { + STsdReaderNotifyInfo info = {0}; + info.duration.fileSetId = fid; + pReader->notifyFn(TSD_READER_NOTIFY_DURATION, &info, pReader->notifyParam); + } *hasNext = true; return TSDB_CODE_SUCCESS; } @@ -4055,6 +4060,8 @@ _err: return code; } +void tsdbReaderSetNotifyFn(STsdbReader* pReader, ) + void tsdbReaderClose2(STsdbReader* pReader) { if (pReader == NULL) { return; @@ -5025,3 +5032,8 @@ void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ } + +void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param) { + pReader->notifyFn = notifyFn; + pReader->notifyParam = param; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 60e6e6960a..fb32190115 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -22,6 +22,7 @@ extern "C" { #include "tsdbDataFileRW.h" #include "tsdbUtil2.h" +#include "storageapi.h" #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -227,6 +228,8 @@ struct STsdbReader { SBlockInfoBuf blockInfoBuf; EContentData step; STsdbReader* innerReader[2]; + TsdReaderNotifyCbFn notifyFn; + void* notifyParam; }; typedef struct SBrinRecordIter { diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index a6673917bf..1729e3ab0b 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -60,6 +60,8 @@ void initTsdbReaderAPI(TsdReader* pReader) { pReader->tsdSetQueryTableList = tsdbSetTableList2; pReader->tsdSetReaderTaskId = (void (*)(void*, const char*))tsdbReaderSetId2; + + pReader->tsdSetSetNotifyCb = (void (*)(void*, TsdReaderNotifyCbFn, void*))tsdbReaderSetNotifyCb; } void initMetadataAPI(SStoreMeta* pMeta) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 28832ffec8..3d2053c0c0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3284,7 +3284,7 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; - + uInfo("getBlockForTableMergeScan retrieved one block"); return pBlock; } @@ -3320,6 +3320,11 @@ int32_t dumpQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* return 0; } +void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param) { + uInfo("tableMergeScanTsdbNotifyCb, %d, %d", type, info->duration.fileSetId); + return; +} + int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -3368,7 +3373,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx); pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, &pInfo->mSkipTables); - + pAPI->tsdReader.tsdSetSetNotifyCb(pInfo->base.dataReader, tableMergeScanTsdbNotifyCb, pInfo); SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = param; ps->onlyRef = false; From 1a7ee3643e7021d916e6c84b6becf4f844159634 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 22 Nov 2023 14:09:09 +0800 Subject: [PATCH 002/102] fix: pass compilation --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 2412639722..f3c5d9ec93 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4060,8 +4060,6 @@ _err: return code; } -void tsdbReaderSetNotifyFn(STsdbReader* pReader, ) - void tsdbReaderClose2(STsdbReader* pReader) { if (pReader == NULL) { return; From d007d31c4d205ef6558d59eeda471b3f31bf6574 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 27 Nov 2023 18:15:09 +0800 Subject: [PATCH 003/102] fix:process snode when drop stream --- source/dnode/mnode/impl/src/mndStream.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 235703428f..bc265db766 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -700,8 +700,17 @@ static int32_t mndPersistTaskDropReq(SMnode* pMnode, STrans *pTrans, SStreamTask STransAction action = {0}; SEpSet epset = {0}; if(pTask->info.nodeId == SNODE_HANDLE){ - SSnodeObj* pObj = mndAcquireSnode(pMnode, pTask->info.nodeId); - addEpIntoEpSet(&epset, pObj->pDnode->fqdn, pObj->pDnode->port); + SSnodeObj *pObj = NULL; + void *pIter = NULL; + while (1) { + pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj); + if (pIter == NULL) { + break; + } + + addEpIntoEpSet(&epset, pObj->pDnode->fqdn, pObj->pDnode->port); + sdbRelease(pMnode->pSdb, pObj); + } }else{ SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); epset = mndGetVgroupEpset(pMnode, pVgObj); From 644dbad920c575c06ed00388c3a0dd0158448531 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 28 Nov 2023 08:28:49 +0800 Subject: [PATCH 004/102] Revert "fix: pass compilation" This reverts commit 1a7ee3643e7021d916e6c84b6becf4f844159634. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index f3c5d9ec93..2412639722 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4060,6 +4060,8 @@ _err: return code; } +void tsdbReaderSetNotifyFn(STsdbReader* pReader, ) + void tsdbReaderClose2(STsdbReader* pReader) { if (pReader == NULL) { return; From 3331e25aafc3efa07825df72e753e7c210c6442f Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 28 Nov 2023 08:29:05 +0800 Subject: [PATCH 005/102] Revert "enhance: duration notification" This reverts commit 9dd306e1ef80ec21863d0ef7c4e06c2899a346e6. --- include/libs/executor/storageapi.h | 14 -------------- source/dnode/vnode/inc/vnode.h | 1 - source/dnode/vnode/src/tsdb/tsdbRead2.c | 12 ------------ source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 3 --- source/dnode/vnode/src/vnd/vnodeInitApi.c | 2 -- source/libs/executor/src/scanoperator.c | 9 ++------- 6 files changed, 2 insertions(+), 39 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index b45103f94a..d5f1da957d 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -165,18 +165,6 @@ typedef struct { // clang-format off /*-------------------------------------------------new api format---------------------------------------------------*/ -typedef enum { - TSD_READER_NOTIFY_DURATION -} ETsdReaderNotifyType; - -typedef union { - struct { - int32_t fileSetId; - } duration; -} STsdReaderNotifyInfo; - -typedef void (*TsdReaderNotifyCbFn)(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param); - typedef struct TsdReader { int32_t (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly, @@ -195,8 +183,6 @@ typedef struct TsdReader { int32_t (*tsdReaderGetDataBlockDistInfo)(); int64_t (*tsdReaderGetNumOfInMemRows)(); void (*tsdReaderNotifyClosing)(); - - void (*tsdSetSetNotifyCb)(void* pReader, TsdReaderNotifyCbFn notifyFn, void* param); } TsdReader; typedef struct SStoreCacheReader { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index db01f7d995..6a0c991be4 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -191,7 +191,6 @@ void *tsdbGetIvtIdx2(SMeta *pMeta); uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader); void tsdbReaderSetCloseFlag(STsdbReader *pReader); int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); -void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param); //====================================================================================================================== int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 2412639722..6169014d9f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -241,11 +241,6 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo tsdbDebug("%p file found fid:%d for qrange:%" PRId64 "-%" PRId64 ", %s", pReader, fid, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); - if (pReader->notifyFn) { - STsdReaderNotifyInfo info = {0}; - info.duration.fileSetId = fid; - pReader->notifyFn(TSD_READER_NOTIFY_DURATION, &info, pReader->notifyParam); - } *hasNext = true; return TSDB_CODE_SUCCESS; } @@ -4060,8 +4055,6 @@ _err: return code; } -void tsdbReaderSetNotifyFn(STsdbReader* pReader, ) - void tsdbReaderClose2(STsdbReader* pReader) { if (pReader == NULL) { return; @@ -5032,8 +5025,3 @@ void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ } - -void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param) { - pReader->notifyFn = notifyFn; - pReader->notifyParam = param; -} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index fb32190115..60e6e6960a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -22,7 +22,6 @@ extern "C" { #include "tsdbDataFileRW.h" #include "tsdbUtil2.h" -#include "storageapi.h" #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -228,8 +227,6 @@ struct STsdbReader { SBlockInfoBuf blockInfoBuf; EContentData step; STsdbReader* innerReader[2]; - TsdReaderNotifyCbFn notifyFn; - void* notifyParam; }; typedef struct SBrinRecordIter { diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 1729e3ab0b..a6673917bf 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -60,8 +60,6 @@ void initTsdbReaderAPI(TsdReader* pReader) { pReader->tsdSetQueryTableList = tsdbSetTableList2; pReader->tsdSetReaderTaskId = (void (*)(void*, const char*))tsdbReaderSetId2; - - pReader->tsdSetSetNotifyCb = (void (*)(void*, TsdReaderNotifyCbFn, void*))tsdbReaderSetNotifyCb; } void initMetadataAPI(SStoreMeta* pMeta) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 3d2053c0c0..28832ffec8 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3284,7 +3284,7 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; - uInfo("getBlockForTableMergeScan retrieved one block"); + return pBlock; } @@ -3320,11 +3320,6 @@ int32_t dumpQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* return 0; } -void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param) { - uInfo("tableMergeScanTsdbNotifyCb, %d, %d", type, info->duration.fileSetId); - return; -} - int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -3373,7 +3368,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx); pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, &pInfo->mSkipTables); - pAPI->tsdReader.tsdSetSetNotifyCb(pInfo->base.dataReader, tableMergeScanTsdbNotifyCb, pInfo); + SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = param; ps->onlyRef = false; From df4f7d6956e229599559a17ffaada117ff3c73a8 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 28 Nov 2023 10:31:18 +0800 Subject: [PATCH 006/102] Revert "Revert "enhance: duration notification"" This reverts commit 3331e25aafc3efa07825df72e753e7c210c6442f. --- include/libs/executor/storageapi.h | 14 ++++++++++++++ source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/tsdb/tsdbRead2.c | 12 ++++++++++++ source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 3 +++ source/dnode/vnode/src/vnd/vnodeInitApi.c | 2 ++ source/libs/executor/src/scanoperator.c | 9 +++++++-- 6 files changed, 39 insertions(+), 2 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index d5f1da957d..b45103f94a 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -165,6 +165,18 @@ typedef struct { // clang-format off /*-------------------------------------------------new api format---------------------------------------------------*/ +typedef enum { + TSD_READER_NOTIFY_DURATION +} ETsdReaderNotifyType; + +typedef union { + struct { + int32_t fileSetId; + } duration; +} STsdReaderNotifyInfo; + +typedef void (*TsdReaderNotifyCbFn)(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param); + typedef struct TsdReader { int32_t (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly, @@ -183,6 +195,8 @@ typedef struct TsdReader { int32_t (*tsdReaderGetDataBlockDistInfo)(); int64_t (*tsdReaderGetNumOfInMemRows)(); void (*tsdReaderNotifyClosing)(); + + void (*tsdSetSetNotifyCb)(void* pReader, TsdReaderNotifyCbFn notifyFn, void* param); } TsdReader; typedef struct SStoreCacheReader { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 6a0c991be4..db01f7d995 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -191,6 +191,7 @@ void *tsdbGetIvtIdx2(SMeta *pMeta); uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader); void tsdbReaderSetCloseFlag(STsdbReader *pReader); int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); +void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param); //====================================================================================================================== int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 6169014d9f..2412639722 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -241,6 +241,11 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo tsdbDebug("%p file found fid:%d for qrange:%" PRId64 "-%" PRId64 ", %s", pReader, fid, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); + if (pReader->notifyFn) { + STsdReaderNotifyInfo info = {0}; + info.duration.fileSetId = fid; + pReader->notifyFn(TSD_READER_NOTIFY_DURATION, &info, pReader->notifyParam); + } *hasNext = true; return TSDB_CODE_SUCCESS; } @@ -4055,6 +4060,8 @@ _err: return code; } +void tsdbReaderSetNotifyFn(STsdbReader* pReader, ) + void tsdbReaderClose2(STsdbReader* pReader) { if (pReader == NULL) { return; @@ -5025,3 +5032,8 @@ void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ } + +void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param) { + pReader->notifyFn = notifyFn; + pReader->notifyParam = param; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 60e6e6960a..fb32190115 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -22,6 +22,7 @@ extern "C" { #include "tsdbDataFileRW.h" #include "tsdbUtil2.h" +#include "storageapi.h" #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -227,6 +228,8 @@ struct STsdbReader { SBlockInfoBuf blockInfoBuf; EContentData step; STsdbReader* innerReader[2]; + TsdReaderNotifyCbFn notifyFn; + void* notifyParam; }; typedef struct SBrinRecordIter { diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index a6673917bf..1729e3ab0b 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -60,6 +60,8 @@ void initTsdbReaderAPI(TsdReader* pReader) { pReader->tsdSetQueryTableList = tsdbSetTableList2; pReader->tsdSetReaderTaskId = (void (*)(void*, const char*))tsdbReaderSetId2; + + pReader->tsdSetSetNotifyCb = (void (*)(void*, TsdReaderNotifyCbFn, void*))tsdbReaderSetNotifyCb; } void initMetadataAPI(SStoreMeta* pMeta) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 28832ffec8..3d2053c0c0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3284,7 +3284,7 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; - + uInfo("getBlockForTableMergeScan retrieved one block"); return pBlock; } @@ -3320,6 +3320,11 @@ int32_t dumpQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* return 0; } +void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param) { + uInfo("tableMergeScanTsdbNotifyCb, %d, %d", type, info->duration.fileSetId); + return; +} + int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -3368,7 +3373,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx); pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, &pInfo->mSkipTables); - + pAPI->tsdReader.tsdSetSetNotifyCb(pInfo->base.dataReader, tableMergeScanTsdbNotifyCb, pInfo); SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = param; ps->onlyRef = false; From cdb7eb462de681913b8c2ee2fee3e6f4fb742835 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 28 Nov 2023 10:31:18 +0800 Subject: [PATCH 007/102] Revert "Revert "fix: pass compilation"" This reverts commit 644dbad920c575c06ed00388c3a0dd0158448531. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 2412639722..f3c5d9ec93 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4060,8 +4060,6 @@ _err: return code; } -void tsdbReaderSetNotifyFn(STsdbReader* pReader, ) - void tsdbReaderClose2(STsdbReader* pReader) { if (pReader == NULL) { return; From 89cd2b6f17ee681594e18800f4e9a1723d45f75e Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 28 Nov 2023 14:39:08 +0800 Subject: [PATCH 008/102] enhance: tsdb output duration order --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 97 +++++++++++++++++----- source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 2 + 2 files changed, 78 insertions(+), 21 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index f3c5d9ec93..d71f1729b6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -58,6 +58,7 @@ static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbRea static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo); static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter); static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order); +static void resetTableListIndex(SReaderStatus* pStatus); static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } @@ -241,11 +242,7 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo tsdbDebug("%p file found fid:%d for qrange:%" PRId64 "-%" PRId64 ", %s", pReader, fid, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); - if (pReader->notifyFn) { - STsdReaderNotifyInfo info = {0}; - info.duration.fileSetId = fid; - pReader->notifyFn(TSD_READER_NOTIFY_DURATION, &info, pReader->notifyParam); - } + *hasNext = true; return TSDB_CODE_SUCCESS; } @@ -436,6 +433,8 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void goto _end; } + pReader->bDurationOrder = true; + tsdbInitReaderLock(pReader); *ppReader = pReader; @@ -2550,6 +2549,8 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr } if (pBlockNum->numOfBlocks + pBlockNum->numOfSttFiles > 0) { + pReader->status.bProcMemPreFileset = true; + resetTableListIndex(&pReader->status); break; } } @@ -2931,7 +2932,7 @@ static int32_t readRowsCountFromMem(STsdbReader* pReader) { return code; } -static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { +static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t endKey) { SReaderStatus* pStatus = &pReader->status; STableUidList* pUidList = &pStatus->uidList; @@ -2948,13 +2949,12 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { if (!hasNexTable) { return TSDB_CODE_SUCCESS; } - pBlockScanInfo = pStatus->pTableIter; + continue; } initMemDataIterator(*pBlockScanInfo, pReader); initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); - int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; int32_t code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); if (code != TSDB_CODE_SUCCESS) { return code; @@ -4350,6 +4350,70 @@ static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { return pBlock->info.rows > 0; } +static int32_t doTsdbNextDataBlockDurationOrder(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + + if (pStatus->loadFromFile) { + if (pStatus->bProcMemPreFileset) { + int32_t fid = pReader->status.pCurrentFileset->fid; + STimeWindow win = {0}; + tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey); + + code = buildBlockFromBufferSequentially(pReader, win.skey); + if (code != TSDB_CODE_SUCCESS || pBlock->info.rows > 0) { + return code; + } else { + pStatus->bProcMemPreFileset = false; + if (pReader->notifyFn) { + STsdReaderNotifyInfo info = {0}; + info.duration.fileSetId = fid; + pReader->notifyFn(TSD_READER_NOTIFY_DURATION, &info, pReader->notifyParam); + } + } + } + + code = buildBlockFromFiles(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (pBlock->info.rows <= 0) { + resetTableListIndex(&pReader->status); + int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; + code = buildBlockFromBufferSequentially(pReader, endKey); + } + } else { // no data in files, let's try the buffer + int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; + code = buildBlockFromBufferSequentially(pReader, endKey); + } + return code; +} + +static int32_t doTsdbNextDataBlockFilesFirst(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + + if (pStatus->loadFromFile) { + code = buildBlockFromFiles(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (pBlock->info.rows <= 0) { + resetTableListIndex(&pReader->status); + int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; + code = buildBlockFromBufferSequentially(pReader, endKey); + } + } else { // no data in files, let's try the buffer + int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; + code = buildBlockFromBufferSequentially(pReader, endKey); + } + return code; +} + static int32_t doTsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { int32_t code = TSDB_CODE_SUCCESS; @@ -4367,19 +4431,10 @@ static int32_t doTsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { if (READ_MODE_COUNT_ONLY == pReader->info.readMode) { return tsdbReadRowsCountOnly(pReader); } - - if (pStatus->loadFromFile) { - code = buildBlockFromFiles(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (pBlock->info.rows <= 0) { - resetTableListIndex(&pReader->status); - code = buildBlockFromBufferSequentially(pReader); - } - } else { // no data in files, let's try the buffer - code = buildBlockFromBufferSequentially(pReader); + if (!pReader->bDurationOrder) { + code = doTsdbNextDataBlockFilesFirst(pReader); + } else { + code = doTsdbNextDataBlockDurationOrder(pReader); } *hasNext = pBlock->info.rows > 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index fb32190115..2d5b6f38a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -206,6 +206,7 @@ typedef struct SReaderStatus { SArray* pLDataIterArray; SRowMerger merger; SColumnInfoData* pPrimaryTsCol; // primary time stamp output col info data + bool bProcMemPreFileset; } SReaderStatus; struct STsdbReader { @@ -228,6 +229,7 @@ struct STsdbReader { SBlockInfoBuf blockInfoBuf; EContentData step; STsdbReader* innerReader[2]; + bool bDurationOrder; // duration by duration output TsdReaderNotifyCbFn notifyFn; void* notifyParam; }; From 4fb0f83f070ef2ea03717f4d132b8e31b7741238 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 28 Nov 2023 14:50:50 +0800 Subject: [PATCH 009/102] fix: core dump without resetTableListIndex --- include/libs/executor/storageapi.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead2.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index b45103f94a..2c9b8a4e5e 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -166,7 +166,7 @@ typedef struct { // clang-format off /*-------------------------------------------------new api format---------------------------------------------------*/ typedef enum { - TSD_READER_NOTIFY_DURATION + TSD_READER_NOTIFY_DURATION_START } ETsdReaderNotifyType; typedef union { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index d71f1729b6..710a076edc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4369,8 +4369,9 @@ static int32_t doTsdbNextDataBlockDurationOrder(STsdbReader* pReader) { if (pReader->notifyFn) { STsdReaderNotifyInfo info = {0}; info.duration.fileSetId = fid; - pReader->notifyFn(TSD_READER_NOTIFY_DURATION, &info, pReader->notifyParam); + pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam); } + resetTableListIndex(pStatus); } } From 5a75d744213839c7180761f75f8bc80ed67c21cd Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 28 Nov 2023 15:58:33 +0800 Subject: [PATCH 010/102] fix:mem table has data in the fileset duration --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 72 +++++++++++++++++++++- source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 2 + 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 710a076edc..492739e407 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -59,6 +59,7 @@ static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFil static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter); static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order); static void resetTableListIndex(SReaderStatus* pStatus); +static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t* pMinKey); static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } @@ -2549,8 +2550,19 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr } if (pBlockNum->numOfBlocks + pBlockNum->numOfSttFiles > 0) { - pReader->status.bProcMemPreFileset = true; - resetTableListIndex(&pReader->status); + if (pReader->bDurationOrder) { + int32_t fid = pReader->status.pCurrentFileset->fid; + STimeWindow win = {0}; + tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey); + + if ((ASCENDING_TRAVERSE(pReader->info.order) && + win.skey >= pReader->status.memTableMinKey && win.skey <= pReader->status.memTableMaxKey) || + (!ASCENDING_TRAVERSE(pReader->info.order) && + win.ekey >= pReader->status.memTableMinKey && win.ekey <= pReader->status.memTableMaxKey)) { + pReader->status.bProcMemPreFileset = true; + resetTableListIndex(&pReader->status); + } + } break; } } @@ -4243,6 +4255,10 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { tsdbUntakeReadSnap2(pReader, pReader->pReadSnap, false); pReader->pReadSnap = NULL; + if (pReader->bDurationOrder) { + pReader->status.memTableMinKey = INT64_MAX; + pReader->status.memTableMaxKey = INT64_MIN; + } pReader->flag = READER_STATUS_SUSPEND; tsdbDebug("reader: %p suspended uid %" PRIu64 " in this query %s", pReader, pBlockScanInfo ? pBlockScanInfo->uid : 0, @@ -4314,6 +4330,10 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { } } + if (pReader->bDurationOrder) { + getMemTableTimeRange(pReader, &pReader->status.memTableMaxKey, &pReader->status.memTableMinKey); + } + pReader->flag = READER_STATUS_NORMAL; tsdbDebug("reader: %p resumed uid %" PRIu64 ", numOfTable:%" PRId32 ", in this query %s", pReader, pBlockScanInfo ? (*pBlockScanInfo)->uid : 0, numOfTables, pReader->idStr); @@ -4899,6 +4919,54 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT return code; } +static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t* pMinKey) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t rows = 0; + + SReaderStatus* pStatus = &pReader->status; + + int32_t iter = 0; + int64_t maxKey = INT64_MIN; + int64_t minKey = INT64_MAX; + + pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, NULL, &iter); + while (pStatus->pTableIter != NULL) { + STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; + + STbData* d = NULL; + if (pReader->pReadSnap->pMem != NULL) { + d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->info.suid, pBlockScanInfo->uid); + if (d != NULL) { + if (d->maxKey > maxKey) { + maxKey = d->maxKey; + } + if (d->minKey < minKey) { + minKey = d->minKey; + } + } + } + + STbData* di = NULL; + if (pReader->pReadSnap->pIMem != NULL) { + di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->info.suid, pBlockScanInfo->uid); + if (di != NULL) { + if (d->maxKey > maxKey) { + maxKey = d->maxKey; + } + if (d->minKey < minKey) { + minKey = d->minKey; + } + } + } + + // current table is exhausted, let's try the next table + pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, pStatus->pTableIter, &iter); + } + + *pMaxKey = maxKey; + *pMinKey = minKey; +} + int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; int64_t rows = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 2d5b6f38a6..31bd9b5c78 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -207,6 +207,8 @@ typedef struct SReaderStatus { SRowMerger merger; SColumnInfoData* pPrimaryTsCol; // primary time stamp output col info data bool bProcMemPreFileset; + int64_t memTableMaxKey; + int64_t memTableMinKey; } SReaderStatus; struct STsdbReader { From b2fa6209d90f5fd09272e565eb9c9884790fb005 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 29 Nov 2023 08:09:00 +0800 Subject: [PATCH 011/102] fix: descending traverse wrong endkey --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 710a076edc..462f385172 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4361,7 +4361,8 @@ static int32_t doTsdbNextDataBlockDurationOrder(STsdbReader* pReader) { STimeWindow win = {0}; tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey); - code = buildBlockFromBufferSequentially(pReader, win.skey); + int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? win.skey : win.ekey; + code = buildBlockFromBufferSequentially(pReader, endKey); if (code != TSDB_CODE_SUCCESS || pBlock->info.rows > 0) { return code; } else { From 8a79df1113fbda4d06834053c2375d1640d893da Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 29 Nov 2023 10:15:43 +0800 Subject: [PATCH 012/102] enhance: prepare duration for next fileset --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 39 ++++++++++++++++------ source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 3 ++ 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 20dcb4b52c..cbfbe782f6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2509,6 +2509,33 @@ TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) } } +static void prepareDurationForNextFileSet(STsdbReader* pReader) { + if (pReader->status.bProcMemFirstFileset) { + pReader->status.prevFilesetStartKey = INT64_MIN; + pReader->status.prevFilesetEndKey = INT64_MAX; + pReader->status.bProcMemFirstFileset = false; + } + + int32_t fid = pReader->status.pCurrentFileset->fid; + STimeWindow winFid = {0}; + tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &winFid.skey, &winFid.ekey); + + bool bProcMemPreFileset = false; + if (ASCENDING_TRAVERSE(pReader->info.order)) { + bProcMemPreFileset = !(pReader->status.prevFilesetStartKey > pReader->status.memTableMaxKey || winFid.skey < pReader->status.memTableMinKey); + } else { + bProcMemPreFileset = !(winFid.ekey > pReader->status.memTableMaxKey || pReader->status.prevFilesetEndKey < pReader->status.memTableMinKey); + } + + if (bProcMemPreFileset) { + pReader->status.bProcMemPreFileset = true; + resetTableListIndex(&pReader->status); + } + + pReader->status.prevFilesetStartKey = winFid.skey; + pReader->status.prevFilesetEndKey = winFid.ekey; +} + static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pTableList) { SReaderStatus* pStatus = &pReader->status; pBlockNum->numOfBlocks = 0; @@ -2551,17 +2578,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr if (pBlockNum->numOfBlocks + pBlockNum->numOfSttFiles > 0) { if (pReader->bDurationOrder) { - int32_t fid = pReader->status.pCurrentFileset->fid; - STimeWindow win = {0}; - tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey); - - if ((ASCENDING_TRAVERSE(pReader->info.order) && - win.skey >= pReader->status.memTableMinKey && win.skey <= pReader->status.memTableMaxKey) || - (!ASCENDING_TRAVERSE(pReader->info.order) && - win.ekey >= pReader->status.memTableMinKey && win.ekey <= pReader->status.memTableMaxKey)) { - pReader->status.bProcMemPreFileset = true; - resetTableListIndex(&pReader->status); - } + prepareDurationForNextFileSet(pReader); } break; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 31bd9b5c78..c7331e1913 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -209,6 +209,9 @@ typedef struct SReaderStatus { bool bProcMemPreFileset; int64_t memTableMaxKey; int64_t memTableMinKey; + int64_t prevFilesetStartKey; + int64_t prevFilesetEndKey; + bool bProcMemFirstFileset; } SReaderStatus; struct STsdbReader { From 4691a7767a8005eebbb7cd683a208f93f4f3cf67 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 29 Nov 2023 10:29:40 +0800 Subject: [PATCH 013/102] restore buildBlockFromBufferSequentially back --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index cbfbe782f6..b999607503 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2519,7 +2519,7 @@ static void prepareDurationForNextFileSet(STsdbReader* pReader) { int32_t fid = pReader->status.pCurrentFileset->fid; STimeWindow winFid = {0}; tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &winFid.skey, &winFid.ekey); - + bool bProcMemPreFileset = false; if (ASCENDING_TRAVERSE(pReader->info.order)) { bProcMemPreFileset = !(pReader->status.prevFilesetStartKey > pReader->status.memTableMaxKey || winFid.skey < pReader->status.memTableMinKey); @@ -2978,7 +2978,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t en if (!hasNexTable) { return TSDB_CODE_SUCCESS; } - continue; + pBlockScanInfo = pStatus->pTableIter; } initMemDataIterator(*pBlockScanInfo, pReader); From ae386c6e345a9d182ed9483d1047203e0968470b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Wed, 29 Nov 2023 11:16:56 +0800 Subject: [PATCH 014/102] opt expired data --- include/libs/executor/storageapi.h | 2 + include/libs/stream/tstreamUpdate.h | 1 + source/dnode/snode/src/snodeInitApi.c | 1 + source/dnode/vnode/src/vnd/vnodeInitApi.c | 1 + source/libs/executor/inc/executorInt.h | 91 +++++----- source/libs/executor/src/scanoperator.c | 9 + .../executor/src/streameventwindowoperator.c | 3 +- .../executor/src/streamtimewindowoperator.c | 13 +- source/libs/stream/src/streamUpdate.c | 15 +- .../script/tsim/stream/ignoreExpiredData.sim | 164 +++++++++++++++++- 10 files changed, 245 insertions(+), 55 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 045f2bad70..1d40274504 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -351,6 +351,8 @@ typedef struct SStateStore { TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol); bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts); bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); + bool (*isIncrementalTimeStamp)(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts); + void (*updateInfoDestroy)(SUpdateInfo* pInfo); void (*windowSBfDelete)(SUpdateInfo *pInfo, uint64_t count); void (*windowSBfAdd)(SUpdateInfo *pInfo, uint64_t count); diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 41ada56904..af93c6ac01 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -55,6 +55,7 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *p int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo); void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count); void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count); +bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts); #ifdef __cplusplus } diff --git a/source/dnode/snode/src/snodeInitApi.c b/source/dnode/snode/src/snodeInitApi.c index 570feffc14..d1e0aefca0 100644 --- a/source/dnode/snode/src/snodeInitApi.c +++ b/source/dnode/snode/src/snodeInitApi.c @@ -80,6 +80,7 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->updateInfoDestroy = updateInfoDestroy; pStore->windowSBfDelete = windowSBfDelete; pStore->windowSBfAdd = windowSBfAdd; + pStore->isIncrementalTimeStamp = isIncrementalTimeStamp; pStore->updateInfoInitP = updateInfoInitP; pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF; diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index a6673917bf..6584b7072f 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -189,6 +189,7 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->updateInfoDestroy = updateInfoDestroy; pStore->windowSBfDelete = windowSBfDelete; pStore->windowSBfAdd = windowSBfAdd; + pStore->isIncrementalTimeStamp = isIncrementalTimeStamp; pStore->updateInfoInitP = updateInfoInitP; pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF; diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index e29583d8fc..865fe98be9 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -338,18 +338,19 @@ enum { }; typedef struct SStreamAggSupporter { - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row - SSDataBlock* pScanBlock; - SStreamState* pState; - int64_t gap; // stream session window gap - SqlFunctionCtx* pDummyCtx; // for combine - SSHashObj* pResultRows; - int32_t stateKeySize; - int16_t stateKeyType; - SDiskbasedBuf* pResultBuf; - SStateStore stateStore; - STimeWindow winRange; - SStorageAPI* pSessionAPI; + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + SSDataBlock* pScanBlock; + SStreamState* pState; + int64_t gap; // stream session window gap + SqlFunctionCtx* pDummyCtx; // for combine + SSHashObj* pResultRows; + int32_t stateKeySize; + int16_t stateKeyType; + SDiskbasedBuf* pResultBuf; + SStateStore stateStore; + STimeWindow winRange; + SStorageAPI* pSessionAPI; + struct SUpdateInfo* pUpdateInfo; } SStreamAggSupporter; typedef struct SWindowSupporter { @@ -502,38 +503,39 @@ typedef struct SOpCheckPointInfo { } SOpCheckPointInfo; typedef struct SStreamIntervalOperatorInfo { - SOptrBasicInfo binfo; // basic info - SAggSupporter aggSup; // aggregate supporter - SExprSupp scalarSupp; // supporter for perform scalar function - SGroupResInfo groupResInfo; // multiple results build supporter - SInterval interval; // interval info - int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. - STimeWindowAggSupp twAggSup; - bool invertible; - bool ignoreExpiredData; - bool ignoreExpiredDataSaved; - SArray* pDelWins; // SWinRes - int32_t delIndex; - SSDataBlock* pDelRes; - SPhysiNode* pPhyNode; // create new child - SHashObj* pPullDataMap; - SArray* pPullWins; // SPullWindowInfo - int32_t pullIndex; - SSDataBlock* pPullDataRes; - SArray* pChildren; - int32_t numOfChild; - SStreamState* pState; // void - SWinKey delKey; - uint64_t numOfDatapack; - SArray* pUpdated; - SSHashObj* pUpdatedMap; - int64_t dataVersion; - SStateStore stateStore; - bool recvGetAll; - SHashObj* pFinalPullDataMap; - SOpCheckPointInfo checkPointInfo; - bool reCkBlock; - SSDataBlock* pCheckpointRes; + SOptrBasicInfo binfo; // basic info + SAggSupporter aggSup; // aggregate supporter + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; // multiple results build supporter + SInterval interval; // interval info + int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. + STimeWindowAggSupp twAggSup; + bool invertible; + bool ignoreExpiredData; + bool ignoreExpiredDataSaved; + SArray* pDelWins; // SWinRes + int32_t delIndex; + SSDataBlock* pDelRes; + SPhysiNode* pPhyNode; // create new child + SHashObj* pPullDataMap; + SArray* pPullWins; // SPullWindowInfo + int32_t pullIndex; + SSDataBlock* pPullDataRes; + SArray* pChildren; + int32_t numOfChild; + SStreamState* pState; // void + SWinKey delKey; + uint64_t numOfDatapack; + SArray* pUpdated; + SSHashObj* pUpdatedMap; + int64_t dataVersion; + SStateStore stateStore; + bool recvGetAll; + SHashObj* pFinalPullDataMap; + SOpCheckPointInfo checkPointInfo; + bool reCkBlock; + SSDataBlock* pCheckpointRes; + struct SUpdateInfo* pUpdateInfo; } SStreamIntervalOperatorInfo; typedef struct SDataGroupInfo { @@ -830,6 +832,7 @@ void compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeW SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap); int32_t releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI); void resetWinRange(STimeWindow* winRange); +bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, TSKEY ts); int32_t encodeSSessionKey(void** buf, SSessionKey* key); void* decodeSSessionKey(void* buf, SSessionKey* key); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 448c585869..4c212afce8 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1619,6 +1619,15 @@ void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKE pBlock->info.rows++; } +bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, TSKEY ts) { + bool isExpired = false; + bool isInc = pAPI->isIncrementalTimeStamp(pUpdateInfo, tableId, ts); + if (!isInc) { + isExpired = isOverdue(ts, pTwSup); + } + return isExpired; +} + static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) { if (out) { blockDataCleanup(pInfo->pUpdateDataRes); diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 8029b9b156..a2394b4690 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -297,7 +297,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl int32_t rows = pSDataBlock->info.rows; blockDataEnsureCapacity(pAggSup->pScanBlock, rows); for (int32_t i = 0; i < rows; i += winRows) { - if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) { + if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, + &pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i])) { i++; continue; } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 86d83e05ad..2df3213c39 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -450,6 +450,7 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt pScanInfo->interval = pInfo->interval; pScanInfo->twAggSup = pInfo->twAggSup; pScanInfo->pState = pInfo->pState; + pInfo->pUpdateInfo = pScanInfo->pUpdateInfo; } void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput, @@ -800,7 +801,9 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat } while (1) { bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); - if ((pInfo->ignoreExpiredData && isClosed && !IS_FINAL_INTERVAL_OP(pOperator)) || + if ((!IS_FINAL_INTERVAL_OP(pOperator) && pInfo->ignoreExpiredData && + checkExpiredData(&pInfo->stateStore, pInfo->pUpdateInfo, &pInfo->twAggSup, pSDataBlock->info.id.uid, + nextWin.ekey)) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); if (startPos < 0) { @@ -1621,6 +1624,7 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin pScanInfo->igCheckUpdate); } pScanInfo->twAggSup = *pTwSup; + pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo; } static TSKEY sesionTs(void* pKey) { @@ -2015,7 +2019,9 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData TSKEY* endTsCols = (int64_t*)pEndTsCol->pData; for (int32_t i = 0; i < rows;) { - if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) { + if (!IS_FINAL_SESSION_OP(pOperator) && pInfo->ignoreExpiredData && + checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, &pInfo->twAggSup, + pSDataBlock->info.id.uid, endTsCols[i])) { i++; continue; } @@ -3327,7 +3333,8 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl blockDataEnsureCapacity(pAggSup->pScanBlock, rows); SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); for (int32_t i = 0; i < rows; i += winRows) { - if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup) || colDataIsNull_s(pKeyColInfo, i)) { + if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, + &pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i]) || colDataIsNull_s(pKeyColInfo, i)) { i++; continue; } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 858667b563..f78f6f4df1 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -129,9 +129,9 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma } pInfo->numBuckets = DEFAULT_BUCKET_SIZE; pInfo->pCloseWinSBF = NULL; - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT); - pInfo->pMap = taosHashInit(DEFAULT_MAP_CAPACITY, hashFn, true, HASH_NO_LOCK); } + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT); + pInfo->pMap = taosHashInit(DEFAULT_MAP_CAPACITY, hashFn, true, HASH_NO_LOCK); pInfo->maxDataVersion = 0; return pInfo; } @@ -384,3 +384,14 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) { tDecoderClear(&decoder); return 0; } + +bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { + TSKEY *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); + bool res = true; + if ( pMapMaxTs && ts < *pMapMaxTs ) { + res = false; + } else { + taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), &ts, sizeof(TSKEY)); + } + return res; +} diff --git a/tests/script/tsim/stream/ignoreExpiredData.sim b/tests/script/tsim/stream/ignoreExpiredData.sim index 884b7cbb5f..864f8caea5 100644 --- a/tests/script/tsim/stream/ignoreExpiredData.sim +++ b/tests/script/tsim/stream/ignoreExpiredData.sim @@ -107,7 +107,7 @@ sql select * from information_schema.ins_databases print ======database=$rows -sql use test1 +sql use test1; sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); @@ -118,9 +118,9 @@ sql insert into ts1 values(1648791211000,1,2,3); sleep 1000 sql insert into ts1 values(1648791222001,2,2,3); sleep 1000 -sql insert into ts2 values(1648791211000,1,2,3); -sleep 1000 sql insert into ts2 values(1648791222001,2,2,3); +sleep 1000 +sql insert into ts2 values(1648791211000,1,2,3); $loop_count = 0 loop4: @@ -132,16 +132,26 @@ if $loop_count == 10 then return -1 endi -if $data01 != 2 then +if $data01 != 1 then print =====data01=$data01 goto loop4 endi -if $data02 != 2 then +if $data02 != 1 then print =====data02=$data02 goto loop4 endi +if $data11 != 2 then + print =====data11=$data11 + goto loop4 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop4 +endi + $loop_count = 0 loop5: sleep 1000 @@ -162,4 +172,148 @@ if $data02 != 1 then goto loop5 endi +if $data11 != 2 then + print =====data11=$data11 + goto loop4 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop4 +endi + + +print =============== create database test2 +sql create database test2 vgroups 4 +sql select * from information_schema.ins_databases + +print ======database=$rows + +sql use test2; + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create table ts3 using st tags(3,3,3); +sql create table ts4 using st tags(4,4,4); +sql create stream streams_21 trigger at_once IGNORE EXPIRED 1 into streamt_21 as select _wstart, count(*) c1 from st interval(10s) ; +sleep 1000 + +sql insert into ts1 values(1648791211000,1,2,3); +sql insert into ts1 values(1648791211001,2,2,3); +sql insert into ts1 values(1648791211002,2,2,3); +sql insert into ts1 values(1648791211003,2,2,3); +sql insert into ts1 values(1648791211004,2,2,3); + +sleep 1000 +sql insert into ts2 values(1648791201000,1,2,3); +sql insert into ts2 values(1648791201001,2,2,3); +sql insert into ts2 values(1648791201002,2,2,3); +sql insert into ts2 values(1648791201003,2,2,3); +sql insert into ts2 values(1648791201004,2,2,3); + +sleep 1000 +sql insert into ts2 values(1648791101000,1,2,3); +sql insert into ts2 values(1648791101001,2,2,3); +sql insert into ts2 values(1648791101002,2,2,3); +sql insert into ts2 values(1648791101003,2,2,3); +sql insert into ts2 values(1648791101004,2,2,3); + + +$loop_count = 0 +loop6: +sleep 1000 +print 1 select * from streamt_21; +sql select * from streamt_21; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop6 +endi + +if $data01 != 5 then + print =====data01=$data01 + goto loop6 +endi + +if $data11 != 5 then + print =====data11=$data11 + goto loop6 +endi + +sleep 1000 +sql insert into ts3 values(1648791241000,1,2,3); + +sleep 1000 +sql insert into ts3 values(1648791231001,2,2,3); +sql insert into ts3 values(1648791231002,2,2,3); +sql insert into ts3 values(1648791231003,2,2,3); +sql insert into ts3 values(1648791231004,2,2,3); + +$loop_count = 0 +loop7: +sleep 1000 +print 2 select * from streamt_21; +sql select * from streamt_21; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 3 then + print =====rows=$rows + goto loop7 +endi + +if $data21 != 1 then + print =====data21=$data21 + goto loop7 +endi + +sleep 1000 +sql insert into ts4 values(1648791231001,2,2,3); +sql insert into ts4 values(1648791231002,2,2,3); +sql insert into ts4 values(1648791231003,2,2,3); +sql insert into ts4 values(1648791231004,2,2,3); + +sleep 1000 +sql insert into ts4 values(1648791211001,2,2,3); +sql insert into ts4 values(1648791211002,2,2,3); +sql insert into ts4 values(1648791211003,2,2,3); +sql insert into ts4 values(1648791211004,2,2,3); + +$loop_count = 0 +loop8: +sleep 1000 +print 3 select * from streamt_21; +sql select * from streamt_21; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 4 then + print =====rows=$rows + goto loop8 +endi + +if $data21 != 4 then + print =====data21=$data21 + goto loop8 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop8 +endi + +print ============================end + system sh/stop_dnodes.sh From 419712494eea8a2305eb9b539fb16b9ba02e68c0 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Wed, 29 Nov 2023 11:22:13 +0800 Subject: [PATCH 015/102] skip scan table --- source/libs/function/src/builtins.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index fbabef8fd3..98fda024fa 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -3425,7 +3425,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_wstart", .type = FUNCTION_TYPE_WSTART, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_SKIP_SCAN_CHECK_FUNC, .translateFunc = translateTimePseudoColumn, .getEnvFunc = getTimePseudoFuncEnv, .initFunc = NULL, @@ -3435,7 +3435,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_wend", .type = FUNCTION_TYPE_WEND, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_SKIP_SCAN_CHECK_FUNC, .translateFunc = translateTimePseudoColumn, .getEnvFunc = getTimePseudoFuncEnv, .initFunc = NULL, @@ -3445,7 +3445,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_wduration", .type = FUNCTION_TYPE_WDURATION, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_SKIP_SCAN_CHECK_FUNC, .translateFunc = translateWduration, .getEnvFunc = getTimePseudoFuncEnv, .initFunc = NULL, From 09e314c3ae6f087adc798292d6dccaea8b7ce0bf Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 29 Nov 2023 12:26:32 +0800 Subject: [PATCH 016/102] fix: pStatus->pTableIter is set to null when getting mem table time range --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index b999607503..e590358267 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2520,18 +2520,15 @@ static void prepareDurationForNextFileSet(STsdbReader* pReader) { STimeWindow winFid = {0}; tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &winFid.skey, &winFid.ekey); - bool bProcMemPreFileset = false; if (ASCENDING_TRAVERSE(pReader->info.order)) { - bProcMemPreFileset = !(pReader->status.prevFilesetStartKey > pReader->status.memTableMaxKey || winFid.skey < pReader->status.memTableMinKey); + pReader->status.bProcMemPreFileset = !(pReader->status.prevFilesetStartKey > pReader->status.memTableMaxKey || winFid.skey < pReader->status.memTableMinKey); } else { - bProcMemPreFileset = !(winFid.ekey > pReader->status.memTableMaxKey || pReader->status.prevFilesetEndKey < pReader->status.memTableMinKey); + pReader->status.bProcMemPreFileset = !(winFid.ekey > pReader->status.memTableMaxKey || pReader->status.prevFilesetEndKey < pReader->status.memTableMinKey); } - - if (bProcMemPreFileset) { - pReader->status.bProcMemPreFileset = true; + + if (pReader->status.bProcMemPreFileset) { resetTableListIndex(&pReader->status); } - pReader->status.prevFilesetStartKey = winFid.skey; pReader->status.prevFilesetEndKey = winFid.ekey; } @@ -2978,7 +2975,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t en if (!hasNexTable) { return TSDB_CODE_SUCCESS; } - pBlockScanInfo = pStatus->pTableIter; + continue; } initMemDataIterator(*pBlockScanInfo, pReader); @@ -4947,9 +4944,9 @@ static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t int64_t maxKey = INT64_MIN; int64_t minKey = INT64_MAX; - pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, NULL, &iter); - while (pStatus->pTableIter != NULL) { - STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; + void* pHashIter = tSimpleHashIterate(pStatus->pTableMap, NULL, &iter); + while (pHashIter!= NULL) { + STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pHashIter; STbData* d = NULL; if (pReader->pReadSnap->pMem != NULL) { @@ -4978,7 +4975,7 @@ static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t } // current table is exhausted, let's try the next table - pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, pStatus->pTableIter, &iter); + pHashIter = tSimpleHashIterate(pStatus->pTableMap, pHashIter, &iter); } *pMaxKey = maxKey; From 1a3e280ea878b491d82694b1935a8f6140595f2e Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 29 Nov 2023 13:50:53 +0800 Subject: [PATCH 017/102] fix: use imem maxKey/minKey --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index e590358267..0320265f7f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4965,11 +4965,11 @@ static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t if (pReader->pReadSnap->pIMem != NULL) { di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->info.suid, pBlockScanInfo->uid); if (di != NULL) { - if (d->maxKey > maxKey) { - maxKey = d->maxKey; + if (di->maxKey > maxKey) { + maxKey = di->maxKey; } - if (d->minKey < minKey) { - minKey = d->minKey; + if (di->minKey < minKey) { + minKey = di->minKey; } } } From e66a1d6c9aa51aa71a9b58ffa0699813f91a5559 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 29 Nov 2023 09:14:44 +0800 Subject: [PATCH 018/102] enh: limit snap replication msg by size of block data in tsdbSnapReadTimeSeriesData --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index a9da0fbcec..8a7d1f8b97 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -278,6 +278,15 @@ _exit: return code; } +static int64_t tBlockDataSize(SBlockData* pBlockData) { + int64_t nData = 0; + for (int32_t iCol = 0; iCol < pBlockData->nColData; iCol++) { + SColData* pColData = tBlockDataGetColDataByIdx(pBlockData, iCol); + nData += pColData->nData; + } + return nData; +} + static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** data) { int32_t code = 0; int32_t lino = 0; @@ -320,8 +329,11 @@ static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** dat code = tsdbIterMergerNext(reader->dataIterMerger); TSDB_CHECK_CODE(code, lino, _exit); - if (reader->blockData->nRow >= 81920) { - break; + if (!(reader->blockData->nRow % 16)) { + int64_t nData = tBlockDataSize(reader->blockData); + if (nData >= 1 * 1024 * 1024) { + break; + } } } From 9e8944d17cf9d7433b956a5073766c1125908bae Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 29 Nov 2023 09:18:05 +0800 Subject: [PATCH 019/102] enh: adjust buffer size of sync snap replication --- include/util/tdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 69d0c1126d..1a440c7268 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -305,7 +305,7 @@ typedef enum ELogicConditionType { #define TSDB_SYNC_APPLYQ_SIZE_LIMIT 512 #define TSDB_SYNC_NEGOTIATION_WIN 512 -#define TSDB_SYNC_SNAP_BUFFER_SIZE 2048 +#define TSDB_SYNC_SNAP_BUFFER_SIZE 1024 #define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta From 4f168b4b2dd579082db6d7b4b449fc400c97afed Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 29 Nov 2023 17:22:43 +0800 Subject: [PATCH 020/102] fix:tmqVnodeSplit-column.py failed in arm64 because of rebalance after unsubscribe while split vnode --- source/dnode/mnode/impl/src/mndStream.c | 31 +------------------ source/dnode/mnode/impl/src/mndSubscribe.c | 1 + source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqPush.c | 2 +- .../system-test/7-tmq/tmqVnodeSplit-column.py | 2 +- 5 files changed, 5 insertions(+), 33 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index bc265db766..08c60e7afa 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -328,35 +328,6 @@ static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) { return 0; } -static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64_t watermark, char **pStr) { - if (NULL == ast) { - return TSDB_CODE_SUCCESS; - } - - SNode * pAst = NULL; - int32_t code = nodesStringToNode(ast, &pAst); - - SQueryPlan *pPlan = NULL; - if (TSDB_CODE_SUCCESS == code) { - SPlanContext cxt = { - .pAstRoot = pAst, - .topicQuery = false, - .streamQuery = true, - .triggerType = (triggerType == STREAM_TRIGGER_MAX_DELAY) ? STREAM_TRIGGER_WINDOW_CLOSE : triggerType, - .watermark = watermark, - }; - code = qCreateQueryPlan(&cxt, &pPlan, NULL); - } - - if (TSDB_CODE_SUCCESS == code) { - code = nodesNodeToString((SNode *)pPlan, false, pStr, NULL); - } - nodesDestroyNode(pAst); - nodesDestroyNode((SNode *)pPlan); - terrno = code; - return code; -} - static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, SCMCreateStreamReq *pCreate) { SNode * pAst = NULL; SQueryPlan *pPlan = NULL; @@ -777,6 +748,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { SDbObj * pDb = NULL; SCMCreateStreamReq createStreamReq = {0}; SStreamObj streamObj = {0}; + char* sql = NULL; if (tDeserializeSCMCreateStreamReq(pReq->pCont, pReq->contLen, &createStreamReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -808,7 +780,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - char* sql = NULL; int32_t sqlLen = 0; if(createStreamReq.sql != NULL){ sqlLen = strlen(createStreamReq.sql); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 408b664e50..44ee804d22 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -161,6 +161,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg, SSubplan* pPlan) { if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { + if(pRebVg->oldConsumerId == -1) return 0; //drop stream, no consumer, while split vnode,all consumerId is -1 terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; return -1; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9d16402ee6..f077caace7 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -337,7 +337,7 @@ int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg) { while (pIter) { STqHandle* pHandle = *(STqHandle**)pIter; - tqInfo("vgId:%d start set submit for pHandle:%p, consumer:0x%" PRIx64, vgId, pHandle, pHandle->consumerId); + tqDebug("vgId:%d start set submit for pHandle:%p, consumer:0x%" PRIx64, vgId, pHandle, pHandle->consumerId); if (ASSERT(pHandle->msg != NULL)) { tqError("pHandle->msg should not be null"); diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index f367bc96f8..8fee1d5904 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -72,7 +72,7 @@ int32_t tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg) { memcpy(pHandle->msg->pCont, pMsg->pCont, pMsg->contLen); pHandle->msg->contLen = pMsg->contLen; int32_t ret = taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey), &pHandle, POINTER_BYTES); - tqInfo("vgId:%d data is over, ret:%d, consumerId:0x%" PRIx64 ", register to pHandle:%p, pCont:%p, len:%d", vgId, ret, + tqDebug("vgId:%d data is over, ret:%d, consumerId:0x%" PRIx64 ", register to pHandle:%p, pCont:%p, len:%d", vgId, ret, pHandle->consumerId, pHandle, pHandle->msg->pCont, pHandle->msg->contLen); return 0; } diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column.py b/tests/system-test/7-tmq/tmqVnodeSplit-column.py index 54a43465e7..87a73e981e 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-column.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column.py @@ -188,7 +188,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) From 80ee5bfb3e11cbce9759c390da3e3834b41d3bf7 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 30 Nov 2023 11:08:18 +0800 Subject: [PATCH 021/102] fix: the initial value of first fileset start / end key --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 13 +++++++------ source/libs/executor/src/scanoperator.c | 1 - 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 0320265f7f..e7018638a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2521,9 +2521,9 @@ static void prepareDurationForNextFileSet(STsdbReader* pReader) { tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &winFid.skey, &winFid.ekey); if (ASCENDING_TRAVERSE(pReader->info.order)) { - pReader->status.bProcMemPreFileset = !(pReader->status.prevFilesetStartKey > pReader->status.memTableMaxKey || winFid.skey < pReader->status.memTableMinKey); + pReader->status.bProcMemPreFileset = !(pReader->status.prevFilesetStartKey > pReader->status.memTableMaxKey || winFid.skey-1 < pReader->status.memTableMinKey); } else { - pReader->status.bProcMemPreFileset = !(winFid.ekey > pReader->status.memTableMaxKey || pReader->status.prevFilesetEndKey < pReader->status.memTableMinKey); + pReader->status.bProcMemPreFileset = !(winFid.ekey+1 > pReader->status.memTableMaxKey || pReader->status.prevFilesetEndKey < pReader->status.memTableMinKey); } if (pReader->status.bProcMemPreFileset) { @@ -3911,6 +3911,11 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; + if (pReader->bDurationOrder) { + getMemTableTimeRange(pReader, &pReader->status.memTableMaxKey, &pReader->status.memTableMinKey); + pReader->status.bProcMemFirstFileset = true; + } + initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); resetDataBlockIterator(&pStatus->blockIter, pReader->info.order); @@ -4344,10 +4349,6 @@ int32_t tsdbReaderResume2(STsdbReader* pReader) { } } - if (pReader->bDurationOrder) { - getMemTableTimeRange(pReader, &pReader->status.memTableMaxKey, &pReader->status.memTableMinKey); - } - pReader->flag = READER_STATUS_NORMAL; tsdbDebug("reader: %p resumed uid %" PRIu64 ", numOfTable:%" PRId32 ", in this query %s", pReader, pBlockScanInfo ? (*pBlockScanInfo)->uid : 0, numOfTables, pReader->idStr); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 3d2053c0c0..8af3765f87 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3284,7 +3284,6 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; - uInfo("getBlockForTableMergeScan retrieved one block"); return pBlock; } From d9c84b4b720a502fe0b7d2b22709dbf3e3debea9 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 30 Nov 2023 11:55:17 +0800 Subject: [PATCH 022/102] enh: tolerate out-of-order of closeTS in walEndSnapshot --- source/libs/wal/src/walWrite.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 33d8d34514..341d989f8f 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -327,15 +327,19 @@ int32_t walEndSnapshot(SWal *pWal) { // iterate files, until the searched result // delete according to file size or close time + SWalFileInfo *pUntil = NULL; for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { if ((pWal->cfg.retentionSize > 0 && newTotSize > pWal->cfg.retentionSize) || (pWal->cfg.retentionPeriod == 0 || pWal->cfg.retentionPeriod > 0 && iter->closeTs >= 0 && iter->closeTs + pWal->cfg.retentionPeriod < ts)) { - deleteCnt++; newTotSize -= iter->fileSize; - taosArrayPush(pWal->toDeleteFiles, iter); + pUntil = iter; } } + for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter <= pUntil; iter++) { + deleteCnt++; + taosArrayPush(pWal->toDeleteFiles, iter); + } // make new array, remove files taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); From f96044e09fcfa0286a31e049cbb9a74d00bf9219 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 30 Nov 2023 13:39:54 +0800 Subject: [PATCH 023/102] Revert "enh: grant and active codes" --- include/common/tgrant.h | 7 --- include/util/taoserror.h | 1 - source/dnode/mnode/impl/inc/mndCluster.h | 2 - source/dnode/mnode/impl/inc/mndDef.h | 2 - source/dnode/mnode/impl/src/mndCluster.c | 62 +------------------ source/dnode/mnode/impl/src/mndDnode.c | 20 ++---- source/util/src/terror.c | 1 - .../0-others/information_schema.py | 9 +-- 8 files changed, 8 insertions(+), 96 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index a5f3ab2e3f..f06fca8014 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -31,8 +31,6 @@ extern "C" { #endif #define GRANT_HEART_BEAT_MIN 2 -#define GRANT_ACTIVE_CODE "activeCode" -#define GRANT_C_ACTIVE_CODE "cActiveCode" typedef enum { TSDB_GRANT_ALL, @@ -52,11 +50,6 @@ typedef enum { TSDB_GRANT_TABLE, } EGrantType; -typedef struct { - int64_t grantedTime; - int64_t connGrantedTime; -} SGrantedInfo; - int32_t grantCheck(EGrantType grant); int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, char* out, int8_t type); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 0256a496df..6ab06d06a3 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -558,7 +558,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_GRANT_GEN_IVLD_KEY TAOS_DEF_ERROR_CODE(0, 0x0812) #define TSDB_CODE_GRANT_GEN_APP_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0813) #define TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN TAOS_DEF_ERROR_CODE(0, 0x0814) -#define TSDB_CODE_GRANT_PAR_IVLD_DIST TAOS_DEF_ERROR_CODE(0, 0x0815) // sync // #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index 2b59d9dbf5..e33ffdb372 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -27,8 +27,6 @@ void mndCleanupCluster(SMnode *pMnode); int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len); int64_t mndGetClusterId(SMnode *pMnode); int64_t mndGetClusterCreateTime(SMnode *pMnode); -int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); -int32_t mndSetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int64_t mndGetClusterUpTime(SMnode *pMnode); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 27fef1e81e..aa82dff7ec 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -192,8 +192,6 @@ typedef struct { int64_t createdTime; int64_t updateTime; int32_t upTime; - int64_t grantedTime; - int64_t connGrantedTime; } SClusterObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 26c678b513..4c799e1e1e 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -19,7 +19,7 @@ #include "mndTrans.h" #define CLUSTER_VER_NUMBE 1 -#define CLUSTER_RESERVE_SIZE 44 +#define CLUSTER_RESERVE_SIZE 60 int64_t tsExpireTime = 0; static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); @@ -112,19 +112,6 @@ int64_t mndGetClusterCreateTime(SMnode *pMnode) { return createTime; } -int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { - void *pIter = NULL; - SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); - if (pCluster != NULL) { - pInfo->grantedTime = pCluster->grantedTime; - pInfo->connGrantedTime = pCluster->connGrantedTime; - mndReleaseCluster(pMnode, pCluster, pIter); - return 0; - } - - return -1; -} - static int32_t mndGetClusterUpTimeImp(SClusterObj *pCluster) { #if 0 int32_t upTime = taosGetTimestampSec() - pCluster->updateTime / 1000; @@ -159,8 +146,6 @@ static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime, _OVER) SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, _OVER) SDB_SET_INT32(pRaw, dataPos, pCluster->upTime, _OVER) - SDB_SET_INT64(pRaw, dataPos, pCluster->grantedTime, _OVER) - SDB_SET_INT64(pRaw, dataPos, pCluster->connGrantedTime, _OVER) SDB_SET_RESERVE(pRaw, dataPos, CLUSTER_RESERVE_SIZE, _OVER) terrno = 0; @@ -201,8 +186,6 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pCluster->updateTime, _OVER) SDB_GET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, _OVER) SDB_GET_INT32(pRaw, dataPos, &pCluster->upTime, _OVER) - SDB_GET_INT64(pRaw, dataPos, &pCluster->grantedTime, _OVER); - SDB_GET_INT64(pRaw, dataPos, &pCluster->connGrantedTime, _OVER); SDB_GET_RESERVE(pRaw, dataPos, CLUSTER_RESERVE_SIZE, _OVER) terrno = 0; @@ -235,8 +218,6 @@ static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOld, SClusterObj mTrace("cluster:%" PRId64 ", perform update action, old row:%p new row:%p, uptime from %d to %d", pOld->id, pOld, pNew, pOld->upTime, pNew->upTime); pOld->upTime = pNew->upTime; - pOld->grantedTime = pNew->grantedTime; - pOld->connGrantedTime = pNew->connGrantedTime; pOld->updateTime = taosGetTimestampMs(); return 0; } @@ -378,44 +359,3 @@ static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) { mndTransDrop(pTrans); return 0; } - -int32_t mndSetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { - SClusterObj clusterObj = {0}; - void *pIter = NULL; - SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); - if (pCluster != NULL) { - if (pCluster->grantedTime >= pInfo->grantedTime && pCluster->connGrantedTime >= pInfo->connGrantedTime) { - mndReleaseCluster(pMnode, pCluster, pIter); - return 0; - } - memcpy(&clusterObj, pCluster, sizeof(SClusterObj)); - if (pCluster->grantedTime < pInfo->grantedTime) clusterObj.grantedTime = pInfo->grantedTime; - if (pCluster->connGrantedTime < pInfo->connGrantedTime) clusterObj.connGrantedTime = pInfo->connGrantedTime; - mndReleaseCluster(pMnode, pCluster, pIter); - } - - if (clusterObj.id <= 0) { - mError("can't get cluster info while update granted info"); - return -1; - } - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "granted-info"); - if (pTrans == NULL) return -1; - - SSdbRaw *pCommitRaw = mndClusterActionEncode(&clusterObj); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - - mndTransDrop(pTrans); - return 0; -} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index b0bffcc83e..e224aceec2 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -790,9 +790,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg if (cfgAll) { // alter all dnodes: if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t)); if (failRecord) taosArrayPush(failRecord, &pDnode->id); - if (0 == cfgAllErr || cfgAllErr == TSDB_CODE_GRANT_PAR_IVLD_ACTIVE) { - cfgAllErr = terrno; // output 1st or more specific error - } + if (0 == cfgAllErr) cfgAllErr = terrno; // output 1st terrno. } } else { terrno = 0; // no action for dup active code @@ -808,9 +806,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg if (cfgAll) { if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t)); if (failRecord) taosArrayPush(failRecord, &pDnode->id); - if (0 == cfgAllErr || cfgAllErr == TSDB_CODE_GRANT_PAR_IVLD_ACTIVE) { - cfgAllErr = terrno; // output 1st or more specific error - } + if (0 == cfgAllErr) cfgAllErr = terrno; } } else { terrno = 0; @@ -1287,12 +1283,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { strcpy(dcfgReq.config, "supportvnodes"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); - } else if (strncasecmp(cfgReq.config, GRANT_ACTIVE_CODE, 10) == 0 || - strncasecmp(cfgReq.config, GRANT_C_ACTIVE_CODE, 11) == 0) { - if (cfgReq.dnodeId != -1) { - terrno = TSDB_CODE_INVALID_CFG; - goto _err_out; - } + } else if (strncasecmp(cfgReq.config, "activeCode", 10) == 0 || strncasecmp(cfgReq.config, "cActiveCode", 11) == 0) { int8_t opt = strncasecmp(cfgReq.config, "a", 1) == 0 ? DND_ACTIVE_CODE : DND_CONN_ACTIVE_CODE; int8_t index = opt == DND_ACTIVE_CODE ? 10 : 11; if (' ' != cfgReq.config[index] && 0 != cfgReq.config[index]) { @@ -1310,11 +1301,12 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { goto _err_out; } - strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? GRANT_ACTIVE_CODE : GRANT_C_ACTIVE_CODE); + strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? "activeCode" : "cActiveCode"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%s", cfgReq.value); - if ((terrno = mndConfigDnode(pMnode, pReq, &cfgReq, opt)) != 0) { + if (mndConfigDnode(pMnode, pReq, &cfgReq, opt) != 0) { mError("dnode:%d, failed to config activeCode since %s", cfgReq.dnodeId, terrstr()); + terrno = TSDB_CODE_INVALID_CFG; goto _err_out; } tFreeSMCfgDnodeReq(&cfgReq); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 8847b7d894..cc6647e463 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -445,7 +445,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_DEC_IVLD_KLEN, "Invalid klen to decod TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_IVLD_KEY, "Invalid key to gen active code") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_APP_LIMIT, "Limited app num to gen active code") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN, "Invalid klen to encode active code") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_IVLD_DIST, "Invalid dist to parse active code") // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 544a966960..2bfe33d0af 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -247,10 +247,7 @@ class TDTestCase: tdSql.error('alter all dnodes "activeCode" "' + self.str510 + '"') tdSql.query(f'select * from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][8],"") - tdSql.error('alter dnode 1 "activeCode" ""') - tdSql.error('alter dnode 1 "activeCode"') - tdSql.execute('alter all dnodes "activeCode" ""') - tdSql.execute('alter all dnodes "activeCode"') + tdSql.execute('alter dnode 1 "activeCode" ""') tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],"") tdSql.checkEqual(tdSql.queryResult[0][1],'') @@ -262,10 +259,6 @@ class TDTestCase: tdSql.error('alter all dnodes "cActiveCode" "' + self.str257 + '"') tdSql.error('alter all dnodes "cActiveCode" "' + self.str254 + '"') tdSql.error('alter dnode 1 "cActiveCode" "' + self.str510 + '"') - tdSql.error('alter dnode 1 "cActiveCode" ""') - tdSql.error('alter dnode 1 "cActiveCode"') - tdSql.execute('alter all dnodes "cActiveCode" ""') - tdSql.execute('alter all dnodes "cActiveCode"') tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],"") tdSql.checkEqual(tdSql.queryResult[0][1],"") From b0908b4aa6f76f95216551e3eecea7a62dcdd29e Mon Sep 17 00:00:00 2001 From: charles Date: Thu, 30 Nov 2023 14:57:02 +0800 Subject: [PATCH 024/102] update test cases for amr64 --- tests/parallel_test/cases.task | 2 +- tests/system-test/0-others/compatibility.py | 5 ++++- .../0-others/test_hot_refresh_configurations.py | 4 +++- tests/system-test/6-cluster/5dnode3mnodeRoll.py | 7 +++++-- tests/system-test/7-tmq/tmqDnodeRestart.py | 4 ++++ tests/system-test/7-tmq/tmqVnodeSplit-column.py | 2 +- 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index d039495351..45616c3f19 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -880,7 +880,7 @@ e ,,y,script,./test.sh -f tsim/dnode/balance2.sim ,,y,script,./test.sh -f tsim/vnode/replica3_repeat.sim ,,y,script,./test.sh -f tsim/parser/col_arithmetic_operation.sim -,,y,script,./test.sh -f tsim/trans/create_db.sim +#,,y,script,./test.sh -f tsim/trans/create_db.sim ,,y,script,./test.sh -f tsim/dnode/balance3.sim ,,y,script,./test.sh -f tsim/vnode/replica3_many.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 83bfb2bed7..d54c676c0d 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -90,7 +90,10 @@ class TDTestCase: packagePath = "/usr/local/src/" dataPath = cPath + "/../data/" - packageName = "TDengine-server-"+ BASEVERSION + "-Linux-x64.tar.gz" + if platform.system() == "Linux" and platform.machine() == "aarch64": + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-arm64.tar.gz" + else: + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-x64.tar.gz" packageTPath = packageName.split("-Linux-")[0] my_file = Path(f"{packagePath}/{packageName}") if not my_file.exists(): diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index 7aed7274a4..cbde8c060e 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -2,7 +2,7 @@ import subprocess import random import time import os - +import platform from util.log import * from util.sql import * from util.cases import * @@ -190,6 +190,8 @@ class TDTestCase: for v in values: dnode = random.choice(p_list) tdSql.execute(f'alter {dnode} "{name} {v}";') + if platform.system() == "Linux" and platform.machine() == "aarch64": + continue value = self.get_param_value_with_gdb(alias, "taosd") if value: tdLog.debug(f"value: {value}") diff --git a/tests/system-test/6-cluster/5dnode3mnodeRoll.py b/tests/system-test/6-cluster/5dnode3mnodeRoll.py index 9d62eb3b4b..11a153c48f 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRoll.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRoll.py @@ -4,7 +4,7 @@ import taos import sys import time import os - +import platform from util.log import * from util.sql import * from util.cases import * @@ -96,7 +96,10 @@ class TDTestCase: packagePath = "/usr/local/src/" dataPath = cPath + "/../data/" - packageName = "TDengine-server-"+ BASEVERSION + "-Linux-x64.tar.gz" + if platform.system() == "Linux" and platform.machine() == "aarch64": + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-arm64.tar.gz" + else: + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-x64.tar.gz" packageTPath = packageName.split("-Linux-")[0] my_file = Path(f"{packagePath}/{packageName}") if not my_file.exists(): diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index 74aba31726..0ac8482163 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -4,6 +4,7 @@ import sys import time import socket import os +import platform import threading from enum import Enum @@ -184,6 +185,9 @@ class TDTestCase: paraDict['vgroups'] = self.vgroups paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl + # ARM64：time cost is so long for stopping taosd, so add the pollDdelay to 120s + if platform.system() == "Linux" and platform.machine() == "aarch64": + paraDict['pollDelay'] = 300 tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column.py b/tests/system-test/7-tmq/tmqVnodeSplit-column.py index 54a43465e7..87a73e981e 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-column.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column.py @@ -188,7 +188,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) From 3f6db938f36293da34f4514d336240a9ed068587 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Nov 2023 16:00:02 +0800 Subject: [PATCH 025/102] fix:minus row num to reduce execute time --- tests/system-test/7-tmq/tmqVnodeSplit-column.py | 6 +++--- tests/system-test/7-tmq/tmqVnodeSplit-db.py | 8 ++++---- .../7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py | 8 ++++---- .../7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py | 8 ++++---- tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py | 8 ++++---- tests/system-test/7-tmq/tmqVnodeSplit-stb.py | 8 ++++---- tests/system-test/7-tmq/tmqVnodeTransform-db.py | 8 ++++---- tests/system-test/7-tmq/tmqVnodeTransform-stb.py | 8 ++++---- tests/system-test/7-tmq/tmqVnodeTransform.py | 8 ++++---- 9 files changed, 35 insertions(+), 35 deletions(-) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column.py b/tests/system-test/7-tmq/tmqVnodeSplit-column.py index 87a73e981e..1fe2b5809a 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-column.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column.py @@ -23,7 +23,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -49,7 +49,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -118,7 +118,7 @@ class TDTestCase: 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-db.py b/tests/system-test/7-tmq/tmqVnodeSplit-db.py index e4353d3268..f66acf4fcd 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-db.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-db.py @@ -23,7 +23,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -49,7 +49,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -118,7 +118,7 @@ class TDTestCase: 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -189,7 +189,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py index 8276ae638b..68fb07b813 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py @@ -25,7 +25,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -51,7 +51,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -120,7 +120,7 @@ class TDTestCase: 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 120, @@ -189,7 +189,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py index 0d247b2848..6140e8a544 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py @@ -25,7 +25,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -51,7 +51,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -120,7 +120,7 @@ class TDTestCase: 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 120, @@ -189,7 +189,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py index cda5a27919..18b80b7f8d 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py @@ -27,7 +27,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -53,7 +53,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -122,7 +122,7 @@ class TDTestCase: 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 120, @@ -192,7 +192,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py index 17a427567e..c203350322 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py @@ -25,7 +25,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -51,7 +51,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -120,7 +120,7 @@ class TDTestCase: 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -190,7 +190,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-db.py b/tests/system-test/7-tmq/tmqVnodeTransform-db.py index 005bca70d6..5c61908d96 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-db.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-db.py @@ -20,7 +20,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -46,7 +46,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 30, @@ -138,7 +138,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 10, @@ -217,7 +217,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 10, diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py index ec1331ae59..64cdf6d153 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -20,7 +20,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -46,7 +46,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -137,7 +137,7 @@ class TDTestCase: 'ctbPrefix': 'ctb1', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -207,7 +207,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform.py index aab94bc7a2..3698297618 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform.py @@ -20,7 +20,7 @@ class TDTestCase: def __init__(self): self.vgroups = 1 self.ctbNum = 10 - self.rowsPerTbl = 10000 + self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -46,7 +46,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -137,7 +137,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 60, @@ -203,7 +203,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) From 8ca00ca70b67996cd3c66f3e09885147cadf6044 Mon Sep 17 00:00:00 2001 From: charles Date: Thu, 30 Nov 2023 14:57:02 +0800 Subject: [PATCH 026/102] remove node.sh test case from CI due to exist issue --- tests/parallel_test/cases.task | 4 ++-- tests/system-test/0-others/compatibility.py | 5 ++++- .../0-others/test_hot_refresh_configurations.py | 4 +++- tests/system-test/6-cluster/5dnode3mnodeRoll.py | 7 +++++-- tests/system-test/7-tmq/tmqDnodeRestart.py | 4 ++++ tests/system-test/7-tmq/tmqVnodeSplit-column.py | 2 +- 6 files changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index d039495351..ce6bcfe105 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -880,7 +880,7 @@ e ,,y,script,./test.sh -f tsim/dnode/balance2.sim ,,y,script,./test.sh -f tsim/vnode/replica3_repeat.sim ,,y,script,./test.sh -f tsim/parser/col_arithmetic_operation.sim -,,y,script,./test.sh -f tsim/trans/create_db.sim +#,,y,script,./test.sh -f tsim/trans/create_db.sim ,,y,script,./test.sh -f tsim/dnode/balance3.sim ,,y,script,./test.sh -f tsim/vnode/replica3_many.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim @@ -1337,7 +1337,7 @@ e #docs-examples test ,,n,docs-examples-test,bash python.sh -,,n,docs-examples-test,bash node.sh +#,,n,docs-examples-test,bash node.sh ,,n,docs-examples-test,bash csharp.sh ,,n,docs-examples-test,bash jdbc.sh ,,n,docs-examples-test,bash go.sh diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 83bfb2bed7..d54c676c0d 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -90,7 +90,10 @@ class TDTestCase: packagePath = "/usr/local/src/" dataPath = cPath + "/../data/" - packageName = "TDengine-server-"+ BASEVERSION + "-Linux-x64.tar.gz" + if platform.system() == "Linux" and platform.machine() == "aarch64": + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-arm64.tar.gz" + else: + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-x64.tar.gz" packageTPath = packageName.split("-Linux-")[0] my_file = Path(f"{packagePath}/{packageName}") if not my_file.exists(): diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index 7aed7274a4..cbde8c060e 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -2,7 +2,7 @@ import subprocess import random import time import os - +import platform from util.log import * from util.sql import * from util.cases import * @@ -190,6 +190,8 @@ class TDTestCase: for v in values: dnode = random.choice(p_list) tdSql.execute(f'alter {dnode} "{name} {v}";') + if platform.system() == "Linux" and platform.machine() == "aarch64": + continue value = self.get_param_value_with_gdb(alias, "taosd") if value: tdLog.debug(f"value: {value}") diff --git a/tests/system-test/6-cluster/5dnode3mnodeRoll.py b/tests/system-test/6-cluster/5dnode3mnodeRoll.py index 9d62eb3b4b..11a153c48f 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRoll.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRoll.py @@ -4,7 +4,7 @@ import taos import sys import time import os - +import platform from util.log import * from util.sql import * from util.cases import * @@ -96,7 +96,10 @@ class TDTestCase: packagePath = "/usr/local/src/" dataPath = cPath + "/../data/" - packageName = "TDengine-server-"+ BASEVERSION + "-Linux-x64.tar.gz" + if platform.system() == "Linux" and platform.machine() == "aarch64": + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-arm64.tar.gz" + else: + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-x64.tar.gz" packageTPath = packageName.split("-Linux-")[0] my_file = Path(f"{packagePath}/{packageName}") if not my_file.exists(): diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index 74aba31726..0ac8482163 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -4,6 +4,7 @@ import sys import time import socket import os +import platform import threading from enum import Enum @@ -184,6 +185,9 @@ class TDTestCase: paraDict['vgroups'] = self.vgroups paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl + # ARM64：time cost is so long for stopping taosd, so add the pollDdelay to 120s + if platform.system() == "Linux" and platform.machine() == "aarch64": + paraDict['pollDelay'] = 300 tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column.py b/tests/system-test/7-tmq/tmqVnodeSplit-column.py index 54a43465e7..87a73e981e 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-column.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column.py @@ -188,7 +188,7 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectrowcnt / 2 >= resultList[0]: + if expectrowcnt / 2 > resultList[0]: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) tdLog.exit("%d tmq consume rows error!"%consumerId) From 77a48f87113f2a8c3ea81296421c61587a88ddff Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Nov 2023 17:36:03 +0800 Subject: [PATCH 027/102] fix:test case --- tests/script/tsim/snode/basic1.sim | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/script/tsim/snode/basic1.sim b/tests/script/tsim/snode/basic1.sim index 86072215f7..6e553591bd 100644 --- a/tests/script/tsim/snode/basic1.sim +++ b/tests/script/tsim/snode/basic1.sim @@ -113,11 +113,7 @@ sql_error drop snode on dnode 2 print =============== create drop snodes sql create snode on dnode 1 -sql create snode on dnode 2 -sql show snodes -if $rows != 2 then - return -1 -endi +sql_error create snode on dnode 2 print =============== restart system sh/exec.sh -n dnode1 -s stop -x SIGINT From 72d6614b7bb4b9bdf15f93d15bc6b80a8e6b5336 Mon Sep 17 00:00:00 2001 From: fullhonest Date: Wed, 29 Nov 2023 20:07:40 +0800 Subject: [PATCH 028/102] Fix TD-27403: insert error about use today + now as timestamp value --- source/common/src/ttime.c | 2 +- source/libs/parser/src/parInsertSql.c | 150 ++++++++++++++---- tests/parallel_test/cases.task | 1 + .../system-test/1-insert/insert_timestamp.py | 70 ++++++++ 4 files changed, 194 insertions(+), 29 deletions(-) create mode 100644 tests/system-test/1-insert/insert_timestamp.py diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 227de7f5fc..565d7f1699 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -646,7 +646,7 @@ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* dura /* get the basic numeric value */ int64_t timestamp = taosStr2Int64(token, &endPtr, 10); - if (timestamp < 0 || errno != 0) { + if ((timestamp == 0 && token[0] != '0') || errno != 0) { return -1; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 2b8516d37b..738574b48b 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -267,26 +267,47 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, E return code; } -static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) { - int32_t index = 0; - int64_t interval; - int64_t ts = 0; - const char* pTokenEnd = *end; - +static int parseTimestampOrInterval(const char** end, SToken* pToken, int16_t timePrec, int64_t* ts, int64_t* interval, SMsgBuf* pMsgBuf, bool* isTs) { if (pToken->type == TK_NOW) { - ts = taosGetTimestamp(timePrec); + *isTs = true; + *ts = taosGetTimestamp(timePrec); } else if (pToken->type == TK_TODAY) { - ts = taosGetTimestampToday(timePrec); + *isTs = true; + *ts = taosGetTimestampToday(timePrec); } else if (pToken->type == TK_NK_INTEGER) { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &ts)) { + *isTs = true; + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, ts)) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); } + } else if (pToken->type == TK_NK_VARIABLE) { + char unit = 0; + *isTs = false; + if (parseAbsoluteDuration(pToken->z, pToken->n, interval, &unit, timePrec) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } } else { // parse the RFC-3339/ISO-8601 timestamp format string - if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { + *isTs = true; + if (taosParseTime(pToken->z, ts, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); } + } - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; +} + +static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) { + int32_t index = 0, i = 0; + int64_t interval = 0, tempInterval = 0; + int64_t ts = 0, tempTs = 0; + bool firstIsTS = false, secondIsTs = false; + const char* pTokenEnd = *end; + + if (TSDB_CODE_SUCCESS != parseTimestampOrInterval(&pTokenEnd, pToken, timePrec, &ts, &interval, pMsgBuf, &firstIsTS)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } + + if (firstIsTS) { + *time = ts; } for (int k = pToken->n; pToken->z[k] != '\0'; k++) { @@ -298,45 +319,98 @@ static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t } if (pToken->z[k] == ',') { *end = pTokenEnd; + if (!firstIsTS) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } *time = ts; - return 0; + return TSDB_CODE_SUCCESS; } - break; } - /* - * time expression: - * e.g., now+12a, now-5h - */ + while (pTokenEnd[i] != '\0') { + if (pTokenEnd[i] == ' ' || pTokenEnd[i] == '\t') { + i++; + continue; + } + else if (pTokenEnd[i] == ',' || pTokenEnd[i] == ')') { + *end = pTokenEnd + i; + if (!firstIsTS) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } + *time = ts; + return TSDB_CODE_SUCCESS; + } else { + break; + } + } + pTokenEnd = pTokenEnd + i; + index = 0; SToken token = tStrGetToken(pTokenEnd, &index, false, NULL); - pTokenEnd += index; if (token.type == TK_NK_MINUS || token.type == TK_NK_PLUS) { + pTokenEnd += index; index = 0; SToken valueToken = tStrGetToken(pTokenEnd, &index, false, NULL); pTokenEnd += index; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW]; + if (TK_NK_STRING == valueToken.type) { + if (valueToken.n >= TSDB_MAX_BYTES_PER_ROW) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", valueToken.z); + } + int32_t len = trimString(valueToken.z, valueToken.n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW); + valueToken.z = tmpTokenBuf; + valueToken.n = len; + } + + if (TSDB_CODE_SUCCESS != parseTimestampOrInterval(&pTokenEnd, &valueToken, timePrec, &tempTs, &tempInterval, pMsgBuf, &secondIsTs)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } if (valueToken.n < 2) { return buildSyntaxErrMsg(pMsgBuf, "value expected in timestamp", token.z); } - char unit = 0; - if (parseAbsoluteDuration(valueToken.z, valueToken.n, &interval, &unit, timePrec) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; + if (secondIsTs) { + // not support operator between tow timestamp, such as today() + now() + if (firstIsTS) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } + ts = tempTs; + }else { + // not support operator between tow interval, such as 2h + 3s + if (!firstIsTS) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } + interval = tempInterval; } - - if (token.type == TK_NK_PLUS) { - ts += interval; + if (token.type == TK_NK_MINUS) { + // not support interval - ts,such as 2h - today() + if (secondIsTs) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } + *time = ts - interval; } else { - ts = ts - interval; + *time = ts + interval; } - *end = pTokenEnd; + for (int k = valueToken.n; valueToken.z[k] != '\0'; k++) { + if (valueToken.z[k] == ' ' || valueToken.z[k] == '\t') continue; + if (valueToken.z[k] == '(' && valueToken.z[k + 1] == ')') { // for insert NOW()/TODAY() + *end = pTokenEnd = &valueToken.z[k + 2]; + k++; + continue; + } + if (valueToken.z[k] == ',' || valueToken.z[k] == ')') { + *end = pTokenEnd; + return TSDB_CODE_SUCCESS; + } + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } } - *time = ts; + *end = pTokenEnd; return TSDB_CODE_SUCCESS; } @@ -666,7 +740,7 @@ static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMs if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER && pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL && pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && - pToken->type != TK_NK_BIN) || + pToken->type != TK_NK_BIN && pToken->type != TK_NK_VARIABLE) || (pToken->n == 0) || (pToken->type == TK_NK_RP)) { return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); } @@ -845,6 +919,9 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt SSchema* pTagSchema = &pSchema[pCxt->tags.pColIndex[i]]; isJson = pTagSchema->type == TSDB_DATA_TYPE_JSON; code = checkAndTrimValue(&token, pCxt->tmpTokenBuf, &pCxt->msg); + if (TK_NK_VARIABLE == token.type) { + code = buildSyntaxErrMsg(&pCxt->msg, "not expected tags values ", token.z); + } if (TSDB_CODE_SUCCESS == code) { code = parseTagValue(pCxt, pStmt, &pStmt->pSql, pTagSchema, &token, pTagName, pTagVals, &pTag); } @@ -1535,6 +1612,10 @@ static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, STo if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { return buildSyntaxErrMsg(&pCxt->msg, "primary timestamp should not be null", pToken->z); } + + if (TK_NK_VARIABLE == pToken->type && pSchema->type != TSDB_DATA_TYPE_TIMESTAMP) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid values", pToken->z); + } pVal->flag = CV_FLAG_NULL; return TSDB_CODE_SUCCESS; } @@ -1587,6 +1668,9 @@ typedef union SRowsDataContext{ static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, bool* pFoundCtbName) { *pFoundCtbName = false; int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); + if (TK_NK_VARIABLE == pToken->type) { + code = buildInvalidOperationMsg(&pCxt->msg, "not expected tbname"); + } if (code == TSDB_CODE_SUCCESS){ if (isNullValue(TSDB_DATA_TYPE_BINARY, pToken)) { return buildInvalidOperationMsg(&pCxt->msg, "tbname can not be null value"); @@ -1624,6 +1708,10 @@ static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModif SToken* pTagToken = (SToken*)(tagTokens + i); SSchema* pTagSchema = tagSchemas[i]; code = checkAndTrimValue(pTagToken, pCxt->tmpTokenBuf, &pCxt->msg); + if (TK_NK_VARIABLE == pTagToken->type) { + code = buildInvalidOperationMsg(&pCxt->msg, "not expected tag"); + } + if (code == TSDB_CODE_SUCCESS) { code = parseTagValue(pCxt, pStmt, NULL, pTagSchema, pTagToken, pStbRowsCxt->aTagNames, pStbRowsCxt->aTagVals, &pStbRowsCxt->pTag); @@ -1668,6 +1756,9 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* const SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; SColVal* pVal = taosArrayGet(pStbRowsCxt->aColVals, pCols->pColIndex[i]); code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)pSchema, getTableInfo(pStbRowsCxt->pStbMeta).precision, pVal); + if (TK_NK_VARIABLE == pToken->type) { + code = buildInvalidOperationMsg(&pCxt->msg, "not expected row value"); + } } else if (pCols->pColIndex[i] < tbnameIdx) { const SSchema* pTagSchema = &pSchemas[pCols->pColIndex[i]]; if (canParseTagsAfter) { @@ -1676,6 +1767,9 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* ++(*pNumOfTagTokens); } else { code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); + if (TK_NK_VARIABLE == pToken->type) { + code = buildInvalidOperationMsg(&pCxt->msg, "not expected row value"); + } if (code == TSDB_CODE_SUCCESS) { code = parseTagValue(pCxt, pStmt, ppSql, (SSchema*)pTagSchema, pToken, pTagNames, pTagVals, &pStbRowsCxt->pTag); } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 7a47df97a9..61cb9e5b35 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -302,6 +302,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4219.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4295.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_td27388.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_timestamp.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py diff --git a/tests/system-test/1-insert/insert_timestamp.py b/tests/system-test/1-insert/insert_timestamp.py new file mode 100644 index 0000000000..621912f664 --- /dev/null +++ b/tests/system-test/1-insert/insert_timestamp.py @@ -0,0 +1,70 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf + + +class TDTestCase: + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + #def prepare_data(self): + + + def run(self): + tdSql.execute("create database test_insert_timestamp;") + tdSql.execute("use test_insert_timestamp;") + tdSql.execute("create stable st(ts timestamp, c1 int) tags(id int);") + tdSql.execute("create table test_t using st tags(1);") + + tdSql.error("insert into test_t values(now + today(), 1 ); ") + tdSql.error("insert into test_t values(now - today(), 1 ); ") + tdSql.error("insert into test_t values(today() + now(), 1 ); ") + tdSql.error("insert into test_t values(today() - now(), 1 ); ") + tdSql.error("insert into test_t values(2h - now(), 1 ); ") + tdSql.error("insert into test_t values(2h - today(), 1 ); ") + tdSql.error("insert into test_t values(2h - 1h, 1 ); ") + tdSql.error("insert into test_t values(2h + 1h, 1 ); ") + tdSql.error("insert into test_t values('2023-11-28 00:00:00.000' + '2023-11-28 00:00:00.000', 1 ); ") + tdSql.error("insert into test_t values('2023-11-28 00:00:00.000' + 1701111600000, 1 ); ") + tdSql.error("insert into test_t values(1701111500000 + 1701111600000, 1 ); ") + tdSql.error("insert into test_insert_timestamp.test_t values(1701111600000 + 1h + 1s, 4); ") + + tdSql.execute("insert into test_insert_timestamp.test_t values(1701111600000 + 1h, 4); ") + tdSql.execute("insert into test_insert_timestamp.test_t values(2h + 1701111600000, 5); ") + tdSql.execute("insert into test_insert_timestamp.test_t values('2023-11-28 00:00:00.000' + 1h, 1); ") + tdSql.execute("insert into test_insert_timestamp.test_t values(3h + '2023-11-28 00:00:00.000', 3); ") + tdSql.execute("insert into test_insert_timestamp.test_t values(1701111600000 - 1h, 2); ") + tdSql.execute("insert into test_insert_timestamp.test_t values(1701122400000, 6); ") + tdSql.execute("insert into test_insert_timestamp.test_t values('2023-11-28 07:00:00.000', 7); ") + + tdSql.query(f'select ts, c1 from test_t order by ts;') + tdSql.checkRows(7) + tdSql.checkEqual(tdSql.queryResult[0][0], datetime.datetime(2023, 11, 28, 1, 0, 0) ) + tdSql.checkEqual(tdSql.queryResult[0][1], 1) + tdSql.checkEqual(tdSql.queryResult[1][0], datetime.datetime(2023, 11, 28, 2, 0, 0) ) + tdSql.checkEqual(tdSql.queryResult[1][1], 2) + tdSql.checkEqual(tdSql.queryResult[2][0], datetime.datetime(2023, 11, 28, 3, 0, 0) ) + tdSql.checkEqual(tdSql.queryResult[2][1], 3) + tdSql.checkEqual(tdSql.queryResult[3][0], datetime.datetime(2023, 11, 28, 4, 0, 0) ) + tdSql.checkEqual(tdSql.queryResult[3][1], 4) + tdSql.checkEqual(tdSql.queryResult[4][0], datetime.datetime(2023, 11, 28, 5, 0, 0) ) + tdSql.checkEqual(tdSql.queryResult[4][1], 5) + tdSql.checkEqual(tdSql.queryResult[5][0], datetime.datetime(2023, 11, 28, 6, 0, 0) ) + tdSql.checkEqual(tdSql.queryResult[5][1], 6) + tdSql.checkEqual(tdSql.queryResult[6][0], datetime.datetime(2023, 11, 28, 7, 0, 0) ) + tdSql.checkEqual(tdSql.queryResult[6][1], 7) + + tdSql.execute("drop table if exists test_t ;") + tdSql.execute("drop stable if exists st;") + tdSql.execute("drop database if exists test_insert_timestamp;") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From acce0852df15aefcd818c0c1388ab3a920a3b3e6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Nov 2023 19:52:27 +0800 Subject: [PATCH 029/102] fix:test case --- tests/script/tsim/snode/basic1.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/tsim/snode/basic1.sim b/tests/script/tsim/snode/basic1.sim index 6e553591bd..e7346b75a0 100644 --- a/tests/script/tsim/snode/basic1.sim +++ b/tests/script/tsim/snode/basic1.sim @@ -123,7 +123,7 @@ system sh/exec.sh -n dnode2 -s start sleep 2000 sql show snodes -if $rows != 2 then +if $rows != 1 then return -1 endi From 353c8d21a9ba89bec6cbc87e746a3c9e3edb6e48 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Fri, 1 Dec 2023 09:25:41 +0800 Subject: [PATCH 030/102] ci:increase timeout of mac test in Jenkinsfile2 --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 1ab6a4a8cd..d3fc05a1d2 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -393,7 +393,7 @@ pipeline { agent{label " Mac_catalina "} steps { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { - timeout(time: 20, unit: 'MINUTES'){ + timeout(time: 30, unit: 'MINUTES'){ pre_test() pre_test_build_mac() } From 3ae0e2d74fee58e9c386e301d935eb2a5e0f1efe Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 1 Dec 2023 14:20:16 +0800 Subject: [PATCH 031/102] valid msg --- include/common/tmsg.h | 145 +++++++++++++-------------- include/common/tmsgdef.h | 97 ++++++++++++------ source/common/src/tmsg.c | 211 ++++++++++++++++++--------------------- 3 files changed, 230 insertions(+), 223 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 73d1ab2473..f7e855db8a 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -35,12 +35,14 @@ extern "C" { #define TD_MSG_NUMBER_ #undef TD_MSG_DICT_ #undef TD_MSG_INFO_ +#undef TD_MSG_RANGE_CODE_ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" #undef TD_MSG_NUMBER_ #undef TD_MSG_DICT_ #undef TD_MSG_INFO_ +#undef TD_MSG_RANGE_CODE_ #define TD_MSG_SEG_CODE_ #include "tmsgdef.h" @@ -48,33 +50,22 @@ extern "C" { #undef TD_MSG_DICT_ #undef TD_MSG_INFO_ #undef TD_MSG_SEG_CODE_ +#undef TD_MSG_RANGE_CODE_ #include "tmsgdef.h" extern char* tMsgInfo[]; extern int32_t tMsgDict[]; - -#define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8) -#define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff) -#define TMSG_INFO(TYPE) \ - ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG || (TYPE) < TDMT_SCH_MAX_MSG || \ - (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || (TYPE) < TDMT_SYNC_MAX_MSG) || \ - (TYPE) < TDMT_VND_STREAM_MSG || (TYPE) < TDMT_VND_TMQ_MSG || (TYPE) < TDMT_VND_TMQ_MAX_MSG \ - ? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] \ - : 0 - -#define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)) +// extern int32_t tMsgRangeDict[]; typedef uint16_t tmsg_t; -static inline bool tmsgIsValid(tmsg_t type) { - if (type < TDMT_DND_MAX_MSG || type < TDMT_MND_MAX_MSG || type < TDMT_VND_MAX_MSG || type < TDMT_SCH_MAX_MSG || - type < TDMT_STREAM_MAX_MSG || type < TDMT_MON_MAX_MSG || type < TDMT_SYNC_MAX_MSG || type < TDMT_VND_STREAM_MSG || - type < TDMT_VND_TMQ_MSG || type < TDMT_VND_TMQ_MAX_MSG) { - return true; - } else { - return false; - } -} +#define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8) +#define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff) +#define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)) + +bool tmsgIsValid(tmsg_t type); +char* TMSG_INFO(tmsg_t type); + static inline bool vnodeIsMsgBlock(tmsg_t type) { return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_COMMIT) || @@ -169,14 +160,14 @@ typedef enum _mgmt_table { #define TSDB_FILL_PREV 6 #define TSDB_FILL_NEXT 7 -#define TSDB_ALTER_USER_PASSWD 0x1 -#define TSDB_ALTER_USER_SUPERUSER 0x2 -#define TSDB_ALTER_USER_ENABLE 0x3 -#define TSDB_ALTER_USER_SYSINFO 0x4 -#define TSDB_ALTER_USER_ADD_PRIVILEGES 0x5 -#define TSDB_ALTER_USER_DEL_PRIVILEGES 0x6 -#define TSDB_ALTER_USER_ADD_WHITE_LIST 0x7 -#define TSDB_ALTER_USER_DROP_WHITE_LIST 0x8 +#define TSDB_ALTER_USER_PASSWD 0x1 +#define TSDB_ALTER_USER_SUPERUSER 0x2 +#define TSDB_ALTER_USER_ENABLE 0x3 +#define TSDB_ALTER_USER_SYSINFO 0x4 +#define TSDB_ALTER_USER_ADD_PRIVILEGES 0x5 +#define TSDB_ALTER_USER_DEL_PRIVILEGES 0x6 +#define TSDB_ALTER_USER_ADD_WHITE_LIST 0x7 +#define TSDB_ALTER_USER_DROP_WHITE_LIST 0x8 #define TSDB_KILL_MSG_LEN 30 @@ -346,7 +337,7 @@ typedef enum ENodeType { QUERY_NODE_RESTORE_MNODE_STMT, QUERY_NODE_RESTORE_VNODE_STMT, QUERY_NODE_PAUSE_STREAM_STMT, - QUERY_NODE_RESUME_STREAM_STMT, + QUERY_NODE_RESUME_STREAM_STMT, QUERY_NODE_CREATE_VIEW_STMT, QUERY_NODE_DROP_VIEW_STMT, @@ -790,7 +781,7 @@ typedef struct { int32_t tSerializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq); int32_t tDeserializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq); -void tFreeSMDropStbReq(SMDropStbReq *pReq); +void tFreeSMDropStbReq(SMDropStbReq* pReq); typedef struct { char name[TSDB_TABLE_FNAME_LEN]; @@ -871,18 +862,18 @@ int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); typedef struct { - char user[TSDB_USER_LEN]; + char user[TSDB_USER_LEN]; int32_t sqlLen; char* sql; } SDropUserReq, SDropAcctReq; int32_t tSerializeSDropUserReq(void* buf, int32_t bufLen, SDropUserReq* pReq); int32_t tDeserializeSDropUserReq(void* buf, int32_t bufLen, SDropUserReq* pReq); -void tFreeSDropUserReq(SDropUserReq *pReq); +void tFreeSDropUserReq(SDropUserReq* pReq); -typedef struct SIpV4Range{ - uint32_t ip; - uint32_t mask; +typedef struct SIpV4Range { + uint32_t ip; + uint32_t mask; } SIpV4Range; typedef struct { @@ -892,21 +883,21 @@ typedef struct { SIpWhiteList* cloneIpWhiteList(SIpWhiteList* pIpWhiteList); typedef struct { - int8_t createType; - int8_t superUser; // denote if it is a super user or not - int8_t sysInfo; - int8_t enable; - char user[TSDB_USER_LEN]; - char pass[TSDB_USET_PASSWORD_LEN]; + int8_t createType; + int8_t superUser; // denote if it is a super user or not + int8_t sysInfo; + int8_t enable; + char user[TSDB_USER_LEN]; + char pass[TSDB_USET_PASSWORD_LEN]; int32_t numIpRanges; SIpV4Range* pIpRanges; - int32_t sqlLen; - char* sql; + int32_t sqlLen; + char* sql; } SCreateUserReq; int32_t tSerializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq); int32_t tDeserializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq); -void tFreeSCreateUserReq(SCreateUserReq *pReq); +void tFreeSCreateUserReq(SCreateUserReq* pReq); typedef struct { int64_t ver; @@ -933,22 +924,22 @@ int32_t tSerializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq int32_t tDeserializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq* pReq); typedef struct { - int8_t alterType; - int8_t superUser; - int8_t sysInfo; - int8_t enable; - int8_t isView; - char user[TSDB_USER_LEN]; - char pass[TSDB_USET_PASSWORD_LEN]; - char objname[TSDB_DB_FNAME_LEN]; // db or topic - char tabName[TSDB_TABLE_NAME_LEN]; - char* tagCond; - int32_t tagCondLen; + int8_t alterType; + int8_t superUser; + int8_t sysInfo; + int8_t enable; + int8_t isView; + char user[TSDB_USER_LEN]; + char pass[TSDB_USET_PASSWORD_LEN]; + char objname[TSDB_DB_FNAME_LEN]; // db or topic + char tabName[TSDB_TABLE_NAME_LEN]; + char* tagCond; + int32_t tagCondLen; int32_t numIpRanges; SIpV4Range* pIpRanges; int64_t privileges; - int32_t sqlLen; - char* sql; + int32_t sqlLen; + char* sql; } SAlterUserReq; int32_t tSerializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq); @@ -978,9 +969,9 @@ typedef struct { SHashObj* alterTbs; SHashObj* readViews; SHashObj* writeViews; - SHashObj* alterViews; + SHashObj* alterViews; SHashObj* useDbs; - int64_t whiteListVer; + int64_t whiteListVer; } SGetUserAuthRsp; int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); @@ -995,8 +986,8 @@ int32_t tSerializeSGetUserWhiteListReq(void* buf, int32_t bufLen, SGetUserWhiteL int32_t tDeserializeSGetUserWhiteListReq(void* buf, int32_t bufLen, SGetUserWhiteListReq* pReq); typedef struct { - char user[TSDB_USER_LEN]; - int32_t numWhiteLists; + char user[TSDB_USER_LEN]; + int32_t numWhiteLists; SIpV4Range* pWhiteLists; } SGetUserWhiteListRsp; @@ -1169,8 +1160,8 @@ int32_t tDeserializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq); void tFreeSAlterDbReq(SAlterDbReq* pReq); typedef struct { - char db[TSDB_DB_FNAME_LEN]; - int8_t ignoreNotExists; + char db[TSDB_DB_FNAME_LEN]; + int8_t ignoreNotExists; int32_t sqlLen; char* sql; } SDropDbReq; @@ -1378,7 +1369,7 @@ typedef struct { int32_t tSerializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq); int32_t tDeserializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq); -void tFreeSCompactDbReq(SCompactDbReq *pReq); +void tFreeSCompactDbReq(SCompactDbReq* pReq); typedef struct { char name[TSDB_FUNC_NAME_LEN]; @@ -1817,7 +1808,6 @@ int32_t tSerializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); int32_t tDeserializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); void tFreeSViewHbRsp(SViewHbRsp* pRsp); - typedef struct { int32_t numOfTables; int32_t numOfVgroup; @@ -2006,7 +1996,7 @@ typedef struct { int32_t tSerializeSRestoreDnodeReq(void* buf, int32_t bufLen, SRestoreDnodeReq* pReq); int32_t tDeserializeSRestoreDnodeReq(void* buf, int32_t bufLen, SRestoreDnodeReq* pReq); -void tFreeSRestoreDnodeReq(SRestoreDnodeReq *pReq); +void tFreeSRestoreDnodeReq(SRestoreDnodeReq* pReq); typedef struct { int32_t dnodeId; @@ -2018,7 +2008,7 @@ typedef struct { int32_t tSerializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); int32_t tDeserializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); -void tFreeSMCfgDnodeReq(SMCfgDnodeReq *pReq); +void tFreeSMCfgDnodeReq(SMCfgDnodeReq* pReq); typedef struct { char config[TSDB_DNODE_CONFIG_LEN]; @@ -2037,7 +2027,7 @@ typedef struct { int32_t tSerializeSCreateDropMQSNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq); int32_t tDeserializeSCreateDropMQSNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq); -void tFreeSMCreateQnodeReq(SMCreateQnodeReq *pReq); +void tFreeSMCreateQnodeReq(SMCreateQnodeReq* pReq); void tFreeSDDropQnodeReq(SDDropQnodeReq* pReq); typedef struct { int8_t replica; @@ -2079,7 +2069,7 @@ typedef struct { int32_t tSerializeSBalanceVgroupReq(void* buf, int32_t bufLen, SBalanceVgroupReq* pReq); int32_t tDeserializeSBalanceVgroupReq(void* buf, int32_t bufLen, SBalanceVgroupReq* pReq); -void tFreeSBalanceVgroupReq(SBalanceVgroupReq *pReq); +void tFreeSBalanceVgroupReq(SBalanceVgroupReq* pReq); typedef struct { int32_t vgId1; @@ -2100,7 +2090,7 @@ typedef struct { int32_t tSerializeSRedistributeVgroupReq(void* buf, int32_t bufLen, SRedistributeVgroupReq* pReq); int32_t tDeserializeSRedistributeVgroupReq(void* buf, int32_t bufLen, SRedistributeVgroupReq* pReq); -void tFreeSRedistributeVgroupReq(SRedistributeVgroupReq *pReq); +void tFreeSRedistributeVgroupReq(SRedistributeVgroupReq* pReq); typedef struct { int32_t useless; @@ -2111,7 +2101,7 @@ typedef struct { int32_t tSerializeSBalanceVgroupLeaderReq(void* buf, int32_t bufLen, SBalanceVgroupLeaderReq* pReq); int32_t tDeserializeSBalanceVgroupLeaderReq(void* buf, int32_t bufLen, SBalanceVgroupLeaderReq* pReq); -void tFreeSBalanceVgroupLeaderReq(SBalanceVgroupLeaderReq *pReq); +void tFreeSBalanceVgroupLeaderReq(SBalanceVgroupLeaderReq* pReq); typedef struct { int32_t vgId; @@ -2503,15 +2493,15 @@ typedef struct { } SMVSubscribeRsp; typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - int8_t igNotExists; + char name[TSDB_TOPIC_FNAME_LEN]; + int8_t igNotExists; int32_t sqlLen; char* sql; } SMDropTopicReq; int32_t tSerializeSMDropTopicReq(void* buf, int32_t bufLen, SMDropTopicReq* pReq); int32_t tDeserializeSMDropTopicReq(void* buf, int32_t bufLen, SMDropTopicReq* pReq); -void tFreeSMDropTopicReq(SMDropTopicReq *pReq); +void tFreeSMDropTopicReq(SMDropTopicReq* pReq); typedef struct { char topic[TSDB_TOPIC_FNAME_LEN]; @@ -3082,8 +3072,8 @@ typedef struct { } SMqVDeleteRsp; typedef struct { - char name[TSDB_STREAM_FNAME_LEN]; - int8_t igNotExists; + char name[TSDB_STREAM_FNAME_LEN]; + int8_t igNotExists; int32_t sqlLen; char* sql; } SMDropStreamReq; @@ -3920,7 +3910,7 @@ int32_t tDeserializeSCMDropViewReq(void* buf, int32_t bufLen, SCMDropViewReq* pR void tFreeSCMDropViewReq(SCMDropViewReq* pReq); typedef struct { - char fullname[TSDB_VIEW_FNAME_LEN]; + char fullname[TSDB_VIEW_FNAME_LEN]; } SViewMetaReq; int32_t tSerializeSViewMetaReq(void* buf, int32_t bufLen, const SViewMetaReq* pReq); int32_t tDeserializeSViewMetaReq(void* buf, int32_t bufLen, SViewMetaReq* pReq); @@ -3942,7 +3932,6 @@ int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pR int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp); void tFreeSViewMetaRsp(SViewMetaRsp* pRsp); - #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index afa0fa2a6e..84dca38108 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -24,48 +24,74 @@ #if defined(TD_MSG_INFO_) -#undef TD_NEW_MSG_SEG -#undef TD_DEF_MSG_TYPE -#define TD_NEW_MSG_SEG(TYPE) "null", -#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) MSG, MSG "-rsp", + #undef TD_NEW_MSG_SEG + #undef TD_DEF_MSG_TYPE + #undef TD_CLOSE_MSG_TYPE + #define TD_NEW_MSG_SEG(TYPE) "null", + #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) MSG, MSG "-rsp", + #define TD_CLOSE_MSG_TYPE(TYPE) -char *tMsgInfo[] = { + char *tMsgInfo[] = { + +#elif defined(TD_MSG_RANGE_CODE_) + + typedef struct { + int32_t start; + int32_t end; + } TMsgRange; + #undef TD_NEW_MSG_SEG + #undef TD_DEF_MSG_TYPE + #undef TD_CLOSE_MSG_TYPE + #define TD_NEW_MSG_SEG(TYPE) {(TYPE##_SEG_CODE) << 8, + #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) + #define TD_CLOSE_MSG_TYPE(TYPE) TYPE}, + TMsgRange tMsgRangeDict[] = { #elif defined(TD_MSG_NUMBER_) -#undef TD_NEW_MSG_SEG -#undef TD_DEF_MSG_TYPE -#define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM, -#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE##_NUM, TYPE##_RSP_NUM, + #undef TD_NEW_MSG_SEG + #undef TD_DEF_MSG_TYPE + #undef TD_CLOSE_MSG_TYPE + #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM, + #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE##_NUM, TYPE##_RSP_NUM, + #define TD_CLOSE_MSG_TYPE(TYPE) -enum { + enum { #elif defined(TD_MSG_DICT_) -#undef TD_NEW_MSG_SEG -#undef TD_DEF_MSG_TYPE -#define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM, -#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) + #undef TD_NEW_MSG_SEG + #undef TD_DEF_MSG_TYPE + #undef TD_CLOSE_MSG_TYPE + #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM, + #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) + #define TD_CLOSE_MSG_TYPE(type) + + int32_t tMsgDict[] = { -int32_t tMsgDict[] = { #elif defined(TD_MSG_SEG_CODE_) -#undef TD_NEW_MSG_SEG -#undef TD_DEF_MSG_TYPE -#define TD_NEW_MSG_SEG(TYPE) TYPE##_SEG_CODE, -#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) + #undef TD_NEW_MSG_SEG + #undef TD_DEF_MSG_TYPE + #undef TD_CLOSE_MSG_TYPE + #define TD_NEW_MSG_SEG(TYPE) TYPE##_SEG_CODE, + #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) + #define TD_CLOSE_MSG_TYPE(TYPE) -enum { + enum { -#else -#undef TD_NEW_MSG_SEG -#undef TD_DEF_MSG_TYPE -#define TD_NEW_MSG_SEG(TYPE) TYPE = ((TYPE##_SEG_CODE) << 8), -#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE, TYPE##_RSP, +#else -enum { // WARN: new msg should be appended to segment tail + #undef TD_NEW_MSG_SEG + #undef TD_DEF_MSG_TYPE + #undef TD_CLOSE_MSG_TYPE + #define TD_NEW_MSG_SEG(TYPE) TYPE = ((TYPE##_SEG_CODE) << 8), + #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE, TYPE##_RSP, + #define TD_CLOSE_MSG_TYPE(TYPE) TYPE, + + enum { // WARN: new msg should be appended to segment tail #endif TD_NEW_MSG_SEG(TDMT_DND_MSG) // 0<<8 TD_DEF_MSG_TYPE(TDMT_DND_CREATE_MNODE, "dnode-create-mnode", NULL, NULL) @@ -82,10 +108,12 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_DND_NET_TEST, "net-test", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "config-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_SYSTABLE_RETRIEVE, "dnode-retrieve", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_UNUSED_CODE, "dnd-unused", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_ALTER_MNODE_TYPE, "dnode-alter-mnode-type", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, "dnode-check-vnode-learner-catchup", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_DND_MSG) TD_NEW_MSG_SEG(TDMT_MND_MSG) // 1<<8 TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL) @@ -194,6 +222,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_MND_MSG) TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8 TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp) @@ -243,6 +272,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_VND_MSG) TD_NEW_MSG_SEG(TDMT_SCH_MSG) // 3<<8 TD_DEF_MSG_TYPE(TDMT_SCH_QUERY, "query", NULL, NULL) @@ -257,6 +287,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_TASK_NOTIFY, "task-notify", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_MAX_MSG, "sch-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_SCH_MSG) TD_NEW_MSG_SEG(TDMT_STREAM_MSG) //4 << 8 @@ -274,9 +305,11 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_STOP, "stream-task-stop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_HTASK_DROP, "stream-htask-drop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_STREAM_MSG) TD_NEW_MSG_SEG(TDMT_MON_MSG) //5 << 8 TD_DEF_MSG_TYPE(TDMT_MON_MAX_MSG, "monitor-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_MON_MSG) TD_NEW_MSG_SEG(TDMT_SYNC_MSG) //6 << 8 TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL) @@ -308,6 +341,8 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_SYNC_PREP_SNAPSHOT_REPLY, "sync-prep-snapshot-reply", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_SYNC_MSG) + TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) //7 << 8 TD_DEF_MSG_TYPE(TDMT_VND_STREAM_SCAN_HISTORY, "vnode-stream-scan-history", NULL, NULL) @@ -317,6 +352,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_VND_STREAM_MSG) TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) //8 << 8 TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp) @@ -330,9 +366,10 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_WALINFO, "vnode-tmq-vg-walinfo", SMqPollReq, SMqDataBlkRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_COMMITTEDINFO, "vnode-tmq-committedinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL) + TD_CLOSE_MSG_TYPE(TDMT_END_TMQ_MSG) -#if defined(TD_MSG_NUMBER_) - TDMT_MAX -#endif + #if defined(TD_MSG_NUMBER_) + TDMT_MAX + #endif }; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 01b1df9d5f..d9be51e04b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -18,42 +18,62 @@ #undef TD_MSG_NUMBER_ #undef TD_MSG_DICT_ +#undef TD_MSG_RANGE_CODE_ #define TD_MSG_INFO_ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" #undef TD_MSG_NUMBER_ #undef TD_MSG_INFO_ +#undef TD_MSG_RANGE_CODE_ #define TD_MSG_DICT_ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" +#undef TD_MSG_NUMBER_ +#undef TD_MSG_INFO_ +#undef TD_MSG_DICT_ +#undef TD_MSG_SEG_CODE_ +#define TD_MSG_RANGE_CODE_ +#include "tmsgdef.h" + #include "tlog.h" -#define DECODESQL() \ - do { \ - if(!tDecodeIsEnd(&decoder)){ \ - if(tDecodeI32(&decoder, &pReq->sqlLen) < 0) return -1; \ - if(pReq->sqlLen > 0){ \ - if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->sql, NULL) < 0) return -1; \ - } \ - } \ +inline bool tmsgIsValid(tmsg_t type) { + static int8_t sz = sizeof(tMsgRangeDict) / sizeof(tMsgRangeDict[0]); + for (int i = 0; i < sz; i++) { + if (type > tMsgRangeDict[i].start && type < tMsgRangeDict[i].end) { + return true; + } + } + return false; +} + +inline char *TMSG_INFO(tmsg_t type) { return (tmsgIsValid(type) ? tMsgInfo[TMSG_INDEX(type)] : "null"); }; +#define DECODESQL() \ + do { \ + if (!tDecodeIsEnd(&decoder)) { \ + if (tDecodeI32(&decoder, &pReq->sqlLen) < 0) return -1; \ + if (pReq->sqlLen > 0) { \ + if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->sql, NULL) < 0) return -1; \ + } \ + } \ } while (0) -#define ENCODESQL() \ - do { \ - if (pReq->sqlLen > 0 && pReq->sql != NULL){ \ - if (tEncodeI32(&encoder, pReq->sqlLen) < 0) return -1; \ - if (tEncodeBinary(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1; \ - } \ +#define ENCODESQL() \ + do { \ + if (pReq->sqlLen > 0 && pReq->sql != NULL) { \ + if (tEncodeI32(&encoder, pReq->sqlLen) < 0) return -1; \ + if (tEncodeBinary(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1; \ + } \ } while (0) -#define FREESQL() \ - do { \ - if(pReq->sql != NULL){ \ - taosMemoryFree(pReq->sql); \ - } \ - pReq->sql = NULL; \ +#define FREESQL() \ + do { \ + if (pReq->sql != NULL) { \ + taosMemoryFree(pReq->sql); \ + } \ + pReq->sql = NULL; \ } while (0) static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq); @@ -742,9 +762,7 @@ int32_t tDeserializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) return 0; } -void tFreeSMDropStbReq(SMDropStbReq *pReq) { - FREESQL(); -} +void tFreeSMDropStbReq(SMDropStbReq *pReq) { FREESQL(); } int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq) { SEncoder encoder = {0}; @@ -1489,9 +1507,7 @@ int32_t tDeserializeSDropUserReq(void *buf, int32_t bufLen, SDropUserReq *pReq) return 0; } -void tFreeSDropUserReq(SDropUserReq *pReq) { - FREESQL(); -} +void tFreeSDropUserReq(SDropUserReq *pReq) { FREESQL(); } SIpWhiteList *cloneIpWhiteList(SIpWhiteList *pIpWhiteList) { if (pIpWhiteList == NULL) return NULL; @@ -1822,7 +1838,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) char *tb = taosHashIterate(pRsp->readTbs, NULL); while (tb != NULL) { size_t keyLen = 0; - void *key = taosHashGetKey(tb, &keyLen); + void * key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1837,7 +1853,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) tb = taosHashIterate(pRsp->writeTbs, NULL); while (tb != NULL) { size_t keyLen = 0; - void *key = taosHashGetKey(tb, &keyLen); + void * key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1852,7 +1868,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) tb = taosHashIterate(pRsp->alterTbs, NULL); while (tb != NULL) { size_t keyLen = 0; - void *key = taosHashGetKey(tb, &keyLen); + void * key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1867,7 +1883,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) tb = taosHashIterate(pRsp->readViews, NULL); while (tb != NULL) { size_t keyLen = 0; - void *key = taosHashGetKey(tb, &keyLen); + void * key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1882,7 +1898,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) tb = taosHashIterate(pRsp->writeViews, NULL); while (tb != NULL) { size_t keyLen = 0; - void *key = taosHashGetKey(tb, &keyLen); + void * key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1897,7 +1913,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) tb = taosHashIterate(pRsp->alterViews, NULL); while (tb != NULL) { size_t keyLen = 0; - void *key = taosHashGetKey(tb, &keyLen); + void * key = taosHashGetKey(tb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1912,7 +1928,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL); while (useDb != NULL) { size_t keyLen = 0; - void *key = taosHashGetKey(useDb, &keyLen); + void * key = taosHashGetKey(useDb, &keyLen); if (tEncodeI32(pEncoder, keyLen) < 0) return -1; if (tEncodeCStr(pEncoder, key) < 0) return -1; @@ -1954,8 +1970,8 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs pRsp->alterViews = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pRsp->useDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pRsp->createdDbs == NULL || pRsp->readDbs == NULL || pRsp->writeDbs == NULL || pRsp->readTbs == NULL || - pRsp->writeTbs == NULL || pRsp->alterTbs == NULL || pRsp->readViews == NULL || - pRsp->writeViews == NULL || pRsp->alterViews == NULL ||pRsp->useDbs == NULL) { + pRsp->writeTbs == NULL || pRsp->alterTbs == NULL || pRsp->readViews == NULL || pRsp->writeViews == NULL || + pRsp->alterViews == NULL || pRsp->useDbs == NULL) { goto _err; } @@ -2219,7 +2235,7 @@ int32_t tDeserializeSGetUserWhiteListReq(void *buf, int32_t bufLen, SGetUserWhit return 0; } -int32_t tSerializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteListRsp* pRsp) { +int32_t tSerializeSGetUserWhiteListRsp(void *buf, int32_t bufLen, SGetUserWhiteListRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2237,7 +2253,7 @@ int32_t tSerializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteL return tlen; } -int32_t tDeserializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteListRsp* pRsp) { +int32_t tDeserializeSGetUserWhiteListRsp(void *buf, int32_t bufLen, SGetUserWhiteListRsp *pRsp) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -2257,9 +2273,7 @@ int32_t tDeserializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhit return 0; } -void tFreeSGetUserWhiteListRsp(SGetUserWhiteListRsp* pRsp) { - taosMemoryFree(pRsp->pWhiteLists); -} +void tFreeSGetUserWhiteListRsp(SGetUserWhiteListRsp *pRsp) { taosMemoryFree(pRsp->pWhiteLists); } int32_t tSerializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { SEncoder encoder = {0}; @@ -2288,13 +2302,9 @@ int32_t tDeserializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQno return 0; } -void tFreeSMCreateQnodeReq(SMCreateQnodeReq *pReq){ - FREESQL(); -} +void tFreeSMCreateQnodeReq(SMCreateQnodeReq *pReq) { FREESQL(); } -void tFreeSDDropQnodeReq(SDDropQnodeReq* pReq) { - FREESQL(); -} +void tFreeSDDropQnodeReq(SDDropQnodeReq *pReq) { FREESQL(); } int32_t tSerializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq) { SEncoder encoder = {0}; @@ -2336,9 +2346,7 @@ int32_t tDeserializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq return 0; } -void tFreeSDropDnodeReq(SDropDnodeReq *pReq) { - FREESQL(); -} +void tFreeSDropDnodeReq(SDropDnodeReq *pReq) { FREESQL(); } int32_t tSerializeSRestoreDnodeReq(void *buf, int32_t bufLen, SRestoreDnodeReq *pReq) { SEncoder encoder = {0}; @@ -2369,9 +2377,7 @@ int32_t tDeserializeSRestoreDnodeReq(void *buf, int32_t bufLen, SRestoreDnodeReq return 0; } -void tFreeSRestoreDnodeReq(SRestoreDnodeReq *pReq) { - FREESQL(); -} +void tFreeSRestoreDnodeReq(SRestoreDnodeReq *pReq) { FREESQL(); } int32_t tSerializeSMCfgDnodeReq(void *buf, int32_t bufLen, SMCfgDnodeReq *pReq) { SEncoder encoder = {0}; @@ -2404,9 +2410,7 @@ int32_t tDeserializeSMCfgDnodeReq(void *buf, int32_t bufLen, SMCfgDnodeReq *pReq return 0; } -void tFreeSMCfgDnodeReq(SMCfgDnodeReq *pReq) { - FREESQL(); -} +void tFreeSMCfgDnodeReq(SMCfgDnodeReq *pReq) { FREESQL(); } int32_t tSerializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) { SEncoder encoder = {0}; @@ -2464,9 +2468,7 @@ int32_t tDeserializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq * return 0; } -void tFreeSCreateDnodeReq(SCreateDnodeReq *pReq) { - FREESQL(); -} +void tFreeSCreateDnodeReq(SCreateDnodeReq *pReq) { FREESQL(); } int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq) { SEncoder encoder = {0}; @@ -3121,9 +3123,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { return 0; } -void tFreeSAlterDbReq(SAlterDbReq *pReq) { - FREESQL(); -} +void tFreeSAlterDbReq(SAlterDbReq *pReq) { FREESQL(); } int32_t tSerializeSDropDbReq(void *buf, int32_t bufLen, SDropDbReq *pReq) { SEncoder encoder = {0}; @@ -3154,9 +3154,7 @@ int32_t tDeserializeSDropDbReq(void *buf, int32_t bufLen, SDropDbReq *pReq) { return 0; } -void tFreeSDropDbReq(SDropDbReq *pReq) { - FREESQL(); -} +void tFreeSDropDbReq(SDropDbReq *pReq) { FREESQL(); } int32_t tSerializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { SEncoder encoder = {0}; @@ -3435,9 +3433,7 @@ int32_t tDeserializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq return 0; } -void tFreeSCompactDbReq(SCompactDbReq *pReq) { - FREESQL(); -} +void tFreeSCompactDbReq(SCompactDbReq *pReq) { FREESQL(); } int32_t tSerializeSUseDbRspImp(SEncoder *pEncoder, const SUseDbRsp *pRsp) { if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1; @@ -4611,9 +4607,7 @@ int32_t tDeserializeSMDropTopicReq(void *buf, int32_t bufLen, SMDropTopicReq *pR return 0; } -void tFreeSMDropTopicReq(SMDropTopicReq *pReq) { - FREESQL(); -} +void tFreeSMDropTopicReq(SMDropTopicReq *pReq) { FREESQL(); } int32_t tSerializeSMDropCgroupReq(void *buf, int32_t bufLen, SMDropCgroupReq *pReq) { SEncoder encoder = {0}; @@ -5501,9 +5495,7 @@ int32_t tDeserializeSBalanceVgroupReq(void *buf, int32_t bufLen, SBalanceVgroupR return 0; } -void tFreeSBalanceVgroupReq(SBalanceVgroupReq *pReq) { - FREESQL(); -} +void tFreeSBalanceVgroupReq(SBalanceVgroupReq *pReq) { FREESQL(); } int32_t tSerializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceVgroupLeaderReq *pReq) { SEncoder encoder = {0}; @@ -5526,7 +5518,7 @@ int32_t tDeserializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceV if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; - if(!tDecodeIsEnd(&decoder)){ + if (!tDecodeIsEnd(&decoder)) { if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; } @@ -5537,9 +5529,7 @@ int32_t tDeserializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceV return 0; } -void tFreeSBalanceVgroupLeaderReq(SBalanceVgroupLeaderReq *pReq) { - FREESQL(); -} +void tFreeSBalanceVgroupLeaderReq(SBalanceVgroupLeaderReq *pReq) { FREESQL(); } int32_t tSerializeSMergeVgroupReq(void *buf, int32_t bufLen, SMergeVgroupReq *pReq) { SEncoder encoder = {0}; @@ -5601,9 +5591,7 @@ int32_t tDeserializeSRedistributeVgroupReq(void *buf, int32_t bufLen, SRedistrib return 0; } -void tFreeSRedistributeVgroupReq(SRedistributeVgroupReq *pReq) { - FREESQL(); -} +void tFreeSRedistributeVgroupReq(SRedistributeVgroupReq *pReq) { FREESQL(); } int32_t tSerializeSSplitVgroupReq(void *buf, int32_t bufLen, SSplitVgroupReq *pReq) { SEncoder encoder = {0}; @@ -7152,9 +7140,7 @@ int32_t tDeserializeSMDropStreamReq(void *buf, int32_t bufLen, SMDropStreamReq * return 0; } -void tFreeMDropStreamReq(SMDropStreamReq *pReq) { - FREESQL(); -} +void tFreeMDropStreamReq(SMDropStreamReq *pReq) { FREESQL(); } int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) { SEncoder encoder = {0}; @@ -7297,8 +7283,8 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { } else { ASSERT(0); } - //ENCODESQL - if(pReq->sqlLen > 0 && pReq->sql != NULL) { + // ENCODESQL + if (pReq->sqlLen > 0 && pReq->sql != NULL) { if (tEncodeI32(pCoder, pReq->sqlLen) < 0) return -1; if (tEncodeBinary(pCoder, pReq->sql, pReq->sqlLen) < 0) return -1; } @@ -7345,11 +7331,11 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { ASSERT(0); } - //DECODESQL - if(!tDecodeIsEnd(pCoder)){ - if(tDecodeI32(pCoder, &pReq->sqlLen) < 0) return -1; - if(pReq->sqlLen > 0){ - if (tDecodeBinaryAlloc(pCoder, (void**)&pReq->sql, NULL) < 0) return -1; + // DECODESQL + if (!tDecodeIsEnd(pCoder)) { + if (tDecodeI32(pCoder, &pReq->sqlLen) < 0) return -1; + if (pReq->sqlLen > 0) { + if (tDecodeBinaryAlloc(pCoder, (void **)&pReq->sql, NULL) < 0) return -1; } } @@ -7375,7 +7361,7 @@ void tDestroySVCreateTbReq(SVCreateTbReq *pReq, int32_t flags) { } } - if(pReq->sql != NULL){ + if (pReq->sql != NULL) { taosMemoryFree(pReq->sql); } pReq->sql = NULL; @@ -8194,7 +8180,7 @@ int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { for (int32_t i = 0; i < pRsp->blockNum; i++) { int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i); - void *data = taosArrayGetP(pRsp->blockData, i); + void * data = taosArrayGetP(pRsp->blockData, i); if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1; if (pRsp->withSchema) { SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i); @@ -8227,7 +8213,7 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { } for (int32_t i = 0; i < pRsp->blockNum; i++) { - void *data; + void * data; uint64_t bLen; if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1; taosArrayPush(pRsp->blockData, &data); @@ -8273,7 +8259,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { for (int32_t i = 0; i < pRsp->createTableNum; i++) { - void *createTableReq = taosArrayGetP(pRsp->createTableReq, i); + void * createTableReq = taosArrayGetP(pRsp->createTableReq, i); int32_t createTableLen = *(int32_t *)taosArrayGet(pRsp->createTableLen, i); if (tEncodeBinary(pEncoder, createTableReq, createTableLen) < 0) return -1; } @@ -8289,7 +8275,7 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) { pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t)); pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *)); for (int32_t i = 0; i < pRsp->createTableNum; i++) { - void *pCreate = NULL; + void * pCreate = NULL; uint64_t len; if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1; int32_t l = (int32_t)len; @@ -8591,7 +8577,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) { taosArrayDestroy(pTbData->aCol); } else { int32_t nRow = TARRAY_SIZE(pTbData->aRowP); - SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP); + SRow ** rows = (SRow **)TARRAY_DATA(pTbData->aRowP); for (int32_t i = 0; i < nRow; ++i) { tRowDestroy(rows[i]); @@ -8855,7 +8841,7 @@ int32_t tDeserializeSCMCreateViewReq(void *buf, int32_t bufLen, SCMCreateViewReq } for (int32_t i = 0; i < pReq->numOfCols; ++i) { - SSchema* pSchema = pReq->pSchema + i; + SSchema *pSchema = pReq->pSchema + i; if (tDecodeSSchema(&decoder, pSchema) < 0) return -1; } } @@ -8866,17 +8852,17 @@ int32_t tDeserializeSCMCreateViewReq(void *buf, int32_t bufLen, SCMCreateViewReq return 0; } -void tFreeSCMCreateViewReq(SCMCreateViewReq* pReq) { +void tFreeSCMCreateViewReq(SCMCreateViewReq *pReq) { if (NULL == pReq) { return; } taosMemoryFreeClear(pReq->querySql); - taosMemoryFreeClear(pReq->sql); + taosMemoryFreeClear(pReq->sql); taosMemoryFreeClear(pReq->pSchema); } -int32_t tSerializeSCMDropViewReq(void* buf, int32_t bufLen, const SCMDropViewReq* pReq) { +int32_t tSerializeSCMDropViewReq(void *buf, int32_t bufLen, const SCMDropViewReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -8894,7 +8880,7 @@ int32_t tSerializeSCMDropViewReq(void* buf, int32_t bufLen, const SCMDropViewReq return tlen; } -int32_t tDeserializeSCMDropViewReq(void* buf, int32_t bufLen, SCMDropViewReq* pReq) { +int32_t tDeserializeSCMDropViewReq(void *buf, int32_t bufLen, SCMDropViewReq *pReq) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -8910,15 +8896,15 @@ int32_t tDeserializeSCMDropViewReq(void* buf, int32_t bufLen, SCMDropViewReq* pR tDecoderClear(&decoder); return 0; } -void tFreeSCMDropViewReq(SCMDropViewReq* pReq) { +void tFreeSCMDropViewReq(SCMDropViewReq *pReq) { if (NULL == pReq) { return; } - + taosMemoryFree(pReq->sql); } -int32_t tSerializeSViewMetaReq(void* buf, int32_t bufLen, const SViewMetaReq* pReq) { +int32_t tSerializeSViewMetaReq(void *buf, int32_t bufLen, const SViewMetaReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -8932,7 +8918,7 @@ int32_t tSerializeSViewMetaReq(void* buf, int32_t bufLen, const SViewMetaReq* pR return tlen; } -int32_t tDeserializeSViewMetaReq(void* buf, int32_t bufLen, SViewMetaReq* pReq) { +int32_t tDeserializeSViewMetaReq(void *buf, int32_t bufLen, SViewMetaReq *pReq) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -8964,8 +8950,7 @@ static int32_t tEncodeSViewMetaRsp(SEncoder *pEncoder, const SViewMetaRsp *pRsp) return 0; } - -int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pRsp) { +int32_t tSerializeSViewMetaRsp(void *buf, int32_t bufLen, const SViewMetaRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -8998,7 +8983,7 @@ static int32_t tDecodeSViewMetaRsp(SDecoder *pDecoder, SViewMetaRsp *pRsp) { } for (int32_t i = 0; i < pRsp->numOfCols; ++i) { - SSchema* pSchema = pRsp->pSchema + i; + SSchema *pSchema = pRsp->pSchema + i; if (tDecodeSSchema(pDecoder, pSchema) < 0) return -1; } } @@ -9006,7 +8991,7 @@ static int32_t tDecodeSViewMetaRsp(SDecoder *pDecoder, SViewMetaRsp *pRsp) { return 0; } -int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp) { +int32_t tDeserializeSViewMetaRsp(void *buf, int32_t bufLen, SViewMetaRsp *pRsp) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -9019,7 +9004,7 @@ int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp) return 0; } -void tFreeSViewMetaRsp(SViewMetaRsp* pRsp) { +void tFreeSViewMetaRsp(SViewMetaRsp *pRsp) { if (NULL == pRsp) { return; } @@ -9064,7 +9049,7 @@ int32_t tDeserializeSViewHbRsp(void *buf, int32_t bufLen, SViewHbRsp *pRsp) { } for (int32_t i = 0; i < numOfMeta; ++i) { - SViewMetaRsp* metaRsp = taosMemoryCalloc(1, sizeof(SViewMetaRsp)); + SViewMetaRsp *metaRsp = taosMemoryCalloc(1, sizeof(SViewMetaRsp)); if (NULL == metaRsp) return -1; if (tDecodeSViewMetaRsp(&decoder, metaRsp) < 0) return -1; taosArrayPush(pRsp->pViewRsp, &metaRsp); @@ -9086,7 +9071,3 @@ void tFreeSViewHbRsp(SViewHbRsp *pRsp) { taosArrayDestroy(pRsp->pViewRsp); } - - - - From 1e106bd10f2f559d90be055f77946bf34dc61cca Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 1 Dec 2023 14:22:47 +0800 Subject: [PATCH 032/102] valid msg --- source/common/src/tmsg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d9be51e04b..11a47142b8 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -49,7 +49,7 @@ inline bool tmsgIsValid(tmsg_t type) { return false; } -inline char *TMSG_INFO(tmsg_t type) { return (tmsgIsValid(type) ? tMsgInfo[TMSG_INDEX(type)] : "null"); }; +inline char *TMSG_INFO(tmsg_t type) { return (tmsgIsValid(type) ? tMsgInfo[TMSG_INDEX(type)] : "unKnown"); }; #define DECODESQL() \ do { \ if (!tDecodeIsEnd(&decoder)) { \ From 5a05f1019f0951d01f98e6f1aa3af698103d1da5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 1 Dec 2023 14:52:37 +0800 Subject: [PATCH 033/102] valid msg --- include/common/tmsg.h | 1 - include/common/tmsgdef.h | 10 +++------- source/common/src/tmsg.c | 8 ++++---- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index f7e855db8a..be5b51d938 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -55,7 +55,6 @@ extern "C" { extern char* tMsgInfo[]; extern int32_t tMsgDict[]; -// extern int32_t tMsgRangeDict[]; typedef uint16_t tmsg_t; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 84dca38108..0c9d2c2192 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -35,17 +35,13 @@ #elif defined(TD_MSG_RANGE_CODE_) - typedef struct { - int32_t start; - int32_t end; - } TMsgRange; #undef TD_NEW_MSG_SEG #undef TD_DEF_MSG_TYPE #undef TD_CLOSE_MSG_TYPE - #define TD_NEW_MSG_SEG(TYPE) {(TYPE##_SEG_CODE) << 8, + #define TD_NEW_MSG_SEG(TYPE) #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) - #define TD_CLOSE_MSG_TYPE(TYPE) TYPE}, - TMsgRange tMsgRangeDict[] = { + #define TD_CLOSE_MSG_TYPE(TYPE) TYPE, + int32_t tMsgRangeDict[] = { #elif defined(TD_MSG_NUMBER_) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 11a47142b8..197750a6a8 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -41,10 +41,10 @@ inline bool tmsgIsValid(tmsg_t type) { static int8_t sz = sizeof(tMsgRangeDict) / sizeof(tMsgRangeDict[0]); - for (int i = 0; i < sz; i++) { - if (type > tMsgRangeDict[i].start && type < tMsgRangeDict[i].end) { - return true; - } + + int segIdx = TMSG_SEG_CODE(type); + if (segIdx < sz) { + return type < tMsgRangeDict[segIdx]; } return false; } From 58e84d86b2716f07f31f79b0b0aeb2a8ebbf9b90 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 1 Dec 2023 14:54:47 +0800 Subject: [PATCH 034/102] valid msg --- source/common/src/tmsg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 197750a6a8..c7e67c81f5 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -43,7 +43,7 @@ inline bool tmsgIsValid(tmsg_t type) { static int8_t sz = sizeof(tMsgRangeDict) / sizeof(tMsgRangeDict[0]); int segIdx = TMSG_SEG_CODE(type); - if (segIdx < sz) { + if (segIdx >= 0 && segIdx < sz) { return type < tMsgRangeDict[segIdx]; } return false; From 82629c26581708c4c0674604bd9a9570a20f8609 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 1 Dec 2023 15:11:43 +0800 Subject: [PATCH 035/102] fix: notify when there are no memory pre fileset --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 853d0d7374..ecc1351698 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2514,6 +2514,15 @@ static void prepareDurationForNextFileSet(STsdbReader* pReader) { if (pReader->status.bProcMemPreFileset) { resetTableListIndex(&pReader->status); } + + if (!pReader->status.bProcMemFirstFileset) { + if (pReader->notifyFn) { + STsdReaderNotifyInfo info = {0}; + info.duration.fileSetId = fid; + pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam); + } + } + pReader->status.prevFilesetStartKey = winFid.skey; pReader->status.prevFilesetEndKey = winFid.ekey; } From fd290709ff45bd99dfb14cf6c6d2c25e37911788 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 1 Dec 2023 14:05:04 +0800 Subject: [PATCH 036/102] fix: state window with block rowIndex overflow --- source/libs/executor/src/timewindowoperator.c | 4 +- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/state_window.py | 203 ++++++++++++++++++ 3 files changed, 205 insertions(+), 3 deletions(-) create mode 100644 tests/system-test/2-query/state_window.py diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index db7c5e2570..0ca91f74ad 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -899,6 +899,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI SWindowRowsSup* pRowSup = &pInfo->winSup; pRowSup->numOfRows = 0; + pRowSup->startRowIndex = 0; struct SColumnDataAgg* pAgg = NULL; for (int32_t j = 0; j < pBlock->info.rows; ++j) { @@ -923,9 +924,6 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI doKeepTuple(pRowSup, tsList[j], gid); } else if (compareVal(val, &pInfo->stateKey)) { doKeepTuple(pRowSup, tsList[j], gid); - if (j == 0 && pRowSup->startRowIndex != 0) { - pRowSup->startRowIndex = 0; - } } else { // a new state window started SResultRow* pResult = NULL; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 7d3efbf181..af835c137a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -856,6 +856,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/odbc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/fill_with_group.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/state_window.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py ,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 diff --git a/tests/system-test/2-query/state_window.py b/tests/system-test/2-query/state_window.py new file mode 100644 index 0000000000..a211cd9dbe --- /dev/null +++ b/tests/system-test/2-query/state_window.py @@ -0,0 +1,203 @@ +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +# from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 4 + self.ctbNum = 1 + self.rowsPerTbl = 10 + self.duraion = '1h' + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d duration %s"%(dbName, vgroups, replica, duration)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, paraDict): + colString = tdCom.gen_column_type_str(colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"]) + tagString = tdCom.gen_tag_type_str(tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"]) + sqlString = f"create table if not exists %s.%s (%s) tags (%s)"%(paraDict["dbName"], paraDict["stbName"], colString, tagString) + tdLog.debug("%s"%(sqlString)) + tsql.execute(sqlString) + return + + def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0): + for i in range(ctbNum): + sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % \ + (dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,(i+ctbStartIdx) % 5,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx) + tsql.execute(sqlString) + + tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,tsStep): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + for i in range(ctbNum): + rowsBatched = 0 + sql += " %s%d values "%(ctbPrefix,i) + for j in range(rowsPerTbl): + if (i < ctbNum/2): + sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, 1, j%10, j%10, j%10, j%10, j%10, j%10) + else: + sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10) + rowsBatched += 1 + if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsBatched = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s%d values " %(ctbPrefix,i) + else: + sql = "insert into " + if sql != pre_insert: + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'test', + 'dropFlag': 1, + 'vgroups': 2, + 'stbName': 'meters', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'FLOAT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'smallint', 'count':1},{'type': 'tinyint', 'count':1},{'type': 'bool', 'count':1},{'type': 'binary', 'len':10, 'count':1},{'type': 'nchar', 'len':10, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'nchar', 'len':20, 'count':1},{'type': 'binary', 'len':20, 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'smallint', 'count':1},{'type': 'DOUBLE', 'count':1}], + 'ctbPrefix': 't', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 10000, + 'batchNum': 3000, + 'startTs': 1537146000000, + 'tsStep': 600000} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdLog.info("create database") + self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"], vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion) + + tdLog.info("create stb") + self.create_stable(tsql=tdSql, paraDict=paraDict) + + tdLog.info("create child tables") + self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"], \ + stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],\ + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict["ctbStartIdx"]) + self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],\ + ctbPrefix=paraDict["ctbPrefix"],ctbNum=paraDict["ctbNum"],\ + rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],\ + startTs=paraDict["startTs"],tsStep=paraDict["tsStep"]) + return + + def prepare_original_data(self): + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,3,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("flush database test", queryTimes=1) + time.sleep(2) + + def test_crash_for_state_window1(self): + tdSql.execute("drop database if exists test") + self.prepareTestEnv() + tdSql.execute("alter local 'queryPolicy' '3'") + self.prepare_original_data() + tdSql.execute("insert into t0 values(now, 4,4,4,4,4,4,4,4,4)", queryTimes=1) + tdSql.execute("select bottom(c1, 1), c2 from t0 state_window(c2) order by ts", queryTimes=1) + + def test_crash_for_state_window2(self): + tdSql.execute("drop database if exists test") + self.prepareTestEnv() + tdSql.execute("alter local 'queryPolicy' '3'") + self.prepare_original_data() + tdSql.execute("insert into t0 values(now, 4,NULL,4,4,4,4,4,4,4)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 4,4,4,4,4,4,4,4,4)", queryTimes=1) + tdSql.execute("select bottom(c1, 1), c2 from t0 state_window(c2) order by ts", queryTimes=1) + + def test_crash_for_state_window3(self): + tdSql.execute("drop database if exists test") + self.prepareTestEnv() + tdSql.execute("alter local 'queryPolicy' '3'") + self.prepare_original_data() + tdSql.execute("insert into t0 values(now, 4,NULL,4,4,4,4,4,4,4)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 4,5,4,4,4,4,4,4,4)", queryTimes=1) + tdSql.execute("select bottom(c1, 1), c2 from t0 state_window(c2) order by ts", queryTimes=1) + + def test_crash_for_state_window4(self): + tdSql.execute("drop database if exists test") + self.prepareTestEnv() + tdSql.execute("alter local 'queryPolicy' '3'") + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,3,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("flush database test", queryTimes=1) + time.sleep(2) + tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("select bottom(c1, 1), c2 from t0 state_window(c2) order by ts", queryTimes=1) + + def test_crash_for_state_window5(self): + tdSql.execute("drop database if exists test") + self.prepareTestEnv() + tdSql.execute("alter local 'queryPolicy' '3'") + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,3,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("flush database test", queryTimes=1) + time.sleep(2) + tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,3,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("select bottom(c1, 1), c2 from t0 state_window(c2) order by ts", queryTimes=1) + + def run(self): + self.test_crash_for_state_window1() + self.test_crash_for_state_window2() + self.test_crash_for_state_window3() + self.test_crash_for_state_window4() + self.test_crash_for_state_window5() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From f4415b070d18f60cd24026b35f5f5b3efb601104 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 1 Dec 2023 16:21:40 +0800 Subject: [PATCH 037/102] valid msg --- source/libs/transport/inc/transComm.h | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 73427446e6..19b2dfc300 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -23,6 +23,7 @@ extern "C" { #include "taoserror.h" #include "theap.h" #include "tmisce.h" +#include "tmsg.h" #include "transLog.h" #include "transportInt.h" #include "trpc.h" From f029995142fc6abf4c7294b5d835d0f350d9ecec Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 1 Dec 2023 16:22:55 +0800 Subject: [PATCH 038/102] valid msg --- include/common/tmsgdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 0c9d2c2192..3b229e4c1d 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -256,7 +256,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_EXEC_RSMA, "vnode-exec-rsma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp) TD_DEF_MSG_TYPE(TDMT_VND_BATCH_DEL, "batch-delete", SBatchDeleteReq, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) +TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL) From 34e79d158cbba5c0d91384758f1b68aae888b0fa Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 1 Dec 2023 16:27:02 +0800 Subject: [PATCH 039/102] case: add stream basic test case --- tests/system-test/8-stream/stream_basic.py | 110 +++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 tests/system-test/8-stream/stream_basic.py diff --git a/tests/system-test/8-stream/stream_basic.py b/tests/system-test/8-stream/stream_basic.py new file mode 100644 index 0000000000..7f4d1d5ee3 --- /dev/null +++ b/tests/system-test/8-stream/stream_basic.py @@ -0,0 +1,110 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +from util.autogen import * + +import random +import time +import traceback +import os +from os import path + + +class TDTestCase: + # init + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + # autoGen + self.autoGen = AutoGen() + + def waitTranslation(self, waitSeconds): + # wait end + for i in range(waitSeconds): + sql ="show transactions;" + rows = tdSql.query(sql) + if rows == 0: + return True + tdLog.info(f"i={i} wait for translation finish ...") + time.sleep(1) + + return False + + def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + if (len(paths) == 0): + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + def taosBenchmark(self, param): + binPath = self.getPath() + cmd = f"{binPath} {param}" + tdLog.info(cmd) + os.system(cmd) + + # run + def run(self): + # gen data + random.seed(int(time.time())) + self.taosBenchmark(" -d db -t 2 -v 2 -n 1000000 -y") + # create stream + tdSql.execute("use db") + tdSql.execute("create stream stream1 fill_history 1 into sta as select count(*) as cnt from meters interval(10a);",show=True) + sql = "select count(*) from sta" + # loop wait max 60s to check count is ok + tdLog.info("loop wait result ...") + tdSql.checkDataLoop(0, 0, 99999, sql, loopCount=120, waitTime=0.5) + + # check all data is correct + sql = "select * from sta where cnt != 20;" + tdSql.query(sql) + tdSql.checkRows(0) + + # check ts interval is correct + sql = "select * from ( select diff(_wstart) as tsdif from sta ) where tsdif != 10;" + tdSql.query(sql) + tdSql.checkRows(0) + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From bff88e0639db40dec27c1bb9512c8e048b7b10d1 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 1 Dec 2023 16:27:36 +0800 Subject: [PATCH 040/102] fix: set duration order api --- include/libs/executor/storageapi.h | 1 + source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/tsdb/tsdbRead2.c | 6 +++++- source/dnode/vnode/src/vnd/vnodeInitApi.c | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index baa5ef8100..84fe62f836 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -183,6 +183,7 @@ typedef struct TsdReader { int64_t (*tsdReaderGetNumOfInMemRows)(); void (*tsdReaderNotifyClosing)(); + void (*tsdSetDurationOrder)(void* pReader); void (*tsdSetSetNotifyCb)(void* pReader, TsdReaderNotifyCbFn notifyFn, void* param); } TsdReader; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index cdf3fb6a2a..876e8767e8 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -191,6 +191,7 @@ void *tsdbGetIvtIdx2(SMeta *pMeta); uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader); void tsdbReaderSetCloseFlag(STsdbReader *pReader); int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); +void tsdbSetDurationOrder(STsdbReader* pReader); void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param); //====================================================================================================================== diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index ecc1351698..42790a9e7a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -423,7 +423,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void goto _end; } - pReader->bDurationOrder = true; + pReader->bDurationOrder = false; tsdbInitReaderLock(pReader); tsem_init(&pReader->resumeAfterSuspend, 0, 0); @@ -5136,6 +5136,10 @@ void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ } +void tsdbSetDurationOrder(STsdbReader* pReader) { + pReader->bDurationOrder = true; +} + void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param) { pReader->notifyFn = notifyFn; pReader->notifyParam = param; diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 1729e3ab0b..fee823083c 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -61,6 +61,7 @@ void initTsdbReaderAPI(TsdReader* pReader) { pReader->tsdSetQueryTableList = tsdbSetTableList2; pReader->tsdSetReaderTaskId = (void (*)(void*, const char*))tsdbReaderSetId2; + pReader->tsdSetDurationOrder = (void (*)(void*))tsdbSetDurationOrder; pReader->tsdSetSetNotifyCb = (void (*)(void*, TsdReaderNotifyCbFn, void*))tsdbReaderSetNotifyCb; } From a769ccb816de54ed2cc3ad6d798236bfeb57643e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 1 Dec 2023 16:29:06 +0800 Subject: [PATCH 041/102] case: add stream basic test case --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 92eaec52b5..b0129428b2 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -6,6 +6,7 @@ ,,y,unit-test,bash test.sh #system test +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_basic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/scalar_function.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/at_once_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/at_once_session.py From f2b38f5a4a8e47ef55c75b19f5584c339e4f18a1 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 1 Dec 2023 16:51:02 +0800 Subject: [PATCH 042/102] refactor: start/stop duration from start/stop table merge scan --- source/libs/executor/src/scanoperator.c | 90 ++++++++++++++----------- 1 file changed, 50 insertions(+), 40 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 34f35c3ba4..b88bf1ff56 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3327,9 +3327,56 @@ int32_t dumpQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param) { uInfo("tableMergeScanTsdbNotifyCb, %d, %d", type, info->duration.fileSetId); + return; } +int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) { + STableMergeScanInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + int32_t code = TSDB_CODE_SUCCESS; + int32_t numOfTable = pInfo->tableEndIndex - pInfo->tableStartIndex + 1; + + pInfo->sortBufSize = 2048 * pInfo->bufPageSize; + int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage, + pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0); + + tsortSetMergeLimit(pInfo->pSortHandle, pInfo->mergeLimit); + tsortSetAbortCheckFn(pInfo->pSortHandle, isTaskKilled, pOperator->pTaskInfo); + + tsortSetFetchRawDataFp(pInfo->pSortHandle, getBlockForTableMergeScan, NULL, NULL); + + STableMergeScanSortSourceParam *param = taosMemoryCalloc(1, sizeof(STableMergeScanSortSourceParam)); + param->pOperator = pOperator; + + SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); + ps->param = param; + ps->onlyRef = false; + tsortAddSource(pInfo->pSortHandle, ps); + + if (numOfTable == 1) { + tsortSetSingleTableMerge(pInfo->pSortHandle); + } else { + code = tsortOpen(pInfo->pSortHandle); + } + return code; +} + +void stopDurationForGroupTableMergeScan(SOperatorInfo* pOperator) { + STableMergeScanInfo* pInfo = pOperator->info; + + SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle); + pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod; + pInfo->sortExecInfo.sortBuffer = sortExecInfo.sortBuffer; + pInfo->sortExecInfo.loops += sortExecInfo.loops; + pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes; + pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes; + + tsortDestroySortHandle(pInfo->pSortHandle); + pInfo->pSortHandle = NULL; +} + int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -3353,43 +3400,14 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { tSimpleHashClear(pInfo->mTableNumRows); - size_t szRow = blockDataGetRowSize(pInfo->pResBlock); -// if (pInfo->mergeLimit != -1) { -// pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, -// NULL, pTaskInfo->id.str, pInfo->mergeLimit, szRow+8, tsPQSortMemThreshold * 1024* 1024); -// } else - { - pInfo->sortBufSize = 2048 * pInfo->bufPageSize; - int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage, - pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0); - - tsortSetMergeLimit(pInfo->pSortHandle, pInfo->mergeLimit); - tsortSetAbortCheckFn(pInfo->pSortHandle, isTaskKilled, pOperator->pTaskInfo); - } - - tsortSetFetchRawDataFp(pInfo->pSortHandle, getBlockForTableMergeScan, NULL, NULL); - - // one table has one data block int32_t numOfTable = tableEndIdx - tableStartIdx + 1; - - STableMergeScanSortSourceParam *param = taosMemoryCalloc(1, sizeof(STableMergeScanSortSourceParam)); - param->pOperator = pOperator; STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx); pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, &pInfo->mSkipTables); + pAPI->tsdReader.tsdSetDurationOrder(pInfo->base.dataReader); pAPI->tsdReader.tsdSetSetNotifyCb(pInfo->base.dataReader, tableMergeScanTsdbNotifyCb, pInfo); - SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); - ps->param = param; - ps->onlyRef = false; - tsortAddSource(pInfo->pSortHandle, ps); - int32_t code = TSDB_CODE_SUCCESS; - if (numOfTable == 1) { - tsortSetSingleTableMerge(pInfo->pSortHandle); - } else { - code = tsortOpen(pInfo->pSortHandle); - } + int32_t code = startDurationForGroupTableMergeScan(pOperator); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, terrno); @@ -3403,21 +3421,13 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle); - pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod; - pInfo->sortExecInfo.sortBuffer = sortExecInfo.sortBuffer; - pInfo->sortExecInfo.loops += sortExecInfo.loops; - pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes; - pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes; + stopDurationForGroupTableMergeScan(pOperator); if (pInfo->base.dataReader != NULL) { pAPI->tsdReader.tsdReaderClose(pInfo->base.dataReader); pInfo->base.dataReader = NULL; } - tsortDestroySortHandle(pInfo->pSortHandle); - pInfo->pSortHandle = NULL; - resetLimitInfoForNextGroup(&pInfo->limitInfo); taosHashCleanup(pInfo->mSkipTables); pInfo->mSkipTables = NULL; From 0aa9db4fd5160936842943b3709f1ab40d91b4c9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 1 Dec 2023 17:16:12 +0800 Subject: [PATCH 043/102] valid msg --- include/common/tmsg.h | 14 ++++++++++++-- include/common/tmsgdef.h | 5 +++++ source/common/src/tmsg.c | 11 ----------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index be5b51d938..b42df78e34 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -55,6 +55,7 @@ extern "C" { extern char* tMsgInfo[]; extern int32_t tMsgDict[]; +extern int32_t tMsgRangeDict[]; typedef uint16_t tmsg_t; @@ -62,8 +63,17 @@ typedef uint16_t tmsg_t; #define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff) #define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)) -bool tmsgIsValid(tmsg_t type); -char* TMSG_INFO(tmsg_t type); +static inline bool tmsgIsValid(tmsg_t type) { + // static int8_t sz = sizeof(tMsgRangeDict) / sizeof(tMsgRangeDict[0]); + int8_t maxSegIdx = TMSG_SEG_CODE(TDMT_MAX_MSG); + int segIdx = TMSG_SEG_CODE(type); + if (segIdx >= 0 && segIdx < maxSegIdx) { + return type < tMsgRangeDict[segIdx]; + } + return false; +} + +#define TMSG_INFO(type) (tmsgIsValid(type) ? tMsgInfo[TMSG_INDEX(type)] : "unKnown") static inline bool vnodeIsMsgBlock(tmsg_t type) { return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 3b229e4c1d..61b471912f 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -364,6 +364,11 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL) TD_CLOSE_MSG_TYPE(TDMT_END_TMQ_MSG) + TD_NEW_MSG_SEG(TDMT_MAX_MSG) // msg end mark + + + + #if defined(TD_MSG_NUMBER_) TDMT_MAX diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index c7e67c81f5..d69542c98b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -39,17 +39,6 @@ #include "tlog.h" -inline bool tmsgIsValid(tmsg_t type) { - static int8_t sz = sizeof(tMsgRangeDict) / sizeof(tMsgRangeDict[0]); - - int segIdx = TMSG_SEG_CODE(type); - if (segIdx >= 0 && segIdx < sz) { - return type < tMsgRangeDict[segIdx]; - } - return false; -} - -inline char *TMSG_INFO(tmsg_t type) { return (tmsgIsValid(type) ? tMsgInfo[TMSG_INDEX(type)] : "unKnown"); }; #define DECODESQL() \ do { \ if (!tDecodeIsEnd(&decoder)) { \ From f06117dbc915a02d086dbc22c46fe85e40858d6c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 1 Dec 2023 17:32:29 +0800 Subject: [PATCH 044/102] fix:[TD-27644]heartbeat closed before snode close leading to snode is hanged --- include/libs/stream/tstream.h | 3 -- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 6 ++++ source/dnode/vnode/src/inc/vnodeInt.h | 2 -- source/dnode/vnode/src/tq/tq.c | 36 ------------------- source/dnode/vnode/src/vnd/vnodeModule.c | 8 ----- source/libs/stream/inc/streamInt.h | 7 +--- source/libs/stream/src/stream.c | 44 +++++++----------------- source/libs/stream/src/streamDispatch.c | 4 +-- source/libs/stream/src/streamMeta.c | 6 ++-- source/libs/stream/src/streamStart.c | 16 ++++----- 10 files changed, 32 insertions(+), 100 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 7f65ef8358..00d2bee880 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -216,9 +216,6 @@ int32_t streamQueuePush(SStreamQueue1* pQueue, SStreamQueueItem* pItem); SStreamQueueRes streamQueueGetRes(SStreamQueue1* pQueue); #endif -int32_t streamInit(); -void streamCleanUp(); - SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type); void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 6f13abcebc..e0503c83c6 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -28,6 +28,9 @@ } \ } while (0) +extern int32_t streamTimerInit(); +extern void streamTimerCleanUp(); + static SDnode globalDnode = {0}; SDnode *dmInstance() { return &globalDnode; } @@ -166,6 +169,7 @@ int32_t dmInit() { #if defined(USE_S3) if (s3Begin() != 0) return -1; #endif + if (streamTimerInit() != 0) return -1; dInfo("dnode env is initialized"); return 0; @@ -194,6 +198,8 @@ void dmCleanup() { #if defined(USE_S3) s3End(); #endif + streamTimerCleanUp(); + dInfo("dnode env is cleaned up"); taosCleanupCfg(); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 8a4cbb5fd0..7ed0b5103f 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -223,8 +223,6 @@ int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg); // tq -int tqInit(); -void tqCleanUp(); STQ* tqOpen(const char* path, SVnode* pVnode); void tqNotifyClose(STQ*); void tqClose(STQ*); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e9943d2abf..ee76a27414 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -17,12 +17,6 @@ #include "vnd.h" #include "tqCommon.h" -typedef struct { - int8_t inited; -} STqMgmt; - -static STqMgmt tqMgmt = {0}; - // 0: not init // 1: already inited // 2: wait to be inited or cleaup @@ -32,36 +26,6 @@ static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_ static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_EXEC; } static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_IDLE; } -int32_t tqInit() { - int8_t old; - while (1) { - old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 2); - if (old != 2) break; - } - - if (old == 0) { - if (streamInit() < 0) { - return -1; - } - atomic_store_8(&tqMgmt.inited, 1); - } - - return 0; -} - -void tqCleanUp() { - int8_t old; - while (1) { - old = atomic_val_compare_exchange_8(&tqMgmt.inited, 1, 2); - if (old != 2) break; - } - - if (old == 1) { - streamCleanUp(); - atomic_store_8(&tqMgmt.inited, 0); - } -} - void tqDestroyTqHandle(void* data) { STqHandle* pData = (STqHandle*)data; qDestroyTask(pData->execHandle.task); diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 4e3cee42c6..44fcbefba7 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -39,12 +39,6 @@ int vnodeInit(int nthreads) { if (walInit() < 0) { return -1; } - if (tqInit() < 0) { - return -1; - } - if (s3Init() < 0) { - return -1; - } return 0; } @@ -58,7 +52,5 @@ void vnodeCleanup() { vnodeAsyncDestroy(&vnodeAsyncHandle[1]); walCleanUp(); - tqCleanUp(); smaCleanUp(); - s3CleanUp(); } diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 7b8dae8be7..0df36ec391 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -57,11 +57,6 @@ typedef struct { SSDataBlock* pBlock; } SStreamTrigger; -typedef struct SStreamGlobalEnv { - int8_t inited; - void* timer; -} SStreamGlobalEnv; - typedef struct SStreamContinueExecInfo { SEpSet epset; int32_t taskId; @@ -92,7 +87,7 @@ struct SStreamQueue { int8_t status; }; -extern SStreamGlobalEnv streamEnv; +extern void* streamTimer; extern int32_t streamBackendId; extern int32_t streamBackendCfWrapperId; extern int32_t taskDbWrapperId; diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 1c874f34de..1bef42bf14 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -16,38 +16,18 @@ #include "streamInt.h" #include "ttimer.h" -SStreamGlobalEnv streamEnv; +void* streamTimer = NULL; -int32_t streamInit() { - int8_t old; - while (1) { - old = atomic_val_compare_exchange_8(&streamEnv.inited, 0, 2); - if (old != 2) break; +int32_t streamTimerInit() { + streamTimer = taosTmrInit(1000, 100, 10000, "STREAM"); + if (streamTimer == NULL) { + return -1; } - - if (old == 0) { - streamEnv.timer = taosTmrInit(1000, 100, 10000, "STREAM"); - if (streamEnv.timer == NULL) { - atomic_store_8(&streamEnv.inited, 0); - return -1; - } - atomic_store_8(&streamEnv.inited, 1); - } - return 0; } -void streamCleanUp() { - int8_t old; - while (1) { - old = atomic_val_compare_exchange_8(&streamEnv.inited, 1, 2); - if (old != 2) break; - } - - if (old == 1) { - taosTmrCleanUp(streamEnv.timer); - atomic_store_8(&streamEnv.inited, 0); - } +void streamTimerCleanUp() { + taosTmrCleanUp(streamTimer); } char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { @@ -77,7 +57,7 @@ static void streamSchedByTimer(void* param, void* tmrId) { if (pTrigger == NULL) { stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory", nextTrigger); - taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamEnv.timer, &pTask->schedInfo.pTimer); + taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pTimer); return; } @@ -88,7 +68,7 @@ static void streamSchedByTimer(void* param, void* tmrId) { stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory", nextTrigger); - taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamEnv.timer, &pTask->schedInfo.pTimer); + taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pTimer); return; } @@ -97,7 +77,7 @@ static void streamSchedByTimer(void* param, void* tmrId) { int32_t code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger); if (code != TSDB_CODE_SUCCESS) { - taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamEnv.timer, &pTask->schedInfo.pTimer); + taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pTimer); return; } @@ -105,7 +85,7 @@ static void streamSchedByTimer(void* param, void* tmrId) { } } - taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamEnv.timer, &pTask->schedInfo.pTimer); + taosTmrReset(streamSchedByTimer, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pTimer); } int32_t streamSetupScheduleTrigger(SStreamTask* pTask) { @@ -115,7 +95,7 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask) { stDebug("s-task:%s setup scheduler trigger, delay:%" PRId64 " ms", pTask->id.idStr, pTask->info.triggerParam); - pTask->schedInfo.pTimer = taosTmrStart(streamSchedByTimer, (int32_t)pTask->info.triggerParam, pTask, streamEnv.timer); + pTask->schedInfo.pTimer = taosTmrStart(streamSchedByTimer, (int32_t)pTask->info.triggerParam, pTask, streamTimer); pTask->schedInfo.status = TASK_TRIGGER_STATUS__INACTIVE; } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 6247d4ed53..1a67b08749 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -506,9 +506,9 @@ void streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration) { waitDuration, pTask->execInfo.dispatch, pTask->msgInfo.retryCount); if (pTask->msgInfo.pTimer != NULL) { - taosTmrReset(doRetryDispatchData, waitDuration, pTask, streamEnv.timer, &pTask->msgInfo.pTimer); + taosTmrReset(doRetryDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pTimer); } else { - pTask->msgInfo.pTimer = taosTmrStart(doRetryDispatchData, waitDuration, pTask, streamEnv.timer); + pTask->msgInfo.pTimer = taosTmrStart(doRetryDispatchData, waitDuration, pTask, streamTimer); } } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index bd23e41a84..807f120cb7 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -369,7 +369,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); metaRefMgtAdd(pMeta->vgId, pRid); - pMeta->pHbInfo->hbTmr = taosTmrStart(metaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamEnv.timer); + pMeta->pHbInfo->hbTmr = taosTmrStart(metaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamTimer); pMeta->pHbInfo->tickCounter = 0; pMeta->pHbInfo->stopFlag = 0; pMeta->qHandle = taosInitScheduler(32, 1, "stream-chkp", NULL); @@ -1099,7 +1099,7 @@ void metaHbToMnode(void* param, void* tmrId) { } if (!waitForEnoughDuration(pMeta->pHbInfo)) { - taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamEnv.timer, &pMeta->pHbInfo->hbTmr); + taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); taosReleaseRef(streamMetaId, rid); return; } @@ -1215,7 +1215,7 @@ void metaHbToMnode(void* param, void* tmrId) { _end: clearHbMsg(&hbMsg, pIdList); - taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamEnv.timer, &pMeta->pHbInfo->hbTmr); + taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); taosReleaseRef(streamMetaId, rid); } diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index a4c448f678..4c748d58bc 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -114,7 +114,7 @@ static void doReExecScanhistory(void* param, void* tmrId) { // release the task. streamMetaReleaseTask(pTask->pMeta, pTask); } else { - taosTmrReset(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamEnv.timer, + taosTmrReset(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, &pTask->schedHistoryInfo.pTimer); } } @@ -137,9 +137,9 @@ int32_t streamReExecScanHistoryFuture(SStreamTask* pTask, int32_t idleDuration) stDebug("s-task:%s scan-history resumed in %.2fs, ref:%d", pTask->id.idStr, numOfTicks*0.1, ref); if (pTask->schedHistoryInfo.pTimer == NULL) { - pTask->schedHistoryInfo.pTimer = taosTmrStart(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamEnv.timer); + pTask->schedHistoryInfo.pTimer = taosTmrStart(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer); } else { - taosTmrReset(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamEnv.timer, &pTask->schedHistoryInfo.pTimer); + taosTmrReset(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, &pTask->schedHistoryInfo.pTimer); } return TSDB_CODE_SUCCESS; @@ -485,7 +485,7 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s downstream taskId:0x%x (vgId:%d) not ready, stage:%"PRId64", retry in 100ms, ref:%d ", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->oldStage, ref); - pInfo->checkTimer = taosTmrStart(recheckDownstreamTasks, CHECK_DOWNSTREAM_INTERVAL, pInfo, streamEnv.timer); + pInfo->checkTimer = taosTmrStart(recheckDownstreamTasks, CHECK_DOWNSTREAM_INTERVAL, pInfo, streamTimer); } } @@ -726,7 +726,7 @@ static void tryLaunchHistoryTask(void* param, void* tmrId) { pHTaskInfo->tickCount -= 1; if (pHTaskInfo->tickCount > 0) { - taosTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamEnv.timer, &pHTaskInfo->pTimer); + taosTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer); streamMetaReleaseTask(pMeta, pTask); return; } @@ -754,7 +754,7 @@ static void tryLaunchHistoryTask(void* param, void* tmrId) { stDebug("s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d", pTask->id.idStr, p, hTaskId, pHTaskInfo->waitInterval, pHTaskInfo->retryTimes); - taosTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamEnv.timer, &pHTaskInfo->pTimer); + taosTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer); streamMetaReleaseTask(pMeta, pTask); return; } @@ -815,7 +815,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { streamTaskInitForLaunchHTask(&pTask->hTaskInfo); if (pTask->hTaskInfo.pTimer == NULL) { int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); - pTask->hTaskInfo.pTimer = taosTmrStart(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamEnv.timer); + pTask->hTaskInfo.pTimer = taosTmrStart(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamTimer); if (pTask->hTaskInfo.pTimer == NULL) { atomic_sub_fetch_32(&pTask->status.timerActive, 1); stError("s-task:%s failed to start timer, related fill-history task not launched, ref:%d", pTask->id.idStr, @@ -828,7 +828,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { } else { // timer exists ASSERT(pTask->status.timerActive >= 1); stDebug("s-task:%s set timer active flag, task timer not null", pTask->id.idStr); - taosTmrReset(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamEnv.timer, &pTask->hTaskInfo.pTimer); + taosTmrReset(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamTimer, &pTask->hTaskInfo.pTimer); } return TSDB_CODE_SUCCESS; From 7e8f3579de3d3e37c9bf9173227bf74bd74f63c4 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 30 Nov 2023 21:06:06 +0800 Subject: [PATCH 045/102] accept float data as int data input --- include/common/ttypes.h | 2 + include/common/tvariant.h | 3 + source/common/src/tvariant.c | 190 ++++++++++++++++++++++++-- source/common/test/commonTests.cpp | 94 ++++++++++++- source/libs/parser/src/parInsertSql.c | 50 ++++--- 5 files changed, 309 insertions(+), 30 deletions(-) diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 279799b172..741e3663db 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -275,9 +275,11 @@ typedef struct { #define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) #define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX) #define IS_VALID_INT(_t) ((_t) >= INT32_MIN && (_t) <= INT32_MAX) +#define IS_VALID_INT64(_t) ((_t) >= INT64_MIN && (_t) <= INT64_MAX) #define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) <= UINT8_MAX) #define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX) #define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) <= UINT32_MAX) +#define IS_VALID_UINT64(_t) ((_t) >= 0 && (_t) <= UINT64_MAX) #define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) #define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) diff --git a/include/common/tvariant.h b/include/common/tvariant.h index 130945cce5..61cf277bb7 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -37,6 +37,9 @@ typedef struct SVariant { }; } SVariant; +int32_t toIntegerEx(const char *z, int32_t n, int64_t *value); +int32_t toUIntegerEx(const char *z, int32_t n, uint64_t *value); + int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value); int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value); diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 5f2796260c..adb0964fe6 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -19,6 +19,181 @@ #include "ttokendef.h" #include "tvariant.h" +int32_t parseBinaryUInteger(const char *z, int32_t n, uint64_t *value) { + // skip head 0b + const char *p = z + 2; + int32_t l = n - 2; + + while (*p == '0' && l > 0) { + p++; + l--; + } + if (l > 64) { // too big + return TSDB_CODE_FAILED; + } + + uint64_t val = 0; + for (int32_t i = 0; i < l; i++) { + val = val << 1; + if (p[i] == '1') { + val |= 1; + } else if (p[i] != '0') { // abnormal char + return TSDB_CODE_FAILED; + } + } + *value = val; + return TSDB_CODE_SUCCESS; +} + +int32_t parseDecimalUInteger(const char *z, int32_t n, uint64_t *value) { + errno = 0; + char *endPtr = NULL; + while (*z == '0' && n > 1) { + z++; + n--; + } + *value = taosStr2UInt64(z, &endPtr, 10); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t parseHexUInteger(const char *z, int32_t n, uint64_t *value) { + errno = 0; + char *endPtr = NULL; + *value = taosStr2UInt64(z, &endPtr, 16); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *value) { + *is_neg = false; + if (n < 1) { + return TSDB_CODE_FAILED; + } + + // parse sign + bool has_sign = false; + if (z[0] == '-') { + *is_neg = true; + has_sign = true; + } else if (z[0] == '+') { + has_sign = true; + } else if (z[0] < '0' || z[0] > '9') { + return TSDB_CODE_FAILED; + } + if (has_sign) { + z++; + n--; + if (n < 1) { + return TSDB_CODE_FAILED; + } + } + + if (n > 2 && z[0] == '0') { + if (z[1] == 'b' || z[1] == 'B') { + // paring as binary + return parseBinaryUInteger(z, n, value); + } + + if (z[1] == 'x' || z[1] == 'X') { + // parsing as hex + return parseHexUInteger(z, n, value); + } + } + + //rm flag u-unsigned, l-long, f-float(if not in hex str) + char last = tolower(z[n-1]); + if (last == 'u' || last == 'l' || last == 'f') { + n--; + if (n < 1) { + return TSDB_CODE_FAILED; + } + } + + // parsing as decimal + bool parse_int = true; + for (int32_t i = 0; i < n; i++) { + if (z[i] < '0' || z[i] > '9') { + parse_int = false; + break; + } + } + if (parse_int) { + return parseDecimalUInteger(z, n, value); + } + + // parsing as double + errno = 0; + char *endPtr = NULL; + double val = taosStr2Double(z, &endPtr); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n || !IS_VALID_UINT64(val)) { + return TSDB_CODE_FAILED; + } + *value = val; + return TSDB_CODE_SUCCESS; +} + +int32_t removeSpace(const char **pp, int32_t n) { + // rm blank space from both head and tail + const char *z = *pp; + while (*z == ' ' && n > 0) { + z++; + n--; + } + if (n > 0) { + for (int32_t i = n - 1; i > 0; i--) { + if (z[i] == ' ') { + n--; + } else { + break; + } + } + } + *pp = z; + return n; +} + +int32_t toIntegerEx(const char *z, int32_t n, int64_t *value) { + n = removeSpace(&z, n); + if (n < 1) { // fail: all char is space + return TSDB_CODE_FAILED; + } + + bool is_neg = false; + uint64_t uv = 0; + int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv); + if (code == TSDB_CODE_SUCCESS) { + // truncate into int64 + if (uv > INT64_MAX) { + *value = is_neg ? INT64_MIN : INT64_MAX; + return TSDB_CODE_FAILED; + } else { + *value = is_neg ? -uv : uv; + } + } + + return code; +} + +int32_t toUIntegerEx(const char *z, int32_t n, uint64_t *value) { + n = removeSpace(&z, n); + if (n < 1) { // fail: all char is space + return TSDB_CODE_FAILED; + } + + bool is_neg = false; + int32_t code = parseSignAndUInteger(z, n, &is_neg, value); + if (is_neg) { + return TSDB_CODE_FAILED; + } + return code; +} + int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) { errno = 0; char *endPtr = NULL; @@ -26,10 +201,10 @@ int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) { *value = taosStr2Int64(z, &endPtr, base); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { errno = 0; - return -1; + return TSDB_CODE_FAILED; } - return 0; + return TSDB_CODE_SUCCESS; } int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) { @@ -39,16 +214,15 @@ int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) { const char *p = z; while (*p == ' ') p++; if (*p == '-') { - return -1; + return TSDB_CODE_FAILED; } *value = taosStr2UInt64(z, &endPtr, base); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { errno = 0; - return -1; + return TSDB_CODE_FAILED; } - - return 0; + return TSDB_CODE_SUCCESS; } /** @@ -147,14 +321,13 @@ void taosVariantDestroy(SVariant *pVar) { taosMemoryFreeClear(pVar->pz); pVar->nLen = 0; } - } void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { if (pSrc == NULL || pDst == NULL) return; pDst->nType = pSrc->nType; - if (pSrc->nType == TSDB_DATA_TYPE_BINARY ||pSrc->nType == TSDB_DATA_TYPE_VARBINARY || + if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_VARBINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR || pSrc->nType == TSDB_DATA_TYPE_JSON || pSrc->nType == TSDB_DATA_TYPE_GEOMETRY) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; @@ -172,7 +345,6 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) { pDst->i = pSrc->i; } - } int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) { diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 107276d7f9..035bca0e15 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -24,6 +24,93 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } + +TEST(testCase, toIntegerEx_test) { + int64_t val = 0; + + char* s = "123"; + int32_t ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 123); + + s = "9223372036854775807"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 9223372036854775807); + + s = "9323372036854775807"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, -1); + + s = "-9323372036854775807"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, -1); + + s = "-1"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -1); + + s = "-9223372036854775807"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -9223372036854775807); + + s = "1000u"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 1000); + + s = "1000l"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 1000); + + s = "1000.0f"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 1000); + + s = "0x1f"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 31); + + s = "-0x40"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -64); + + s = "0b110"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 6); + + s = "-0b10010"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -18); + + s = "-5.2343544534e10"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -52343544534); + + s = "1.869895343e4"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 18698); + + // UINT64_MAX + s = "18446744073709551615"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, -1); + + s = "18446744073709551616"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, -1); +} + TEST(testCase, toInteger_test) { char* s = "123"; uint32_t type = 0; @@ -39,10 +126,9 @@ TEST(testCase, toInteger_test) { ASSERT_EQ(ret, 0); ASSERT_EQ(val, 9223372036854775807); - s = "9323372036854775807"; + s = "9323372036854775807"; // out of range, > int64_max ret = toInteger(s, strlen(s), 10, &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 9323372036854775807u); + ASSERT_EQ(ret, -1); s = "-9323372036854775807"; ret = toInteger(s, strlen(s), 10, &val); @@ -139,7 +225,7 @@ TEST(testCase, Datablock_test) { printf("binary column length:%d\n", *(int32_t*)p1->pData); - ASSERT_EQ(blockDataGetNumOfCols(b), 2); + ASSERT_EQ(blockDataGetNumOfCols(b), 3); ASSERT_EQ(blockDataGetNumOfRows(b), 40); char* pData = colDataGetData(p1, 3); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 31b016458a..c993efb9ae 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -433,7 +433,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_TINYINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + code = toIntegerEx(pToken->z, pToken->n, &iv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z); @@ -444,7 +445,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UTINYINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + code = toUIntegerEx(pToken->z, pToken->n, &uv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); } else if (uv > UINT8_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); @@ -454,7 +456,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_SMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + code = toIntegerEx(pToken->z, pToken->n, &iv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z); @@ -464,7 +467,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_USMALLINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + code = toUIntegerEx(pToken->z, pToken->n, &uv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); } else if (uv > UINT16_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); @@ -474,7 +478,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_INT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + code = toIntegerEx(pToken->z, pToken->n, &iv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z); } else if (!IS_VALID_INT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z); @@ -484,7 +489,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + code = toUIntegerEx(pToken->z, pToken->n, &uv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); } else if (uv > UINT32_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); @@ -494,7 +500,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_BIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + code = toIntegerEx(pToken->z, pToken->n, &iv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); } val->i64 = iv; @@ -502,7 +509,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UBIGINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + code = toUIntegerEx(pToken->z, pToken->n, &uv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); } *(uint64_t*)(&val->i64) = uv; @@ -1351,7 +1359,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_TINYINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "tinyint data overflow", pToken->z); @@ -1359,7 +1368,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_UTINYINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); } else if (pVal->value.val > UINT8_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); @@ -1367,7 +1377,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_SMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); @@ -1375,7 +1386,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_USMALLINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); } else if (pVal->value.val > UINT16_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); @@ -1383,7 +1395,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_INT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); } else if (!IS_VALID_INT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); @@ -1391,7 +1404,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_UINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); } else if (pVal->value.val > UINT32_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); @@ -1399,14 +1413,16 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_BIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); } break; } case TSDB_DATA_TYPE_UBIGINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); + int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); } break; } From 3ce2a2157afef414b7dea47906062f3c489728d2 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 10:13:45 +0800 Subject: [PATCH 046/102] docs: remove desc of rsma --- docs/en/12-taos-sql/02-database.md | 2 -- docs/en/12-taos-sql/03-table.md | 7 ++----- docs/zh/12-taos-sql/02-database.md | 2 -- docs/zh/12-taos-sql/03-table.md | 7 ++----- 4 files changed, 4 insertions(+), 14 deletions(-) diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index 255263ecfd..a0e036664a 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -26,7 +26,6 @@ database_option: { | PAGESIZE value | PRECISION {'ms' | 'us' | 'ns'} | REPLICA value - | RETENTIONS ingestion_duration:keep_duration ... | WAL_LEVEL {1 | 2} | VGROUPS value | SINGLE_STABLE {0 | 1} @@ -61,7 +60,6 @@ database_option: { - PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode. The default value is 4. Enter a value between 1 and 16384. - PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms. - REPLICA: specifies the number of replicas that are made of the database. Enter 1 or 3. The default value is 1. The value of the REPLICA parameter cannot exceed the number of dnodes in the cluster. -- RETENTIONS: specifies the retention period for data aggregated at various intervals. For example, RETENTIONS 15s:7d,1m:21d,15m:50d indicates that data aggregated every 15 seconds is retained for 7 days, data aggregated every 1 minute is retained for 21 days, and data aggregated every 15 minutes is retained for 50 days. You must enter three aggregation intervals and corresponding retention periods. - WAL_LEVEL: specifies whether fsync is enabled. The default value is 1. - 1: WAL is enabled but fsync is disabled. - 2: WAL and fsync are both enabled. diff --git a/docs/en/12-taos-sql/03-table.md b/docs/en/12-taos-sql/03-table.md index 10c44848c9..8c6ab6489f 100644 --- a/docs/en/12-taos-sql/03-table.md +++ b/docs/en/12-taos-sql/03-table.md @@ -54,11 +54,8 @@ table_option: { **Parameter description** 1. COMMENT: specifies comments for the table. This parameter can be used with supertables, standard tables, and subtables. -2. WATERMARK: specifies the time after which the window is closed. The default value is 5 seconds. Enter a value between 0 and 15 minutes in milliseconds, seconds, or minutes. You can enter multiple values separated by commas (,). This parameter applies only to supertables and takes effect only when the RETENTIONS parameter has been specified for the database. -3. MAX_DELAY: specifies the maximum latency for pushing computation results. The default value is 15 minutes or the value of the INTERVAL parameter, whichever is smaller. Enter a value between 0 and 15 minutes in milliseconds, seconds, or minutes. You can enter multiple values separated by commas (,). Note: Retain the default value if possible. Configuring a small MAX_DELAY may cause results to be frequently pushed, affecting storage and query performance. This parameter applies only to supertables and takes effect only when the RETENTIONS parameter has been specified for the database. -4. ROLLUP: specifies aggregate functions to roll up. Rolling up a function provides downsampled results based on multiple axes. This parameter applies only to supertables and takes effect only when the RETENTIONS parameter has been specified for the database. You can specify only one function to roll up. The rollup takes effect on all columns except TS. Enter one of the following values: avg, sum, min, max, last, or first. -5. SMA: specifies functions on which to enable small materialized aggregates (SMA). SMA is user-defined precomputation of aggregates based on data blocks. Enter one of the following values: max, min, or sum This parameter can be used with supertables and standard tables. -6. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The default value is 0, i.e. never expire. +2. SMA: specifies functions on which to enable small materialized aggregates (SMA). SMA is user-defined precomputation of aggregates based on data blocks. Enter one of the following values: max, min, or sum This parameter can be used with supertables and standard tables. +3. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The default value is 0, i.e. never expire. ## Create Subtables diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index bd33281bc0..c3e0e9a07a 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -26,7 +26,6 @@ database_option: { | PAGESIZE value | PRECISION {'ms' | 'us' | 'ns'} | REPLICA value - | RETENTIONS ingestion_duration:keep_duration ... | WAL_LEVEL {1 | 2} | VGROUPS value | SINGLE_STABLE {0 | 1} @@ -61,7 +60,6 @@ database_option: { - PAGESIZE：一个 VNODE 中元数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB 到 16 MB。 - PRECISION：数据库的时间戳精度。ms 表示毫秒，us 表示微秒，ns 表示纳秒，默认 ms 毫秒。 - REPLICA：表示数据库副本数，取值为 1 或 3，默认为 1。在集群中使用，副本数必须小于或等于 DNODE 的数目。 -- RETENTIONS：表示数据的聚合周期和保存时长，如 RETENTIONS 15s:7d,1m:21d,15m:50d 表示数据原始采集周期为 15 秒，原始数据保存 7 天；按 1 分钟聚合的数据保存 21 天；按 15 分钟聚合的数据保存 50 天。目前支持且只支持三级存储周期。 - WAL_LEVEL：WAL 级别，默认为 1。 - 1：写 WAL，但不执行 fsync。 - 2：写 WAL，而且执行 fsync。 diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index 9258258263..adfadef981 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -52,11 +52,8 @@ table_option: { **参数说明** 1. COMMENT：表注释。可用于超级表、子表和普通表。 -2. WATERMARK：指定窗口的关闭时间，默认值为 5 秒，最小单位毫秒，范围为 0 到 15 分钟，多个以逗号分隔。只可用于超级表，且只有当数据库使用了 RETENTIONS 参数时，才可以使用此表参数。 -3. MAX_DELAY：用于控制推送计算结果的最大延迟，默认值为 interval 的值(但不能超过最大值)，最小单位毫秒，范围为 1 毫秒到 15 分钟，多个以逗号分隔。注：不建议 MAX_DELAY 设置太小，否则会过于频繁的推送结果，影响存储和查询性能，如无特殊需求，取默认值即可。只可用于超级表，且只有当数据库使用了 RETENTIONS 参数时，才可以使用此表参数。 -4. ROLLUP：Rollup 指定的聚合函数，提供基于多层级的降采样聚合结果。只可用于超级表。只有当数据库使用了 RETENTIONS 参数时，才可以使用此表参数。作用于超级表除 TS 列外的其它所有列，但是只能定义一个聚合函数。 聚合函数支持 avg, sum, min, max, last, first。 -5. SMA：Small Materialized Aggregates，提供基于数据块的自定义预计算功能。预计算类型包括 MAX、MIN 和 SUM。可用于超级表/普通表。 -6. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联，如果 KEEP 比 TTL 小，在表被删除之前数据也可能已经被删除。 +2. SMA：Small Materialized Aggregates，提供基于数据块的自定义预计算功能。预计算类型包括 MAX、MIN 和 SUM。可用于超级表/普通表。 +3. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联，如果 KEEP 比 TTL 小，在表被删除之前数据也可能已经被删除。 ## 创建子表 From b69d17c751e37ba42688e8459d1101a6d86e8b10 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 10:16:57 +0800 Subject: [PATCH 047/102] docs: remove desc of rsma --- docs/en/12-taos-sql/03-table.md | 3 --- docs/zh/12-taos-sql/03-table.md | 3 --- 2 files changed, 6 deletions(-) diff --git a/docs/en/12-taos-sql/03-table.md b/docs/en/12-taos-sql/03-table.md index 8c6ab6489f..a10abd28a5 100644 --- a/docs/en/12-taos-sql/03-table.md +++ b/docs/en/12-taos-sql/03-table.md @@ -32,9 +32,6 @@ table_options: table_option: { COMMENT 'string_value' - | WATERMARK duration[,duration] - | MAX_DELAY duration[,duration] - | ROLLUP(func_name [, func_name] ...) | SMA(col_name [, col_name] ...) | TTL value } diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index adfadef981..7e20f20574 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -30,9 +30,6 @@ table_options: table_option: { COMMENT 'string_value' - | WATERMARK duration[,duration] - | MAX_DELAY duration[,duration] - | ROLLUP(func_name [, func_name] ...) | SMA(col_name [, col_name] ...) | TTL value } From 0fe26f1507dfefc561de32f5b73fc82c93e02112 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Mon, 4 Dec 2023 10:40:43 +0800 Subject: [PATCH 048/102] fix error msg if failed get ip from fqdn --- source/common/src/tglobal.c | 2 +- source/libs/transport/src/transCli.c | 2 +- source/os/test/osTests.cpp | 29 ++++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index ced68c134e..39cd2d604b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1282,7 +1282,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi static int32_t taosCheckGlobalCfg() { uint32_t ipv4 = taosGetIpv4FromFqdn(tsLocalFqdn); if (ipv4 == 0xffffffff) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TSDB_CODE_RPC_FQDN_ERROR; uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, terrstr()); return -1; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index e51c61c49d..936d11c151 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1468,7 +1468,7 @@ static FORCE_INLINE uint32_t cliGetIpFromFqdnCache(SHashObj* cache, char* fqdn) if (v == NULL) { addr = taosGetIpv4FromFqdn(fqdn); if (addr == 0xffffffff) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TSDB_CODE_RPC_FQDN_ERROR; tError("failed to get ip from fqdn:%s since %s", fqdn, terrstr()); return addr; } diff --git a/source/os/test/osTests.cpp b/source/os/test/osTests.cpp index e2185aeac2..62d3e9ec36 100644 --- a/source/os/test/osTests.cpp +++ b/source/os/test/osTests.cpp @@ -32,6 +32,35 @@ #ifdef WINDOWS #include +#else + +#include + +TEST(osTest, osFQDNSuccess) { + char fqdn[1024]; + char ipString[INET_ADDRSTRLEN]; + int code = taosGetFqdn(fqdn); + uint32_t ipv4 = taosGetIpv4FromFqdn(fqdn); + ASSERT_NE(ipv4, 0xffffffff); + + struct in_addr addr; + addr.s_addr = htonl(ipv4); + snprintf(ipString, INET_ADDRSTRLEN, "%u.%u.%u.%u", (unsigned int)(addr.s_addr >> 24) & 0xFF, + (unsigned int)(addr.s_addr >> 16) & 0xFF, (unsigned int)(addr.s_addr >> 8) & 0xFF, + (unsigned int)(addr.s_addr) & 0xFF); + printf("fqdn:%s ip:%s\n", fqdn, ipString); +} + +TEST(osTest, osFQDNFailed) { + char fqdn[1024] = "fqdn_test_not_found"; + char ipString[24]; + uint32_t ipv4 = taosGetIpv4FromFqdn(fqdn); + ASSERT_EQ(ipv4, 0xffffffff); + + terrno = TSDB_CODE_RPC_FQDN_ERROR; + printf("fqdn:%s transfer to ip failed!\n", fqdn); +} + #endif // WINDOWS TEST(osTest, osSystem) { From 3f8b2e826fdd775cda9217cf5083aaa4b9ab77d6 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 4 Dec 2023 10:49:40 +0800 Subject: [PATCH 049/102] fix: add new duration processing to table merge scan --- source/libs/executor/inc/executorInt.h | 2 ++ source/libs/executor/src/scanoperator.c | 45 ++++++++++++++++++------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index e29583d8fc..a3ec2cca74 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -297,6 +297,8 @@ typedef struct STableMergeScanInfo { SHashObj* mSkipTables; int64_t mergeLimit; SSortExecInfo sortExecInfo; + bool bNewDuration; + SSDataBlock* pNewDurationBlock; } STableMergeScanInfo; typedef struct STagScanFilterContext { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b88bf1ff56..e8213d0597 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3242,6 +3242,12 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { STsdbReader* reader = pInfo->base.dataReader; while (true) { + if (pInfo->pNewDurationBlock) { + SSDataBlock* pNewDurationBlock = pInfo->pNewDurationBlock; + pInfo->pNewDurationBlock = NULL; + return pNewDurationBlock; + } + code = pAPI->tsdReader.tsdNextDataBlock(reader, &hasNext); if (code != 0) { pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); @@ -3267,7 +3273,6 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { uint32_t status = 0; code = loadDataBlock(pOperator, &pInfo->base, pBlock, &status); - // code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { qInfo("table merge scan load datablock code %d, %s", code, GET_TASKID(pTaskInfo)); T_LONG_JMP(pTaskInfo->env, code); @@ -3290,6 +3295,11 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; + + if (pInfo->bNewDuration) { + pInfo->pNewDurationBlock = pBlock; + return NULL; + } return pBlock; } @@ -3327,7 +3337,8 @@ int32_t dumpQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param) { uInfo("tableMergeScanTsdbNotifyCb, %d, %d", type, info->duration.fileSetId); - + STableMergeScanInfo* pTmsInfo = param; + pTmsInfo->bNewDuration = true; return; } @@ -3337,6 +3348,8 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t code = TSDB_CODE_SUCCESS; int32_t numOfTable = pInfo->tableEndIndex - pInfo->tableStartIndex + 1; + pInfo->bNewDuration = false; + pInfo->sortBufSize = 2048 * pInfo->bufPageSize; int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage, @@ -3354,7 +3367,7 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) { ps->param = param; ps->onlyRef = false; tsortAddSource(pInfo->pSortHandle, ps); - + if (numOfTable == 1) { tsortSetSingleTableMerge(pInfo->pSortHandle); } else { @@ -3510,17 +3523,23 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { pOperator->resultInfo.totalRows += pBlock->info.rows; return pBlock; } else { - // Data of this group are all dumped, let's try the next group - stopGroupTableMergeScan(pOperator); - if (pInfo->tableEndIndex >= tableListSize - 1) { - setOperatorCompleted(pOperator); - break; - } + if (pInfo->bNewDuration) { + stopDurationForGroupTableMergeScan(pOperator); + startDurationForGroupTableMergeScan(pOperator); - pInfo->tableStartIndex = pInfo->tableEndIndex + 1; - pInfo->groupId = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex)->groupId; - startGroupTableMergeScan(pOperator); - resetLimitInfoForNextGroup(&pInfo->limitInfo); + } else { + // Data of this group are all dumped, let's try the next group + stopGroupTableMergeScan(pOperator); + if (pInfo->tableEndIndex >= tableListSize - 1) { + setOperatorCompleted(pOperator); + break; + } + + pInfo->tableStartIndex = pInfo->tableEndIndex + 1; + pInfo->groupId = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex)->groupId; + startGroupTableMergeScan(pOperator); + resetLimitInfoForNextGroup(&pInfo->limitInfo); + } } } From d368956e3fc438c809338294b899f930351aae95 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 24 Nov 2023 16:24:12 +0800 Subject: [PATCH 050/102] fix timezone range --- source/common/src/ttime.c | 6 +++++- source/os/src/osFile.c | 18 +++++++++--------- source/os/test/osTests.cpp | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 227de7f5fc..cbe700fa2d 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -170,10 +170,14 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { hour = strnatoi(&str[i], len); i += len + 1; } else { - hour = strnatoi(&str[i], 2); + hour = strnatoi(&str[i], strlen(&str[i])); i += 2; } + if (hour > 12 || hour < 0) { + return -1; + } + // return error if there're illegal charaters after min(2 Digits) char* minStr = &str[i]; if (minStr[1] != '\0' && minStr[2] != '\0') { diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 15aca85fc2..39de117339 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -348,15 +348,15 @@ HANDLE taosOpenFileNotStream(const char *path, int32_t tdFileOptions) { if (h != INVALID_HANDLE_VALUE && (tdFileOptions & TD_FILE_APPEND) && (tdFileOptions & TD_FILE_WRITE)) { SetFilePointer(h, 0, NULL, FILE_END); } - if (h == INVALID_HANDLE_VALUE) { - DWORD dwError = GetLastError(); - LPVOID lpMsgBuf; - FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError, - 0, - (LPTSTR)&lpMsgBuf, 0, NULL); - printf("CreateFile failed with error %d: %s", dwError, (char*)lpMsgBuf); - LocalFree(lpMsgBuf); - } + // if (h == INVALID_HANDLE_VALUE) { + // DWORD dwError = GetLastError(); + // LPVOID lpMsgBuf; + // FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError, + // 0, + // (LPTSTR)&lpMsgBuf, 0, NULL); + // printf("CreateFile failed with error %d: %s", dwError, (char*)lpMsgBuf); + // LocalFree(lpMsgBuf); + // } return h; } diff --git a/source/os/test/osTests.cpp b/source/os/test/osTests.cpp index e2185aeac2..9ab58a180a 100644 --- a/source/os/test/osTests.cpp +++ b/source/os/test/osTests.cpp @@ -314,6 +314,6 @@ TEST(osTest, osFilePerformance) { printf("Test OpenForRead & Close file %d times, cost: %" PRId64 "us\n", TESTTIMES, OpenForReadCloseFileCost); } -#endif OSFILE_PERFORMANCE_TEST +#endif // OSFILE_PERFORMANCE_TEST #pragma GCC diagnostic pop From 8180aaeb6739520d80fb852e2373ae2b6761b2fe Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 24 Nov 2023 17:56:50 +0800 Subject: [PATCH 051/102] fix time format --- source/common/src/ttime.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index cbe700fa2d..cdc380dee0 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -170,7 +170,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { hour = strnatoi(&str[i], len); i += len + 1; } else { - hour = strnatoi(&str[i], strlen(&str[i])); + hour = strnatoi(&str[i], 2); i += 2; } @@ -185,7 +185,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { } int64_t minute = strnatoi(&str[i], 2); - if (minute > 59) { + if (minute > 59 || (hour == 12 && minute > 0)) { return -1; } From cea4be6f3c62963c53abf7925dc81a4fce29c3da Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 4 Dec 2023 13:40:28 +0800 Subject: [PATCH 052/102] fix: bg task memory waste --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbFS2.c | 31 +++++++++++++++------ source/dnode/vnode/src/tsdb/tsdbFSet2.c | 1 + source/dnode/vnode/src/tsdb/tsdbFSet2.h | 1 + source/dnode/vnode/src/tsdb/tsdbMerge.c | 2 ++ source/dnode/vnode/src/tsdb/tsdbRetention.c | 5 ++++ source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 5 +--- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 22 ++++++++++++--- 8 files changed, 52 insertions(+), 16 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 88362239f5..197952d190 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -377,6 +377,7 @@ struct STsdb { SVnode *pVnode; STsdbKeepCfg keepCfg; TdThreadMutex mutex; + bool bgTaskDisabled; SMemTable *mem; SMemTable *imem; STsdbFS fs; // old diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index add8da52e0..7322da6a23 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -745,31 +745,45 @@ _exit: return code; } -int32_t tsdbFSCancelAllBgTask(STFileSystem *fs) { +static int32_t tsdbFSSetBlockCommit(STFileSet *fset, bool block); + +int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) { + STFileSystem *fs = pTsdb->pFS; TARRAY2(int64_t) channelArr = {0}; - // collect all open channels taosThreadMutexLock(&fs->tsdb->mutex); + + // disable + pTsdb->bgTaskDisabled = true; + + // collect channel STFileSet *fset; TARRAY2_FOREACH(fs->fSetArr, fset) { if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) { TARRAY2_APPEND(&channelArr, fset->bgTaskChannel); fset->bgTaskChannel = 0; } + fset->mergeScheduled = false; + tsdbFSSetBlockCommit(fset, false); } + taosThreadMutexUnlock(&fs->tsdb->mutex); // destroy all channels - int64_t channel; - TARRAY2_FOREACH(&channelArr, channel) { vnodeAChannelDestroy(vnodeAsyncHandle[1], channel, true); } - TARRAY2_DESTROY(&channelArr, NULL); + return 0; +} + +int32_t tsdbEnableBgTask(STsdb *pTsdb) { + taosThreadMutexLock(&pTsdb->mutex); + pTsdb->bgTaskDisabled = false; + taosThreadMutexUnlock(&pTsdb->mutex); return 0; } int32_t tsdbCloseFS(STFileSystem **fs) { if (fs[0] == NULL) return 0; - tsdbFSCancelAllBgTask(*fs); + tsdbDisableAndCancelAllBgTask((*fs)->tsdb); close_file_system(fs[0]); destroy_fs(fs); return 0; @@ -857,7 +871,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { // schedule merge int32_t sttTrigger = fs->tsdb->pVnode->config.sttTrigger; - if (sttTrigger > 1) { + if (sttTrigger > 1 && !fs->tsdb->bgTaskDisabled) { STFileSet *fset; TARRAY2_FOREACH_REVERSE(fs->fSetArr, fset) { if (TARRAY2_SIZE(fset->lvlArr) == 0) { @@ -873,7 +887,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { bool skipMerge = false; int32_t numFile = TARRAY2_SIZE(lvl->fobjArr); - if (numFile >= sttTrigger) { + if (numFile >= sttTrigger && (!fset->mergeScheduled)) { // launch merge { extern int8_t tsS3Enabled; @@ -917,6 +931,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg, NULL); TSDB_CHECK_CODE(code, lino, _exit); + fset->mergeScheduled = true; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 025671ff3d..1bf886e3b0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -453,6 +453,7 @@ int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) { // background task queue fset[0]->bgTaskChannel = 0; + fset[0]->mergeScheduled = false; // block commit variables taosThreadCondInit(&fset[0]->canCommit, NULL); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index 32028db352..83f5b1e83c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -93,6 +93,7 @@ struct STFileSet { // background task channel int64_t bgTaskChannel; + bool mergeScheduled; // block commit variables TdThreadCond canCommit; diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index b47b951b2b..7f33f08794 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -514,6 +514,8 @@ static int32_t tsdbMergeGetFSet(SMerger *merger) { return 0; } + fset->mergeScheduled = false; + int32_t code = tsdbTFileSetInitCopy(merger->tsdb, fset, &merger->fset); if (code) { taosThreadMutexUnlock(&merger->tsdb->mutex); diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index d8f1ad7c6c..dc98f46ac5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -430,6 +430,11 @@ int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) { taosThreadMutexLock(&tsdb->mutex); + if (tsdb->bgTaskDisabled) { + taosThreadMutexUnlock(&tsdb->mutex); + return 0; + } + STFileSet *fset; TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { code = tsdbTFileSetOpenChannel(fset); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index faea881f72..e757daa0af 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1602,7 +1602,4 @@ _out: } return code; -} - -extern int32_t tsdbFSCancelAllBgTask(STFileSystem* fs); -int32_t tsdbCancelAllBgTask(STsdb* tsdb) { return tsdbFSCancelAllBgTask(tsdb->pFS); } \ No newline at end of file +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index f2ef11e9ed..34b508388f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -519,7 +519,21 @@ _out: return code; } -extern int32_t tsdbCancelAllBgTask(STsdb *tsdb); +extern int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb); +extern int32_t tsdbEnableBgTask(STsdb *pTsdb); + +static int32_t vnodeCancelAndDisableAllBgTask(SVnode *pVnode) { + tsdbDisableAndCancelAllBgTask(pVnode->pTsdb); + vnodeSyncCommit(pVnode); + vnodeAChannelDestroy(vnodeAsyncHandle[0], pVnode->commitChannel, true); + return 0; +} + +static int32_t vnodeEnableBgTask(SVnode *pVnode) { + tsdbEnableBgTask(pVnode->pTsdb); + vnodeAChannelInit(vnodeAsyncHandle[0], &pVnode->commitChannel); + return 0; +} int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter) { int32_t code = 0; @@ -527,9 +541,8 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter int64_t sver = pParam->start; int64_t ever = pParam->end; - // commit memory data - vnodeSyncCommit(pVnode); - tsdbCancelAllBgTask(pVnode->pTsdb); + // cancel and disable all bg task + vnodeCancelAndDisableAllBgTask(pVnode); // alloc pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter)); @@ -659,6 +672,7 @@ _exit: vInfo("vgId:%d, vnode snapshot writer closed, rollback:%d", TD_VID(pVnode), rollback); taosMemoryFree(pWriter); } + vnodeEnableBgTask(pVnode); return code; } From fc905406d52d79266f27fbf0a9d0e96f6c260994 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 14:08:24 +0800 Subject: [PATCH 053/102] chore: validate tag val --- source/common/src/tdataformat.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index f97f9c0c11..5eebf05596 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1247,6 +1247,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { if (isJson) { n += tPutCStr(p ? p + n : p, pTagVal->pKey); } else { + ASSERTS(pTagVal->cid > 0, "Invalid tag cid %" PRIi16, pTagVal->cid); n += tPutI16v(p ? p + n : p, pTagVal->cid); } From 7d8864ec89ff4f18ff31aba677935500f8300349 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 14:22:49 +0800 Subject: [PATCH 054/102] chore: validate tag val --- source/common/src/tdataformat.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 5eebf05596..fe5374d5da 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1247,8 +1247,8 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { if (isJson) { n += tPutCStr(p ? p + n : p, pTagVal->pKey); } else { - ASSERTS(pTagVal->cid > 0, "Invalid tag cid %" PRIi16, pTagVal->cid); n += tPutI16v(p ? p + n : p, pTagVal->cid); + ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); } // type @@ -1273,6 +1273,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { n += tGetCStr(p + n, &pTagVal->pKey); } else { n += tGetI16v(p + n, &pTagVal->cid); + ASSERTS(pTagVal->cid < 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); } // type From 4cc22a1b90b8bc2860d8694361f05c7444d76189 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 14:26:34 +0800 Subject: [PATCH 055/102] chore: validate tag val --- source/common/src/tdataformat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index fe5374d5da..cf0dd9ecdb 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1273,7 +1273,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { n += tGetCStr(p + n, &pTagVal->pKey); } else { n += tGetI16v(p + n, &pTagVal->cid); - ASSERTS(pTagVal->cid < 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); + ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); } // type From 210a304cc123df9ce01266f9871f3028fba078f0 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 4 Dec 2023 14:41:28 +0800 Subject: [PATCH 056/102] Update tdataformat.c --- source/common/src/tdataformat.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index cf0dd9ecdb..02dfbfebfe 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1273,7 +1273,6 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { n += tGetCStr(p + n, &pTagVal->pKey); } else { n += tGetI16v(p + n, &pTagVal->cid); - ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); } // type From c83b7b80de4559e41d930371ffbf5832d62e310d Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 4 Dec 2023 14:46:28 +0800 Subject: [PATCH 057/102] test case --- tests/system-test/2-query/timezone.py | 51 ++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/timezone.py b/tests/system-test/2-query/timezone.py index 316e776cb1..970ee0ff3b 100644 --- a/tests/system-test/2-query/timezone.py +++ b/tests/system-test/2-query/timezone.py @@ -112,10 +112,59 @@ class TDTestCase: self.data_check(timezone,self.stbname,'stable') for i in range(self.tbnum): self.data_check(timezone,f'{self.stbname}_{i}','child_table') + tdSql.execute(f'drop database {self.dbname}') + + def timezone_format_test(self): + tdSql.execute(f'create database {self.dbname}') + tdSql.execute(self.setsql.set_create_stable_sql(f'{self.dbname}.stb', {'ts':'timestamp','id':'int'}, {'status':'int'})) + + tdSql.execute(f"insert into {self.dbname}.d0 using {self.dbname}.stb tags (1) values ('2021-07-01 00:00:00.000',0);") + tdSql.query(f"select ts from {self.dbname}.d0;") + tdSql.checkData(0, 0, "2021-07-01 00:00:00.000") + + tdSql.execute(f"insert into {self.dbname}.d1 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+07:50',1)") + tdSql.query(f"select ts from {self.dbname}.d1") + tdSql.checkData(0, 0, "2021-07-01 00:10:00.000") + + tdSql.execute(f"insert into {self.dbname}.d2 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+12:00',1)") + tdSql.query(f"select ts from {self.dbname}.d2") + tdSql.checkData(0, 0, "2021-06-30 20:00:00.000") + + tdSql.execute(f"insert into {self.dbname}.d3 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-00:10',1)") + tdSql.query(f"select ts from {self.dbname}.d3") + tdSql.checkData(0, 0, "2021-07-01 08:10:00.000") + + tdSql.execute(f"insert into {self.dbname}.d4 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-12:00',1)") + tdSql.query(f"select ts from {self.dbname}.d4") + tdSql.checkData(0, 0, "2021-07-01 20:00:00.000") + + tdSql.execute(f"insert into {self.dbname}.d5 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-1200',1)") + tdSql.query(f"select ts from {self.dbname}.d5") + tdSql.checkData(0, 0, "2021-07-01 20:00:00.000") + + tdSql.execute(f"insert into {self.dbname}.d6 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-115',1)") + tdSql.query(f"select ts from {self.dbname}.d6") + tdSql.checkData(0, 0, "2021-07-01 19:05:00.000") + + tdSql.execute(f"insert into {self.dbname}.d7 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-1105',1)") + tdSql.query(f"select ts from {self.dbname}.d7") + tdSql.checkData(0, 0, "2021-07-01 19:05:00.000") + + tdSql.error(f"insert into {self.dbname}.d21 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+12:10',1)") + tdSql.error(f"insert into {self.dbname}.d22 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-24:10',1)") + tdSql.error(f"insert into {self.dbname}.d23 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+12:10',1)") + tdSql.error(f"insert into {self.dbname}.d24 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-24:10',1)") + tdSql.error(f"insert into {self.dbname}.d24 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-24100',1)") + tdSql.error(f"insert into {self.dbname}.d24 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-1210',1)") + + tdSql.execute(f'drop database {self.dbname}') + + def run(self): # sourcery skip: extract-duplicate-method timezone = self.get_system_timezone() self.timezone_check_ntb(timezone) - self.timezone_check_stb(timezone) + #self.timezone_check_stb(timezone) + self.timezone_format_test() def stop(self): tdSql.close() From 07e43fe56b92113504d5ca0746bcc8465d1bc470 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 4 Dec 2023 14:54:15 +0800 Subject: [PATCH 058/102] fix: two intervals are intesected or not --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index ecc1351698..a17d664232 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2506,9 +2506,11 @@ static void prepareDurationForNextFileSet(STsdbReader* pReader) { tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &winFid.skey, &winFid.ekey); if (ASCENDING_TRAVERSE(pReader->info.order)) { - pReader->status.bProcMemPreFileset = !(pReader->status.prevFilesetStartKey > pReader->status.memTableMaxKey || winFid.skey-1 < pReader->status.memTableMinKey); + pReader->status.bProcMemPreFileset = !(pReader->status.memTableMaxKey < pReader->status.prevFilesetStartKey || + (winFid.skey-1) < pReader->status.memTableMinKey); } else { - pReader->status.bProcMemPreFileset = !(winFid.ekey+1 > pReader->status.memTableMaxKey || pReader->status.prevFilesetEndKey < pReader->status.memTableMinKey); + pReader->status.bProcMemPreFileset = !( pReader->status.memTableMaxKey < (winFid.ekey+1) || + pReader->status.prevFilesetEndKey < pReader->status.memTableMinKey); } if (pReader->status.bProcMemPreFileset) { From 8c290587b78249373dc990f51aa3058efe15a856 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 4 Dec 2023 16:19:07 +0800 Subject: [PATCH 059/102] fix: typo error --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 725679f079..ee50526883 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2517,7 +2517,7 @@ static void prepareDurationForNextFileSet(STsdbReader* pReader) { resetTableListIndex(&pReader->status); } - if (!pReader->status.bProcMemFirstFileset) { + if (!pReader->status.bProcMemPreFileset) { if (pReader->notifyFn) { STsdReaderNotifyInfo info = {0}; info.duration.fileSetId = fid; From ac2d9ebc271a99e4cb012a852db0b5446c71d0f2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 4 Dec 2023 16:48:43 +0800 Subject: [PATCH 060/102] fix: memory leak --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 7322da6a23..635c53bbed 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -770,6 +770,9 @@ int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) { taosThreadMutexUnlock(&fs->tsdb->mutex); // destroy all channels + int64_t channel; + TARRAY2_FOREACH(&channelArr, channel) { vnodeAChannelDestroy(vnodeAsyncHandle[1], channel, true); } + TARRAY2_DESTROY(&channelArr, NULL); return 0; } From 02ff87b3a3eae2c15c19b51ad338aaad4af2cf7b Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Mon, 4 Dec 2023 19:18:20 +0800 Subject: [PATCH 061/102] fix: revert test case --- tests/system-test/2-query/timezone.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/timezone.py b/tests/system-test/2-query/timezone.py index 970ee0ff3b..33bcb16595 100644 --- a/tests/system-test/2-query/timezone.py +++ b/tests/system-test/2-query/timezone.py @@ -163,7 +163,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method timezone = self.get_system_timezone() self.timezone_check_ntb(timezone) - #self.timezone_check_stb(timezone) + self.timezone_check_stb(timezone) self.timezone_format_test() def stop(self): From 08be3d790cc906727f2c4a3bfa250b37664d4664 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 20:08:46 +0800 Subject: [PATCH 062/102] fix: ignore NULL tag --- source/libs/parser/src/parInsertSql.c | 7 +++++-- tests/script/tsim/parser/columnValue_bigint.sim | 2 ++ tests/script/tsim/parser/columnValue_bool.sim | 2 ++ tests/script/tsim/parser/columnValue_float.sim | 2 ++ tests/script/tsim/parser/columnValue_int.sim | 14 ++++++++++++++ tests/script/tsim/parser/columnValue_smallint.sim | 2 ++ tests/script/tsim/parser/columnValue_tinyint.sim | 2 ++ tests/script/tsim/parser/null_char.sim | 2 ++ 8 files changed, 31 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 9347f94e8d..a3b6f01e3a 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -700,7 +700,8 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SSchema* pTagSchema, SToken* pToken, SArray* pTagName, SArray* pTagVals, STag** pTag) { - if (!isNullValue(pTagSchema->type, pToken)) { + bool isNull = isNullValue(pTagSchema->type, pToken); + if (!isNull) { taosArrayPush(pTagName, pTagSchema->name); } @@ -709,13 +710,15 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z); } - if (isNullValue(pTagSchema->type, pToken)) { + if (isNull) { return tTagNew(pTagVals, 1, true, pTag); } else { return parseJsontoTagData(pToken->z, pTagVals, pTag, &pCxt->msg); } } + if (isNull) return 0; + STagVal val = {0}; int32_t code = parseTagToken(ppSql, pToken, pTagSchema, pStmt->pTableMeta->tableInfo.precision, &val, &pCxt->msg); diff --git a/tests/script/tsim/parser/columnValue_bigint.sim b/tests/script/tsim/parser/columnValue_bigint.sim index 0a024029a5..056855eea2 100644 --- a/tests/script/tsim/parser/columnValue_bigint.sim +++ b/tests/script/tsim/parser/columnValue_bigint.sim @@ -17,6 +17,7 @@ sql create table mt_bigint (ts timestamp, c bigint) tags (tagname bigint) ## case 00: static create table for test tag values sql create table st_bigint_0 using mt_bigint tags (NULL) +sql show create table st_bigint_0 sql show tags from st_bigint_0 if $data05 != NULL then return -1 @@ -173,6 +174,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_bigint_16 using mt_bigint tags (NULL) values (now, NULL) +sql show create table st_bigint_16 sql show tags from st_bigint_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_bool.sim b/tests/script/tsim/parser/columnValue_bool.sim index ad4232d884..2553e6805a 100644 --- a/tests/script/tsim/parser/columnValue_bool.sim +++ b/tests/script/tsim/parser/columnValue_bool.sim @@ -17,6 +17,7 @@ sql create table mt_bool (ts timestamp, c bool) tags (tagname bool) ## case 00: static create table for test tag values sql create table st_bool_0 using mt_bool tags (NULL) +sql show create table st_bool_0 sql show tags from st_bool_0 if $data05 != NULL then print ==1== expect: NULL, actually: $data05 @@ -291,6 +292,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_bool_16 using mt_bool tags (NULL) values (now, NULL) +sql show create table st_bool_16 sql show tags from st_bool_16 if $data05 != NULL then print ==33== expect: NULL, actually: $data00 diff --git a/tests/script/tsim/parser/columnValue_float.sim b/tests/script/tsim/parser/columnValue_float.sim index b2db7dff2b..4dcda33224 100644 --- a/tests/script/tsim/parser/columnValue_float.sim +++ b/tests/script/tsim/parser/columnValue_float.sim @@ -17,6 +17,7 @@ sql create table mt_float (ts timestamp, c float) tags (tagname float) ## case 00: static create table for test tag values sql create table st_float_0 using mt_float tags (NULL) +sql show create table st_float_0 sql show tags from st_float_0 if $data05 != NULL then return -1 @@ -299,6 +300,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_float_16 using mt_float tags (NULL) values (now, NULL) +sql show create table st_float_16 sql show tags from st_float_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_int.sim b/tests/script/tsim/parser/columnValue_int.sim index 4a3b8ebd0b..9f0fb6e294 100644 --- a/tests/script/tsim/parser/columnValue_int.sim +++ b/tests/script/tsim/parser/columnValue_int.sim @@ -17,6 +17,7 @@ sql create table mt_int (ts timestamp, c int) tags (tagname int) ## case 00: static create table for test tag values sql create table st_int_0 using mt_int tags (NULL) +sql show create table st_int_0 sql show tags from st_int_0 if $data05 != NULL then return -1 @@ -27,6 +28,18 @@ if $data05 != NULL then return -1 endi +sql insert into mt_int(tbname,tagname,ts,c) values("st_int_00",NULL,now,1) values("st_int_01", NULL, now,2) +sql show create table st_int_00 +sql show create table st_int_01 +sql show tags from st_int_00 +if $data05 != NULL then + return -1 +endi +sql show tags from st_int_01 +if $data05 != NULL then + return -1 +endi + sql_error create table st_int_2 using mt_int tags ('NULL') sql_error create table st_int_3 using mt_int tags ('NULL') sql_error create table st_int_4 using mt_int tags ("NULL") @@ -172,6 +185,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_int_16 using mt_int tags (NULL) values (now, NULL) +sql show create table st_int_16 sql show tags from st_int_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_smallint.sim b/tests/script/tsim/parser/columnValue_smallint.sim index eb364f3630..f9be6ebd52 100644 --- a/tests/script/tsim/parser/columnValue_smallint.sim +++ b/tests/script/tsim/parser/columnValue_smallint.sim @@ -20,6 +20,7 @@ sql create table mt_smallint (ts timestamp, c smallint) tags (tagname smallint) ## case 00: static create table for test tag values sql create table st_smallint_0 using mt_smallint tags (NULL) +sql show create table st_smallint_0 sql show tags from st_smallint_0 if $data05 != NULL then return -1 @@ -175,6 +176,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_smallint_16 using mt_smallint tags (NULL) values (now, NULL) +sql show create table st_smallint_16 sql show tags from st_smallint_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_tinyint.sim b/tests/script/tsim/parser/columnValue_tinyint.sim index d7938aa739..7d0f10a30d 100644 --- a/tests/script/tsim/parser/columnValue_tinyint.sim +++ b/tests/script/tsim/parser/columnValue_tinyint.sim @@ -17,6 +17,7 @@ sql create table mt_tinyint (ts timestamp, c tinyint) tags (tagname tinyint) ## case 00: static create table for test tag values sql create table st_tinyint_0 using mt_tinyint tags (NULL) +sql show create table st_tinyint_0 sql show tags from st_tinyint_0 if $data05 != NULL then print expect NULL, actually: $data05 @@ -173,6 +174,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_tinyint_16 using mt_tinyint tags (NULL) values (now, NULL) +sql show create table st_tinyint_16 sql show tags from st_tinyint_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/null_char.sim b/tests/script/tsim/parser/null_char.sim index 098f4061c1..9d476b5227 100644 --- a/tests/script/tsim/parser/null_char.sim +++ b/tests/script/tsim/parser/null_char.sim @@ -13,6 +13,7 @@ sql use $db #### case 0: field NULL, or 'NULL' sql create table mt1 (ts timestamp, col1 int, col2 bigint, col3 float, col4 double, col5 binary(8), col6 bool, col7 smallint, col8 tinyint, col9 nchar(8)) tags (tag1 binary(8), tag2 nchar(8), tag3 int, tag4 bigint, tag5 bool, tag6 float) sql create table st1 using mt1 tags (NULL, 'NULL', 100, 1000, 'false', 9.123) +sql show create table st1 sql insert into st1 values ('2019-01-01 09:00:00.000', 123, -123, 3.0, 4.0, 'binary', true, 1000, 121, 'nchar') sql insert into st1 values ('2019-01-01 09:00:01.000', '456', '456', '3.33', '4.444', 'binary', 'true', '1001', '122', 'nchar') sql insert into st1 values ('2019-01-01 09:00:02.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) @@ -117,6 +118,7 @@ if $rows != 0 then endi sql create table st3 using mt2 tags (NULL, 'ABC', 103, 'FALSE') +sql show create table st3 sql insert into st3 (ts, col1) values(now, 1) sql select tag1, tag2, tag3, tag5 from st3 if $rows != 1 then From f3a075ee9e5b897c949fba69e3c2d2f46ff65264 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 21:00:13 +0800 Subject: [PATCH 063/102] test: check null json tag --- tests/system-test/2-query/json_tag.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index ebd580efd4..8db92f38a2 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -394,9 +394,12 @@ class TDTestCase: tdSql.execute(f"create table if not exists {dbname}.jsons3(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") tdSql.execute(f"insert into {dbname}.jsons3_1 using {dbname}.jsons3 tags('{{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}}') values(1591060618000, 3, false, 'json3', '你是3')") tdSql.execute(f"insert into {dbname}.jsons3_2 using {dbname}.jsons3 tags('{{\"tag1\":5,\"tag2\":\"beijing\"}}') values (1591060638000, 2, true, 'json3', 'sss')") + tdSql.execute(f"insert into {dbname}.jsons3_3 using {dbname}.jsons3 tags(NULL) values (1591060638000, 2, true, 'json3', 'sss')") tdSql.query(f"select 'sss',33,a.jtag->'tag3' from {dbname}.jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") tdSql.checkData(0, 0, "sss") tdSql.checkData(0, 2, "true") + tdSql.query(f"show create table jsons3_3") + tdSql.checkNotEqual(tdSql.queryResult[0][1].find("TAGS (null)"), 0) res = tdSql.getColNameList(f"select 'sss',33,a.jtag->'tag3' from {dbname}.jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") cname_list = [] From 646d807a28acd2549c0b6fd731d05e3be7f30e93 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 22:43:04 +0800 Subject: [PATCH 064/102] test: check null tag when insert by stb --- tests/script/tsim/insert/insert_stb.sim | 23 ++++++++++++++++++++ tests/script/tsim/parser/columnValue_int.sim | 12 ---------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/script/tsim/insert/insert_stb.sim b/tests/script/tsim/insert/insert_stb.sim index 4c0797e2a7..14ff2261c2 100644 --- a/tests/script/tsim/insert/insert_stb.sim +++ b/tests/script/tsim/insert/insert_stb.sim @@ -66,6 +66,29 @@ if $rows != 1 then return -1 endi +sql insert into st(tbname, ts, f, t) values('ct2',now,20,NULL)('ct3',now,30,NULL) +sql insert into st(tbname, t, ts, f) values('ct4',NULL, now,20)('ct5',NULL, now,30) +sql show create table ct2 +sql show create table ct3 +sql show create table ct4 +sql show create table ct5 +sql show tags from ct2 +if $data05 != NULL then + return -1 +endi +sql show tags from ct3 +if $data05 != NULL then + return -1 +endi +sql show tags from ct4 +if $data05 != NULL then + return -1 +endi +sql show tags from ct5 +if $data05 != NULL then + return -1 +endi + sql_error insert into d2.st values(now, 1, 1) sql_error insert into d2.st(ts, f) values(now, 1); sql_error insert into d2.st(ts, f, tbname) values(now, 1); diff --git a/tests/script/tsim/parser/columnValue_int.sim b/tests/script/tsim/parser/columnValue_int.sim index 9f0fb6e294..e68ae6f13f 100644 --- a/tests/script/tsim/parser/columnValue_int.sim +++ b/tests/script/tsim/parser/columnValue_int.sim @@ -28,18 +28,6 @@ if $data05 != NULL then return -1 endi -sql insert into mt_int(tbname,tagname,ts,c) values("st_int_00",NULL,now,1) values("st_int_01", NULL, now,2) -sql show create table st_int_00 -sql show create table st_int_01 -sql show tags from st_int_00 -if $data05 != NULL then - return -1 -endi -sql show tags from st_int_01 -if $data05 != NULL then - return -1 -endi - sql_error create table st_int_2 using mt_int tags ('NULL') sql_error create table st_int_3 using mt_int tags ('NULL') sql_error create table st_int_4 using mt_int tags ("NULL") From 538f6fc722a6ca0a9d4be92ea13b8af2ced415c2 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 5 Dec 2023 09:51:14 +0800 Subject: [PATCH 065/102] fix: no block after new duration --- source/libs/executor/inc/executorInt.h | 2 +- source/libs/executor/src/scanoperator.c | 49 ++++++++++++------------- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index a3ec2cca74..b190cd815c 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -298,7 +298,7 @@ typedef struct STableMergeScanInfo { int64_t mergeLimit; SSortExecInfo sortExecInfo; bool bNewDuration; - SSDataBlock* pNewDurationBlock; + bool bOnlyRetrieveBlock; } STableMergeScanInfo; typedef struct STagScanFilterContext { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e8213d0597..bb3cf7b937 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3242,29 +3242,28 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { STsdbReader* reader = pInfo->base.dataReader; while (true) { - if (pInfo->pNewDurationBlock) { - SSDataBlock* pNewDurationBlock = pInfo->pNewDurationBlock; - pInfo->pNewDurationBlock = NULL; - return pNewDurationBlock; - } + if (!pInfo->bOnlyRetrieveBlock) { + code = pAPI->tsdReader.tsdNextDataBlock(reader, &hasNext); + if (code != 0) { + pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); + qError("table merge scan fetch next data block error code: %d, %s", code, GET_TASKID(pTaskInfo)); + T_LONG_JMP(pTaskInfo->env, code); + } - code = pAPI->tsdReader.tsdNextDataBlock(reader, &hasNext); - if (code != 0) { - pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); - qError("table merge scan fetch next data block error code: %d, %s", code, GET_TASKID(pTaskInfo)); - T_LONG_JMP(pTaskInfo->env, code); - } + if (!hasNext || isTaskKilled(pTaskInfo)) { + pInfo->bNewDuration = false; + if (isTaskKilled(pTaskInfo)) { + qInfo("table merge scan fetch next data block found task killed. %s", GET_TASKID(pTaskInfo)); + pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); + } + break; + } - if (!hasNext) { - break; + if (pInfo->bNewDuration) { + pInfo->bOnlyRetrieveBlock = true; + return NULL; + } } - - if (isTaskKilled(pTaskInfo)) { - qInfo("table merge scan fetch next data block found task killed. %s", GET_TASKID(pTaskInfo)); - pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); - break; - } - // process this data block based on the probabilities bool processThisBlock = processBlockWithProbability(&pInfo->sample); if (!processThisBlock) { @@ -3273,6 +3272,9 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { uint32_t status = 0; code = loadDataBlock(pOperator, &pInfo->base, pBlock, &status); + if (pInfo->bOnlyRetrieveBlock) { + pInfo->bOnlyRetrieveBlock = false; + } if (code != TSDB_CODE_SUCCESS) { qInfo("table merge scan load datablock code %d, %s", code, GET_TASKID(pTaskInfo)); T_LONG_JMP(pTaskInfo->env, code); @@ -3295,11 +3297,7 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; - - if (pInfo->bNewDuration) { - pInfo->pNewDurationBlock = pBlock; - return NULL; - } + return pBlock; } @@ -3336,7 +3334,6 @@ int32_t dumpQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* } void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param) { - uInfo("tableMergeScanTsdbNotifyCb, %d, %d", type, info->duration.fileSetId); STableMergeScanInfo* pTmsInfo = param; pTmsInfo->bNewDuration = true; return; From dd53d570219ade0329a8c6760dc1ffe3f7827c66 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 5 Dec 2023 10:51:19 +0800 Subject: [PATCH 066/102] fix: strncpy replace with memcpy on 3.0 --- source/client/src/clientEnv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 212c42125e..85624c31c5 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -732,7 +732,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { return -1; } newstr[0] = '"'; - strncpy(newstr+1, str, len); + memcpy(newstr+1, str, len); newstr[len + 1] = '"'; newstr[len + 2] = '\0'; str = newstr; From be6daee280cda58e441fc2e55b315fbaf7146f53 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 5 Dec 2023 11:01:57 +0800 Subject: [PATCH 067/102] feature: switch duration mode randomly --- source/libs/executor/src/scanoperator.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6f2de2fbe2..9a556a4ea1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3423,7 +3423,11 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx); pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, &pInfo->mSkipTables); - pAPI->tsdReader.tsdSetDurationOrder(pInfo->base.dataReader); + int32_t r = taosRand() % 2; + if (r == 1) { + uInfo("zsl: set duration order"); + pAPI->tsdReader.tsdSetDurationOrder(pInfo->base.dataReader); + } pAPI->tsdReader.tsdSetSetNotifyCb(pInfo->base.dataReader, tableMergeScanTsdbNotifyCb, pInfo); int32_t code = startDurationForGroupTableMergeScan(pOperator); From 07301130c5e19676f0b620d0153d392edbdea6f2 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Tue, 5 Dec 2023 11:18:07 +0800 Subject: [PATCH 068/102] enh: event_window + super table need partition by tbname --- source/libs/parser/src/parTranslater.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5c30384a6b..1ef74349b0 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7388,6 +7388,11 @@ static int32_t subtableExprHasColumnOrPseudoColumn(SNode* pNode) { static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + int8_t tableType = ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType; + if (TSDB_SUPER_TABLE == tableType && !hasPartitionByTbname(((SSelectStmt*)pStmt->pQuery)->pPartitionByList)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Stream query on super tables must patitioned by table name"); + } if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || !isTimeLineQuery(pStmt->pQuery) || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList || crossTableWithUdaf(pSelect) || hasJsonTypeProjection(pSelect)) { From e0a14bd359f359c00a8420ad2c422f2e51fa542d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 5 Dec 2023 14:05:16 +0800 Subject: [PATCH 069/102] fix(tsdb/cache): remove debug log --- source/dnode/vnode/src/tsdb/tsdbCache.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 5076599753..4b26697464 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1797,8 +1797,8 @@ static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsRea } if (record.version <= pReader->info.verRange.maxVer) { - tsdbError("tomb xx load/cache: vgId:%d fid:%d commit %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records", - TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid); + /*tsdbError("tomb xx load/cache: vgId:%d fid:%d record %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records", + TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid);*/ SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey}; taosArrayPush(pInfo->pTombData, &delData); From 2019e95049ceb18c41b8ca3b9aec48e6b4e3cad6 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 5 Dec 2023 14:08:17 +0800 Subject: [PATCH 070/102] fix: stream for event window --- source/libs/parser/src/parTranslater.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1ef74349b0..ac92c9a427 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7388,8 +7388,10 @@ static int32_t subtableExprHasColumnOrPseudoColumn(SNode* pNode) { static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; - int8_t tableType = ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType; - if (TSDB_SUPER_TABLE == tableType && !hasPartitionByTbname(((SSelectStmt*)pStmt->pQuery)->pPartitionByList)) { + if ( (SRealTableNode*)pSelect->pFromTable && ((SRealTableNode*)pSelect->pFromTable)->pMeta + && TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType + && !hasPartitionByTbname(pSelect->pPartitionByList) + && pSelect->pWindow != NULL && pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Stream query on super tables must patitioned by table name"); } From f91d98df2588b1a8b7db8753506d123edfb2cb88 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 5 Dec 2023 14:11:58 +0800 Subject: [PATCH 071/102] fix: error message --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ac92c9a427..88bfcec682 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7393,7 +7393,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm && !hasPartitionByTbname(pSelect->pPartitionByList) && pSelect->pWindow != NULL && pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Stream query on super tables must patitioned by table name"); + "Event window for stream on super table must patitioned by table name"); } if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || !isTimeLineQuery(pStmt->pQuery) || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList || From a6c876bbc15f082bc144306c8eaee03256fdb03e Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 5 Dec 2023 14:16:01 +0800 Subject: [PATCH 072/102] fix: last/last_row crash with dropped column --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 53 ++++++-- source/libs/executor/src/cachescanoperator.c | 6 + tests/system-test/2-query/last_cache_scan.py | 135 +++++++++++++++++++ 3 files changed, 185 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index b6aa791cf0..f668ea5f72 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -22,19 +22,34 @@ #define HASTYPE(_type, _t) (((_type) & (_t)) == (_t)) +static void setFirstLastResColToNull(SColumnInfoData* pCol, int32_t row) { + char *buf = taosMemoryCalloc(1, pCol->info.bytes); + SFirstLastRes* pRes = (SFirstLastRes*)((char*)buf + VARSTR_HEADER_SIZE); + pRes->bytes = 0; + pRes->hasResult = true; + pRes->isNull = true; + varDataSetLen(buf, pCol->info.bytes - VARSTR_HEADER_SIZE); + colDataSetVal(pCol, row, buf, false); + taosMemoryFree(buf); +} + static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* pReader, const int32_t* slotIds, const int32_t* dstSlotIds, void** pRes, const char* idStr) { int32_t numOfRows = pBlock->info.rows; // bool allNullRow = true; if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST)) { - uint64_t ts = 0; - SFirstLastRes* p; - col_id_t colId; + uint64_t ts = TSKEY_MIN; + SFirstLastRes* p = NULL; + col_id_t colId = -1; for (int32_t i = 0; i < pReader->numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]); - int32_t slotId = slotIds[i]; - SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); + if (slotIds[i] == -1) { + setFirstLastResColToNull(pColInfoData, numOfRows); + continue; + } + int32_t slotId = slotIds[i]; + SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); colId = pColVal->colVal.cid; p = (SFirstLastRes*)varDataVal(pRes[i]); @@ -63,10 +78,14 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p for (int32_t idx = 0; idx < taosArrayGetSize(pBlock->pDataBlock); ++idx) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, idx); if (pCol->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID && pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - colDataSetVal(pCol, numOfRows, (const char*)&ts, false); + if (ts == TSKEY_MIN) { + colDataSetNULL(pCol, numOfRows); + } else { + colDataSetVal(pCol, numOfRows, (const char*)&ts, false); + } continue; - } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && pCol->info.colId == colId) { - if (!p->isNull) { + } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && (pCol->info.colId == colId || colId == -1)) { + if (p && !p->isNull) { colDataSetVal(pCol, numOfRows, p->buf, false); } else { colDataSetNULL(pCol, numOfRows); @@ -81,6 +100,10 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]); int32_t slotId = slotIds[i]; + if (slotId == -1) { + colDataSetNULL(pColInfoData, numOfRows); + continue; + } SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); SColVal* pVal = &pColVal->colVal; @@ -300,7 +323,13 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } for (int32_t j = 0; j < pr->numOfCols; ++j) { - pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + pr->pSchema->columns[slotIds[j]].bytes + VARSTR_HEADER_SIZE); + int32_t bytes; + if (slotIds[j] == -1) + bytes = 1; + else + bytes = pr->pSchema->columns[slotIds[j]].bytes; + + pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + bytes + VARSTR_HEADER_SIZE); SFirstLastRes* p = (SFirstLastRes*)varDataVal(pRes[j]); p->ts = INT64_MIN; } @@ -324,6 +353,11 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 for (int32_t i = 0; i < pr->numOfCols; ++i) { int32_t slotId = slotIds[i]; + if (slotId == -1) { + SLastCol p = {.ts = INT64_MIN, .colVal.type = TSDB_DATA_TYPE_BOOL, .colVal.flag = CV_FLAG_NULL}; + taosArrayPush(pLastCols, &p); + continue; + } struct STColumn* pCol = &pr->pSchema->columns[slotId]; SLastCol p = {.ts = INT64_MIN, .colVal.type = pCol->type, .colVal.flag = CV_FLAG_NULL}; @@ -348,6 +382,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 bool hasNotNullRow = true; int64_t singleTableLastTs = INT64_MAX; for (int32_t k = 0; k < pr->numOfCols; ++k) { + if (slotIds[k] == -1) continue; SLastCol* p = taosArrayGet(pLastCols, k); SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, k); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 6d59698855..75fe3c51dd 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -352,6 +352,7 @@ int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTask for (int32_t i = 0; i < numOfCols; ++i) { SColMatchItem* pColMatch = taosArrayGet(pColMatchInfo, i); + bool found = false; for (int32_t j = 0; j < pWrapper->nCols; ++j) { /* if (pColMatch->colId == pWrapper->pSchema[j].colId && pColMatch->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { (*pSlotIds)[pColMatch->dstSlotId] = -1; @@ -361,9 +362,14 @@ int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTask if (pColMatch->colId == pWrapper->pSchema[j].colId) { (*pSlotIds)[i] = j; (*pDstSlotIds)[i] = pColMatch->dstSlotId; + found = true; break; } } + if (!found) { + (*pSlotIds)[i] = -1; + (*pDstSlotIds)[i] = pColMatch->dstSlotId; + } } return TSDB_CODE_SUCCESS; diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 0f0936ebab..01795f6eef 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -1,3 +1,4 @@ +from sqlite3 import ProgrammingError import taos import sys import time @@ -319,10 +320,144 @@ class TDTestCase: tdSql.checkData(0, 0, '2018-11-25 19:30:00.000') tdSql.checkData(0, 1, '2018-11-25 19:30:01.000') + def test_cache_scan_with_drop_and_add_column(self): + tdSql.query("select last(c10) from meters") + tdSql.checkData(0, 0, '2018-11-25 19:30:01') + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c10; alter table test.meters add column c11 int"]) + p.check_returncode() + tdSql.query("select last(c10) from meters", queryTimes=1) + tdSql.checkData(0, 0, None) + tdSql.query('select last(*) from meters', queryTimes=1) + tdSql.checkData(0, 10, None) + tdSql.query('select last(c10), c10, ts from meters', queryTimes=1) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + + def test_cache_scan_with_drop_and_add_column2(self): + tdSql.query("select last(c1) from meters") + tdSql.checkData(0, 0, '999') + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c12 int"]) + p.check_returncode() + tdSql.query("select last(c1) from meters", queryTimes=1) + tdSql.checkData(0, 0, None) + tdSql.query('select last(*) from meters', queryTimes=1) + print(str(tdSql.queryResult)) + tdSql.checkData(0, 1, None) + tdSql.query('select last(c1), c1, ts from meters', queryTimes=1) + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + + try: + tdSql.query('select ts, last(c1), c1, ts, c1 from meters', queryTimes=1) + except Exception as e: + if str(e).count('Invalid column name') == 1: + print('column has been dropped, the cache has been updated: %s' % (str(e))) + return + else: + raise + tdSql.checkRows(1) + tdSql.checkCols(5) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(0, 3, None) + tdSql.checkData(0, 4, None) + + try: + tdSql.query('select last(c1), last(c2), last(c3) from meters', queryTimes=1) + except Exception as e: + if str(e).count('Invalid column name') == 1: + print('column has been dropped, the cache has been updated: %s' % (str(e))) + return + else: + raise + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, None) + + def test_cache_scan_with_drop_column(self): + tdSql.query('select last(*) from meters') + print(str(tdSql.queryResult)) + tdSql.checkCols(11) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c9"]) + p.check_returncode() + tdSql.query('select last(*) from meters') + print(str(tdSql.queryResult)) + tdSql.checkCols(11) + tdSql.checkData(0, 9, None) + + def test_cache_scan_last_row_with_drop_column(self): + tdSql.query('select last_row(*) from meters') + print(str(tdSql.queryResult)) + tdSql.checkCols(11) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c10; alter table test.meters add column c11 int"]) + p.check_returncode() + tdSql.query('select last_row(*) from meters') + print(str(tdSql.queryResult)) + tdSql.checkCols(11) + tdSql.checkData(0, 10, None) + + def test_cache_scan_last_row_with_drop_column2(self): + tdSql.query('select last_row(c1) from meters') + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) + p.check_returncode() + tdSql.query('select last_row(c1) from meters', queryTimes=1) + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + tdSql.checkData(0, 0, None) + + def test_cache_scan_last_row_with_partition_by(self): + tdSql.query('select last(c1) from meters partition by t1') + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + tdSql.checkRows(5) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) + p.check_returncode() + tdSql.query('select last_row(c1) from meters partition by t1', queryTimes=1) + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + + def test_cache_scan_last_row_with_partition_by_tbname(self): + tdSql.query('select last(c1) from meters partition by tbname', queryTimes=1) + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + tdSql.checkRows(10) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) + p.check_returncode() + tdSql.query('select last_row(c1) from meters partition by tbname', queryTimes=1) + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + tdSql.checkRows(10) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + + + def run(self): self.prepareTestEnv() #time.sleep(99999999) self.test_last_cache_scan() + #self.test_cache_scan_with_drop_and_add_column() + self.test_cache_scan_with_drop_and_add_column2() + #self.test_cache_scan_with_drop_column() + #self.test_cache_scan_last_row_with_drop_column() + #self.test_cache_scan_last_row_with_drop_column2() + #self.test_cache_scan_last_row_with_partition_by() + #self.test_cache_scan_last_row_with_partition_by_tbname() def stop(self): tdSql.close() From cb3d9551117f81ab37ff70306938c65307cbb45c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 5 Dec 2023 16:19:56 +0800 Subject: [PATCH 073/102] code coverage --- tests/parallel_test/cases.task | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index cf4559100b..ef222db0f9 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1132,6 +1132,9 @@ e ,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim ,,y,script,./test.sh -f tsim/stream/distributeSession0.sim ,,y,script,./test.sh -f tsim/stream/drop_stream.sim +,,y,script,./test.sh -f tsim/stream/event0.sim +,,y,script,./test.sh -f tsim/stream/event1.sim +,,y,script,./test.sh -f tsim/stream/event2.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic1.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic2.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim From b642f0fc83060c7bb7961d9e6cbadc073242c3cd Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 5 Dec 2023 17:51:48 +0800 Subject: [PATCH 074/102] enh: adjust ENodeType for show statements nodes --- include/common/tmsg.h | 56 +++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b42df78e34..aff1bd55e4 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -295,7 +295,36 @@ typedef enum ENodeType { QUERY_NODE_SYNCDB_STMT, QUERY_NODE_GRANT_STMT, QUERY_NODE_REVOKE_STMT, - QUERY_NODE_SHOW_DNODES_STMT, + // placeholder for [152, 180] + QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181, + QUERY_NODE_SHOW_CREATE_DATABASE_STMT, + QUERY_NODE_SHOW_CREATE_TABLE_STMT, + QUERY_NODE_SHOW_CREATE_STABLE_STMT, + QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT, + QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT, + QUERY_NODE_SHOW_SCORES_STMT, + QUERY_NODE_SHOW_TABLE_TAGS_STMT, + QUERY_NODE_KILL_CONNECTION_STMT, + QUERY_NODE_KILL_QUERY_STMT, + QUERY_NODE_KILL_TRANSACTION_STMT, + QUERY_NODE_DELETE_STMT, + QUERY_NODE_INSERT_STMT, + QUERY_NODE_QUERY, + QUERY_NODE_SHOW_DB_ALIVE_STMT, + QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT, + QUERY_NODE_BALANCE_VGROUP_LEADER_STMT, + QUERY_NODE_RESTORE_DNODE_STMT, + QUERY_NODE_RESTORE_QNODE_STMT, + QUERY_NODE_RESTORE_MNODE_STMT, + QUERY_NODE_RESTORE_VNODE_STMT, + QUERY_NODE_PAUSE_STREAM_STMT, + QUERY_NODE_RESUME_STREAM_STMT, + QUERY_NODE_CREATE_VIEW_STMT, + QUERY_NODE_DROP_VIEW_STMT, + + // show statement nodes + // see 'sysTableShowAdapter', 'SYSTABLE_SHOW_TYPE_OFFSET' + QUERY_NODE_SHOW_DNODES_STMT = 400, QUERY_NODE_SHOW_MNODES_STMT, QUERY_NODE_SHOW_MODULES_STMT, QUERY_NODE_SHOW_QNODES_STMT, @@ -324,31 +353,6 @@ typedef enum ENodeType { QUERY_NODE_SHOW_VNODES_STMT, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT, QUERY_NODE_SHOW_VIEWS_STMT, - QUERY_NODE_SHOW_CREATE_VIEW_STMT, - QUERY_NODE_SHOW_CREATE_DATABASE_STMT, - QUERY_NODE_SHOW_CREATE_TABLE_STMT, - QUERY_NODE_SHOW_CREATE_STABLE_STMT, - QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT, - QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT, - QUERY_NODE_SHOW_SCORES_STMT, - QUERY_NODE_SHOW_TABLE_TAGS_STMT, - QUERY_NODE_KILL_CONNECTION_STMT, - QUERY_NODE_KILL_QUERY_STMT, - QUERY_NODE_KILL_TRANSACTION_STMT, - QUERY_NODE_DELETE_STMT, - QUERY_NODE_INSERT_STMT, - QUERY_NODE_QUERY, - QUERY_NODE_SHOW_DB_ALIVE_STMT, - QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT, - QUERY_NODE_BALANCE_VGROUP_LEADER_STMT, - QUERY_NODE_RESTORE_DNODE_STMT, - QUERY_NODE_RESTORE_QNODE_STMT, - QUERY_NODE_RESTORE_MNODE_STMT, - QUERY_NODE_RESTORE_VNODE_STMT, - QUERY_NODE_PAUSE_STREAM_STMT, - QUERY_NODE_RESUME_STREAM_STMT, - QUERY_NODE_CREATE_VIEW_STMT, - QUERY_NODE_DROP_VIEW_STMT, // logic plan node QUERY_NODE_LOGIC_PLAN_SCAN = 1000, From cc51fd47d0bc7812cc8cc638177c0a23d523ef54 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 5 Dec 2023 19:09:29 +0800 Subject: [PATCH 075/102] fix:offset out of range if poll offset is read from offsetRestored while wal is deleted --- source/dnode/vnode/src/tq/tqUtil.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 8f62928d22..110cf79b4e 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -168,6 +168,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, taosWUnLockLatch(&pTq->lock); } + dataRsp.reqOffset = *pOffset; // reqOffset represents the current date offset, may be changed if wal not exists code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); end : { From 9bb82ac03be31a8b43d5af1395985e18a0433fc9 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 5 Dec 2023 19:43:00 +0800 Subject: [PATCH 076/102] fix merge key --- source/libs/parser/src/parTranslater.c | 11 ++++++++++- source/libs/planner/src/planSpliter.c | 5 +++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5c30384a6b..f689ebc820 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4622,11 +4622,20 @@ static int32_t addOrderByPrimaryKeyToQueryImpl(STranslateContext* pCxt, SNode* p return TSDB_CODE_OUT_OF_MEMORY; } ((SExprNode*)pOrderByExpr->pExpr)->orderAlias = true; - NODES_DESTORY_LIST(*pOrderByList); + // NODES_DESTORY_LIST(*pOrderByList); return nodesListMakeStrictAppend(pOrderByList, (SNode*)pOrderByExpr); } static int32_t addOrderByPrimaryKeyToQuery(STranslateContext* pCxt, SNode* pPrimaryKeyExpr, SNode* pStmt) { + SNode* pOrderNode = NULL; + SNodeList* pOrederList = ((SSelectStmt*)pStmt)->pOrderByList; + if (pOrederList != NULL) { + FOREACH(pOrderNode, pOrederList) { + if (((SColumnNode*)pPrimaryKeyExpr)->colId == ((SColumnNode*)((SOrderByExprNode*)pOrderNode)->pExpr)->colId) { + return TSDB_CODE_SUCCESS; + } + } + } if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { return addOrderByPrimaryKeyToQueryImpl(pCxt, pPrimaryKeyExpr, &((SSelectStmt*)pStmt)->pOrderByList); } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 43bd8a5589..bb88beba72 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -1062,8 +1062,9 @@ static int32_t stbSplCreateMergeKeys(SNodeList* pSortKeys, SNodeList* pTargets, SNode* pTarget = NULL; bool found = false; FOREACH(pTarget, pTargets) { - if ((QUERY_NODE_COLUMN == nodeType(pSortExpr) && nodesEqualNode((SNode*)pSortExpr, pTarget)) || - (0 == strcmp(pSortExpr->aliasName, ((SColumnNode*)pTarget)->colName))) { + if ((QUERY_NODE_COLUMN == nodeType(pSortExpr) && nodesEqualNode((SNode*)pSortExpr, pTarget)) + // || (0 == strcmp(pSortExpr->aliasName, ((SColumnNode*)pTarget)->colName)) + ) { code = nodesListMakeStrictAppend(&pMergeKeys, stbSplCreateOrderByExpr(pSortKey, pTarget)); if (TSDB_CODE_SUCCESS != code) { break; From 82f54f094a4077b6c7f7f0502e6d598b0d6c544b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 5 Dec 2023 20:08:18 +0800 Subject: [PATCH 077/102] fix:add log for time cost --- source/client/inc/clientSml.h | 1 + source/client/src/clientSml.c | 19 ++++++++++++++----- source/client/src/clientSmlLine.c | 25 +++++++++++++++++-------- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 30d4792279..0a5b91e039 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -199,6 +199,7 @@ typedef struct { STableMeta *currSTableMeta; STableDataCxt *currTableDataCtx; bool needModifySchema; + SArray *parseTimeList; } SSmlHandle; #define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 6ab0bfc563..46e88b4a00 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1261,6 +1261,7 @@ void smlDestroyInfo(SSmlHandle *info) { taosArrayDestroy(info->valueJsonArray); taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); + taosArrayDestroy(info->parseTimeList); if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { @@ -1309,6 +1310,7 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); + info->parseTimeList = taosArrayInit(8, LONG_BYTES); if (NULL == info->pVgHash || NULL == info->childTables || NULL == info->superTables || NULL == info->tableUids) { uError("create SSmlHandle failed"); @@ -1523,16 +1525,23 @@ static int32_t smlInsertData(SSmlHandle *info) { } static void smlPrintStatisticInfo(SSmlHandle *info) { - uDebug( + char parseTimeStr[102400] = {0}; + char* tmp = parseTimeStr; + for(int i = 0; i < taosArrayGetSize(info->parseTimeList); i++){ + int64_t *t = (int64_t *)taosArrayGet(info->parseTimeList, i); + tmp += sprintf(tmp, ":%d", (int32_t)(*t)); + } + uError( "SML:0x%" PRIx64 " smlInsertLines result, code:%d, msg:%s, lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64 - "", + ", parList:%s", info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime, info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime, - info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime); + info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime, parseTimeStr); + } int32_t smlClearForRerun(SSmlHandle *info) { @@ -1574,7 +1583,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { } static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { - uDebug("SML:0x%" PRIx64 " smlParseLine start", info->id); + uError("SML:0x%" PRIx64 " smlParseLine start", info->id); int32_t code = TSDB_CODE_SUCCESS; if (info->protocol == TSDB_SML_JSON_PROTOCOL) { if (lines) { @@ -1647,7 +1656,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } i++; } - uDebug("SML:0x%" PRIx64 " smlParseLine end", info->id); + uError("SML:0x%" PRIx64 " smlParseLine end", info->id); return code; } diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 006475654a..91dd6ea700 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -237,14 +237,21 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); - for (int i = pTableMeta->tableInfo.numOfColumns; - i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { + for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { SSchema *tag = pTableMeta->schema + i; - SSmlKv kv = {.key = tag->name, - .keyLen = strlen(tag->name), - .type = tag->type, - .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE}; - taosArrayPush(sMeta->tags, &kv); + + if(i < pTableMeta->tableInfo.numOfColumns){ + SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type}; + if (tag->type == TSDB_DATA_TYPE_NCHAR) { + kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + } else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY || tag->type == TSDB_DATA_TYPE_VARBINARY) { + kv.length = tag->bytes - VARSTR_HEADER_SIZE; + } + taosArrayPush(sMeta->cols, &kv); + }else{ + SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE}; + taosArrayPush(sMeta->tags, &kv); + } } tmp = &sMeta; } @@ -625,7 +632,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine JUMP_SPACE(sql, sqlEnd) if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA; elements->measure = sql; - + int64_t t1 = taosGetTimestampUs(); // parse measure size_t measureLenEscaped = 0; while (sql < sqlEnd) { @@ -718,6 +725,8 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); return TSDB_CODE_INVALID_TIMESTAMP; } + int64_t t2 = taosGetTimestampUs() - t1; + taosArrayPush(info->parseTimeList, &t2); // add ts to SSmlKv kv = {.key = tsSmlTsDefaultName, .keyLen = strlen(tsSmlTsDefaultName), From d2b2dee256812085a464ba3d170779f106e46efd Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 1 Dec 2023 23:44:05 +0800 Subject: [PATCH 078/102] opt & add test case --- include/common/tvariant.h | 5 +- source/common/src/tvariant.c | 185 +++++++++++------- source/common/test/commonTests.cpp | 203 +++++++++++++------- source/libs/parser/src/parInsertSql.c | 60 +++--- source/libs/parser/src/parTokenizer.c | 7 +- tests/parallel_test/cases.task | 1 + tests/system-test/1-insert/insert_double.py | 133 +++++++++++++ 7 files changed, 422 insertions(+), 172 deletions(-) create mode 100644 tests/system-test/1-insert/insert_double.py diff --git a/include/common/tvariant.h b/include/common/tvariant.h index 61cf277bb7..66b7302f4e 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -37,8 +37,9 @@ typedef struct SVariant { }; } SVariant; -int32_t toIntegerEx(const char *z, int32_t n, int64_t *value); -int32_t toUIntegerEx(const char *z, int32_t n, uint64_t *value); +int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value); +int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value); +int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double *value); int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value); int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value); diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index adb0964fe6..609bcf1382 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -45,21 +45,6 @@ int32_t parseBinaryUInteger(const char *z, int32_t n, uint64_t *value) { return TSDB_CODE_SUCCESS; } -int32_t parseDecimalUInteger(const char *z, int32_t n, uint64_t *value) { - errno = 0; - char *endPtr = NULL; - while (*z == '0' && n > 1) { - z++; - n--; - } - *value = taosStr2UInt64(z, &endPtr, 10); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - return TSDB_CODE_FAILED; - } - - return TSDB_CODE_SUCCESS; -} - int32_t parseHexUInteger(const char *z, int32_t n, uint64_t *value) { errno = 0; char *endPtr = NULL; @@ -71,10 +56,6 @@ int32_t parseHexUInteger(const char *z, int32_t n, uint64_t *value) { } int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *value) { - *is_neg = false; - if (n < 1) { - return TSDB_CODE_FAILED; - } // parse sign bool has_sign = false; @@ -83,18 +64,18 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v has_sign = true; } else if (z[0] == '+') { has_sign = true; - } else if (z[0] < '0' || z[0] > '9') { + } else if (z[0] != '.' && (z[0] < '0' || z[0] > '9')) { return TSDB_CODE_FAILED; } if (has_sign) { - z++; - n--; - if (n < 1) { + if (n < 2) { return TSDB_CODE_FAILED; } + z++; + n--; } - if (n > 2 && z[0] == '0') { + if (z[0] == '0' && n > 2) { if (z[1] == 'b' || z[1] == 'B') { // paring as binary return parseBinaryUInteger(z, n, value); @@ -106,89 +87,157 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v } } - //rm flag u-unsigned, l-long, f-float(if not in hex str) - char last = tolower(z[n-1]); - if (last == 'u' || last == 'l' || last == 'f') { - n--; - if (n < 1) { - return TSDB_CODE_FAILED; - } - } - - // parsing as decimal - bool parse_int = true; - for (int32_t i = 0; i < n; i++) { - if (z[i] < '0' || z[i] > '9') { - parse_int = false; - break; - } - } - if (parse_int) { - return parseDecimalUInteger(z, n, value); - } - // parsing as double errno = 0; char *endPtr = NULL; double val = taosStr2Double(z, &endPtr); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n || !IS_VALID_UINT64(val)) { + if (errno == ERANGE || errno == EINVAL || endPtr - z != n || val > UINT64_MAX) { return TSDB_CODE_FAILED; } - *value = val; + *value = round(val); return TSDB_CODE_SUCCESS; } int32_t removeSpace(const char **pp, int32_t n) { - // rm blank space from both head and tail + // rm blank space from both head and tail, keep at least one char const char *z = *pp; - while (*z == ' ' && n > 0) { + while (n > 1 && *z == ' ') { z++; n--; } - if (n > 0) { - for (int32_t i = n - 1; i > 0; i--) { - if (z[i] == ' ') { - n--; - } else { - break; - } - } + while (n > 1 && z[n-1] == ' ') { + n--; } *pp = z; return n; } -int32_t toIntegerEx(const char *z, int32_t n, int64_t *value) { - n = removeSpace(&z, n); - if (n < 1) { // fail: all char is space +int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { + if (type != TK_NK_FLOAT) { + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + // rm tail space + while (n > 1 && z[n-1] == ' ') { + n--; + } + } + + errno = 0; + char* endPtr = NULL; + *value = taosStr2Double(z, &endPtr); + + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { return TSDB_CODE_FAILED; } + return TSDB_CODE_SUCCESS; +} + +int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { + errno = 0; + char *endPtr = NULL; + switch (type) + { + case TK_NK_INTEGER: { + *value = taosStr2Int64(z, &endPtr, 10); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + case TK_NK_HEX: { + *value = taosStr2Int64(z, &endPtr, 16); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + case TK_NK_BIN: { + *value = taosStr2Int64(z, &endPtr, 2); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + default: + break; + } + + // parse string + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + n = removeSpace(&z, n); bool is_neg = false; uint64_t uv = 0; int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv); if (code == TSDB_CODE_SUCCESS) { // truncate into int64 - if (uv > INT64_MAX) { - *value = is_neg ? INT64_MIN : INT64_MAX; - return TSDB_CODE_FAILED; + if (is_neg) { + if (uv > 1ull + INT64_MAX) { + *value = INT64_MIN; + return TSDB_CODE_FAILED; + } else { + *value = -uv; + } } else { - *value = is_neg ? -uv : uv; + if (uv > INT64_MAX) { + *value = INT64_MAX; + return TSDB_CODE_FAILED; + } + *value = uv; } } return code; } -int32_t toUIntegerEx(const char *z, int32_t n, uint64_t *value) { - n = removeSpace(&z, n); - if (n < 1) { // fail: all char is space - return TSDB_CODE_FAILED; +int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { + errno = 0; + char *endPtr = NULL; + switch (type) + { + case TK_NK_INTEGER: { + *value = taosStr2UInt64(z, &endPtr, 10); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + case TK_NK_HEX: { + *value = taosStr2UInt64(z, &endPtr, 16); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + case TK_NK_BIN: { + *value = taosStr2UInt64(z, &endPtr, 2); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + default: + break; } + // parse string + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + n = removeSpace(&z, n); + bool is_neg = false; int32_t code = parseSignAndUInteger(z, n, &is_neg, value); if (is_neg) { + if (TSDB_CODE_SUCCESS == code && 0 == *value) { + return TSDB_CODE_SUCCESS; + } return TSDB_CODE_FAILED; } return code; diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 035bca0e15..291941d9b2 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -14,6 +14,7 @@ #include "tdef.h" #include "tvariant.h" #include "ttime.h" +#include "ttokendef.h" namespace { // @@ -25,125 +26,176 @@ int main(int argc, char** argv) { } +TEST(testCase, toUIntegerEx_test) { + uint64_t val = 0; + + char* s = "123"; + int32_t ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 123); + + s = "1000u"; + ret = toUIntegerEx(s, strlen(s), 0, &val); + ASSERT_EQ(ret, -1); + + s = "0x1f"; + ret = toUIntegerEx(s, strlen(s), TK_NK_HEX, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 31); + + s = "0b110"; + ret = toUIntegerEx(s, strlen(s), 0, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 6); + + s = "2567.4787"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 2567); + + s = "1.869895343e4"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 18699); + + s = "-1"; + ret = toUIntegerEx(s, strlen(s),TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); + + s = "-0b10010"; + ret = toUIntegerEx(s, strlen(s), 0, &val); + ASSERT_EQ(ret, -1); + + s = "-0x40"; + ret = toUIntegerEx(s, strlen(s), TK_NK_HEX, &val); + ASSERT_EQ(ret, -1); + + s = "-80.9999"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, -1); + + s = "-5.2343544534e10"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, -1); + + // INT64_MAX + s = "9223372036854775807"; + ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 9223372036854775807); + + // UINT64_MAX + s = "18446744073709551615"; + ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 18446744073709551615u); + + // out of range + s = "18446744073709551616"; + ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); + + s = "5.23e25"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, -1); +} + TEST(testCase, toIntegerEx_test) { int64_t val = 0; char* s = "123"; - int32_t ret = toIntegerEx(s, strlen(s), &val); + int32_t ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 123); - s = "9223372036854775807"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 9223372036854775807); - - s = "9323372036854775807"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, -1); - - s = "-9323372036854775807"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, -1); - s = "-1"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -1); - s = "-9223372036854775807"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, -9223372036854775807); - s = "1000u"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 1000); - - s = "1000l"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 1000); - - s = "1000.0f"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 1000); + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); s = "0x1f"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_HEX, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 31); s = "-0x40"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_HEX, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -64); s = "0b110"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), 0, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 6); s = "-0b10010"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), 0, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -18); + s = "-80.9999"; + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -81); + + s = "2567.8787"; + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 2568); + s = "-5.2343544534e10"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -52343544534); s = "1.869895343e4"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 18698); + ASSERT_EQ(val, 18699); + + // INT64_MAX + s = "9223372036854775807"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 9223372036854775807LL); + + s = "-9223372036854775808"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -9223372036854775808); + + // out of range + s = "9323372036854775807"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); + + s = "-9323372036854775807"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); // UINT64_MAX s = "18446744073709551615"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, -1); - - s = "18446744073709551616"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); ASSERT_EQ(ret, -1); } TEST(testCase, toInteger_test) { - char* s = "123"; - uint32_t type = 0; - int64_t val = 0; + char* s = "123"; int32_t ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 123); - s = "9223372036854775807"; - ret = toInteger(s, strlen(s), 10, &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 9223372036854775807); - - s = "9323372036854775807"; // out of range, > int64_max - ret = toInteger(s, strlen(s), 10, &val); - ASSERT_EQ(ret, -1); - - s = "-9323372036854775807"; - ret = toInteger(s, strlen(s), 10, &val); - ASSERT_EQ(ret, -1); - s = "-1"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -1); - s = "-9223372036854775807"; - ret = toInteger(s, strlen(s), 10, &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, -9223372036854775807); - s = "1000u"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, -1); @@ -163,13 +215,22 @@ TEST(testCase, toInteger_test) { ASSERT_EQ(ret, 0); ASSERT_EQ(val, 72); - // 18446744073709551615 UINT64_MAX - s = "18446744073709551615"; + s = "9223372036854775807"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 18446744073709551615u); + ASSERT_EQ(val, 9223372036854775807); - s = "18446744073709551616"; + s = "-9223372036854775808"; + ret = toInteger(s, strlen(s), 10, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -9223372036854775808); + + // out of range + s = "9323372036854775807"; + ret = toInteger(s, strlen(s), 10, &val); + ASSERT_EQ(ret, -1); + + s = "-9323372036854775807"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, -1); } @@ -225,7 +286,7 @@ TEST(testCase, Datablock_test) { printf("binary column length:%d\n", *(int32_t*)p1->pData); - ASSERT_EQ(blockDataGetNumOfCols(b), 3); + ASSERT_EQ(blockDataGetNumOfCols(b), 2); ASSERT_EQ(blockDataGetNumOfRows(b), 40); char* pData = colDataGetData(p1, 3); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index c993efb9ae..dd62d05440 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -433,7 +433,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_TINYINT: { - code = toIntegerEx(pToken->z, pToken->n, &iv); + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(iv)) { @@ -445,7 +445,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UTINYINT: { - code = toUIntegerEx(pToken->z, pToken->n, &uv); + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); } else if (uv > UINT8_MAX) { @@ -456,7 +456,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_SMALLINT: { - code = toIntegerEx(pToken->z, pToken->n, &iv); + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(iv)) { @@ -467,7 +467,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_USMALLINT: { - code = toUIntegerEx(pToken->z, pToken->n, &uv); + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); } else if (uv > UINT16_MAX) { @@ -478,7 +478,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_INT: { - code = toIntegerEx(pToken->z, pToken->n, &iv); + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z); } else if (!IS_VALID_INT(iv)) { @@ -489,7 +489,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UINT: { - code = toUIntegerEx(pToken->z, pToken->n, &uv); + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); } else if (uv > UINT32_MAX) { @@ -500,7 +500,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_BIGINT: { - code = toIntegerEx(pToken->z, pToken->n, &iv); + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); } @@ -509,7 +509,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UBIGINT: { - code = toUIntegerEx(pToken->z, pToken->n, &uv); + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); } @@ -519,11 +519,11 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, case TSDB_DATA_TYPE_FLOAT: { double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || - isnan(dv)) { + if (dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } *(float*)(&val->i64) = dv; @@ -532,8 +532,9 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, case TSDB_DATA_TYPE_DOUBLE: { double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); + code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { + return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); @@ -1359,7 +1360,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_TINYINT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(pVal->value.val)) { @@ -1368,7 +1369,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_UTINYINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); } else if (pVal->value.val > UINT8_MAX) { @@ -1377,7 +1378,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_SMALLINT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(pVal->value.val)) { @@ -1386,7 +1387,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_USMALLINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); } else if (pVal->value.val > UINT16_MAX) { @@ -1395,7 +1396,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_INT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); } else if (!IS_VALID_INT(pVal->value.val)) { @@ -1404,7 +1405,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_UINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); } else if (pVal->value.val > UINT32_MAX) { @@ -1413,27 +1414,26 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_BIGINT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); } break; } case TSDB_DATA_TYPE_UBIGINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); } break; } case TSDB_DATA_TYPE_FLOAT: { - char* endptr = NULL; double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + int32_t code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || - isnan(dv)) { + if (dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } float f = dv; @@ -1441,12 +1441,12 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_DOUBLE: { - char* endptr = NULL; double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); + int32_t code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { + if (isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); } pVal->value.val = *(int64_t*)&dv; diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 5193bdc47f..5988c68e17 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -609,6 +609,11 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) { break; } + // support float with no decimal part after the decimal point + if (z[i] == '.' && seg == 1) { + *tokenId = TK_NK_FLOAT; + i++; + } if ((z[i] == 'e' || z[i] == 'E') && (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) { i += 2; @@ -751,7 +756,7 @@ SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr, bool* pIgnoreC // support parse the -/+number format if ((isPrevOptr) && (t0.type == TK_NK_MINUS || t0.type == TK_NK_PLUS)) { len = tGetToken(&str[*i + t0.n], &type); - if (type == TK_NK_INTEGER || type == TK_NK_FLOAT) { + if (type == TK_NK_INTEGER || type == TK_NK_FLOAT || type == TK_NK_BIN || type == TK_NK_HEX) { t0.type = type; t0.n += len; } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 981a2411eb..15464eb1b3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -249,6 +249,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_double.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py new file mode 100644 index 0000000000..ce482287fb --- /dev/null +++ b/tests/system-test/1-insert/insert_double.py @@ -0,0 +1,133 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * +import random + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + self.database = "db1" + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepare_db(self): + tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database {self.database}") + tdSql.execute(f"use {self.database}") + + def test_value(self, table_name, dtype, bits): + tdSql.execute(f"drop table if exists {table_name}") + tdSql.execute(f"create table {table_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned)") + + tdSql.execute(f"insert into {table_name} values(now, -16, +6)") + tdSql.execute(f"insert into {table_name} values(now, 80.99 , +0042 )") + tdSql.execute(f"insert into {table_name} values(now, -0042 , +80.99 )") + tdSql.execute(f"insert into {table_name} values(now, 52.34354, 18.6)") + tdSql.execute(f"insert into {table_name} values(now, -12., +3.)") + tdSql.execute(f"insert into {table_name} values(now, -.12, +.3)") + tdSql.execute(f"insert into {table_name} values(now, -2.3e1, +2.324e2)") + tdSql.execute(f"insert into {table_name} values(now, -2e1, +2e2)") + tdSql.execute(f"insert into {table_name} values(now, -2.e1, +2.e2)") + tdSql.execute(f"insert into {table_name} values(now, -0x40, +0b10000)") + tdSql.execute(f"insert into {table_name} values(now, -0b10000, +0x40)") + + # str support + tdSql.execute(f"insert into {table_name} values(now, '-16', '+6')") + tdSql.execute(f"insert into {table_name} values(now, ' -80.99 ', ' +0042 ')") + tdSql.execute(f"insert into {table_name} values(now, ' -0042 ', ' +80.99 ')") + tdSql.execute(f"insert into {table_name} values(now, '52.34354', '18.6')") + tdSql.execute(f"insert into {table_name} values(now, '-12.', '+5.')") + tdSql.execute(f"insert into {table_name} values(now, '-.12', '+.5')") + tdSql.execute(f"insert into {table_name} values(now, '-2.e1', '+2.e2')") + tdSql.execute(f"insert into {table_name} values(now, '-2e1', '+2e2')") + tdSql.execute(f"insert into {table_name} values(now, '-2.3e1', '+2.324e2')") + tdSql.execute(f"insert into {table_name} values(now, '-0x40', '+0b10010')") + tdSql.execute(f"insert into {table_name} values(now, '-0b10010', '+0x40')") + + tdSql.query(f"select * from {table_name}") + tdSql.checkRows(22) + + baseval = 2**(bits/2) + negval = -baseval + 1.645 + posval = baseval + 4.323 + bigval = 2**(bits-1) + max_i = bigval - 1 + min_i = -bigval + max_u = 2*bigval - 1 + min_u = 0 + print("val:", baseval, negval, posval, max_i) + + tdSql.execute(f"insert into {table_name} values(now, {negval}, {posval})") + tdSql.execute(f"insert into {table_name} values(now, -{baseval}, {baseval})") + tdSql.execute(f"insert into {table_name} values(now, {max_i}, {max_u})") + tdSql.execute(f"insert into {table_name} values(now, {min_i}, {min_u})") + + tdSql.query(f"select * from {table_name}") + tdSql.checkRows(26) + + # fail + tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})", "error") + tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})", "error") + tdSql.error(f"insert into {table_name} values(now, {max_i+1}, -1)", "error") + + + def test_tags(self, stable_name, dtype, bits): + tdSql.execute(f"create stable {stable_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned) tags(id {dtype})") + + baseval = 2**(bits/2) + negval = -baseval + 1.645 + posval = baseval + 4.323 + bigval = 2**(bits-1) + max_i = bigval - 1 + min_i = -bigval + max_u = 2*bigval - 1 + min_u = 0 + + tdSql.execute(f"insert into {stable_name}_1 using {stable_name} tags('{negval}') values(now, {negval}, {posval})") + tdSql.execute(f"insert into {stable_name}_2 using {stable_name} tags({posval}) values(now, -{baseval} , {baseval})") + tdSql.execute(f"insert into {stable_name}_3 using {stable_name} tags('0x40') values(now, {max_i}, {max_u})") + tdSql.execute(f"insert into {stable_name}_4 using {stable_name} tags(0b10000) values(now, {min_i}, {min_u})") + + tdSql.execute(f"insert into {stable_name}_5 using {stable_name} tags({max_i}) values(now, '{negval}', '{posval}')") + tdSql.execute(f"insert into {stable_name}_6 using {stable_name} tags('{min_i}') values(now, '-{baseval}' , '{baseval}')") + tdSql.execute(f"insert into {stable_name}_7 using {stable_name} tags(-0x40) values(now, '{max_i}', '{max_u}')") + tdSql.execute(f"insert into {stable_name}_8 using {stable_name} tags('-0b10000') values(now, '{min_i}', '{min_u}')") + + tdSql.execute(f"insert into {stable_name}_9 using {stable_name} tags(12.) values(now, {negval}, {posval})") + tdSql.execute(f"insert into {stable_name}_10 using {stable_name} tags('-8.3') values(now, -{baseval} , {baseval})") + tdSql.execute(f"insert into {stable_name}_11 using {stable_name} tags(2.e1) values(now, {max_i}, {max_u})") + tdSql.execute(f"insert into {stable_name}_12 using {stable_name} tags('-2.3e1') values(now, {min_i}, {min_u})") + + tdSql.query(f"select * from {stable_name}") + tdSql.checkRows(12) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare(replica = self.replicaVar) + self.prepare_db() + + self.test_value("t1", "bigint", 64) + self.test_value("t2", "int", 32) + self.test_value("t3", "smallint", 16) + self.test_value("t4", "tinyint", 8) + tdLog.printNoPrefix("==========end case1 run ...............") + + self.test_tags("t_big", "bigint", 64) + self.test_tags("t_int", "int", 32) + self.test_tags("t_small", "smallint", 16) + self.test_tags("t_tiny", "tinyint", 8) + tdLog.printNoPrefix("==========end case2 run ...............") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 9fa884a693ab876cd0c8949870baa5f82cfd6bad Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 6 Dec 2023 01:24:23 +0800 Subject: [PATCH 079/102] toUIntegerEx --- source/common/src/tvariant.c | 122 ++++++++++---------- source/common/test/commonTests.cpp | 2 +- tests/system-test/1-insert/insert_double.py | 4 +- 3 files changed, 67 insertions(+), 61 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 609bcf1382..25f025ad3d 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -45,18 +45,7 @@ int32_t parseBinaryUInteger(const char *z, int32_t n, uint64_t *value) { return TSDB_CODE_SUCCESS; } -int32_t parseHexUInteger(const char *z, int32_t n, uint64_t *value) { - errno = 0; - char *endPtr = NULL; - *value = taosStr2UInt64(z, &endPtr, 16); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; -} - int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *value) { - // parse sign bool has_sign = false; if (z[0] == '-') { @@ -75,6 +64,9 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v n--; } + errno = 0; + char *endPtr = NULL; + bool parsed = false; if (z[0] == '0' && n > 2) { if (z[1] == 'b' || z[1] == 'B') { // paring as binary @@ -83,18 +75,35 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v if (z[1] == 'x' || z[1] == 'X') { // parsing as hex - return parseHexUInteger(z, n, value); + *value = taosStr2UInt64(z, &endPtr, 16); + parsed = true; } } - // parsing as double - errno = 0; - char *endPtr = NULL; - double val = taosStr2Double(z, &endPtr); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n || val > UINT64_MAX) { + if (!parsed) { + bool parse_int = true; + for (int32_t i = 0; i < n; i++) { + if (z[i] < '0' || z[i] > '9') { + parse_int = false; + break; + } + } + if (parse_int) { + // parsing as decimal + *value = taosStr2UInt64(z, &endPtr, 10); + } else { + // parsing as double + double val = taosStr2Double(z, &endPtr); + if (val > UINT64_MAX) { + return TSDB_CODE_FAILED; + } + *value = round(val); + } + } + + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { return TSDB_CODE_FAILED; } - *value = round(val); return TSDB_CODE_SUCCESS; } @@ -113,15 +122,13 @@ int32_t removeSpace(const char **pp, int32_t n) { } int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { - if (type != TK_NK_FLOAT) { - if (n == 0) { - *value = 0; - return TSDB_CODE_SUCCESS; - } - // rm tail space - while (n > 1 && z[n-1] == ' ') { - n--; - } + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + // rm tail space + while (n > 1 && z[n-1] == ' ') { + n--; } errno = 0; @@ -137,33 +144,38 @@ int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { errno = 0; char *endPtr = NULL; + bool parsed = false; switch (type) { case TK_NK_INTEGER: { *value = taosStr2Int64(z, &endPtr, 10); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + parsed = true; + } break; + case TK_NK_FLOAT: { + double val = taosStr2Double(z, &endPtr); + parsed = true; + if (!IS_VALID_INT64(val)) { return TSDB_CODE_FAILED; } - return TSDB_CODE_SUCCESS; - } - case TK_NK_HEX: { - *value = taosStr2Int64(z, &endPtr, 16); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; - } - case TK_NK_BIN: { - *value = taosStr2Int64(z, &endPtr, 2); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; - } + *value = round(val); + } break; default: break; } + if (parsed) { + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } + while (n > 0 && z[n-1] == ' ') { + n--; + } + if (endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + // parse string if (n == 0) { *value = 0; @@ -196,29 +208,23 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { } int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { - errno = 0; - char *endPtr = NULL; + switch (type) { case TK_NK_INTEGER: { - *value = taosStr2UInt64(z, &endPtr, 10); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; + return toUInteger(z, n, 10, value); } - case TK_NK_HEX: { - *value = taosStr2UInt64(z, &endPtr, 16); + case TK_NK_FLOAT: { + errno = 0; + char *endPtr = NULL; + double val = round(taosStr2Double(z, &endPtr)); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { return TSDB_CODE_FAILED; } - return TSDB_CODE_SUCCESS; - } - case TK_NK_BIN: { - *value = taosStr2UInt64(z, &endPtr, 2); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + if (!IS_VALID_UINT64(val)) { return TSDB_CODE_FAILED; } + *value = val; return TSDB_CODE_SUCCESS; } default: diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 291941d9b2..9f7ee165ac 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -128,7 +128,7 @@ TEST(testCase, toIntegerEx_test) { ASSERT_EQ(val, -64); s = "0b110"; - ret = toIntegerEx(s, strlen(s), 0, &val); + ret = toIntegerEx(s, strlen(s), TK_NK_BIN, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 6); diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py index ce482287fb..8a7180325b 100644 --- a/tests/system-test/1-insert/insert_double.py +++ b/tests/system-test/1-insert/insert_double.py @@ -27,8 +27,8 @@ class TDTestCase: tdSql.execute(f"create table {table_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned)") tdSql.execute(f"insert into {table_name} values(now, -16, +6)") - tdSql.execute(f"insert into {table_name} values(now, 80.99 , +0042 )") - tdSql.execute(f"insert into {table_name} values(now, -0042 , +80.99 )") + tdSql.execute(f"insert into {table_name} values(now, 80.99, +0042)") + tdSql.execute(f"insert into {table_name} values(now, -0042, +80.99)") tdSql.execute(f"insert into {table_name} values(now, 52.34354, 18.6)") tdSql.execute(f"insert into {table_name} values(now, -12., +3.)") tdSql.execute(f"insert into {table_name} values(now, -.12, +.3)") From 17866b2b855190dc016f4d316a3dd9376d292421 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 6 Dec 2023 02:44:35 +0800 Subject: [PATCH 080/102] case adjust --- tests/system-test/1-insert/insert_double.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py index 8a7180325b..54df7e0352 100644 --- a/tests/system-test/1-insert/insert_double.py +++ b/tests/system-test/1-insert/insert_double.py @@ -31,7 +31,7 @@ class TDTestCase: tdSql.execute(f"insert into {table_name} values(now, -0042, +80.99)") tdSql.execute(f"insert into {table_name} values(now, 52.34354, 18.6)") tdSql.execute(f"insert into {table_name} values(now, -12., +3.)") - tdSql.execute(f"insert into {table_name} values(now, -.12, +.3)") + tdSql.execute(f"insert into {table_name} values(now, -0.12, +3.0)") tdSql.execute(f"insert into {table_name} values(now, -2.3e1, +2.324e2)") tdSql.execute(f"insert into {table_name} values(now, -2e1, +2e2)") tdSql.execute(f"insert into {table_name} values(now, -2.e1, +2.e2)") From 8ff3f8e25ef0f3d0c414c1fed4e4849b2d44bb26 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 6 Dec 2023 08:48:29 +0800 Subject: [PATCH 081/102] enhance: add duration order change by file /tmp/duration --- source/libs/executor/src/scanoperator.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 9a556a4ea1..3fc228503a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -12,6 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#define ALLOW_FORBID_FUNC #include "executorInt.h" #include "filter.h" @@ -3423,10 +3424,17 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx); pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, &pInfo->mSkipTables); - int32_t r = taosRand() % 2; + int r = 1; + FILE* f = fopen("/tmp/duration", "r"); + if (f) { + fscanf(f, "%d", &r); + fclose(f); + } if (r == 1) { - uInfo("zsl: set duration order"); + uInfo("zsl: DURATION ORDER"); pAPI->tsdReader.tsdSetDurationOrder(pInfo->base.dataReader); + } else { + uInfo("zsl: NO DURATION"); } pAPI->tsdReader.tsdSetSetNotifyCb(pInfo->base.dataReader, tableMergeScanTsdbNotifyCb, pInfo); From 7bb06cb1cc67a5fc5ebd41f283d1a2d0d59f87b9 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Wed, 6 Dec 2023 09:13:18 +0800 Subject: [PATCH 082/102] fix: userAlias --- source/libs/planner/src/planSpliter.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index bb88beba72..15efaae7f4 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -1033,6 +1033,7 @@ static SNode* stbSplCreateColumnNode(SExprNode* pExpr) { strcpy(pCol->colName, pExpr->aliasName); } strcpy(pCol->node.aliasName, pExpr->aliasName); + strcpy(pCol->node.userAlias, pExpr->userAlias); pCol->node.resType = pExpr->resType; return (SNode*)pCol; } From a2e7c78d27b20583ca5f435a5e429e2d463596c2 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 6 Dec 2023 09:39:38 +0800 Subject: [PATCH 083/102] refactor: rename variables --- include/libs/executor/storageapi.h | 4 ++-- include/libs/nodes/plannodes.h | 1 + source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead2.c | 22 +++++++++++----------- source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 2 +- source/dnode/vnode/src/vnd/vnodeInitApi.c | 2 +- source/libs/executor/inc/executorInt.h | 2 +- source/libs/executor/src/scanoperator.c | 12 ++++++------ 8 files changed, 24 insertions(+), 23 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 3f5bd7cf23..32b60b55ae 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -158,7 +158,7 @@ typedef enum { typedef union { struct { - int32_t fileSetId; + int32_t filesetId; } duration; } STsdReaderNotifyInfo; @@ -183,7 +183,7 @@ typedef struct TsdReader { int64_t (*tsdReaderGetNumOfInMemRows)(); void (*tsdReaderNotifyClosing)(); - void (*tsdSetDurationOrder)(void* pReader); + void (*tsdSetFilesetDelimited)(void* pReader); void (*tsdSetSetNotifyCb)(void* pReader, TsdReaderNotifyCbFn notifyFn, void* param); } TsdReader; diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index e29750d8a0..ce6ea32e2e 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -118,6 +118,7 @@ typedef struct SScanLogicNode { bool igLastNull; bool groupOrderScan; bool onlyMetaCtbIdx; // for tag scan with no tbname + bool filesetDelimited; // returned blocks delimited by fileset } SScanLogicNode; typedef struct SJoinLogicNode { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index bd840c5975..74d62da5c5 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -171,7 +171,7 @@ void *tsdbGetIvtIdx2(SMeta *pMeta); uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader); void tsdbReaderSetCloseFlag(STsdbReader *pReader); int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); -void tsdbSetDurationOrder(STsdbReader* pReader); +void tsdbSetFilesetDelimited(STsdbReader* pReader); void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param); int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index c108bbe03e..823e2e1786 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -423,7 +423,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void goto _end; } - pReader->bDurationOrder = false; + pReader->bFilesetDelimited = false; tsdbInitReaderLock(pReader); tsem_init(&pReader->resumeAfterSuspend, 0, 0); @@ -2520,7 +2520,7 @@ static void prepareDurationForNextFileSet(STsdbReader* pReader) { if (!pReader->status.bProcMemPreFileset) { if (pReader->notifyFn) { STsdReaderNotifyInfo info = {0}; - info.duration.fileSetId = fid; + info.duration.filesetId = fid; pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam); } } @@ -2570,7 +2570,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr } if (pBlockNum->numOfBlocks + pBlockNum->numOfSttFiles > 0) { - if (pReader->bDurationOrder) { + if (pReader->bFilesetDelimited) { prepareDurationForNextFileSet(pReader); } break; @@ -3915,7 +3915,7 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; - if (pReader->bDurationOrder) { + if (pReader->bFilesetDelimited) { getMemTableTimeRange(pReader, &pReader->status.memTableMaxKey, &pReader->status.memTableMinKey); pReader->status.bProcMemFirstFileset = true; } @@ -4227,7 +4227,7 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { tsdbUntakeReadSnap2(pReader, pReader->pReadSnap, false); pReader->pReadSnap = NULL; - if (pReader->bDurationOrder) { + if (pReader->bFilesetDelimited) { pReader->status.memTableMinKey = INT64_MAX; pReader->status.memTableMaxKey = INT64_MIN; } @@ -4342,7 +4342,7 @@ static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { return pBlock->info.rows > 0; } -static int32_t doTsdbNextDataBlockDurationOrder(STsdbReader* pReader) { +static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; int32_t code = TSDB_CODE_SUCCESS; SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; @@ -4361,7 +4361,7 @@ static int32_t doTsdbNextDataBlockDurationOrder(STsdbReader* pReader) { pStatus->bProcMemPreFileset = false; if (pReader->notifyFn) { STsdReaderNotifyInfo info = {0}; - info.duration.fileSetId = fid; + info.duration.filesetId = fid; pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam); } resetTableListIndex(pStatus); @@ -4425,10 +4425,10 @@ static int32_t doTsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { if (READ_MODE_COUNT_ONLY == pReader->info.readMode) { return tsdbReadRowsCountOnly(pReader); } - if (!pReader->bDurationOrder) { + if (!pReader->bFilesetDelimited) { code = doTsdbNextDataBlockFilesFirst(pReader); } else { - code = doTsdbNextDataBlockDurationOrder(pReader); + code = doTsdbNextDataBlockFilesetDelimited(pReader); } *hasNext = pBlock->info.rows > 0; @@ -5152,8 +5152,8 @@ void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ } -void tsdbSetDurationOrder(STsdbReader* pReader) { - pReader->bDurationOrder = true; +void tsdbSetFilesetDelimited(STsdbReader* pReader) { + pReader->bFilesetDelimited = true; } void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index c5904ea03e..f4fa7bc2c8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -230,7 +230,7 @@ struct STsdbReader { SBlockInfoBuf blockInfoBuf; EContentData step; STsdbReader* innerReader[2]; - bool bDurationOrder; // duration by duration output + bool bFilesetDelimited; // duration by duration output TsdReaderNotifyCbFn notifyFn; void* notifyParam; }; diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index f495b0a3e0..6b7b778cd5 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -61,7 +61,7 @@ void initTsdbReaderAPI(TsdReader* pReader) { pReader->tsdSetQueryTableList = tsdbSetTableList2; pReader->tsdSetReaderTaskId = (void (*)(void*, const char*))tsdbReaderSetId2; - pReader->tsdSetDurationOrder = (void (*)(void*))tsdbSetDurationOrder; + pReader->tsdSetFilesetDelimited = (void (*)(void*))tsdbSetFilesetDelimited; pReader->tsdSetSetNotifyCb = (void (*)(void*, TsdReaderNotifyCbFn, void*))tsdbReaderSetNotifyCb; } diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 4fa1dcdf72..c91701788d 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -297,7 +297,7 @@ typedef struct STableMergeScanInfo { SHashObj* mSkipTables; int64_t mergeLimit; SSortExecInfo sortExecInfo; - bool bNewDuration; + bool bNewFileset; bool bOnlyRetrieveBlock; } STableMergeScanInfo; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 3fc228503a..9911366166 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3261,7 +3261,7 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { } if (!hasNext || isTaskKilled(pTaskInfo)) { - pInfo->bNewDuration = false; + pInfo->bNewFileset = false; if (isTaskKilled(pTaskInfo)) { qInfo("table merge scan fetch next data block found task killed. %s", GET_TASKID(pTaskInfo)); pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); @@ -3269,7 +3269,7 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { break; } - if (pInfo->bNewDuration) { + if (pInfo->bNewFileset) { pInfo->bOnlyRetrieveBlock = true; return NULL; } @@ -3345,7 +3345,7 @@ int32_t dumpQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param) { STableMergeScanInfo* pTmsInfo = param; - pTmsInfo->bNewDuration = true; + pTmsInfo->bNewFileset = true; return; } @@ -3355,7 +3355,7 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t code = TSDB_CODE_SUCCESS; int32_t numOfTable = pInfo->tableEndIndex - pInfo->tableStartIndex + 1; - pInfo->bNewDuration = false; + pInfo->bNewFileset = false; pInfo->sortBufSize = 2048 * pInfo->bufPageSize; int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; @@ -3432,7 +3432,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { } if (r == 1) { uInfo("zsl: DURATION ORDER"); - pAPI->tsdReader.tsdSetDurationOrder(pInfo->base.dataReader); + pAPI->tsdReader.tsdSetFilesetDelimited(pInfo->base.dataReader); } else { uInfo("zsl: NO DURATION"); } @@ -3541,7 +3541,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { pOperator->resultInfo.totalRows += pBlock->info.rows; return pBlock; } else { - if (pInfo->bNewDuration) { + if (pInfo->bNewFileset) { stopDurationForGroupTableMergeScan(pOperator); startDurationForGroupTableMergeScan(pOperator); From 1ec8bf9ef959782d6976548fc9d41013a5e38bf6 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 6 Dec 2023 10:11:52 +0800 Subject: [PATCH 084/102] add error case --- source/common/src/tvariant.c | 23 ++++++++++++--------- tests/system-test/1-insert/insert_double.py | 13 +++++++----- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 25f025ad3d..d78c76a543 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -126,16 +126,19 @@ int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { *value = 0; return TSDB_CODE_SUCCESS; } - // rm tail space - while (n > 1 && z[n-1] == ' ') { - n--; - } errno = 0; char* endPtr = NULL; *value = taosStr2Double(z, &endPtr); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } + // rm tail space + while (n > 1 && z[n-1] == ' ') { + n--; + } + if (endPtr - z != n) { return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; @@ -152,12 +155,12 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { parsed = true; } break; case TK_NK_FLOAT: { - double val = taosStr2Double(z, &endPtr); + double val = round(taosStr2Double(z, &endPtr)); parsed = true; if (!IS_VALID_INT64(val)) { return TSDB_CODE_FAILED; } - *value = round(val); + *value = val; } break; default: break; @@ -167,7 +170,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { if (errno == ERANGE || errno == EINVAL) { return TSDB_CODE_FAILED; } - while (n > 0 && z[n-1] == ' ') { + while (n > 1 && z[n-1] == ' ') { n--; } if (endPtr - z != n) { @@ -176,7 +179,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { return TSDB_CODE_SUCCESS; } - // parse string + // parse as string if (n == 0) { *value = 0; return TSDB_CODE_SUCCESS; @@ -231,7 +234,7 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { break; } - // parse string + // parse as string if (n == 0) { *value = 0; return TSDB_CODE_SUCCESS; diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py index 54df7e0352..56861f3cd4 100644 --- a/tests/system-test/1-insert/insert_double.py +++ b/tests/system-test/1-insert/insert_double.py @@ -72,11 +72,14 @@ class TDTestCase: tdSql.query(f"select * from {table_name}") tdSql.checkRows(26) - # fail - tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})", "error") - tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})", "error") - tdSql.error(f"insert into {table_name} values(now, {max_i+1}, -1)", "error") - + # error case + tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})") + tdSql.error(f"insert into {table_name} values(now, 0, -1)") + tdSql.error(f"insert into {table_name} values(now, 0, -2.0)") + tdSql.error(f"insert into {table_name} values(now, 0, '-2.0')") + tdSql.error(f"insert into {table_name} values(now, {max_i+1}, 0)") + tdSql.error(f"insert into {table_name} values(now, {min_i-1}, 0)") + tdSql.error(f"insert into {table_name} values(now, '{min_i-1}', 0)") def test_tags(self, stable_name, dtype, bits): tdSql.execute(f"create stable {stable_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned) tags(id {dtype})") From ac531c6d7f4369213f04f37833db6de68aebfa0c Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 6 Dec 2023 10:45:12 +0800 Subject: [PATCH 085/102] enhance: add fileset delimited front end --- include/libs/nodes/plannodes.h | 1 + source/libs/executor/inc/executorInt.h | 2 ++ source/libs/executor/src/scanoperator.c | 19 +++++++------------ source/libs/nodes/src/nodesCloneFuncs.c | 2 ++ source/libs/nodes/src/nodesCodeFuncs.c | 15 ++++++++++++++- source/libs/nodes/src/nodesMsgFuncs.c | 7 ++++++- source/libs/planner/src/planOptimizer.c | 1 + source/libs/planner/src/planPhysiCreater.c | 1 + source/libs/planner/src/planSpliter.c | 2 ++ source/libs/planner/src/planUtil.c | 1 + 10 files changed, 37 insertions(+), 14 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index ce6ea32e2e..b99a97a194 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -433,6 +433,7 @@ typedef struct STableScanPhysiNode { int8_t igExpired; bool assignBlockUid; int8_t igCheckUpdate; + bool filesetDelimited; } STableScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode; diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index c91701788d..cba26c46b5 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -270,6 +270,7 @@ typedef struct STableScanInfo { bool hasGroupByTag; bool countOnly; // TsdReader readerAPI; + bool filesetDelimited; } STableScanInfo; typedef struct STableMergeScanInfo { @@ -299,6 +300,7 @@ typedef struct STableMergeScanInfo { SSortExecInfo sortExecInfo; bool bNewFileset; bool bOnlyRetrieveBlock; + bool filesetDelimited; } STableMergeScanInfo; typedef struct STagScanFilterContext { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 9911366166..7d06f5dab6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -894,7 +894,9 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } - + if (pInfo->filesetDelimited) { + pAPI->tsdReader.tsdSetFilesetDelimited(pInfo->base.dataReader); + } if (pInfo->pResBlock->info.capacity > pOperator->resultInfo.capacity) { pOperator->resultInfo.capacity = pInfo->pResBlock->info.capacity; } @@ -1086,6 +1088,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->countOnly = true; } + pInfo->filesetDelimited = pTableScanNode->filesetDelimited; + taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, optrDefaultBufFn, getTableScannerExecInfo, optrDefaultGetNextExtFn, NULL); @@ -3424,17 +3428,8 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx); pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, &pInfo->mSkipTables); - int r = 1; - FILE* f = fopen("/tmp/duration", "r"); - if (f) { - fscanf(f, "%d", &r); - fclose(f); - } - if (r == 1) { - uInfo("zsl: DURATION ORDER"); + if (pInfo->filesetDelimited) { pAPI->tsdReader.tsdSetFilesetDelimited(pInfo->base.dataReader); - } else { - uInfo("zsl: NO DURATION"); } pAPI->tsdReader.tsdSetSetNotifyCb(pInfo->base.dataReader, tableMergeScanTsdbNotifyCb, pInfo); @@ -3544,7 +3539,6 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { if (pInfo->bNewFileset) { stopDurationForGroupTableMergeScan(pOperator); startDurationForGroupTableMergeScan(pOperator); - } else { // Data of this group are all dumped, let's try the next group stopGroupTableMergeScan(pOperator); @@ -3683,6 +3677,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN uint32_t nCols = taosArrayGetSize(pInfo->pResBlock->pDataBlock); pInfo->bufPageSize = getProperSortPageSize(rowSize, nCols); + pInfo->filesetDelimited = pTableScanNode->filesetDelimited; setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index ce23928268..97438b84a6 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -423,6 +423,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_SCALAR_FIELD(igLastNull); COPY_SCALAR_FIELD(groupOrderScan); COPY_SCALAR_FIELD(onlyMetaCtbIdx); + COPY_SCALAR_FIELD(filesetDelimited); return TSDB_CODE_SUCCESS; } @@ -650,6 +651,7 @@ static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhy COPY_SCALAR_FIELD(triggerType); COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(igExpired); + COPY_SCALAR_FIELD(filesetDelimited); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index f3087dd5d4..d26cdcf401 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -677,6 +677,7 @@ static const char* jkScanLogicPlanDataRequired = "DataRequired"; static const char* jkScanLogicPlanTagCond = "TagCond"; static const char* jkScanLogicPlanGroupTags = "GroupTags"; static const char* jkScanLogicPlanOnlyMetaCtbIdx = "OnlyMetaCtbIdx"; +static const char* jkScanLogicPlanFilesetDelimited = "FilesetDelimited"; static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; @@ -721,6 +722,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkScanLogicPlanOnlyMetaCtbIdx, pNode->onlyMetaCtbIdx); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkScanLogicPlanFilesetDelimited, pNode->filesetDelimited); + } return code; } @@ -768,7 +772,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkScanLogicPlanOnlyMetaCtbIdx, &pNode->onlyMetaCtbIdx); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkScanLogicPlanFilesetDelimited, &pNode->filesetDelimited); + } return code; } @@ -1830,6 +1836,7 @@ static const char* jkTableScanPhysiPlanTags = "Tags"; static const char* jkTableScanPhysiPlanSubtable = "Subtable"; static const char* jkTableScanPhysiPlanAssignBlockUid = "AssignBlockUid"; static const char* jkTableScanPhysiPlanIgnoreUpdate = "IgnoreUpdate"; +static const char* jkTableScanPhysiPlanFilesetDelimited = "FilesetDelimited"; static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; @@ -1898,6 +1905,9 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanIgnoreUpdate, pNode->igCheckUpdate); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanFilesetDelimited, pNode->filesetDelimited); + } return code; } @@ -1969,6 +1979,9 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkTableScanPhysiPlanIgnoreUpdate, &pNode->igCheckUpdate); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanFilesetDelimited, &pNode->filesetDelimited); + } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 9804f2075b..d6eb3360aa 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2167,7 +2167,9 @@ static int32_t physiTableScanNodeInlineToMsg(const void* pObj, STlvEncoder* pEnc if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueI8(pEncoder, pNode->igCheckUpdate); } - + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueBool(pEncoder, pNode->filesetDelimited); + } return code; } @@ -2246,6 +2248,9 @@ static int32_t msgToPhysiTableScanNodeInline(STlvDecoder* pDecoder, void* pObj) if (TSDB_CODE_SUCCESS == code) { code = tlvDecodeValueI8(pDecoder, &pNode->igCheckUpdate); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueBool(pDecoder, &pNode->filesetDelimited); + } return code; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index e71e18d37d..aa3181e166 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1400,6 +1400,7 @@ static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicS pScan->node.outputTsOrder = order; if (TSDB_SUPER_TABLE == pScan->tableType) { pScan->scanType = SCAN_TYPE_TABLE_MERGE; + pScan->filesetDelimited = true; pScan->node.resultDataOrder = DATA_ORDER_LEVEL_GLOBAL; pScan->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 598bce3133..d1fbd0681d 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -622,6 +622,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp pTableScan->igExpired = pScanLogicNode->igExpired; pTableScan->igCheckUpdate = pScanLogicNode->igCheckUpdate; pTableScan->assignBlockUid = pCxt->pPlanCxt->rSmaQuery ? true : false; + pTableScan->filesetDelimited = pScanLogicNode->filesetDelimited; int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 43bd8a5589..9b9f701e2b 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -692,6 +692,7 @@ static void stbSplSetTableMergeScan(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { SScanLogicNode* pScan = (SScanLogicNode*)pNode; pScan->scanType = SCAN_TYPE_TABLE_MERGE; + pScan->filesetDelimited = true; if (NULL != pScan->pGroupTags) { pScan->groupSort = true; } @@ -1241,6 +1242,7 @@ static int32_t stbSplCreateMergeScanNode(SScanLogicNode* pScan, SLogicNode** pOu SNodeList* pMergeKeys = NULL; if (TSDB_CODE_SUCCESS == code) { pMergeScan->scanType = SCAN_TYPE_TABLE_MERGE; + pMergeScan->filesetDelimited = true; pMergeScan->node.pChildren = pChildren; splSetParent((SLogicNode*)pMergeScan); code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pMergeScan), diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index 1c7c937b7f..2da270e42d 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -164,6 +164,7 @@ static int32_t adjustScanDataRequirement(SScanLogicNode* pScan, EDataOrderLevel pScan->scanType = SCAN_TYPE_TABLE; } else if (TSDB_SUPER_TABLE == pScan->tableType) { pScan->scanType = SCAN_TYPE_TABLE_MERGE; + pScan->filesetDelimited = true; } if (TSDB_NORMAL_TABLE != pScan->tableType && TSDB_CHILD_TABLE != pScan->tableType) { From a0e356bf849ef3f7b200f1b40df61a1d1d1fe077 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 6 Dec 2023 15:20:05 +0800 Subject: [PATCH 086/102] Keep the last data with the same timestamp when inserting --- source/libs/executor/src/dataInserter.c | 22 ++-- source/libs/parser/src/parTranslater.c | 9 +- tests/system-test/2-query/insert_select.py | 113 ++++++++++++++++++++- 3 files changed, 124 insertions(+), 20 deletions(-) diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index f301ddf4be..00b58263e2 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -189,7 +189,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } int64_t lastTs = TSKEY_MIN; - bool ignoreRow = false; + bool updateLastRow = false; bool disorderTs = false; for (int32_t j = 0; j < rows; ++j) { // iterate by row @@ -249,7 +249,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } else { if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) { if (*(int64_t*)var == lastTs) { - ignoreRow = true; + updateLastRow = true; } else if (*(int64_t*)var < lastTs) { disorderTs = true; } else { @@ -269,15 +269,6 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } break; } - - if (ignoreRow) { - break; - } - } - - if (ignoreRow) { - ignoreRow = false; - continue; } SRow* pRow = NULL; @@ -285,7 +276,14 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); goto _end; } - taosArrayPush(tbData.aRowP, &pRow); + if (updateLastRow) { + updateLastRow = false; + SRow** lastRow = taosArrayPop(tbData.aRowP); + tRowDestroy(*lastRow); + taosArrayPush(tbData.aRowP, &pRow); + } else { + taosArrayPush(tbData.aRowP, &pRow); + } } if (disorderTs) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f689ebc820..9a0a560a1a 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4627,14 +4627,9 @@ static int32_t addOrderByPrimaryKeyToQueryImpl(STranslateContext* pCxt, SNode* p } static int32_t addOrderByPrimaryKeyToQuery(STranslateContext* pCxt, SNode* pPrimaryKeyExpr, SNode* pStmt) { - SNode* pOrderNode = NULL; SNodeList* pOrederList = ((SSelectStmt*)pStmt)->pOrderByList; - if (pOrederList != NULL) { - FOREACH(pOrderNode, pOrederList) { - if (((SColumnNode*)pPrimaryKeyExpr)->colId == ((SColumnNode*)((SOrderByExprNode*)pOrderNode)->pExpr)->colId) { - return TSDB_CODE_SUCCESS; - } - } + if (pOrederList && pOrederList->length > 0) { + return TSDB_CODE_SUCCESS; } if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { return addOrderByPrimaryKeyToQueryImpl(pCxt, pPrimaryKeyExpr, &((SSelectStmt*)pStmt)->pOrderByList); diff --git a/tests/system-test/2-query/insert_select.py b/tests/system-test/2-query/insert_select.py index e74cf7a8d1..cbdcf366d2 100644 --- a/tests/system-test/2-query/insert_select.py +++ b/tests/system-test/2-query/insert_select.py @@ -70,7 +70,116 @@ class TDTestCase: tdSql.error('''insert into %s.tb1 (c8, c9) values(now, 1);'''%(database)) - + def use_select_sort(self,database): + ts = 1604298064000 + + tdSql.execute('''drop database if exists %s ;''' %database) + tdSql.execute('''create database %s keep 36500 ;'''%(database)) + tdSql.execute('''use %s;'''%database) + + tdSql.execute('''create stable %s.st (ts timestamp, val int, vt timestamp) tags (location NCHAR(100));'''%(database)) + tdSql.execute('''create table %s.t1 using %s.st (location) tags ("0001");'''%(database,database)) + tdSql.execute('''create table %s.t2 using %s.st (location) tags ("0002");'''%(database,database)) + tdSql.execute('''create table %s.mt (ts timestamp, val int);'''%(database)) + + + tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) ({ts}, 2, {ts});'''%(database)) + tdSql.query("select ts, val from %s.t1;"%database) + tdSql.checkData(0,1,2) + + ts += 1 + tdSql.execute(f'''insert into %s.t2 values({ts}, 1, {ts}) ({ts}, 5, {ts}) ({ts}, 2, {ts});'''%(database)) + tdSql.query("select ts, val from %s.t2;"%database) + tdSql.checkData(0,1,2) + + tdSql.execute('''delete from %s.t2;'''%(database)) + tdSql.execute('''delete from %s.t1;'''%(database)) + + ts -= 10 + tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) %s.t2 values({ts}, 2, {ts});'''%(database,database)) + ts += 11 + tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) %s.t2 values({ts}, 2, {ts});'''%(database,database)) + ts += 1 + tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) %s.t2 values({ts}, 2, {ts});'''%(database,database)) + + tdSql.query("select count(*) from %s.st;"%database) + tdSql.checkData(0,0,6) + + tdSql.query('''select vt, val from %s.st order by vt, val desc;'''%(database)) + tdSql.checkData(0,1,2) + tdSql.checkData(1,1,1) + tdSql.checkData(2,1,2) + tdSql.checkData(3,1,1) + tdSql.checkData(4,1,2) + tdSql.checkData(5,1,1) + + tdSql.execute('''insert into %s.mt select vt, val from %s.st order by vt, val desc;'''%(database,database)) + tdSql.query("select count(*) from %s.mt;"%database) + tdSql.checkData(0,0,3) + + tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database)) + tdSql.checkData(0,1,1) + tdSql.checkData(1,1,1) + tdSql.checkData(2,1,1) + + tdSql.execute('''delete from %s.mt;'''%(database)) + tdSql.query('''select vt, val from %s.st order by vt, val asc;'''%(database)) + tdSql.checkData(0,1,1) + tdSql.checkData(1,1,2) + tdSql.checkData(2,1,1) + tdSql.checkData(3,1,2) + tdSql.checkData(4,1,1) + tdSql.checkData(5,1,2) + + tdSql.execute('''insert into %s.mt select vt, val from %s.st order by vt, val asc;'''%(database,database)) + tdSql.query("select count(*) from %s.mt;"%database) + tdSql.checkData(0,0,3) + + tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database)) + tdSql.checkData(0,1,2) + tdSql.checkData(1,1,2) + tdSql.checkData(2,1,2) + + tdSql.execute('''delete from %s.mt;'''%(database)) + tdSql.query('''select vt, val from %s.st order by ts, val asc;'''%(database)) + tdSql.checkData(0,1,1) + tdSql.checkData(1,1,2) + tdSql.checkData(2,1,1) + tdSql.checkData(3,1,2) + tdSql.checkData(4,1,1) + tdSql.checkData(5,1,2) + + tdSql.execute('''insert into %s.mt select vt, val from %s.st order by ts, val asc;'''%(database,database)) + tdSql.query("select count(*) from %s.mt;"%database) + tdSql.checkData(0,0,3) + + tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database)) + tdSql.checkData(0,1,2) + tdSql.checkData(1,1,2) + tdSql.checkData(2,1,2) + + tdSql.execute('''delete from %s.mt;'''%(database)) + ts += 1 + tdSql.execute(f'''insert into %s.t1 values({ts}, -1, {ts}) %s.t2 values({ts}, -2, {ts});'''%(database,database)) + tdSql.query('''select vt, val from %s.st order by val asc;'''%(database)) + tdSql.checkData(0,1,-2) + tdSql.checkData(1,1,-1) + tdSql.checkData(2,1,1) + tdSql.checkData(3,1,1) + tdSql.checkData(4,1,1) + tdSql.checkData(5,1,2) + tdSql.checkData(6,1,2) + tdSql.checkData(7,1,2) + + tdSql.execute('''insert into %s.mt select vt, val from %s.st order by val asc;'''%(database,database)) + tdSql.query("select count(*) from %s.mt;"%database) + tdSql.checkData(0,0,4) + + tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database)) + tdSql.checkData(0,1,2) + tdSql.checkData(1,1,2) + tdSql.checkData(2,1,2) + tdSql.checkData(3,1,-1) def run(self): @@ -88,6 +197,8 @@ class TDTestCase: self.users_bug_TD_20592("%s" %self.db) + self.use_select_sort("%s" %self.db) + #taos -f sql print("taos -f sql start!") taos_cmd1 = "taos -f %s/%s.sql" % (self.testcasePath,self.testcaseFilename) From f4b0e7baf520f90e6bf10e744ab3ca6d4e3b118f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 6 Dec 2023 15:46:45 +0800 Subject: [PATCH 087/102] fix:remove debug info --- source/client/inc/clientSml.h | 1 - source/client/src/clientSml.c | 19 +++++-------------- source/client/src/clientSmlLine.c | 3 --- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 0a5b91e039..30d4792279 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -199,7 +199,6 @@ typedef struct { STableMeta *currSTableMeta; STableDataCxt *currTableDataCtx; bool needModifySchema; - SArray *parseTimeList; } SSmlHandle; #define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 46e88b4a00..6bcaae3bf0 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1261,7 +1261,6 @@ void smlDestroyInfo(SSmlHandle *info) { taosArrayDestroy(info->valueJsonArray); taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); - taosArrayDestroy(info->parseTimeList); if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { @@ -1310,7 +1309,6 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); - info->parseTimeList = taosArrayInit(8, LONG_BYTES); if (NULL == info->pVgHash || NULL == info->childTables || NULL == info->superTables || NULL == info->tableUids) { uError("create SSmlHandle failed"); @@ -1525,22 +1523,15 @@ static int32_t smlInsertData(SSmlHandle *info) { } static void smlPrintStatisticInfo(SSmlHandle *info) { - char parseTimeStr[102400] = {0}; - char* tmp = parseTimeStr; - for(int i = 0; i < taosArrayGetSize(info->parseTimeList); i++){ - int64_t *t = (int64_t *)taosArrayGet(info->parseTimeList, i); - tmp += sprintf(tmp, ":%d", (int32_t)(*t)); - } - uError( + uDebug( "SML:0x%" PRIx64 " smlInsertLines result, code:%d, msg:%s, lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ - parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64 - ", parList:%s", + parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64, info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime, info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime, - info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime, parseTimeStr); + info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime); } @@ -1583,7 +1574,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { } static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { - uError("SML:0x%" PRIx64 " smlParseLine start", info->id); + uDebug("SML:0x%" PRIx64 " smlParseLine start", info->id); int32_t code = TSDB_CODE_SUCCESS; if (info->protocol == TSDB_SML_JSON_PROTOCOL) { if (lines) { @@ -1656,7 +1647,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } i++; } - uError("SML:0x%" PRIx64 " smlParseLine end", info->id); + uDebug("SML:0x%" PRIx64 " smlParseLine end", info->id); return code; } diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 91dd6ea700..7ad43d90c7 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -632,7 +632,6 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine JUMP_SPACE(sql, sqlEnd) if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA; elements->measure = sql; - int64_t t1 = taosGetTimestampUs(); // parse measure size_t measureLenEscaped = 0; while (sql < sqlEnd) { @@ -725,8 +724,6 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); return TSDB_CODE_INVALID_TIMESTAMP; } - int64_t t2 = taosGetTimestampUs() - t1; - taosArrayPush(info->parseTimeList, &t2); // add ts to SSmlKv kv = {.key = tsSmlTsDefaultName, .keyLen = strlen(tsSmlTsDefaultName), From bfc12f03cb3e50018b35df3caab0527d5e079d71 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 6 Dec 2023 21:26:29 +0800 Subject: [PATCH 088/102] optimize & perf test --- source/common/src/tvariant.c | 126 ++++++++++++++-------- tests/system-test/1-insert/insert_perf.py | 118 ++++++++++++++++++++ 2 files changed, 198 insertions(+), 46 deletions(-) create mode 100644 tests/system-test/1-insert/insert_perf.py diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index d78c76a543..384df73342 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -81,24 +81,12 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v } if (!parsed) { - bool parse_int = true; - for (int32_t i = 0; i < n; i++) { - if (z[i] < '0' || z[i] > '9') { - parse_int = false; - break; - } - } - if (parse_int) { - // parsing as decimal - *value = taosStr2UInt64(z, &endPtr, 10); - } else { - // parsing as double - double val = taosStr2Double(z, &endPtr); - if (val > UINT64_MAX) { - return TSDB_CODE_FAILED; - } - *value = round(val); + // parsing as double + double val = taosStr2Double(z, &endPtr); + if (val > UINT64_MAX) { + return TSDB_CODE_FAILED; } + *value = round(val); } if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { @@ -145,6 +133,11 @@ int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { } int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + errno = 0; char *endPtr = NULL; bool parsed = false; @@ -179,13 +172,21 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { return TSDB_CODE_SUCCESS; } - // parse as string - if (n == 0) { - *value = 0; + // rm tail space + while (n > 1 && z[n-1] == ' ') { + n--; + } + + // 1. try to parse as integer + *value = taosStr2Int64(z, &endPtr, 10); + if (endPtr - z == n) { + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } return TSDB_CODE_SUCCESS; } - n = removeSpace(&z, n); + // 2. parse as other bool is_neg = false; uint64_t uv = 0; int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv); @@ -211,36 +212,69 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { } int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { - - switch (type) - { - case TK_NK_INTEGER: { - return toUInteger(z, n, 10, value); - } - case TK_NK_FLOAT: { - errno = 0; - char *endPtr = NULL; - double val = round(taosStr2Double(z, &endPtr)); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - return TSDB_CODE_FAILED; - } - if (!IS_VALID_UINT64(val)) { - return TSDB_CODE_FAILED; - } - *value = val; - return TSDB_CODE_SUCCESS; - } - default: - break; - } - - // parse as string if (n == 0) { *value = 0; return TSDB_CODE_SUCCESS; } - n = removeSpace(&z, n); + errno = 0; + const char *p = z; + char *endPtr = NULL; + bool parsed = false; + while (*p == ' ') { + p++; + } + switch (type) { + case TK_NK_INTEGER: { + *value = taosStr2UInt64(p, &endPtr, 10); + if (*p == '-' && *value) { + return TSDB_CODE_FAILED; + } + parsed = true; + } break; + case TK_NK_FLOAT: { + double val = round(taosStr2Double(p, &endPtr)); + if (!IS_VALID_UINT64(val)) { + return TSDB_CODE_FAILED; + } + *value = val; + parsed = true; + } break; + default: + break; + } + + if (parsed) { + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } + while (n > 1 && z[n-1] == ' ') { + n--; + } + if (endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + + // rm tail space + while (n > 1 && z[n-1] == ' ') { + n--; + } + + // 1. parse as integer + *value = taosStr2UInt64(p, &endPtr, 10); + if (endPtr - z == n) { + if (*p == '-' && *value) { + return TSDB_CODE_FAILED; + } + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + + // 2. parse as other bool is_neg = false; int32_t code = parseSignAndUInteger(z, n, &is_neg, value); if (is_neg) { diff --git a/tests/system-test/1-insert/insert_perf.py b/tests/system-test/1-insert/insert_perf.py new file mode 100644 index 0000000000..bf6fb6799d --- /dev/null +++ b/tests/system-test/1-insert/insert_perf.py @@ -0,0 +1,118 @@ +import taos +import sys +import random +import time +import csv + +from datetime import datetime + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + + self.testcasePath = os.path.split(__file__)[0] + self.testcasefilename = os.path.split(__file__)[-1] + self.file1 = f"{self.testcasePath}/int.csv" + self.file2 = f"{self.testcasePath}/double.csv" + self.ts = 1700638570000 # 2023-11-22T07:36:10.000Z + self.database = "db1" + self.tb1 = "t1" + self.tb2 = "t2" + self.stable1 = "st1" + self.stable2 = "st2" + self.tag1 = f'using {self.stable1}(groupId) tags(1)' + self.tag2 = f'using {self.stable2}(groupId) tags(2)' + self.once = 1000 + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepare_db(self): + #tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database if not exists {self.database}") + tdSql.execute(f"use {self.database}") + tdSql.execute(f"create stable if not exists {self.stable1} (ts timestamp, q_bigint bigint , i1 int, i2 int, i3 int, i4 int, i5 int, i6 int, i7 int, i8 int, i9 int) tags (location binary(64), groupId int);") + tdSql.execute(f"create stable if not exists {self.stable2} (ts timestamp, q_double double, f1 double, f2 double, f3 double, f4 double, f5 double, f6 double, f7 double, f8 double, f9 double) tags (location binary(64), groupId int);") + tdSql.execute(f"drop table if exists {self.tb1};") + tdSql.execute(f"drop table if exists {self.tb2};") + tdSql.execute(f"create table {self.tb1} {self.tag1};") + tdSql.execute(f"create table {self.tb2} {self.tag2};") + + def make_csv(self, filepath, once, isint): + f = open(filepath, 'w') + with f: + writer = csv.writer(f) + rows = [] + for i in range(once): + r = [] + if isint: + for k in range(10): + r.append(random.randint(-2147483648, 2147483647)) + else: + for k in range(10): + r.append(random.randint(-2147483648, 2147483647) + random.random()) + rows.append(r) + writer.writerows(rows) + f.close() + print(f"{filepath} ready!") + + def test_insert(self, tbname, qtime, startts, filepath, isint): + f = open(filepath, 'r') + rows = [] + with f: + reader = csv.reader(f, delimiter=',', quotechar='|') + for row in reader: + rows.append(row) + f.close() + self.once = len(rows) + + sum = 0 + for j in range(qtime): + offset = j * self.once + ts = startts + offset + sql = f"insert into {self.database}.{tbname} values" + for i in range(self.once): + r = rows[i] + sql +=f"({ts + i},'{r[0]}','{r[1]}','{r[2]}','{r[3]}','{r[4]}','{r[5]}','{r[6]}','{r[7]}','{r[8]}','{r[9]}')" + + t1 = time.time() + tdSql.execute(f"{sql};", 1) + t2 = time.time() + #print(f"{t2} insert test {j}.") + #print(sql) + sum += t2 - t1 + + sum = sum + tbtype = "10 double col/per row" + if isint: + tbtype = "10 int col/per row" + print(f" insert {self.once} * {qtime} rows: {sum} s, {tbtype}") + + # tdSql.query(f"select count(*) from {self.database}.{tbname};") + # tdSql.checkData(0, 0, once*qtime) + + def run(self): + tdSql.prepare(replica = self.replicaVar) + # self.make_csv(self.file1, self.once, True) + # self.make_csv(self.file2, self.once, False) + self.prepare_db() + self.test_insert(self.tb1, 10000, self.ts, self.file1, True) + self.test_insert(self.tb2, 10000, self.ts, self.file2, False) + self.test_insert(self.tb2, 10000, self.ts, self.file1, False) + + self.test_insert(self.tb1, 10000, self.ts, self.file2, True) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 8f5df0b6a3aba15831f8372051315f4005c582a1 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 7 Dec 2023 01:51:23 +0800 Subject: [PATCH 089/102] opt double -> int --- source/common/src/tvariant.c | 68 +++++++++++++----- tests/system-test/1-insert/insert_perf.py | 86 +++++++++++++++-------- 2 files changed, 108 insertions(+), 46 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 384df73342..5c7567777f 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -95,19 +95,6 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v return TSDB_CODE_SUCCESS; } -int32_t removeSpace(const char **pp, int32_t n) { - // rm blank space from both head and tail, keep at least one char - const char *z = *pp; - while (n > 1 && *z == ' ') { - z++; - n--; - } - while (n > 1 && z[n-1] == ' ') { - n--; - } - *pp = z; - return n; -} int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { if (n == 0) { @@ -184,6 +171,36 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; + } else if (errno == 0 && *endPtr == '.') { + // pure decimal part + const char *s = endPtr + 1; + const char *end = z + n; + bool pure = true; + while (s < end) { + if (*s < '0' || *s > '9') { + pure = false; + break; + } + s++; + } + if (pure) { + if (endPtr+1 < end && endPtr[1] > '4') { + const char *p = z; + while (*p == ' ') { + p++; + } + if (*p == '-') { + if ( *value > INT64_MIN) { + (*value)--; + } + } else { + if ( *value < INT64_MAX) { + (*value)++; + } + } + } + return TSDB_CODE_SUCCESS; + } } // 2. parse as other @@ -218,9 +235,9 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { } errno = 0; - const char *p = z; char *endPtr = NULL; bool parsed = false; + const char *p = z; while (*p == ' ') { p++; } @@ -264,14 +281,31 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { // 1. parse as integer *value = taosStr2UInt64(p, &endPtr, 10); + if (*p == '-' && *value) { + return TSDB_CODE_FAILED; + } if (endPtr - z == n) { - if (*p == '-' && *value) { - return TSDB_CODE_FAILED; - } if (errno == ERANGE || errno == EINVAL) { return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; + } else if (errno == 0 && *endPtr == '.') { + const char *s = endPtr + 1; + const char *end = z + n; + bool pure = true; + while (s < end) { + if (*s < '0' || *s > '9') { + pure = false; + break; + } + s++; + } + if (pure) { + if (endPtr + 1 < end && endPtr[1] > '4' && *value < UINT64_MAX) { + (*value)++; + } + return TSDB_CODE_SUCCESS; + } } // 2. parse as other diff --git a/tests/system-test/1-insert/insert_perf.py b/tests/system-test/1-insert/insert_perf.py index bf6fb6799d..d96a031458 100644 --- a/tests/system-test/1-insert/insert_perf.py +++ b/tests/system-test/1-insert/insert_perf.py @@ -21,48 +21,70 @@ class TDTestCase: self.testcasefilename = os.path.split(__file__)[-1] self.file1 = f"{self.testcasePath}/int.csv" self.file2 = f"{self.testcasePath}/double.csv" + self.file3 = f"{self.testcasePath}/d+.csv" + self.file4 = f"{self.testcasePath}/uint.csv" self.ts = 1700638570000 # 2023-11-22T07:36:10.000Z self.database = "db1" self.tb1 = "t1" self.tb2 = "t2" - self.stable1 = "st1" - self.stable2 = "st2" - self.tag1 = f'using {self.stable1}(groupId) tags(1)' - self.tag2 = f'using {self.stable2}(groupId) tags(2)' + self.tb3 = "t3" self.once = 1000 tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) def prepare_db(self): - #tdSql.execute(f"drop database if exists {self.database}") - tdSql.execute(f"create database if not exists {self.database}") + tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database {self.database}") tdSql.execute(f"use {self.database}") - tdSql.execute(f"create stable if not exists {self.stable1} (ts timestamp, q_bigint bigint , i1 int, i2 int, i3 int, i4 int, i5 int, i6 int, i7 int, i8 int, i9 int) tags (location binary(64), groupId int);") - tdSql.execute(f"create stable if not exists {self.stable2} (ts timestamp, q_double double, f1 double, f2 double, f3 double, f4 double, f5 double, f6 double, f7 double, f8 double, f9 double) tags (location binary(64), groupId int);") - tdSql.execute(f"drop table if exists {self.tb1};") - tdSql.execute(f"drop table if exists {self.tb2};") - tdSql.execute(f"create table {self.tb1} {self.tag1};") - tdSql.execute(f"create table {self.tb2} {self.tag2};") - - def make_csv(self, filepath, once, isint): + tdSql.execute(f"create table {self.tb1} (ts timestamp, i0 bigint , i1 bigint, i2 bigint, i3 bigint, i4 bigint, i5 bigint, i6 bigint, i7 bigint, i8 bigint, i9 bigint)") + tdSql.execute(f"create table {self.tb2} (ts timestamp, f0 double, f1 double, f2 double, f3 double, f4 double, f5 double, f6 double, f7 double, f8 double, f9 double)") + tdSql.execute(f"create table {self.tb3} (ts timestamp, i0 int unsigned , i1 int unsigned, i2 int unsigned, i3 int unsigned, i4 int unsigned, i5 int unsigned, i6 int unsigned, i7 int unsigned, i8 int unsigned, i9 int unsigned)") + + def make_csv(self, once, intype): + filepath = self.file1 + if intype == 2: + filepath = self.file2 + elif intype == 3: + filepath = self.file3 + elif intype == 4: + filepath = self.file4 + f = open(filepath, 'w') with f: writer = csv.writer(f) rows = [] for i in range(once): r = [] - if isint: + if intype == 1: for k in range(10): r.append(random.randint(-2147483648, 2147483647)) + elif intype == 2: + for k in range(10): + r.append(random.randint(-2147483648, 2147483646) + random.random()) + elif intype == 3: + for k in range(10): + r.append(random.randint(0, 4294967294) + random.random()) else: for k in range(10): - r.append(random.randint(-2147483648, 2147483647) + random.random()) + r.append(random.randint(0, 4294967295)) rows.append(r) writer.writerows(rows) f.close() print(f"{filepath} ready!") - def test_insert(self, tbname, qtime, startts, filepath, isint): + def test_insert(self, tbname, qtime, startts, intype, outtype): + filepath = self.file1 + dinfo = "int" + if intype == 2: + filepath = self.file2 + dinfo = "double" + elif intype == 3: + filepath = self.file3 + dinfo = "+double" + elif intype == 4: + filepath = self.file4 + dinfo = "uint" + f = open(filepath, 'r') rows = [] with f: @@ -89,25 +111,31 @@ class TDTestCase: sum += t2 - t1 sum = sum - tbtype = "10 double col/per row" - if isint: - tbtype = "10 int col/per row" - print(f" insert {self.once} * {qtime} rows: {sum} s, {tbtype}") + tbinfo = "10 bigint col/per row" + if outtype == 2: + tbinfo = "10 double col/per row" + elif outtype == 3: + tbinfo = "10 uint col/per row" + print(f" insert {self.once} * {qtime} rows: {sum} s, {dinfo} -> {tbinfo}") # tdSql.query(f"select count(*) from {self.database}.{tbname};") # tdSql.checkData(0, 0, once*qtime) def run(self): tdSql.prepare(replica = self.replicaVar) - # self.make_csv(self.file1, self.once, True) - # self.make_csv(self.file2, self.once, False) + # self.make_csv(self.once, 1) + # self.make_csv(self.once, 2) + # self.make_csv(self.once, 3) + # self.make_csv(self.once, 4) + self.prepare_db() - self.test_insert(self.tb1, 10000, self.ts, self.file1, True) - self.test_insert(self.tb2, 10000, self.ts, self.file2, False) - self.test_insert(self.tb2, 10000, self.ts, self.file1, False) - - self.test_insert(self.tb1, 10000, self.ts, self.file2, True) - + self.test_insert(self.tb1, 1000, self.ts-10000000, 1, 1) + self.test_insert(self.tb2, 1000, self.ts-10000000, 2, 2) + self.test_insert(self.tb3, 1000, self.ts-10000000, 4, 3) + self.test_insert(self.tb2, 1000, self.ts, 1, 2) + + self.test_insert(self.tb1, 1000, self.ts, 2, 1) + self.test_insert(self.tb3, 1000, self.ts, 3, 3) def stop(self): tdSql.close() From 515d671e00621620dc7940fda4cd04552d10c617 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 7 Dec 2023 10:04:47 +0800 Subject: [PATCH 090/102] test:modify the keep parameter of db in python.sh cases --- docs/examples/python/bind_param_example.py | 2 +- docs/examples/python/conn_websocket_pandas.py | 2 +- docs/examples/python/connect_rest_examples.py | 2 +- docs/examples/python/connect_rest_with_req_id_examples.py | 2 +- docs/examples/python/connect_websocket_examples.py | 6 +++--- .../python/connect_websocket_with_req_id_examples.py | 2 +- docs/examples/python/connection_usage_native_reference.py | 2 +- .../python/connection_usage_native_reference_with_req_id.py | 2 +- docs/examples/python/cursor_usage_native_reference.py | 2 +- .../python/cursor_usage_native_reference_with_req_id.py | 2 +- docs/examples/python/fast_write_example.py | 2 +- docs/examples/python/json_protocol_example.py | 2 +- docs/examples/python/kafka_example_common.py | 2 +- docs/examples/python/line_protocol_example.py | 2 +- docs/examples/python/multi_bind_example.py | 2 +- docs/examples/python/native_insert_example.py | 2 +- docs/examples/python/result_set_examples.py | 2 +- docs/examples/python/result_set_with_req_id_examples.py | 2 +- docs/examples/python/schemaless_insert.py | 2 +- docs/examples/python/schemaless_insert_raw.py | 4 ++-- docs/examples/python/schemaless_insert_raw_req_id.py | 4 ++-- docs/examples/python/schemaless_insert_raw_ttl.py | 4 ++-- docs/examples/python/schemaless_insert_req_id.py | 2 +- docs/examples/python/schemaless_insert_ttl.py | 2 +- docs/examples/python/sql_writer.py | 2 +- docs/examples/python/stmt_example.py | 2 +- docs/examples/python/stmt_websocket_example.py | 2 +- docs/examples/python/telnet_line_protocol_example.py | 2 +- docs/examples/python/tmq_assignment_example.py | 2 +- docs/examples/python/tmq_example.py | 2 +- docs/examples/python/tmq_websocket_assgnment_example.py | 2 +- tests/docs-examples-test/python.sh | 2 +- 32 files changed, 37 insertions(+), 37 deletions(-) diff --git a/docs/examples/python/bind_param_example.py b/docs/examples/python/bind_param_example.py index 6a67434f87..e3df9f7d25 100644 --- a/docs/examples/python/bind_param_example.py +++ b/docs/examples/python/bind_param_example.py @@ -20,7 +20,7 @@ def get_ts(ts: str): def create_stable(): conn = taos.connect() try: - conn.execute("CREATE DATABASE power") + conn.execute("CREATE DATABASE power keep 36500") conn.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " "TAGS (location BINARY(64), groupId INT)") finally: diff --git a/docs/examples/python/conn_websocket_pandas.py b/docs/examples/python/conn_websocket_pandas.py index 5cad5384b2..2986aace9f 100644 --- a/docs/examples/python/conn_websocket_pandas.py +++ b/docs/examples/python/conn_websocket_pandas.py @@ -4,7 +4,7 @@ import taos taos_conn = taos.connect() taos_conn.execute('drop database if exists power') -taos_conn.execute('create database if not exists power wal_retention_period 3600') +taos_conn.execute('create database if not exists power wal_retention_period 3600 keep 36500 ') taos_conn.execute("use power") taos_conn.execute( "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") diff --git a/docs/examples/python/connect_rest_examples.py b/docs/examples/python/connect_rest_examples.py index 1c432dcc65..c8a9292547 100644 --- a/docs/examples/python/connect_rest_examples.py +++ b/docs/examples/python/connect_rest_examples.py @@ -11,7 +11,7 @@ conn = connect(url="http://localhost:6041", # create STable cursor = conn.cursor() cursor.execute("DROP DATABASE IF EXISTS power") -cursor.execute("CREATE DATABASE power") +cursor.execute("CREATE DATABASE power keep 36500 ") cursor.execute( "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") diff --git a/docs/examples/python/connect_rest_with_req_id_examples.py b/docs/examples/python/connect_rest_with_req_id_examples.py index f1b5915ea3..568cbea168 100644 --- a/docs/examples/python/connect_rest_with_req_id_examples.py +++ b/docs/examples/python/connect_rest_with_req_id_examples.py @@ -11,7 +11,7 @@ conn = connect(url="http://localhost:6041", # create STable cursor = conn.cursor() cursor.execute("DROP DATABASE IF EXISTS power", req_id=1) -cursor.execute("CREATE DATABASE power", req_id=2) +cursor.execute("CREATE DATABASE power keep 36500", req_id=2) cursor.execute( "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)", req_id=3) diff --git a/docs/examples/python/connect_websocket_examples.py b/docs/examples/python/connect_websocket_examples.py index 29452bbf9d..75e7422a90 100644 --- a/docs/examples/python/connect_websocket_examples.py +++ b/docs/examples/python/connect_websocket_examples.py @@ -6,13 +6,13 @@ conn = taosws.connect("taosws://root:taosdata@localhost:6041") # ANCHOR: basic conn.execute("drop database if exists connwspy") -conn.execute("create database if not exists connwspy wal_retention_period 3600") +conn.execute("create database if not exists connwspy wal_retention_period 3600 keep 36500 ") conn.execute("use connwspy") conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)") conn.execute("create table if not exists tb1 using stb tags (1)") conn.execute("insert into tb1 values (now, 1)") -conn.execute("insert into tb1 values (now, 2)") -conn.execute("insert into tb1 values (now, 3)") +conn.execute("insert into tb1 values (now+1s, 2)") +conn.execute("insert into tb1 values (now+2s, 3)") r = conn.execute("select * from stb") result = conn.query("select * from stb") diff --git a/docs/examples/python/connect_websocket_with_req_id_examples.py b/docs/examples/python/connect_websocket_with_req_id_examples.py index f5f76c8446..3588b8e41f 100644 --- a/docs/examples/python/connect_websocket_with_req_id_examples.py +++ b/docs/examples/python/connect_websocket_with_req_id_examples.py @@ -6,7 +6,7 @@ conn = taosws.connect("taosws://root:taosdata@localhost:6041") # ANCHOR: basic conn.execute("drop database if exists connwspy", req_id=1) -conn.execute("create database if not exists connwspy", req_id=2) +conn.execute("create database if not exists connwspy keep 36500", req_id=2) conn.execute("use connwspy", req_id=3) conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)", req_id=4) conn.execute("create table if not exists tb1 using stb tags (1)", req_id=5) diff --git a/docs/examples/python/connection_usage_native_reference.py b/docs/examples/python/connection_usage_native_reference.py index 0a23c5f95b..3610087e7f 100644 --- a/docs/examples/python/connection_usage_native_reference.py +++ b/docs/examples/python/connection_usage_native_reference.py @@ -4,7 +4,7 @@ import taos conn = taos.connect() # Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement. conn.execute("DROP DATABASE IF EXISTS test") -conn.execute("CREATE DATABASE test") +conn.execute("CREATE DATABASE test keep 36500") # change database. same as execute "USE db" conn.select_db("test") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") diff --git a/docs/examples/python/connection_usage_native_reference_with_req_id.py b/docs/examples/python/connection_usage_native_reference_with_req_id.py index 24d0914ad5..3d568a1e1e 100644 --- a/docs/examples/python/connection_usage_native_reference_with_req_id.py +++ b/docs/examples/python/connection_usage_native_reference_with_req_id.py @@ -4,7 +4,7 @@ import taos conn = taos.connect() # Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement. conn.execute("DROP DATABASE IF EXISTS test", req_id=1) -conn.execute("CREATE DATABASE test", req_id=2) +conn.execute("CREATE DATABASE test keep 36500", req_id=2) # change database. same as execute "USE db" conn.select_db("test") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3) diff --git a/docs/examples/python/cursor_usage_native_reference.py b/docs/examples/python/cursor_usage_native_reference.py index a5103810f0..32ee51354d 100644 --- a/docs/examples/python/cursor_usage_native_reference.py +++ b/docs/examples/python/cursor_usage_native_reference.py @@ -4,7 +4,7 @@ conn = taos.connect() cursor = conn.cursor() cursor.execute("DROP DATABASE IF EXISTS test") -cursor.execute("CREATE DATABASE test") +cursor.execute("CREATE DATABASE test keep 36500") cursor.execute("USE test") cursor.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") diff --git a/docs/examples/python/cursor_usage_native_reference_with_req_id.py b/docs/examples/python/cursor_usage_native_reference_with_req_id.py index 15207ee6bc..345a804923 100644 --- a/docs/examples/python/cursor_usage_native_reference_with_req_id.py +++ b/docs/examples/python/cursor_usage_native_reference_with_req_id.py @@ -4,7 +4,7 @@ conn = taos.connect() cursor = conn.cursor() cursor.execute("DROP DATABASE IF EXISTS test", req_id=1) -cursor.execute("CREATE DATABASE test", req_id=2) +cursor.execute("CREATE DATABASE test keep 36500", req_id=2) cursor.execute("USE test", req_id=3) cursor.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=4) diff --git a/docs/examples/python/fast_write_example.py b/docs/examples/python/fast_write_example.py index 626e3310b1..76e84e97ac 100644 --- a/docs/examples/python/fast_write_example.py +++ b/docs/examples/python/fast_write_example.py @@ -160,7 +160,7 @@ def main(infinity): conn = get_connection() conn.execute("DROP DATABASE IF EXISTS test") - conn.execute("CREATE DATABASE IF NOT EXISTS test") + conn.execute("CREATE DATABASE IF NOT EXISTS test keep 36500") conn.execute("CREATE STABLE IF NOT EXISTS test.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " "TAGS (location BINARY(64), groupId INT)") conn.close() diff --git a/docs/examples/python/json_protocol_example.py b/docs/examples/python/json_protocol_example.py index 58b38f3ff6..a38dcbf0ad 100644 --- a/docs/examples/python/json_protocol_example.py +++ b/docs/examples/python/json_protocol_example.py @@ -16,7 +16,7 @@ def get_connection(): def create_database(conn): - conn.execute("CREATE DATABASE test") + conn.execute("CREATE DATABASE test keep 36500") conn.execute("USE test") diff --git a/docs/examples/python/kafka_example_common.py b/docs/examples/python/kafka_example_common.py index 1c735abfc0..ed0540574f 100644 --- a/docs/examples/python/kafka_example_common.py +++ b/docs/examples/python/kafka_example_common.py @@ -5,7 +5,7 @@ LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanD 'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale', 'California.SantaClara', 'California.Cupertino'] -CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1 wal_retention_period 3600' +CREATE_DATABASE_SQL = 'create database if not exists {} keep 36500 duration 10 buffer 16 wal_level 1 wal_retention_period 3600' USE_DATABASE_SQL = 'use {}' DROP_TABLE_SQL = 'drop table if exists meters' DROP_DATABASE_SQL = 'drop database if exists {}' diff --git a/docs/examples/python/line_protocol_example.py b/docs/examples/python/line_protocol_example.py index 735e8e7eb8..6482032e6e 100644 --- a/docs/examples/python/line_protocol_example.py +++ b/docs/examples/python/line_protocol_example.py @@ -15,7 +15,7 @@ def get_connection(): def create_database(conn): # the default precision is ms (microsecond), but we use us(microsecond) here. - conn.execute("CREATE DATABASE test precision 'us'") + conn.execute("CREATE DATABASE test precision 'us' keep 36500") conn.execute("USE test") diff --git a/docs/examples/python/multi_bind_example.py b/docs/examples/python/multi_bind_example.py index 205ba69fb2..b29e1cd17e 100644 --- a/docs/examples/python/multi_bind_example.py +++ b/docs/examples/python/multi_bind_example.py @@ -71,7 +71,7 @@ def insert_data(): def create_stable(): conn = taos.connect() try: - conn.execute("CREATE DATABASE power") + conn.execute("CREATE DATABASE power keep 36500") conn.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " "TAGS (location BINARY(64), groupId INT)") finally: diff --git a/docs/examples/python/native_insert_example.py b/docs/examples/python/native_insert_example.py index cdde7d23d2..0fba375678 100644 --- a/docs/examples/python/native_insert_example.py +++ b/docs/examples/python/native_insert_example.py @@ -18,7 +18,7 @@ def get_connection() -> taos.TaosConnection: def create_stable(conn: taos.TaosConnection): - conn.execute("CREATE DATABASE power") + conn.execute("CREATE DATABASE power keep 36500") conn.execute("USE power") conn.execute("CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " "TAGS (location BINARY(64), groupId INT)") diff --git a/docs/examples/python/result_set_examples.py b/docs/examples/python/result_set_examples.py index 6cba0d3f73..234c624a4d 100644 --- a/docs/examples/python/result_set_examples.py +++ b/docs/examples/python/result_set_examples.py @@ -2,7 +2,7 @@ import taos conn = taos.connect() conn.execute("DROP DATABASE IF EXISTS test") -conn.execute("CREATE DATABASE test") +conn.execute("CREATE DATABASE test keep 36500") conn.select_db("test") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") # prepare data diff --git a/docs/examples/python/result_set_with_req_id_examples.py b/docs/examples/python/result_set_with_req_id_examples.py index 90ae2f4f26..f46a3697b3 100644 --- a/docs/examples/python/result_set_with_req_id_examples.py +++ b/docs/examples/python/result_set_with_req_id_examples.py @@ -2,7 +2,7 @@ import taos conn = taos.connect() conn.execute("DROP DATABASE IF EXISTS test", req_id=1) -conn.execute("CREATE DATABASE test", req_id=2) +conn.execute("CREATE DATABASE test keep 36500", req_id=2) conn.select_db("test") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3) # prepare data diff --git a/docs/examples/python/schemaless_insert.py b/docs/examples/python/schemaless_insert.py index 334a4b728f..74ab4b15fe 100644 --- a/docs/examples/python/schemaless_insert.py +++ b/docs/examples/python/schemaless_insert.py @@ -3,7 +3,7 @@ import taos conn = taos.connect() dbname = "pytest_line" conn.execute("drop database if exists %s" % dbname) -conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname) conn.select_db(dbname) lines = [ diff --git a/docs/examples/python/schemaless_insert_raw.py b/docs/examples/python/schemaless_insert_raw.py index 0fda7dc505..b5ef5833a6 100644 --- a/docs/examples/python/schemaless_insert_raw.py +++ b/docs/examples/python/schemaless_insert_raw.py @@ -10,9 +10,9 @@ try: conn.execute("drop database if exists %s" % dbname) if taos.IS_V3: - conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname) else: - conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname) conn.select_db(dbname) diff --git a/docs/examples/python/schemaless_insert_raw_req_id.py b/docs/examples/python/schemaless_insert_raw_req_id.py index 606e510986..5488e2ddc0 100644 --- a/docs/examples/python/schemaless_insert_raw_req_id.py +++ b/docs/examples/python/schemaless_insert_raw_req_id.py @@ -10,9 +10,9 @@ try: conn.execute("drop database if exists %s" % dbname) if taos.IS_V3: - conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname) else: - conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname) conn.select_db(dbname) diff --git a/docs/examples/python/schemaless_insert_raw_ttl.py b/docs/examples/python/schemaless_insert_raw_ttl.py index cf57792534..73909e4290 100644 --- a/docs/examples/python/schemaless_insert_raw_ttl.py +++ b/docs/examples/python/schemaless_insert_raw_ttl.py @@ -10,9 +10,9 @@ try: conn.execute("drop database if exists %s" % dbname) if taos.IS_V3: - conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname) else: - conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname) conn.select_db(dbname) diff --git a/docs/examples/python/schemaless_insert_req_id.py b/docs/examples/python/schemaless_insert_req_id.py index ee1472db69..a5091d80a8 100644 --- a/docs/examples/python/schemaless_insert_req_id.py +++ b/docs/examples/python/schemaless_insert_req_id.py @@ -4,7 +4,7 @@ from taos import SmlProtocol, SmlPrecision conn = taos.connect() dbname = "pytest_line" conn.execute("drop database if exists %s" % dbname) -conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname) conn.select_db(dbname) lines = [ diff --git a/docs/examples/python/schemaless_insert_ttl.py b/docs/examples/python/schemaless_insert_ttl.py index 85050439f2..6ad134fae1 100644 --- a/docs/examples/python/schemaless_insert_ttl.py +++ b/docs/examples/python/schemaless_insert_ttl.py @@ -4,7 +4,7 @@ from taos import SmlProtocol, SmlPrecision conn = taos.connect() dbname = "pytest_line" conn.execute("drop database if exists %s" % dbname) -conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname) conn.select_db(dbname) lines = [ diff --git a/docs/examples/python/sql_writer.py b/docs/examples/python/sql_writer.py index 3456981a7b..d62f4c8a8d 100644 --- a/docs/examples/python/sql_writer.py +++ b/docs/examples/python/sql_writer.py @@ -10,7 +10,7 @@ class SQLWriter: self._tb_tags = {} self._conn = get_connection_func() self._max_sql_length = self.get_max_sql_length() - self._conn.execute("create database if not exists test") + self._conn.execute("create database if not exists test keep 36500") self._conn.execute("USE test") def get_max_sql_length(self): diff --git a/docs/examples/python/stmt_example.py b/docs/examples/python/stmt_example.py index 83197a777a..cfdd81c90c 100644 --- a/docs/examples/python/stmt_example.py +++ b/docs/examples/python/stmt_example.py @@ -10,7 +10,7 @@ db_name = 'test_ws_stmt' def before(): taos_conn = taos.connect() taos_conn.execute("drop database if exists %s" % db_name) - taos_conn.execute("create database %s" % db_name) + taos_conn.execute("create database %s keep 36500" % db_name) taos_conn.select_db(db_name) taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))") taos_conn.execute( diff --git a/docs/examples/python/stmt_websocket_example.py b/docs/examples/python/stmt_websocket_example.py index d0824cfa9f..2acab188f3 100644 --- a/docs/examples/python/stmt_websocket_example.py +++ b/docs/examples/python/stmt_websocket_example.py @@ -9,7 +9,7 @@ import taos def before_test(db_name): taos_conn = taos.connect() taos_conn.execute("drop database if exists %s" % db_name) - taos_conn.execute("create database %s" % db_name) + taos_conn.execute("create database %s keep 36500" % db_name) taos_conn.select_db(db_name) taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))") taos_conn.execute( diff --git a/docs/examples/python/telnet_line_protocol_example.py b/docs/examples/python/telnet_line_protocol_example.py index d812e186af..0d524e8d4a 100644 --- a/docs/examples/python/telnet_line_protocol_example.py +++ b/docs/examples/python/telnet_line_protocol_example.py @@ -19,7 +19,7 @@ def get_connection(): def create_database(conn): - conn.execute("CREATE DATABASE test") + conn.execute("CREATE DATABASE test keep 36500") conn.execute("USE test") diff --git a/docs/examples/python/tmq_assignment_example.py b/docs/examples/python/tmq_assignment_example.py index c370db47a5..c063768af4 100644 --- a/docs/examples/python/tmq_assignment_example.py +++ b/docs/examples/python/tmq_assignment_example.py @@ -7,7 +7,7 @@ def prepare(): conn = taos.connect() conn.execute("drop topic if exists tmq_assignment_demo_topic") conn.execute("drop database if exists tmq_assignment_demo_db") - conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600") + conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600 keep 36500") conn.select_db("tmq_assignment_demo_db") conn.execute( "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)") diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py index 5b462fa153..e0bddd9911 100644 --- a/docs/examples/python/tmq_example.py +++ b/docs/examples/python/tmq_example.py @@ -6,7 +6,7 @@ def init_tmq_env(db, topic): conn = taos.connect() conn.execute("drop topic if exists {}".format(topic)) conn.execute("drop database if exists {}".format(db)) - conn.execute("create database if not exists {} wal_retention_period 3600".format(db)) + conn.execute("create database if not exists {} wal_retention_period 3600 keep 36500".format(db)) conn.select_db(db) conn.execute( "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))") diff --git a/docs/examples/python/tmq_websocket_assgnment_example.py b/docs/examples/python/tmq_websocket_assgnment_example.py index a180ef840e..ca50015162 100644 --- a/docs/examples/python/tmq_websocket_assgnment_example.py +++ b/docs/examples/python/tmq_websocket_assgnment_example.py @@ -6,7 +6,7 @@ def prepare(): conn = taos.connect() conn.execute("drop topic if exists tmq_assignment_demo_topic") conn.execute("drop database if exists tmq_assignment_demo_db") - conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600") + conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600 keep 36500") conn.select_db("tmq_assignment_demo_db") conn.execute( "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)") diff --git a/tests/docs-examples-test/python.sh b/tests/docs-examples-test/python.sh index 5de7261e02..84f0771ec5 100644 --- a/tests/docs-examples-test/python.sh +++ b/tests/docs-examples-test/python.sh @@ -86,7 +86,7 @@ pip3 install kafka-python python3 kafka_example_consumer.py # 21 -pip3 install taos-ws-py==0.2.6 +pip3 install taos-ws-py==0.3.1 python3 conn_websocket_pandas.py # 22 From 1ba1fbfabc9403f9ce5d7b111b99689227cd21e2 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 7 Dec 2023 10:45:26 +0800 Subject: [PATCH 091/102] fix: remove allow_forbid_func define --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7d06f5dab6..a92770b1f9 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -12,7 +12,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#define ALLOW_FORBID_FUNC #include "executorInt.h" #include "filter.h" @@ -3228,6 +3227,7 @@ static int32_t tableMergeScanDoSkipTable(STableMergeScanInfo* pInfo, SSDataBlock tSimpleHashPut(pInfo->mTableNumRows, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid), &nRows, sizeof(nRows)); } else { *(int64_t*)pNum = *(int64_t*)pNum + pBlock->info.rows; + nRows = *(int64_t*)pNum; } if (nRows >= pInfo->mergeLimit) { From d3f3ebfe0def31fa04d06bb4287c3dd0f49dae93 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 7 Dec 2023 12:01:59 +0800 Subject: [PATCH 092/102] fix:memory access out of bounds in doGeomFromTextFunc --- source/libs/geometry/src/geomFunc.c | 19 +++++++++++-------- tests/system-test/pytest.sh | 5 +++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/source/libs/geometry/src/geomFunc.c b/source/libs/geometry/src/geomFunc.c index 3588bf8b7d..2ac1761737 100644 --- a/source/libs/geometry/src/geomFunc.c +++ b/source/libs/geometry/src/geomFunc.c @@ -67,15 +67,19 @@ int32_t doGeomFromTextFunc(const char *input, unsigned char **output) { return TSDB_CODE_SUCCESS; } - // make input as a zero ending string - char *end = varDataVal(input) + varDataLen(input); - char endValue = *end; - *end = 0; - + char *inputGeom = NULL; unsigned char *outputGeom = NULL; size_t size = 0; - code = doGeomFromText(varDataVal(input), &outputGeom, &size); + // make a zero ending string + inputGeom = taosMemoryCalloc(1, varDataLen(input) + 1); + if (inputGeom == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(inputGeom, varDataVal(input), varDataLen(input)); + + code = doGeomFromText(inputGeom, &outputGeom, &size); if (code != TSDB_CODE_SUCCESS) { goto _exit; } @@ -92,8 +96,7 @@ int32_t doGeomFromTextFunc(const char *input, unsigned char **output) { _exit: geosFreeBuffer(outputGeom); - - *end = endValue; //recover the input string + geosFreeBuffer(inputGeom); return code; } diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index dfdfc32ed0..2c95516d10 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -79,7 +79,8 @@ else unset LD_PRELOAD #export LD_PRELOAD=libasan.so.5 - export LD_PRELOAD=$(gcc -print-file-name=libasan.so) + #export LD_PRELOAD=$(gcc -print-file-name=libasan.so) + export LD_PRELOAD="$(realpath "$(gcc -print-file-name=libasan.so)") $(realpath "$(gcc -print-file-name=libstdc++.so)")" echo "Preload AsanSo:" $? $* -a 2>$AsanFile @@ -104,4 +105,4 @@ else echo "Execute script failure" exit 1 fi -fi \ No newline at end of file +fi From 07af4e9a08e80935bcdefd5a36ac36bcaf62ebf0 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 7 Dec 2023 12:02:04 +0800 Subject: [PATCH 093/102] adjust --- source/common/src/tvariant.c | 56 +++++++++++++----------------------- 1 file changed, 20 insertions(+), 36 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 5c7567777f..40de0428ff 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -120,42 +120,34 @@ int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { } int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { - if (n == 0) { - *value = 0; - return TSDB_CODE_SUCCESS; - } - errno = 0; char *endPtr = NULL; - bool parsed = false; switch (type) { case TK_NK_INTEGER: { *value = taosStr2Int64(z, &endPtr, 10); - parsed = true; + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; } break; case TK_NK_FLOAT: { double val = round(taosStr2Double(z, &endPtr)); - parsed = true; if (!IS_VALID_INT64(val)) { return TSDB_CODE_FAILED; } + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } *value = val; + return TSDB_CODE_SUCCESS; } break; default: break; } - if (parsed) { - if (errno == ERANGE || errno == EINVAL) { - return TSDB_CODE_FAILED; - } - while (n > 1 && z[n-1] == ' ') { - n--; - } - if (endPtr - z != n) { - return TSDB_CODE_FAILED; - } + if (n == 0) { + *value = 0; return TSDB_CODE_SUCCESS; } @@ -229,14 +221,8 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { } int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { - if (n == 0) { - *value = 0; - return TSDB_CODE_SUCCESS; - } - errno = 0; char *endPtr = NULL; - bool parsed = false; const char *p = z; while (*p == ' ') { p++; @@ -247,30 +233,28 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { if (*p == '-' && *value) { return TSDB_CODE_FAILED; } - parsed = true; + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; } break; case TK_NK_FLOAT: { double val = round(taosStr2Double(p, &endPtr)); if (!IS_VALID_UINT64(val)) { return TSDB_CODE_FAILED; } + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } *value = val; - parsed = true; + return TSDB_CODE_SUCCESS; } break; default: break; } - if (parsed) { - if (errno == ERANGE || errno == EINVAL) { - return TSDB_CODE_FAILED; - } - while (n > 1 && z[n-1] == ' ') { - n--; - } - if (endPtr - z != n) { - return TSDB_CODE_FAILED; - } + if (n == 0) { + *value = 0; return TSDB_CODE_SUCCESS; } From 4737b00879433b5bc7892077ea4a6bf87643c658 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 7 Dec 2023 14:00:50 +0800 Subject: [PATCH 094/102] cancel rm tail space --- source/common/src/tvariant.c | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 40de0428ff..dd5e8240b9 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -109,10 +109,7 @@ int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { if (errno == ERANGE || errno == EINVAL) { return TSDB_CODE_FAILED; } - // rm tail space - while (n > 1 && z[n-1] == ' ') { - n--; - } + if (endPtr - z != n) { return TSDB_CODE_FAILED; } @@ -151,11 +148,6 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { return TSDB_CODE_SUCCESS; } - // rm tail space - while (n > 1 && z[n-1] == ' ') { - n--; - } - // 1. try to parse as integer *value = taosStr2Int64(z, &endPtr, 10); if (endPtr - z == n) { @@ -258,11 +250,6 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { return TSDB_CODE_SUCCESS; } - // rm tail space - while (n > 1 && z[n-1] == ' ') { - n--; - } - // 1. parse as integer *value = taosStr2UInt64(p, &endPtr, 10); if (*p == '-' && *value) { From 26855848f61cd350d7a413fcbceeec8899bb5eba Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 7 Dec 2023 14:07:09 +0800 Subject: [PATCH 095/102] opti:schemaless logic --- source/client/inc/clientSml.h | 85 ++--- source/client/src/clientSml.c | 560 ++++++++++++++++------------ source/client/src/clientSmlJson.c | 416 +++------------------ source/client/src/clientSmlLine.c | 311 +++------------ source/client/src/clientSmlTelnet.c | 204 +++------- 5 files changed, 522 insertions(+), 1054 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 30d4792279..26c0a38c8a 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -65,14 +65,32 @@ extern "C" { #define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) #define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) -//#define TS "_ts" -//#define TS_LEN 3 #define VALUE "_value" -#define VALUE_LEN 6 +#define VALUE_LEN (sizeof(VALUE)-1) #define OTD_JSON_FIELDS_NUM 4 #define MAX_RETRY_TIMES 10 -typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; + +#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0) + +#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0) + +#define IS_SAME_KEY (maxKV->type == kv->type && maxKV->keyLen == kv->keyLen && memcmp(maxKV->key, kv->key, kv->keyLen) == 0) + +#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE)) + +#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) + +#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) { \ + MOVE_FORWARD_ONE(key + i, keyLen - i); \ + keyLen--; \ + } \ + } typedef enum { SCHEMA_ACTION_NULL, @@ -83,18 +101,6 @@ typedef enum { SCHEMA_ACTION_CHANGE_TAG_SIZE, } ESchemaAction; -typedef struct { - const void *key; - int32_t keyLen; - void *value; - bool used; -}Node; - -typedef struct NodeList{ - Node data; - struct NodeList* next; -}NodeList; - typedef struct { char *measure; char *tags; @@ -117,7 +123,6 @@ typedef struct { int32_t sTableNameLen; char childTableName[TSDB_TABLE_NAME_LEN]; uint64_t uid; -// void *key; // for openTsdb SArray *tags; @@ -161,7 +166,8 @@ typedef struct { typedef struct { int64_t id; - SMLProtocolType protocol; + TSDB_SML_PROTOCOL_TYPE protocol; + int8_t precision; bool reRun; bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) @@ -201,29 +207,8 @@ typedef struct { bool needModifySchema; } SSmlHandle; -#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ -&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0) - -#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \ -&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0) - -#define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0) - -#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ - (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE)) - -#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) - -#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \ - for (int i = 1; i < keyLen; ++i) { \ - if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) { \ - MOVE_FORWARD_ONE(key + i, keyLen - i); \ - keyLen--; \ - } \ - } - -extern int64_t smlFactorNS[3]; -extern int64_t smlFactorS[3]; +extern int64_t smlFactorNS[]; +extern int64_t smlFactorS[]; typedef int32_t (*_equal_fn_sml)(const void *, const void *); @@ -231,11 +216,7 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos); void smlDestroyInfo(SSmlHandle *info); int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); -//SArray *smlJsonParseTags(char *start, char *end); bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); -//void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn); -//int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn); -//int nodeListSize(NodeList* list); bool smlDoubleToInt64OverFlow(double num); int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); @@ -253,12 +234,22 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg); uint8_t smlGetTimestampLen(int64_t num); void smlDestroyTableInfo(void *para); -void freeSSmlKv(void* data); +void freeSSmlKv(void* data); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseJSON(SSmlHandle *info, char *payload); - void smlStrReplace(char* src, int32_t len); + +SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement); +bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv); +bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv); +int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements); +int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements); +int32_t smlJoinMeasureTag(SSmlLineInfo *elements); +void smlBuildTsKv(SSmlKv *kv, int64_t ts); +int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); +int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs); + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 6bcaae3bf0..aff8f4f43f 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -20,75 +20,93 @@ #include "clientSml.h" -int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL}; -int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; -int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL}; +#define RETURN_FALSE \ + smlBuildInvalidDataMsg(msg, "invalid data", pVal); \ + return false; -//void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) { -// NodeList *tmp = list; -// while (tmp) { -// if (fn == NULL) { -// if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { -// return tmp->data.value; -// } -// } else { -// if (tmp->data.used && fn(tmp->data.key, key) == 0) { -// return tmp->data.value; -// } -// } -// -// tmp = tmp->next; -// } -// return NULL; -//} -// -//int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) { -// NodeList *tmp = *list; -// while (tmp) { -// if (!tmp->data.used) break; -// if (fn == NULL) { -// if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { -// return -1; -// } -// } else { -// if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) { -// return -1; -// } -// } -// -// tmp = tmp->next; -// } -// if (tmp) { -// tmp->data.key = key; -// tmp->data.keyLen = len; -// tmp->data.value = value; -// tmp->data.used = true; -// } else { -// NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList)); -// if (newNode == NULL) { -// return -1; -// } -// newNode->data.key = key; -// newNode->data.keyLen = len; -// newNode->data.value = value; -// newNode->data.used = true; -// newNode->next = *list; -// *list = newNode; -// } -// return 0; -//} -// -//int nodeListSize(NodeList *list) { -// int cnt = 0; -// while (list) { -// if (list->data.used) -// cnt++; -// else -// break; -// list = list->next; -// } -// return cnt; -//} +#define SET_DOUBLE \ + kvVal->type = TSDB_DATA_TYPE_DOUBLE; \ + kvVal->d = result; + +#define SET_FLOAT \ + if (!IS_VALID_FLOAT(result)) { \ + smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_FLOAT; \ + kvVal->f = (float)result; + +#define SET_BIGINT \ + errno = 0; \ + int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ + if (errno == ERANGE) { \ + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_BIGINT; \ + kvVal->i = tmp; + +#define SET_INT \ + if (!IS_VALID_INT(result)) { \ + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_INT; \ + kvVal->i = result; + +#define SET_SMALL_INT \ + if (!IS_VALID_SMALLINT(result)) { \ + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_SMALLINT; \ + kvVal->i = result; + +#define SET_UBIGINT \ + errno = 0; \ + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ + if (errno == ERANGE || result < 0) { \ + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ + kvVal->u = tmp; + +#define SET_UINT \ + if (!IS_VALID_UINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UINT; \ + kvVal->u = result; + +#define SET_USMALL_INT \ + if (!IS_VALID_USMALLINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_USMALLINT; \ + kvVal->u = result; + +#define SET_TINYINT \ + if (!IS_VALID_TINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_TINYINT; \ + kvVal->i = result; + +#define SET_UTINYINT \ + if (!IS_VALID_UTINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UTINYINT; \ + kvVal->u = result; + +int64_t smlToMilli[] = {3600000LL, 60000LL, 1000LL}; +int64_t smlFactorNS[] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; +int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL}; static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo* conn, const char* pTabName, AUTH_TYPE type){ SUserAuthInfo pAuth = {0}; @@ -114,7 +132,7 @@ inline bool smlDoubleToInt64OverFlow(double num) { return false; } -void smlStrReplace(char* src, int32_t len){ +inline void smlStrReplace(char* src, int32_t len){ if (!tsSmlDot2Underline) return; for(int i = 0; i < len; i++){ if(src[i] == '.'){ @@ -155,7 +173,7 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u return convertTimePrecision(tsInt64, fromPrecision, toPrecision); } -int8_t smlGetTsTypeByLen(int32_t len) { +inline int8_t smlGetTsTypeByLen(int32_t len) { if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { return TSDB_TIME_PRECISION_SECONDS; } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { @@ -180,11 +198,6 @@ SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measu goto cleanup; } - // tag->tags = taosArrayInit(16, sizeof(SSmlKv)); - // if (tag->tags == NULL) { - // uError("SML:smlBuildTableInfo failed to allocate memory"); - // goto cleanup; - // } return tag; cleanup: @@ -192,6 +205,242 @@ cleanup: return NULL; } +void smlBuildTsKv(SSmlKv *kv, int64_t ts){ + kv->key = tsSmlTsDefaultName; + kv->keyLen = strlen(tsSmlTsDefaultName); + kv->type = TSDB_DATA_TYPE_TIMESTAMP; + kv->i = ts; + kv->length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; +} + +SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement){ + SSmlSTableMeta* sMeta = NULL; + char *measure = currElement->measure; + int measureLen = currElement->measureLen; + if (currElement->measureEscaped) { + measure = (char *)taosMemoryMalloc(measureLen); + memcpy(measure, currElement->measure, measureLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + smlStrReplace(measure, measureLen); + } + STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); + if (currElement->measureEscaped) { + taosMemoryFree(measure); + } + if (pTableMeta == NULL) { + info->dataFormat = false; + info->reRun = true; + terrno = TSDB_CODE_SUCCESS; + return sMeta; + } + sMeta = smlBuildSTableMeta(info->dataFormat); + if(sMeta == NULL){ + taosMemoryFreeClear(pTableMeta); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return sMeta; + } + sMeta->tableMeta = pTableMeta; + taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); + for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { + SSchema *col = pTableMeta->schema + i; + SSmlKv kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type}; + if (col->type == TSDB_DATA_TYPE_NCHAR) { + kv.length = (col->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + } else if (col->type == TSDB_DATA_TYPE_BINARY || col->type == TSDB_DATA_TYPE_GEOMETRY || col->type == TSDB_DATA_TYPE_VARBINARY) { + kv.length = col->bytes - VARSTR_HEADER_SIZE; + } else{ + kv.length = col->bytes; + } + + if(i < pTableMeta->tableInfo.numOfColumns){ + taosArrayPush(sMeta->cols, &kv); + }else{ + taosArrayPush(sMeta->tags, &kv); + } + } + return sMeta; +} + +bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv){ + // cnt begin 0, add ts so + 2 + if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { + info->dataFormat = false; + info->reRun = true; + return false; + } + // bind data + int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uDebug("smlBuildCol error, retry"); + info->dataFormat = false; + info->reRun = true; + return false; + } + if (cnt >= taosArrayGetSize(info->maxColKVs)) { + info->dataFormat = false; + info->reRun = true; + return false; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt); + + if (unlikely(!IS_SAME_KEY)) { + info->dataFormat = false; + info->reRun = true; + return false; + } + + if (unlikely(IS_VAR_DATA_TYPE(kv->type) && kv->length > maxKV->length)) { + maxKV->length = kv->length; + info->needModifySchema = true; + } + return true; +} + +bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv){ + if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { + goto END; + } + + if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) { + goto END; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); + + if (unlikely(!IS_SAME_KEY)) { + goto END; + } + + if (unlikely(kv->length > maxKV->length)) { + maxKV->length = kv->length; + info->needModifySchema = true; + } + return true; + +END: + info->dataFormat = false; + info->reRun = true; + return false; +} + +int32_t smlJoinMeasureTag(SSmlLineInfo *elements){ + elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen); + if(elements->measureTag == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(elements->measureTag, elements->measure, elements->measureLen); + memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); + elements->measureTagsLen = elements->measureLen + elements->tagsLen; + return TSDB_CODE_SUCCESS; +} + +int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { + bool isSameMeasure = IS_SAME_SUPER_TABLE; + if(isSameMeasure) { + return 0; + } + SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); + + SSmlSTableMeta *sMeta = NULL; + if (unlikely(tmp == NULL)) { + sMeta = smlBuildSuperTableInfo(info, elements); + if(sMeta == NULL) return -1; + }else{ + sMeta = *tmp; + } + ASSERT(sMeta != NULL); + info->currSTableMeta = sMeta->tableMeta; + info->maxTagKVs = sMeta->tags; + info->maxColKVs = sMeta->cols; + return 0; +} + +int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements){ + SSmlTableInfo **oneTable = + (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); + SSmlTableInfo *tinfo = NULL; + if (unlikely(oneTable == NULL)) { + tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); + if (unlikely(!tinfo)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES); + + tinfo->tags = taosArrayDup(info->preLineTagKV, NULL); + for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i); + if (kv->keyEscaped) kv->key = NULL; + if (kv->valueEscaped) kv->value = NULL; + } + + smlSetCTableName(tinfo); + getTableUid(info, elements, tinfo); + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + smlDestroyTableInfo(&tinfo); + return TSDB_CODE_SML_INVALID_DATA; + } + } + }else{ + tinfo = *oneTable; + } + ASSERT(tinfo != NULL); + if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv){ + if (info->dataFormat) { + uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i); + int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1); + } + if (ret == TSDB_CODE_SUCCESS) { + ret = smlBuildRow(info->currTableDataCtx); + } + clearColValArraySml(info->currTableDataCtx->pValues); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + } else { + uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i); + if (elements->colArray == NULL) { + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, kvTs); + taosArrayPush(elements->colArray, kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs){ + if (info->dataFormat) { + uDebug("SML:0x%" PRIx64 " smlParseEndLine format true, ts:%" PRId64, info->id, kvTs->i); + int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlBuildRow(info->currTableDataCtx); + } + + clearColValArraySml(info->currTableDataCtx->pValues); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + } else { + uDebug("SML:0x%" PRIx64 " smlParseEndLine format false, ts:%" PRId64, info->id, kvTs->i); + taosArraySet(elements->colArray, 0, &kvTs); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} + static int32_t smlParseTableName(SArray *tags, char *childTableName) { bool autoChildName = false; size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); @@ -328,98 +577,6 @@ cleanup: return NULL; } -// uint16_t smlCalTypeSum(char* endptr, int32_t left){ -// uint16_t sum = 0; -// for(int i = 0; i < left; i++){ -// sum += endptr[i]; -// } -// return sum; -// } - -#define RETURN_FALSE \ - smlBuildInvalidDataMsg(msg, "invalid data", pVal); \ - return false; - -#define SET_DOUBLE \ - kvVal->type = TSDB_DATA_TYPE_DOUBLE; \ - kvVal->d = result; - -#define SET_FLOAT \ - if (!IS_VALID_FLOAT(result)) { \ - smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_FLOAT; \ - kvVal->f = (float)result; - -#define SET_BIGINT \ - errno = 0; \ - int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ - if (errno == ERANGE) { \ - smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_BIGINT; \ - kvVal->i = tmp; - -#define SET_INT \ - if (!IS_VALID_INT(result)) { \ - smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_INT; \ - kvVal->i = result; - -#define SET_SMALL_INT \ - if (!IS_VALID_SMALLINT(result)) { \ - smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_SMALLINT; \ - kvVal->i = result; - -#define SET_UBIGINT \ - errno = 0; \ - uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ - if (errno == ERANGE || result < 0) { \ - smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ - kvVal->u = tmp; - -#define SET_UINT \ - if (!IS_VALID_UINT(result)) { \ - smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_UINT; \ - kvVal->u = result; - -#define SET_USMALL_INT \ - if (!IS_VALID_USMALLINT(result)) { \ - smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_USMALLINT; \ - kvVal->u = result; - -#define SET_TINYINT \ - if (!IS_VALID_TINYINT(result)) { \ - smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_TINYINT; \ - kvVal->i = result; - -#define SET_UTINYINT \ - if (!IS_VALID_UTINYINT(result)) { \ - smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_UTINYINT; \ - kvVal->u = result; - bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->length; @@ -765,8 +922,6 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO return TSDB_CODE_SUCCESS; } -// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData, -// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, ESchemaAction action) { SRequestObj *pRequest = NULL; @@ -1121,35 +1276,6 @@ end: return code; } -/* -static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){ - for(int i = 0; i < taosArrayGetSize(tags); i++) { - SSmlKv *tag = taosArrayGet(tags, i); - if (smlCheckDuplicateKey(tag->key, tag->keyLen, dumplicateKey)) { - smlBuildInvalidDataMsg(msg, "dumplicate key", tag->key); - return TSDB_CODE_TSC_DUP_NAMES; - } - } - return TSDB_CODE_SUCCESS; -} - -static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) { - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - int ret = smlCheckDupUnit(dumplicateKey, cols, msg); - if(ret != TSDB_CODE_SUCCESS){ - goto end; - } - ret = smlCheckDupUnit(dumplicateKey, tags, msg); - if(ret != TSDB_CODE_SUCCESS){ - goto end; - } - - end: - taosHashCleanup(dumplicateKey); - return ret; -} -*/ - static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); @@ -1206,11 +1332,6 @@ void smlDestroyTableInfo(void *para) { taosHashCleanup(kvHash); } - // if (info->parseJsonByLib) { - // SSmlLineInfo *key = (SSmlLineInfo *)(tag->key); - // if (key != NULL) taosMemoryFree(key->tags); - // } - // taosMemoryFree(tag->key); taosArrayDestroy(tag->cols); taosArrayDestroyEx(tag->tags, freeSSmlKv); taosMemoryFree(tag); @@ -1228,21 +1349,6 @@ void smlDestroyInfo(SSmlHandle *info) { if (!info) return; qDestroyQuery(info->pQuery); - // destroy info->childTables -// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); -// while (oneTable) { -// smlDestroyTableInfo(oneTable); -// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); -// } - - // destroy info->superTables -// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); -// while (oneSTable) { -// smlDestroySTableMeta(*oneSTable); -// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); -// } - - // destroy info->pVgHash taosHashCleanup(info->pVgHash); taosHashCleanup(info->childTables); taosHashCleanup(info->superTables); @@ -1399,11 +1505,6 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { return ret; } } else { - // ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf); - // if (ret != TSDB_CODE_SUCCESS) { - // uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); - // return ret; - // } uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); @@ -1537,19 +1638,6 @@ static void smlPrintStatisticInfo(SSmlHandle *info) { int32_t smlClearForRerun(SSmlHandle *info) { info->reRun = false; - // clear info->childTables -// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); -// while (oneTable) { -// smlDestroyTableInfo(info, *oneTable); -// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); -// } - - // clear info->superTables -// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); -// while (oneSTable) { -// smlDestroySTableMeta(*oneSTable); -// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); -// } taosHashClear(info->childTables); taosHashClear(info->superTables); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 5e656c71a7..bf53d3c2ab 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -29,199 +29,6 @@ (start)++; \ } -// SArray *smlJsonParseTags(char *start, char *end){ -// SArray *tags = taosArrayInit(4, sizeof(SSmlKv)); -// while(start < end){ -// SSmlKv kv = {0}; -// kv.type = TSDB_DATA_TYPE_NCHAR; -// bool isInQuote = false; -// while(start < end){ -// if(unlikely(!isInQuote && *start == '"')){ -// start++; -// kv.key = start; -// isInQuote = true; -// continue; -// } -// if(unlikely(isInQuote && *start == '"')){ -// kv.keyLen = start - kv.key; -// start++; -// break; -// } -// start++; -// } -// bool hasColon = false; -// while(start < end){ -// if(unlikely(!hasColon && *start == ':')){ -// start++; -// hasColon = true; -// continue; -// } -// if(unlikely(hasColon && kv.value == NULL && (*start > 32 && *start != '"'))){ -// kv.value = start; -// start++; -// continue; -// } -// -// if(unlikely(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}'))){ -// kv.length = start - kv.value; -// taosArrayPush(tags, &kv); -// start++; -// break; -// } -// start++; -// } -// } -// return tags; -// } - -// static int32_t smlParseTagsFromJSON(SSmlHandle *info, SSmlLineInfo *elements) { -// int32_t ret = TSDB_CODE_SUCCESS; -// -// if(is_same_child_table_telnet(elements, &info->preLine) == 0){ -// return TSDB_CODE_SUCCESS; -// } -// -// bool isSameMeasure = IS_SAME_SUPER_TABLE; -// -// int cnt = 0; -// SArray *preLineKV = info->preLineTagKV; -// bool isSuperKVInit = true; -// SArray *superKV = NULL; -// if(info->dataFormat){ -// if(unlikely(!isSameMeasure)){ -// SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, -// elements->measureLen, NULL); -// -// if(unlikely(sMeta == NULL)){ -// sMeta = smlBuildSTableMeta(info->dataFormat); -// STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); -// sMeta->tableMeta = pTableMeta; -// if(pTableMeta == NULL){ -// info->dataFormat = false; -// info->reRun = true; -// return TSDB_CODE_SUCCESS; -// } -// nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); -// } -// info->currSTableMeta = sMeta->tableMeta; -// superKV = sMeta->tags; -// -// if(unlikely(taosArrayGetSize(superKV) == 0)){ -// isSuperKVInit = false; -// } -// taosArraySetSize(preLineKV, 0); -// } -// }else{ -// taosArraySetSize(preLineKV, 0); -// } -// -// SArray *tags = smlJsonParseTags(elements->tags, elements->tags + elements->tagsLen); -// int32_t tagNum = taosArrayGetSize(tags); -// if (tagNum == 0) { -// uError("SML:tag is empty:%s", elements->tags) -// taosArrayDestroy(tags); -// return TSDB_CODE_SML_INVALID_DATA; -// } -// for (int32_t i = 0; i < tagNum; ++i) { -// SSmlKv kv = *(SSmlKv*)taosArrayGet(tags, i); -// -// if(info->dataFormat){ -// if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ -// info->dataFormat = false; -// info->reRun = true; -// taosArrayDestroy(tags); -// return TSDB_CODE_SUCCESS; -// } -// -// if(isSameMeasure){ -// if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { -// info->dataFormat = false; -// info->reRun = true; -// taosArrayDestroy(tags); -// return TSDB_CODE_SUCCESS; -// } -// SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); -// if(unlikely(kv.length > preKV->length)){ -// preKV->length = kv.length; -// SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, -// elements->measureLen, NULL); -// if(unlikely(NULL == tableMeta)){ -// uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); -// return TSDB_CODE_SML_INTERNAL_ERROR; -// } -// -// SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); -// oldKV->length = kv.length; -// info->needModifySchema = true; -// } -// if(unlikely(!IS_SAME_KEY)){ -// info->dataFormat = false; -// info->reRun = true; -// taosArrayDestroy(tags); -// return TSDB_CODE_SUCCESS; -// } -// }else{ -// if(isSuperKVInit){ -// if(unlikely(cnt >= taosArrayGetSize(superKV))) { -// info->dataFormat = false; -// info->reRun = true; -// taosArrayDestroy(tags); -// return TSDB_CODE_SUCCESS; -// } -// SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt); -// if(unlikely(kv.length > preKV->length)) { -// preKV->length = kv.length; -// }else{ -// kv.length = preKV->length; -// } -// info->needModifySchema = true; -// -// if(unlikely(!IS_SAME_KEY)){ -// info->dataFormat = false; -// info->reRun = true; -// taosArrayDestroy(tags); -// return TSDB_CODE_SUCCESS; -// } -// }else{ -// taosArrayPush(superKV, &kv); -// } -// taosArrayPush(preLineKV, &kv); -// } -// }else{ -// taosArrayPush(preLineKV, &kv); -// } -// cnt++; -// } -// taosArrayDestroy(tags); -// -// SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, -// is_same_child_table_telnet); if (unlikely(tinfo == NULL)) { -// tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); -// if (unlikely(!tinfo)) { -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// tinfo->tags = taosArrayDup(preLineKV, NULL); -// -// smlSetCTableName(tinfo); -// if (info->dataFormat) { -// info->currSTableMeta->uid = tinfo->uid; -// tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); -// if (tinfo->tableDataCtx == NULL) { -// smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); -// return TSDB_CODE_SML_INVALID_DATA; -// } -// } -// -// SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); -// *key = *elements; -// tinfo->key = key; -// nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); -// } -// if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; -// -// return ret; -// } - static char *smlJsonGetObj(char *payload) { int leftBracketCnt = 0; bool isInQuote = false; @@ -659,12 +466,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { break; } case cJSON_String: { - /* set default JSON type to binary/nchar according to - * user configured parameter tsDefaultJSONStrType - */ - - char *tsDefaultJSONStrType = "binary"; // todo - smlConvertJSONString(kv, tsDefaultJSONStrType, root); + smlConvertJSONString(kv, "binary", root); break; } case cJSON_Object: { @@ -682,138 +484,71 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { return TSDB_CODE_SUCCESS; } -static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { - int32_t ret = TSDB_CODE_SUCCESS; - - bool isSameMeasure = IS_SAME_SUPER_TABLE; - - int cnt = 0; +static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ SArray *preLineKV = info->preLineTagKV; - if (info->dataFormat) { - if (unlikely(!isSameMeasure)) { - SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); - SSmlSTableMeta *sMeta = NULL; - if (unlikely(tmp == NULL)) { - STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); - if (pTableMeta == NULL) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - sMeta = smlBuildSTableMeta(info->dataFormat); - if(sMeta == NULL){ - taosMemoryFreeClear(pTableMeta); - return TSDB_CODE_OUT_OF_MEMORY; - } - sMeta->tableMeta = pTableMeta; - taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); - for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ - SSchema *tag = pTableMeta->schema + i; - SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE }; - taosArrayPush(sMeta->tags, &kv); - } - tmp = &sMeta; - } - info->currSTableMeta = (*tmp)->tableMeta; - info->maxTagKVs = (*tmp)->tags; - } - } - taosArrayClear(preLineKV); + taosArrayClearEx(preLineKV, freeSSmlKv); + int cnt = 0; int32_t tagNum = cJSON_GetArraySize(tags); if (unlikely(tagNum == 0)) { uError("SML:Tag should not be empty"); - return TSDB_CODE_TSC_INVALID_JSON; + terrno = TSDB_CODE_TSC_INVALID_JSON; + return -1; } for (int32_t i = 0; i < tagNum; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); if (unlikely(tag == NULL)) { - return TSDB_CODE_TSC_INVALID_JSON; + terrno = TSDB_CODE_TSC_INVALID_JSON; + return -1; } - // if(unlikely(tag == cMeasure)) continue; size_t keyLen = strlen(tag->string); if (unlikely(IS_INVALID_COL_LEN(keyLen))) { uError("OTD:Tag key length is 0 or too large than 64"); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + return -1; } // add kv to SSmlKv - SSmlKv kv = {.key = tag->string, .keyLen = keyLen}; - // value - ret = smlParseValueFromJSON(tag, &kv); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - return ret; - } - - if (info->dataFormat) { - if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - - if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - if (unlikely(kv.length > maxKV->length)) { - maxKV->length = kv.length; - info->needModifySchema = true; - } - } + SSmlKv kv = {0}; + kv.key = tag->string; + kv.keyLen = keyLen; taosArrayPush(preLineKV, &kv); + + // value + int32_t ret = smlParseValueFromJSON(tag, &kv); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + terrno = ret; + return -1; + } + + + if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { + terrno = TSDB_CODE_SUCCESS; + return -1; + } + cnt++; } + return 0; +} - elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen); - memcpy(elements->measureTag, elements->measure, elements->measureLen); - memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); - elements->measureTagsLen = elements->measureLen + elements->tagsLen; - - SSmlTableInfo **tmp = - (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen); - SSmlTableInfo *tinfo = NULL; - if (unlikely(tmp == NULL)) { - tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); - if (unlikely(!tinfo)) { - return TSDB_CODE_OUT_OF_MEMORY; +static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { + int32_t ret = 0; + if(info->dataFormat){ + ret = smlProcessSuperTable(info, elements); + if(ret != 0){ + return terrno; } - tinfo->tags = taosArrayDup(preLineKV, NULL); - - smlSetCTableName(tinfo); - getTableUid(info, elements, tinfo); - if (info->dataFormat) { - info->currSTableMeta->uid = tinfo->uid; - tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); - if (tinfo->tableDataCtx == NULL) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(&tinfo); - return TSDB_CODE_SML_INVALID_DATA; - } - } - - // SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); - // *key = *elements; - // if(info->parseJsonByLib){ - // key->tags = taosMemoryMalloc(elements->tagsLen + 1); - // memcpy(key->tags, elements->tags, elements->tagsLen); - // key->tags[elements->tagsLen] = 0; - // } - // tinfo->key = key; - taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo, - POINTER_BYTES); - tmp = &tinfo; } - if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx; - - return ret; + ret = smlProcessTagJson(info, tags); + if(ret != 0){ + return terrno; + } + ret = smlJoinMeasureTag(elements); + if(ret != 0){ + return ret; + } + return smlProcessChildTable(info, elements); } static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) { @@ -998,35 +733,10 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); return TSDB_CODE_INVALID_TIMESTAMP; } - SSmlKv kvTs = {.key = tsSmlTsDefaultName, - .keyLen = strlen(tsSmlTsDefaultName), - .type = TSDB_DATA_TYPE_TIMESTAMP, - .i = ts, - .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + SSmlKv kvTs = {0}; + smlBuildTsKv(&kvTs, ts); - if (info->dataFormat) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); - } - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildRow(info->currTableDataCtx); - } - clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - return ret; - } - } else { - if (elements->colArray == NULL) { - elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); - } - taosArrayPush(elements->colArray, &kvTs); - taosArrayPush(elements->colArray, &kv); - } - info->preLine = *elements; - - return TSDB_CODE_SUCCESS; + return smlParseEndTelnetJson(info, elements, &kvTs, &kv); } static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { @@ -1054,7 +764,6 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { return TSDB_CODE_TSC_INVALID_JSON; } - if (unlikely(info->lines != NULL)) { for (int i = 0; i < info->lineNum; i++) { taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); @@ -1202,35 +911,10 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * return TSDB_CODE_INVALID_TIMESTAMP; } } - SSmlKv kvTs = {.key = tsSmlTsDefaultName, - .keyLen = strlen(tsSmlTsDefaultName), - .type = TSDB_DATA_TYPE_TIMESTAMP, - .i = ts, - .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + SSmlKv kvTs = {0}; + smlBuildTsKv(&kvTs, ts); - if (info->dataFormat) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); - } - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildRow(info->currTableDataCtx); - } - clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - return ret; - } - } else { - if (elements->colArray == NULL) { - elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); - } - taosArrayPush(elements->colArray, &kvTs); - taosArrayPush(elements->colArray, &kv); - } - info->preLine = *elements; - - return TSDB_CODE_SUCCESS; + return smlParseEndTelnetJson(info, elements, &kvTs, &kv); } int32_t smlParseJSON(SSmlHandle *info, char *payload) { diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 7ad43d90c7..0c610a4611 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -20,15 +20,9 @@ #include "clientSml.h" -// comma , #define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) -// space #define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) -// equal = #define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) -// quote " -// #define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) -// SLASH #define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH)) @@ -51,10 +45,10 @@ } \ } -#define BINARY_ADD_LEN 2 // "binary" 2 means " " -#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " +#define BINARY_ADD_LEN (sizeof("\"\"")-1) // "binary" 2 means length of ("") +#define NCHAR_ADD_LEN (sizeof("L\"\"")-1) // L"nchar" 3 means length of (L"") -uint8_t smlPrecisionConvert[7] = {TSDB_TIME_PRECISION_NANO, TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES, +uint8_t smlPrecisionConvert[] = {TSDB_TIME_PRECISION_NANO, TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES, TSDB_TIME_PRECISION_SECONDS, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO}; @@ -198,68 +192,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { return TSDB_CODE_TSC_INVALID_VALUE; } -static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure, - bool isSameCTable) { - if (isSameCTable) { - return TSDB_CODE_SUCCESS; - } - - int cnt = 0; +static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ SArray *preLineKV = info->preLineTagKV; - if (info->dataFormat) { - if (unlikely(!isSameMeasure)) { - SSmlSTableMeta **tmp = - (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); - - SSmlSTableMeta *sMeta = NULL; - if (unlikely(tmp == NULL)) { - char *measure = currElement->measure; - int measureLen = currElement->measureLen; - if (currElement->measureEscaped) { - measure = (char *)taosMemoryMalloc(currElement->measureLen); - memcpy(measure, currElement->measure, currElement->measureLen); - PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); - smlStrReplace(measure, measureLen); - } - STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); - if (currElement->measureEscaped) { - taosMemoryFree(measure); - } - if (pTableMeta == NULL) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - sMeta = smlBuildSTableMeta(info->dataFormat); - if(sMeta == NULL){ - taosMemoryFreeClear(pTableMeta); - return TSDB_CODE_OUT_OF_MEMORY; - } - sMeta->tableMeta = pTableMeta; - taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); - for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { - SSchema *tag = pTableMeta->schema + i; - - if(i < pTableMeta->tableInfo.numOfColumns){ - SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type}; - if (tag->type == TSDB_DATA_TYPE_NCHAR) { - kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; - } else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY || tag->type == TSDB_DATA_TYPE_VARBINARY) { - kv.length = tag->bytes - VARSTR_HEADER_SIZE; - } - taosArrayPush(sMeta->cols, &kv); - }else{ - SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE}; - taosArrayPush(sMeta->tags, &kv); - } - } - tmp = &sMeta; - } - info->currSTableMeta = (*tmp)->tableMeta; - info->maxTagKVs = (*tmp)->tags; - } - } taosArrayClearEx(preLineKV, freeSSmlKv); + int cnt = 0; while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql))) { @@ -274,7 +210,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - return TSDB_CODE_SML_INVALID_DATA; + terrno = TSDB_CODE_SML_INVALID_DATA; + return -1; } if (unlikely(IS_EQUAL(*sql))) { keyLen = *sql - key; @@ -290,7 +227,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + return -1; } // parse value @@ -304,7 +242,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin break; } else if (unlikely(IS_EQUAL(*sql))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - return TSDB_CODE_SML_INVALID_DATA; + terrno = TSDB_CODE_SML_INVALID_DATA; + return -1; } if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { @@ -318,11 +257,13 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin if (unlikely(valueLen == 0)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); - return TSDB_CODE_SML_INVALID_DATA; + terrno = TSDB_CODE_SML_INVALID_DATA; + return -1; } if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + return -1; } if (keyEscaped) { @@ -338,37 +279,17 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin value = tmp; } SSmlKv kv = {.key = key, - .keyLen = keyLen, - .type = TSDB_DATA_TYPE_NCHAR, - .value = value, - .length = valueLen, - .keyEscaped = keyEscaped, - .valueEscaped = valueEscaped}; + .keyLen = keyLen, + .type = TSDB_DATA_TYPE_NCHAR, + .value = value, + .length = valueLen, + .keyEscaped = keyEscaped, + .valueEscaped = valueEscaped}; taosArrayPush(preLineKV, &kv); - if (info->dataFormat) { - if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); - - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - - if (unlikely(kv.length > maxKV->length)) { - maxKV->length = kv.length; - info->needModifySchema = true; - } + if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { + terrno = TSDB_CODE_SUCCESS; + return -1; } cnt++; @@ -377,99 +298,33 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } (*sql)++; } + return 0; +} - void *oneTable = taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen); - if ((oneTable != NULL)) { +static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) { + bool isSameCTable = IS_SAME_CHILD_TABLE; + if(isSameCTable){ return TSDB_CODE_SUCCESS; } - SSmlTableInfo *tinfo = smlBuildTableInfo(1, currElement->measure, currElement->measureLen); - if (unlikely(!tinfo)) { - return TSDB_CODE_OUT_OF_MEMORY; - } - tinfo->tags = taosArrayDup(preLineKV, NULL); - for (size_t i = 0; i < taosArrayGetSize(preLineKV); i++) { - SSmlKv *kv = (SSmlKv *)taosArrayGet(preLineKV, i); - if (kv->keyEscaped) kv->key = NULL; - if (kv->valueEscaped) kv->value = NULL; - } - - smlSetCTableName(tinfo); - getTableUid(info, currElement, tinfo); - if (info->dataFormat) { - info->currSTableMeta->uid = tinfo->uid; - tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); - if (tinfo->tableDataCtx == NULL) { - smlDestroyTableInfo(&tinfo); - smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - return TSDB_CODE_SML_INVALID_DATA; + int32_t ret = 0; + if(info->dataFormat){ + ret = smlProcessSuperTable(info, elements); + if(ret != 0){ + return terrno; } } - taosHashPut(info->childTables, currElement->measure, currElement->measureTagsLen, &tinfo, POINTER_BYTES); + ret = smlProcessTagLine(info, sql, sqlEnd); + if(ret != 0){ + return terrno; + } - return TSDB_CODE_SUCCESS; + return smlProcessChildTable(info, elements); } -static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure, - bool isSameCTable) { +static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) { int cnt = 0; - if (info->dataFormat) { - if (unlikely(!isSameCTable)) { - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen); - if (unlikely(oneTable == NULL)) { - smlBuildInvalidDataMsg(&info->msgBuf, "child table should inside", currElement->measure); - return TSDB_CODE_SML_INVALID_DATA; - } - info->currTableDataCtx = (*oneTable)->tableDataCtx; - } - - if (unlikely(!isSameMeasure)) { - SSmlSTableMeta **tmp = - (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); - if (unlikely(tmp == NULL)) { - char *measure = currElement->measure; - int measureLen = currElement->measureLen; - if (currElement->measureEscaped) { - measure = (char *)taosMemoryMalloc(currElement->measureLen); - memcpy(measure, currElement->measure, currElement->measureLen); - PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); - smlStrReplace(measure, measureLen); - } - STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); - if (currElement->measureEscaped) { - taosMemoryFree(measure); - } - if (pTableMeta == NULL) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - *tmp = smlBuildSTableMeta(info->dataFormat); - if(*tmp == NULL){ - taosMemoryFreeClear(pTableMeta); - return TSDB_CODE_OUT_OF_MEMORY; - } - (*tmp)->tableMeta = pTableMeta; - taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES); - - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema *tag = pTableMeta->schema + i; - SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type}; - if (tag->type == TSDB_DATA_TYPE_NCHAR) { - kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; - } else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY || tag->type == TSDB_DATA_TYPE_VARBINARY) { - kv.length = tag->bytes - VARSTR_HEADER_SIZE; - } - taosArrayPush((*tmp)->cols, &kv); - } - } - info->currSTableMeta = (*tmp)->tableMeta; - info->maxColKVs = (*tmp)->cols; - } - } - while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql))) { break; @@ -569,47 +424,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } if (info->dataFormat) { - // cnt begin 0, add ts so + 2 - if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { - info->dataFormat = false; - info->reRun = true; - freeSSmlKv(&kv); - return TSDB_CODE_SUCCESS; - } - // bind data - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, cnt + 1); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - uDebug("smlBuildCol error, retry"); - info->dataFormat = false; - info->reRun = true; - freeSSmlKv(&kv); - return TSDB_CODE_SUCCESS; - } - if (cnt >= taosArrayGetSize(info->maxColKVs)) { - info->dataFormat = false; - info->reRun = true; - freeSSmlKv(&kv); - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt); - if (kv.type != maxKV->type) { - info->dataFormat = false; - info->reRun = true; - freeSSmlKv(&kv); - return TSDB_CODE_SUCCESS; - } - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - freeSSmlKv(&kv); - return TSDB_CODE_SUCCESS; - } - - if (unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)) { - maxKV->length = kv.length; - info->needModifySchema = true; - } + bool isAligned = isSmlColAligned(info, cnt, &kv); freeSSmlKv(&kv); + if(!isAligned){ + return TSDB_CODE_SUCCESS; + } } else { if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); @@ -665,20 +484,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine tmp++; } elements->measureTagsLen = tmp - elements->measure; - - bool isSameCTable = false; - bool isSameMeasure = false; - if (IS_SAME_CHILD_TABLE) { - isSameCTable = true; - isSameMeasure = true; - } else if (info->dataFormat) { - isSameMeasure = IS_SAME_SUPER_TABLE; - } + elements->measureTag = elements->measure; // parse tag if (*sql == COMMA) sql++; elements->tags = sql; - int ret = smlParseTagKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + int ret = smlParseTagLine(info, &sql, sqlEnd, elements); if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } @@ -693,7 +504,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine JUMP_SPACE(sql, sqlEnd) elements->cols = sql; - ret = smlParseColKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + ret = smlParseColLine(info, &sql, sqlEnd, elements); if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } @@ -724,31 +535,9 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); return TSDB_CODE_INVALID_TIMESTAMP; } - // add ts to - SSmlKv kv = {.key = tsSmlTsDefaultName, - .keyLen = strlen(tsSmlTsDefaultName), - .type = TSDB_DATA_TYPE_TIMESTAMP, - .i = ts, - .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, - .keyEscaped = false, - .valueEscaped = false}; - if (info->dataFormat) { - uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts); - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildRow(info->currTableDataCtx); - } - clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - return ret; - } - } else { - uDebug("SML:0x%" PRIx64 " smlParseInfluxString format false, ts:%" PRId64, info->id, ts); - taosArraySet(elements->colArray, 0, &kv); - } - info->preLine = *elements; + SSmlKv kvTs = {0}; + smlBuildTsKv(&kvTs, ts); - return ret; + return smlParseEndLine(info, elements, &kvTs); } diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index f9f8602c5a..f715f32556 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -23,8 +23,6 @@ int32_t is_same_child_table_telnet(const void *a, const void *b) { SSmlLineInfo *t1 = (SSmlLineInfo *)a; SSmlLineInfo *t2 = (SSmlLineInfo *)b; - // uError("is_same_child_table_telnet len:%d,%d %s,%s @@@ len:%d,%d %s,%s", t1->measureLen, t2->measureLen, - // t1->measure, t2->measure, t1->tagsLen, t2->tagsLen, t1->tags, t2->tags); if (t1 == NULL || t2 == NULL || t1->measure == NULL || t2->measure == NULL || t1->tags == NULL || t2->tags == NULL) return 1; return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) && @@ -69,47 +67,11 @@ static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t } } -static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { - if (is_same_child_table_telnet(elements, &info->preLine) == 0) { - elements->measureTag = info->preLine.measureTag; - return TSDB_CODE_SUCCESS; - } - - bool isSameMeasure = IS_SAME_SUPER_TABLE; - - int cnt = 0; +static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ SArray *preLineKV = info->preLineTagKV; - if (info->dataFormat) { - if (!isSameMeasure) { - SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); - SSmlSTableMeta *sMeta = NULL; - if (unlikely(tmp == NULL)) { - STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); - if (pTableMeta == NULL) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - sMeta = smlBuildSTableMeta(info->dataFormat); - if(sMeta == NULL){ - taosMemoryFreeClear(pTableMeta); - return TSDB_CODE_OUT_OF_MEMORY; - } - sMeta->tableMeta = pTableMeta; - taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); - for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ - SSchema *tag = pTableMeta->schema + i; - SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE }; - taosArrayPush(sMeta->tags, &kv); - } - tmp = &sMeta; - } - info->currSTableMeta = (*tmp)->tableMeta; - info->maxTagKVs = (*tmp)->tags; - } - } + taosArrayClearEx(preLineKV, freeSSmlKv); + int cnt = 0; - taosArrayClear(preLineKV); const char *sql = data; while (sql < sqlEnd) { JUMP_SPACE(sql, sqlEnd) @@ -121,8 +83,9 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS // parse key while (sql < sqlEnd) { if (unlikely(*sql == SPACE)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); + terrno = TSDB_CODE_SML_INVALID_DATA; + return -1; } if (unlikely(*sql == EQUAL)) { keyLen = sql - key; @@ -133,13 +96,10 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS } if (unlikely(IS_INVALID_COL_LEN(keyLen))) { - smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + return -1; } - // if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { - // smlBuildInvalidDataMsg(msg, "dumplicate key", key); - // return TSDB_CODE_TSC_DUP_NAMES; - // } // parse value const char *value = sql; @@ -150,87 +110,67 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS break; } if (unlikely(*sql == EQUAL)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); + terrno = TSDB_CODE_SML_INVALID_DATA; + return -1; } sql++; } valueLen = sql - value; if (unlikely(valueLen == 0)) { - smlBuildInvalidDataMsg(msg, "invalid value", value); - return TSDB_CODE_TSC_INVALID_VALUE; + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + terrno = TSDB_CODE_TSC_INVALID_VALUE; + return -1; } if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + return -1; } - SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; - - if (info->dataFormat) { - if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - if (unlikely(kv.length > maxKV->length)) { - maxKV->length = kv.length; - info->needModifySchema = true; - } - } + SSmlKv kv = {.key = key, + .keyLen = keyLen, + .type = TSDB_DATA_TYPE_NCHAR, + .value = value, + .length = valueLen, + .keyEscaped = false, + .valueEscaped = false}; taosArrayPush(preLineKV, &kv); + if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { + terrno = TSDB_CODE_SUCCESS; + return -1; + } cnt++; } + return 0; +} - elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen); - memcpy(elements->measureTag, elements->measure, elements->measureLen); - memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); - elements->measureTagsLen = elements->measureLen + elements->tagsLen; - - SSmlTableInfo **tmp = - (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen); - SSmlTableInfo *tinfo = NULL; - if (unlikely(tmp == NULL)) { - tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); - if (!tinfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - tinfo->tags = taosArrayDup(preLineKV, NULL); - - smlSetCTableName(tinfo); - getTableUid(info, elements, tinfo); - if (info->dataFormat) { - info->currSTableMeta->uid = tinfo->uid; - tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); - if (tinfo->tableDataCtx == NULL) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(&tinfo); - return TSDB_CODE_SML_INVALID_DATA; - } - } - - // SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); - // *key = *elements; - // tinfo->key = key; - taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo, - POINTER_BYTES); - tmp = &tinfo; +static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) { + if (is_same_child_table_telnet(elements, &info->preLine) == 0) { + elements->measureTag = info->preLine.measureTag; + return TSDB_CODE_SUCCESS; } - if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx; - return TSDB_CODE_SUCCESS; + int32_t ret = 0; + if(info->dataFormat){ + ret = smlProcessSuperTable(info, elements); + if(ret != 0){ + return terrno; + } + } + + ret = smlProcessTagTelnet(info, data, sqlEnd); + if(ret != 0){ + return terrno; + } + + ret = smlJoinMeasureTag(elements); + if(ret != 0){ + return ret; + } + + return smlProcessChildTable(info, elements); } // format: =[ =] @@ -251,7 +191,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine return TSDB_CODE_SML_INVALID_DATA; } - bool needConverTime = false; // get TS before parse tag(get meta), so need conver time + bool needConverTime = false; // get TS before parse tag(get meta), so need convert time if (info->dataFormat && info->currSTableMeta == NULL) { needConverTime = true; } @@ -260,11 +200,6 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); return TSDB_CODE_INVALID_TIMESTAMP; } - SSmlKv kvTs = {.key = tsSmlTsDefaultName, - .keyLen = strlen(tsSmlTsDefaultName), - .type = TSDB_DATA_TYPE_TIMESTAMP, - .i = ts, - .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; // parse value smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen); @@ -287,7 +222,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine return TSDB_CODE_TSC_INVALID_VALUE; } - int ret = smlParseTelnetTags(info, sql, sqlEnd, elements, &info->msgBuf); + int ret = smlParseTelnetTags(info, sql, sqlEnd, elements); if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } @@ -296,30 +231,11 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine return TSDB_CODE_SUCCESS; } - if (info->dataFormat && info->currSTableMeta != NULL) { - if (needConverTime) { - kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); - } - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); - } - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildRow(info->currTableDataCtx); - } - clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - return ret; - } - } else { - if (elements->colArray == NULL) { - elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); - } - taosArrayPush(elements->colArray, &kvTs); - taosArrayPush(elements->colArray, &kv); + SSmlKv kvTs = {0}; + smlBuildTsKv(&kvTs, ts); + if (needConverTime) { + kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); } - info->preLine = *elements; - return TSDB_CODE_SUCCESS; + return smlParseEndTelnetJson(info, elements, &kvTs, &kv); } \ No newline at end of file From 8488f25d204d2a35a458d939672921326381fd0a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 7 Dec 2023 15:01:38 +0800 Subject: [PATCH 096/102] fix: add write_raw_block api check ret error message --- .../7-tmq/raw_block_interface_test.py | 16 ++- utils/test/c/write_raw_block_test.c | 114 +++++++++++++++--- 2 files changed, 114 insertions(+), 16 deletions(-) diff --git a/tests/system-test/7-tmq/raw_block_interface_test.py b/tests/system-test/7-tmq/raw_block_interface_test.py index 1e89de1cce..1c9798421d 100644 --- a/tests/system-test/7-tmq/raw_block_interface_test.py +++ b/tests/system-test/7-tmq/raw_block_interface_test.py @@ -36,7 +36,21 @@ class TDTestCase: buildPath = tdCom.getBuildPath() cmdStr = '%s/build/bin/write_raw_block_test'%(buildPath) tdLog.info(cmdStr) - os.system(cmdStr) + retCode = os.system(cmdStr) + # run program code from system return , 0 is success + runCode = retCode & 0xFF + # program retur code from main function + progCode = retCode >> 8 + + tdLog.info(f"{cmdStr} ret={retCode} runCode={runCode} progCode={progCode}") + + if runCode != 0: + tdLog.exit(f"run {cmdStr} failed, have system error.") + return + + if progCode != 0: + tdLog.exit(f"{cmdStr} found problem, return code = {progCode}.") + return self.checkData() diff --git a/utils/test/c/write_raw_block_test.c b/utils/test/c/write_raw_block_test.c index ee2594af7a..b2b56c19dd 100644 --- a/utils/test/c/write_raw_block_test.c +++ b/utils/test/c/write_raw_block_test.c @@ -19,8 +19,8 @@ #include "taos.h" #include "types.h" -int buildStable(TAOS* pConn, TAOS_RES* pRes) { - pRes = taos_query(pConn, +int buildStable(TAOS* pConn) { + TAOS_RES* pRes = taos_query(pConn, "CREATE STABLE `meters` (`ts` TIMESTAMP, `current` INT, `voltage` INT, `phase` FLOAT) TAGS " "(`groupid` INT, `location` VARCHAR(16))"); if (taos_errno(pRes) != 0) { @@ -57,6 +57,34 @@ int buildStable(TAOS* pConn, TAOS_RES* pRes) { } taos_free_result(pRes); + pRes = taos_query(pConn, "create table ntba(ts timestamp, addr binary(32))"); + if (taos_errno(pRes) != 0) { + printf("failed to create ntba, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table ntbb(ts timestamp, addr binary(8))"); + if (taos_errno(pRes) != 0) { + printf("failed to create ntbb, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ntba values(now,'123456789abcdefg123456789')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert table ntba, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ntba values(now,'hello')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert table ntba, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + return 0; } @@ -65,40 +93,43 @@ int32_t init_env() { if (pConn == NULL) { return -1; } + int32_t ret = -1; TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_raw"); if (taos_errno(pRes) != 0) { printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; + goto END; } taos_free_result(pRes); pRes = taos_query(pConn, "create database if not exists db_raw vgroups 2"); if (taos_errno(pRes) != 0) { printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; + goto END; } taos_free_result(pRes); pRes = taos_query(pConn, "use db_raw"); if (taos_errno(pRes) != 0) { printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; + goto END; } taos_free_result(pRes); - buildStable(pConn, pRes); + buildStable(pConn); pRes = taos_query(pConn, "select * from d0"); if (taos_errno(pRes) != 0) { printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; + goto END; } void *data = NULL; int32_t numOfRows = 0; int error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); - ASSERT(error_code == 0); - ASSERT(numOfRows == 1); + if(error_code !=0 ){ + printf("error fetch raw block, reason:%s\n", taos_errstr(pRes)); + goto END; + } taos_write_raw_block(pConn, numOfRows, data, "d1"); taos_free_result(pRes); @@ -106,23 +137,76 @@ int32_t init_env() { pRes = taos_query(pConn, "select ts,phase from d0"); if (taos_errno(pRes) != 0) { printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; + goto END; } error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); - ASSERT(error_code == 0); - ASSERT(numOfRows == 1); + if(error_code !=0 ){ + printf("error fetch raw block, reason:%s\n", taos_errstr(pRes)); + goto END; + } int numFields = taos_num_fields(pRes); TAOS_FIELD *fields = taos_fetch_fields(pRes); taos_write_raw_block_with_fields(pConn, numOfRows, data, "d2", fields, numFields); taos_free_result(pRes); - taos_close(pConn); - return 0; + // check error msg + pRes = taos_query(pConn, "select * from ntba"); + if (taos_errno(pRes) != 0) { + printf("error select * from ntba, reason:%s\n", taos_errstr(pRes)); + goto END; + } + + data = NULL; + numOfRows = 0; + error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); + if(error_code !=0 ){ + printf("error fetch select * from ntba, reason:%s\n", taos_errstr(pRes)); + goto END; + } + error_code = taos_write_raw_block(pConn, numOfRows, data, "ntbb"); + if(error_code == 0) { + printf(" taos_write_raw_block to ntbb expect failed , but success!\n"); + goto END; + } + + // pass NULL return last error code describe + const char* err = taos_errstr(NULL); + printf("write_raw_block return code =0x%x err=%s\n", error_code, err); + if(strcmp(err, "success") == 0) { + printf("expect failed , but error string is success! err=%s\n", err); + goto END; + } + + // no exist table + error_code = taos_write_raw_block(pConn, numOfRows, data, "no-exist-table"); + if(error_code == 0) { + printf(" taos_write_raw_block to no-exist-table expect failed , but success!\n"); + goto END; + } + + err = taos_errstr(NULL); + printf("write_raw_block no exist table return code =0x%x err=%s\n", error_code, err); + if(strcmp(err, "success") == 0) { + printf("expect failed write no exist table, but error string is success! err=%s\n", err); + goto END; + } + + // success + ret = 0; + +END: + // free + if(pRes) taos_free_result(pRes); + if(pConn) taos_close(pConn); + return ret; } int main(int argc, char* argv[]) { + printf("test write_raw_block...\n"); if (init_env() < 0) { + printf("test write_raw_block failed.\n"); return -1; } -} + printf("test write_raw_block ok.\n"); +} \ No newline at end of file From 5143def1778349423cb34dccae4c106aef6b0fce Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 7 Dec 2023 15:05:56 +0800 Subject: [PATCH 097/102] fix: add write_raw_block api check ret error message --- utils/test/c/write_raw_block_test.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils/test/c/write_raw_block_test.c b/utils/test/c/write_raw_block_test.c index b2b56c19dd..162ecd229c 100644 --- a/utils/test/c/write_raw_block_test.c +++ b/utils/test/c/write_raw_block_test.c @@ -204,9 +204,11 @@ END: int main(int argc, char* argv[]) { printf("test write_raw_block...\n"); - if (init_env() < 0) { + int ret = init_env(); + if (ret < 0) { printf("test write_raw_block failed.\n"); - return -1; + return ret; } printf("test write_raw_block ok.\n"); + return 0; } \ No newline at end of file From 60500f4a81e3f41f086cf77a6a0cb2fc811d5fa8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 7 Dec 2023 15:17:15 +0800 Subject: [PATCH 098/102] fix:error --- source/client/src/clientSml.c | 8 ++- source/client/src/clientSmlJson.c | 3 +- tests/system-test/2-query/sml.py | 4 +- utils/test/c/sml_test.c | 101 ++++++++++++++++++++++++++++-- 4 files changed, 105 insertions(+), 11 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index aff8f4f43f..a066bd8014 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -422,7 +422,7 @@ int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv * int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs){ if (info->dataFormat) { uDebug("SML:0x%" PRIx64 " smlParseEndLine format true, ts:%" PRId64, info->id, kvTs->i); - int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); if (ret == TSDB_CODE_SUCCESS) { ret = smlBuildRow(info->currTableDataCtx); } @@ -434,7 +434,7 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs){ } } else { uDebug("SML:0x%" PRIx64 " smlParseEndLine format false, ts:%" PRId64, info->id, kvTs->i); - taosArraySet(elements->colArray, 0, &kvTs); + taosArraySet(elements->colArray, 0, kvTs); } info->preLine = *elements; @@ -1374,7 +1374,9 @@ void smlDestroyInfo(SSmlHandle *info) { if (info->parseJsonByLib) { taosMemoryFree(info->lines[i].tags); } - if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag); + if (info->lines[i].measureTagsLen != 0 && info->protocol != TSDB_SML_LINE_PROTOCOL) { + taosMemoryFree(info->lines[i].measureTag); + } } taosMemoryFree(info->lines); } diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index bf53d3c2ab..845884d5ac 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -512,7 +512,6 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ SSmlKv kv = {0}; kv.key = tag->string; kv.keyLen = keyLen; - taosArrayPush(preLineKV, &kv); // value int32_t ret = smlParseValueFromJSON(tag, &kv); @@ -520,7 +519,7 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ terrno = ret; return -1; } - + taosArrayPush(preLineKV, &kv); if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { terrno = TSDB_CODE_SUCCESS; diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index 0369f45723..e28f3b1edd 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -80,12 +80,12 @@ class TDTestCase: tdSql.checkData(0, 1, 13.000000000) tdSql.checkData(0, 2, "web01") tdSql.checkData(0, 3, None) - tdSql.checkData(0, 4, "lga") + tdSql.checkData(0, 4, 1) tdSql.checkData(1, 1, 9.000000000) tdSql.checkData(1, 2, "web02") tdSql.checkData(3, 3, "t1") - tdSql.checkData(0, 4, "lga") + tdSql.checkData(2, 4, 4) tdSql.query(f"select * from {dbname}.macylr") tdSql.checkRows(2) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 9153706d23..2c334eb67b 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -116,8 +116,7 @@ int smlProcess_json1_Test() { const char *sql[] = { "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":" - "\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":" - "\"lga\"}}]"}; + "34}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}}]"}; char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { @@ -142,8 +141,8 @@ int smlProcess_json1_Test() { const char *sql2[] = { - "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":\"lga\"}" - "},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}" + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":1}" + "},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}" "}]", }; @@ -270,6 +269,98 @@ int smlProcess_json3_Test() { return code; } +int smlProcess_json_tag_not_same_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "[{\"metric\":\"jstable\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":" + "\"lga\"}},{\"metric\":\"jstable\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":" + "\"lga\"}}]"}; + + char *sql1[1] = {0}; + for (int i = 0; i < 1; i++) { + sql1[i] = taosMemoryCalloc(1, 1024); + strncpy(sql1[i], sql[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + int code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql1[i]); + } + ASSERT(code == 0); + + + const char *sql2[] = { + "[{\"metric\":\"jstable\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":6,\"dc\":\"lga\"}}]", + }; + + char *sql3[1] = {0}; + for (int i = 0; i < 1; i++) { + sql3[i] = taosMemoryCalloc(1, 1024); + strncpy(sql3[i], sql2[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql3[i]); + } + + ASSERT(code != 0); + + const char *sql4[] = { + "[{\"metric\":\"jstable\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":6,\"dc\":\"lga\"}}," + "{\"metric\":\"jstable\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":false,\"dc\":\"lga\"}}]", + }; + char *sql5[1] = {0}; + for (int i = 0; i < 1; i++) { + sql5[i] = taosMemoryCalloc(1, 1024); + strncpy(sql5[i], sql4[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql5[i]); + } + ASSERT(code != 0); + + taos_close(taos); + + return code; +} + int sml_TD15662_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1729,6 +1820,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_19221_Test(); ASSERT(!ret); + ret = smlProcess_json_tag_not_same_Test(); + ASSERT(ret); ret = sml_ts3724_Test(); ASSERT(!ret); From f23171d980e798468d97e4bf8152ccccc0f33341 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 7 Dec 2023 16:06:09 +0800 Subject: [PATCH 099/102] adjust case --- tests/system-test/1-insert/insert_double.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py index 56861f3cd4..b47b22ab44 100644 --- a/tests/system-test/1-insert/insert_double.py +++ b/tests/system-test/1-insert/insert_double.py @@ -40,8 +40,8 @@ class TDTestCase: # str support tdSql.execute(f"insert into {table_name} values(now, '-16', '+6')") - tdSql.execute(f"insert into {table_name} values(now, ' -80.99 ', ' +0042 ')") - tdSql.execute(f"insert into {table_name} values(now, ' -0042 ', ' +80.99 ')") + tdSql.execute(f"insert into {table_name} values(now, ' -80.99', ' +0042')") + tdSql.execute(f"insert into {table_name} values(now, ' -0042', ' +80.99')") tdSql.execute(f"insert into {table_name} values(now, '52.34354', '18.6')") tdSql.execute(f"insert into {table_name} values(now, '-12.', '+5.')") tdSql.execute(f"insert into {table_name} values(now, '-.12', '+.5')") From ff23e9afae6d0dce3d78197cc519c2247f4239c6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 7 Dec 2023 17:03:48 +0800 Subject: [PATCH 100/102] fix:inline error --- source/client/inc/clientSml.h | 27 ++++++++++++++++++++++++--- source/client/src/clientSml.c | 23 ----------------------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 26c0a38c8a..9ae28dd55e 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -217,11 +217,9 @@ void smlDestroyInfo(SSmlHandle *info); int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); -bool smlDoubleToInt64OverFlow(double num); int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); -int8_t smlGetTsTypeByLen(int32_t len); SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen); SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat); int32_t smlSetCTableName(SSmlTableInfo *oneTable); @@ -238,7 +236,6 @@ void freeSSmlKv(void* data); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseJSON(SSmlHandle *info, char *payload); -void smlStrReplace(char* src, int32_t len); SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement); bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv); @@ -250,6 +247,30 @@ void smlBuildTsKv(SSmlKv *kv, int64_t ts); int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs); +static inline bool smlDoubleToInt64OverFlow(double num) { + if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; + return false; +} + +static inline void smlStrReplace(char* src, int32_t len){ + if (!tsSmlDot2Underline) return; + for(int i = 0; i < len; i++){ + if(src[i] == '.'){ + src[i] = '_'; + } + } +} + +static inline int8_t smlGetTsTypeByLen(int32_t len) { + if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { + return TSDB_TIME_PRECISION_SECONDS; + } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { + return TSDB_TIME_PRECISION_MILLI; + } else { + return -1; + } +} + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index a066bd8014..67b23792ad 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -127,19 +127,6 @@ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo* conn, const cha return (code == TSDB_CODE_SUCCESS) ? (authRes.pass[AUTH_RES_BASIC] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code; } -inline bool smlDoubleToInt64OverFlow(double num) { - if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; - return false; -} - -inline void smlStrReplace(char* src, int32_t len){ - if (!tsSmlDot2Underline) return; - for(int i = 0; i < len; i++){ - if(src[i] == '.'){ - src[i] = '_'; - } - } -} int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { if (pBuf->buf) { @@ -173,16 +160,6 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u return convertTimePrecision(tsInt64, fromPrecision, toPrecision); } -inline int8_t smlGetTsTypeByLen(int32_t len) { - if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { - return TSDB_TIME_PRECISION_SECONDS; - } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { - return TSDB_TIME_PRECISION_MILLI; - } else { - return -1; - } -} - SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) { SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); if (!tag) { From 4cc47a2edf7fc5976e5546a43293932f861c525b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 7 Dec 2023 17:19:10 +0800 Subject: [PATCH 101/102] fix:test case error --- tests/system-test/2-query/sml_TS-3724.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/sml_TS-3724.py b/tests/system-test/2-query/sml_TS-3724.py index 410e266f10..b537ad9b9a 100644 --- a/tests/system-test/2-query/sml_TS-3724.py +++ b/tests/system-test/2-query/sml_TS-3724.py @@ -85,12 +85,12 @@ class TDTestCase: tdSql.checkData(0, 1, 13.000000000) tdSql.checkData(0, 2, "web01") tdSql.checkData(0, 3, None) - tdSql.checkData(0, 4, "lga") + tdSql.checkData(0, 4, 1) tdSql.checkData(1, 1, 9.000000000) tdSql.checkData(1, 2, "web02") tdSql.checkData(3, 3, "t1") - tdSql.checkData(0, 4, "lga") + tdSql.checkData(2, 4, 4) tdSql.query(f"select * from {dbname}.macylr") tdSql.checkRows(2) From bd3fd91a50c591d6583dba7eac7a86ddb1ac7c0f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 7 Dec 2023 19:24:19 +0800 Subject: [PATCH 102/102] fix: too long multiple command line --- tools/shell/src/shellEngine.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 115abdcd36..e8a5b04178 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1091,7 +1091,12 @@ void shellSourceFile(const char *file) { char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1); while ((read_len = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) { - if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; + if ( cmd_len + read_len >= TSDB_MAX_ALLOWED_SQL_LEN) { + printf("read command line too long over 1M, ignore this line. cmd_len = %d read_len=%d \n", (int32_t)cmd_len, read_len); + cmd_len = 0; + memset(line, 0, TSDB_MAX_ALLOWED_SQL_LEN + 1); + continue; + } line[--read_len] = '\0'; if (read_len == 0 || shellIsCommentLine(line)) { // line starts with #