fix(query): set the last key of each tablescan info

This commit is contained in:
Haojun Liao 2022-11-03 19:00:13 +08:00
parent 7a14bc05ea
commit 4959529127
3 changed files with 340 additions and 324 deletions

View File

@ -185,11 +185,11 @@ static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STabl
SRowMerger* pMerger, SVersionRange* pVerRange); SRowMerger* pMerger, SVersionRange* pVerRange);
static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger,
STsdbReader* pReader); STsdbReader* pReader);
static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid); static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, STableBlockScanInfo* pInfo);
static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData,
int32_t rowIndex); int32_t rowIndex);
static void setComposedBlockFlag(STsdbReader* pReader, bool composed); static void setComposedBlockFlag(STsdbReader* pReader, bool composed);
static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, SVersionRange* pRange); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, SVersionRange* pVerRange);
static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList,
STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow); STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow);
@ -208,7 +208,6 @@ static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader);
static int32_t doBuildDataBlock(STsdbReader* pReader); static int32_t doBuildDataBlock(STsdbReader* pReader);
static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader);
static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo); static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo);
static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader);
static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
@ -1529,8 +1528,8 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB
// opt version // opt version
// 1. it is not a border point // 1. it is not a border point
// 2. the direct next point is not an duplicated timestamp // 2. the direct next point is not an duplicated timestamp
if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && pReader->order == TSDB_ORDER_ASC) || bool asc = (pReader->order == TSDB_ORDER_ASC);
(pDumpInfo->rowIndex > 0 && pReader->order == TSDB_ORDER_DESC)) { if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) {
int32_t step = pReader->order == TSDB_ORDER_ASC ? 1 : -1; int32_t step = pReader->order == TSDB_ORDER_ASC ? 1 : -1;
int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step]; int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step];
@ -1749,7 +1748,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
return code; return code;
} }
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
taosMemoryFree(pTSRow); taosMemoryFree(pTSRow);
tRowMergerClear(&merge); tRowMergerClear(&merge);
@ -1770,6 +1769,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
// only last block exists // only last block exists
if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) {
if (tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader)) { if (tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader)) {
pBlockScanInfo->lastKey = tsLastBlock;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else { } else {
int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema);
@ -1786,7 +1786,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
return code; return code;
} }
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
taosMemoryFree(pTSRow); taosMemoryFree(pTSRow);
tRowMergerClear(&merge); tRowMergerClear(&merge);
@ -1810,7 +1810,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
return code; return code;
} }
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
taosMemoryFree(pTSRow); taosMemoryFree(pTSRow);
tRowMergerClear(&merge); tRowMergerClear(&merge);
@ -1858,7 +1858,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader
return code; return code;
} }
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
taosMemoryFree(pTSRow); taosMemoryFree(pTSRow);
tRowMergerClear(&merge); tRowMergerClear(&merge);
@ -2082,7 +2082,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
return code; return code;
} }
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
taosMemoryFree(pTSRow); taosMemoryFree(pTSRow);
tRowMergerClear(&merge); tRowMergerClear(&merge);
@ -2233,6 +2233,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc
STsdbReader* pReader) { STsdbReader* pReader) {
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) {
pBlockScanInfo->lastKey = key;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else { } else {
TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
@ -2251,7 +2252,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc
return code; return code;
} }
doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
taosMemoryFree(pTSRow); taosMemoryFree(pTSRow);
tRowMergerClear(&merge); tRowMergerClear(&merge);
@ -2299,29 +2300,32 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader;
bool asc = ASCENDING_TRAVERSE(pReader->order);
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
int32_t step = asc ? 1 : -1;
STableBlockScanInfo* pBlockScanInfo = NULL; STableBlockScanInfo* pBlockScanInfo = NULL;
if (pBlockInfo != NULL) { if (pBlockInfo != NULL) {
pBlockScanInfo = *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); void* p = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
if (pBlockScanInfo == NULL) { if (p == NULL) {
code = TSDB_CODE_INVALID_PARA; code = TSDB_CODE_INVALID_PARA;
tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid, tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid,
taosHashGetSize(pReader->status.pTableMap), pReader->idStr); taosHashGetSize(pReader->status.pTableMap), pReader->idStr);
goto _end; goto _end;
} }
pBlockScanInfo = *(STableBlockScanInfo**) p;
SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader);
// it is a clean block, load it directly // it is a clean block, load it directly
if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader)) { if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader)) {
if (pReader->order == TSDB_ORDER_ASC || if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) {
(pReader->order == TSDB_ORDER_DESC && (!hasDataInLastBlock(pLastBlockReader)))) {
copyBlockDataToSDataBlock(pReader, pBlockScanInfo); copyBlockDataToSDataBlock(pReader, pBlockScanInfo);
// record the last key value // record the last key value
pBlockScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->order)? pBlock->maxKey.ts:pBlock->minKey.ts; pBlockScanInfo->lastKey = asc? pBlock->maxKey.ts:pBlock->minKey.ts;
goto _end; goto _end;
} }
} }
@ -2331,7 +2335,6 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
SBlockData* pBlockData = &pReader->status.fileBlockData; SBlockData* pBlockData = &pReader->status.fileBlockData;
int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1;
while (1) { while (1) {
bool hasBlockData = false; bool hasBlockData = false;
@ -3220,7 +3223,6 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc
int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts,
SRowMerger* pMerger, SVersionRange* pVerRange) { SRowMerger* pMerger, SVersionRange* pVerRange) {
pScanInfo->lastKey = ts;
while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo, pVerRange)) { while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo, pVerRange)) {
int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader);
if (next1 == ts) { if (next1 == ts) {
@ -3413,9 +3415,10 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid) { int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, STableBlockScanInfo* pScanInfo) {
int32_t numOfRows = pBlock->info.rows; int32_t numOfRows = pBlock->info.rows;
int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
int64_t uid = pScanInfo->uid;
SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid);
@ -3454,6 +3457,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
} }
pBlock->info.rows += 1; pBlock->info.rows += 1;
pScanInfo->lastKey = pTSRow->ts;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -3517,7 +3521,8 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
break; break;
} }
doAppendRowFromTSRow(pBlock, pReader, pTSRow, pBlockScanInfo->uid); doAppendRowFromTSRow(pBlock, pReader, pTSRow, pBlockScanInfo);
if (freeTSRow) { if (freeTSRow) {
taosMemoryFree(pTSRow); taosMemoryFree(pTSRow);
} }

View File

@ -2973,7 +2973,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
return pOperator; return pOperator;
_error: _error:
if (pInfo != NULL) { if (pInfo != NULL) {
destroyAggOperatorInfo(pInfo); destroyAggOperatorInfo(pInfo);
} }
@ -3189,11 +3189,12 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
return pOperator; return pOperator;
_error: _error:
if (pInfo != NULL) { if (pInfo != NULL) {
destroyFillOperatorInfo(pInfo); destroyFillOperatorInfo(pInfo);
} }
pTaskInfo->code = code;
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
return NULL; return NULL;
} }

View File

@ -1,302 +1,312 @@
################################################################### ###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc. # Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved. # All rights reserved.
# #
# This file is proprietary and confidential to TAOS Technologies. # This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted, # No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as # disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao # expressly provided by the written permission from Jianhui Tao
# #
################################################################### ###################################################################
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import random import random
import string import string
from numpy import logspace from numpy import logspace
from util import constant from util import constant
from util.log import * from util.log import *
from util.cases import * from util.cases import *
from util.sql import * from util.sql import *
from util.common import * from util.common import *
from util.sqlset import TDSetSql from util.sqlset import TDSetSql
class TDTestCase: class TDTestCase:
def init(self, conn, logSql, replicaVar=1): def init(self, conn, logSql, replicaVar=1):
tdLog.debug("start to execute %s" % __file__) self.replicaVar = int(replicaVar)
tdSql.init(conn.cursor(), True) tdLog.debug("start to execute %s" % __file__)
self.dbname = 'db_test' tdSql.init(conn.cursor(), True)
self.setsql = TDSetSql() self.dbname = 'db_test'
self.stbname = 'stb' self.setsql = TDSetSql()
self.ntbname = 'ntb' self.stbname = 'stb'
self.rowNum = 5 self.ntbname = 'ntb'
self.tbnum = 2 self.rowNum = 10
self.ts = 1537146000000 self.tbnum = 3
self.binary_str = 'taosdata' self.ts = 1537146000000
self.nchar_str = '涛思数据' self.binary_str = 'taosdata'
self.str_length = 20 self.nchar_str = '涛思数据'
self.column_dict = { self.str_length = 20
'col1': 'tinyint', self.column_dict = {
'col2': 'smallint', 'col1': 'tinyint',
'col3': 'int', 'col2': 'smallint',
'col4': 'bigint', 'col3': 'int',
'col5': 'tinyint unsigned', 'col4': 'bigint',
'col6': 'smallint unsigned', 'col5': 'tinyint unsigned',
'col7': 'int unsigned', 'col6': 'smallint unsigned',
'col8': 'bigint unsigned', 'col7': 'int unsigned',
'col9': 'float', 'col8': 'bigint unsigned',
'col10': 'double', 'col9': 'float',
'col11': 'bool', 'col10': 'double',
'col12': f'binary({self.str_length})', 'col11': 'bool',
'col13': f'nchar({self.str_length})', 'col12': f'binary({self.str_length})',
'col13': f'nchar({self.str_length})',
}
}
self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX)
self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX)
self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX) self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX)
self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX)
self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX)
self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX)
self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX) self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX)
self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX)
self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX)
self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX)
self.bool_val = random.randint(0,100)%2 self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300))
self.binary_val = tdCom.getLongName(random.randint(0,self.str_length)) self.bool_val = random.randint(0,100)%2
self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length)) self.binary_val = tdCom.getLongName(random.randint(0,self.str_length))
self.base_data = { self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length))
'tinyint':self.tinyint_val, self.base_data = {
'smallint':self.smallint_val, 'tinyint':self.tinyint_val,
'int':self.int_val, 'smallint':self.smallint_val,
'bigint':self.bigint_val, 'int':self.int_val,
'tinyint unsigned':self.untingint_val, 'bigint':self.bigint_val,
'smallint unsigned':self.unsmallint_val, 'tinyint unsigned':self.untingint_val,
'int unsigned':self.unint_val, 'smallint unsigned':self.unsmallint_val,
'bigint unsigned':self.unbigint_val, 'int unsigned':self.unint_val,
'bool':self.bool_val, 'bigint unsigned':self.unbigint_val,
'float':self.float_val, 'bool':self.bool_val,
'double':self.double_val, 'float':self.float_val,
'binary':self.binary_val, 'double':self.double_val,
'nchar':self.nchar_val 'binary':self.binary_val,
} 'nchar':self.nchar_val
}
def insert_base_data(self,col_type,tbname,rows,base_data):
for i in range(rows): def insert_base_data(self,col_type,tbname,rows,base_data):
if col_type.lower() == 'tinyint': for i in range(rows):
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})') if col_type.lower() == 'tinyint':
elif col_type.lower() == 'smallint': tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})')
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})') elif col_type.lower() == 'smallint':
elif col_type.lower() == 'int': tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})')
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})') elif col_type.lower() == 'int':
elif col_type.lower() == 'bigint': tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})')
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})') elif col_type.lower() == 'bigint':
elif col_type.lower() == 'tinyint unsigned': tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})')
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})') elif col_type.lower() == 'tinyint unsigned':
elif col_type.lower() == 'smallint unsigned': tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})')
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})') elif col_type.lower() == 'smallint unsigned':
elif col_type.lower() == 'int unsigned': tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})')
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})') elif col_type.lower() == 'int unsigned':
elif col_type.lower() == 'bigint unsigned': tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})')
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})') elif col_type.lower() == 'bigint unsigned':
elif col_type.lower() == 'bool': tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})')
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})') elif col_type.lower() == 'bool':
elif col_type.lower() == 'float': tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})')
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})') elif col_type.lower() == 'float':
elif col_type.lower() == 'double': tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})')
tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})') elif col_type.lower() == 'double':
elif 'binary' in col_type.lower(): tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})')
tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''') elif 'binary' in col_type.lower():
elif 'nchar' in col_type.lower(): tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''')
tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''') elif 'nchar' in col_type.lower():
def delete_all_data(self,tbname,col_type,row_num,base_data,dbname,tb_type,tb_num=1): tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''')
tdSql.execute(f'delete from {tbname}') def delete_all_data(self,tbname,col_type,row_num,base_data,dbname,tb_type,tb_num=1,stbname=''):
tdSql.execute(f'flush database {dbname}') tdSql.query(f'select count(*) from {tbname}')
tdSql.execute('reset query cache') tdSql.execute(f'delete from {tbname}')
tdSql.query(f'select * from {tbname}') tdSql.execute(f'flush database {dbname}')
tdSql.checkRows(0) tdSql.execute('reset query cache')
if tb_type == 'ntb' or tb_type == 'ctb': tdSql.query(f'select * from {tbname}')
self.insert_base_data(col_type,tbname,row_num,base_data) tdSql.checkRows(0)
elif tb_type == 'stb': if tb_type == 'ntb' or tb_type == 'ctb':
for i in range(tb_num): if tb_type == 'ctb':
self.insert_base_data(col_type,f'{tbname}_{i}',row_num,base_data) tdSql.query(f'select count(*) from {stbname}')
tdSql.execute(f'flush database {dbname}') if tb_num <= 1:
tdSql.execute('reset query cache') if len(tdSql.queryResult) != 0:
tdSql.query(f'select * from {tbname}') tdLog.exit('delete case failure!')
if tb_type == 'ntb' or tb_type == 'ctb': else:
tdSql.checkRows(row_num) tdSql.checkEqual(tdSql.queryResult[0][0],(tb_num-1)*row_num)
elif tb_type =='stb':
tdSql.checkRows(row_num*tb_num) self.insert_base_data(col_type,tbname,row_num,base_data)
def delete_one_row(self,tbname,column_type,column_name,base_data,row_num,dbname,tb_type,tb_num=1): elif tb_type == 'stb':
tdSql.execute(f'delete from {tbname} where ts={self.ts}') for i in range(tb_num):
tdSql.execute(f'flush database {dbname}') self.insert_base_data(col_type,f'{tbname}_{i}',row_num,base_data)
tdSql.execute('reset query cache') tdSql.execute(f'flush database {dbname}')
tdSql.query(f'select {column_name} from {tbname}') tdSql.execute('reset query cache')
if tb_type == 'ntb' or tb_type == 'ctb': tdSql.query(f'select * from {tbname}')
tdSql.checkRows(row_num-1) if tb_type == 'ntb' or tb_type == 'ctb':
elif tb_type == 'stb': tdSql.checkRows(row_num)
tdSql.checkRows((row_num-1)*tb_num) elif tb_type =='stb':
tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') tdSql.checkRows(row_num*tb_num)
tdSql.checkRows(0) def delete_one_row(self,tbname,column_type,column_name,base_data,row_num,dbname,tb_type,tb_num=1):
if tb_type == 'ntb' or tb_type == 'ctb': tdSql.execute(f'delete from {tbname} where ts={self.ts}')
if 'binary' in column_type.lower(): tdSql.execute(f'flush database {dbname}')
tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['binary']}")''') tdSql.execute('reset query cache')
elif 'nchar' in column_type.lower(): tdSql.query(f'select {column_name} from {tbname}')
tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['nchar']}")''') if tb_type == 'ntb' or tb_type == 'ctb':
else: tdSql.checkRows(row_num-1)
tdSql.execute(f'insert into {tbname} values({self.ts},{base_data[column_type]})') elif tb_type == 'stb':
elif tb_type == 'stb': tdSql.checkRows((row_num-1)*tb_num)
for i in range(tb_num): tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}')
if 'binary' in column_type.lower(): tdSql.checkRows(0)
tdSql.execute(f'''insert into {tbname}_{i} values({self.ts},"{base_data['binary']}")''') if tb_type == 'ntb' or tb_type == 'ctb':
elif 'nchar' in column_type.lower(): if 'binary' in column_type.lower():
tdSql.execute(f'''insert into {tbname}_{i} values({self.ts},"{base_data['nchar']}")''') tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['binary']}")''')
else: elif 'nchar' in column_type.lower():
tdSql.execute(f'insert into {tbname}_{i} values({self.ts},{base_data[column_type]})') tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['nchar']}")''')
tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') else:
if column_type.lower() == 'float' or column_type.lower() == 'double': tdSql.execute(f'insert into {tbname} values({self.ts},{base_data[column_type]})')
if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001: elif tb_type == 'stb':
tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) for i in range(tb_num):
else: if 'binary' in column_type.lower():
tdLog.exit(f'{column_type} data check failure') tdSql.execute(f'''insert into {tbname}_{i} values({self.ts},"{base_data['binary']}")''')
elif 'binary' in column_type.lower(): elif 'nchar' in column_type.lower():
tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary']) tdSql.execute(f'''insert into {tbname}_{i} values({self.ts},"{base_data['nchar']}")''')
elif 'nchar' in column_type.lower(): else:
tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar']) tdSql.execute(f'insert into {tbname}_{i} values({self.ts},{base_data[column_type]})')
else: tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}')
tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type]) if column_type.lower() == 'float' or column_type.lower() == 'double':
def delete_rows(self,dbname,tbname,col_name,col_type,base_data,row_num,tb_type,tb_num=1): if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001:
for i in range(row_num): tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0])
tdSql.execute(f'delete from {tbname} where ts>{self.ts+i}') else:
tdSql.execute(f'flush database {dbname}') tdLog.exit(f'{column_type} data check failure')
tdSql.execute('reset query cache') elif 'binary' in column_type.lower():
tdSql.query(f'select {col_name} from {tbname}') tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary'])
if tb_type == 'ntb' or tb_type == 'ctb': elif 'nchar' in column_type.lower():
tdSql.checkRows(i+1) tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar'])
self.insert_base_data(col_type,tbname,row_num,base_data) else:
elif tb_type == 'stb': tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type])
tdSql.checkRows((i+1)*tb_num) def delete_rows(self,dbname,tbname,col_name,col_type,base_data,row_num,tb_type,tb_num=1):
for j in range(tb_num): for i in range(row_num):
self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) tdSql.execute(f'delete from {tbname} where ts>{self.ts+i}')
for i in range(row_num): tdSql.execute(f'flush database {dbname}')
tdSql.execute(f'delete from {tbname} where ts>={self.ts+i}') tdSql.execute('reset query cache')
tdSql.execute(f'flush database {dbname}') tdSql.query(f'select {col_name} from {tbname}')
tdSql.execute('reset query cache') if tb_type == 'ntb' or tb_type == 'ctb':
tdSql.query(f'select {col_name} from {tbname}') tdSql.checkRows(i+1)
if tb_type == 'ntb' or tb_type == 'ctb': self.insert_base_data(col_type,tbname,row_num,base_data)
tdSql.checkRows(i) elif tb_type == 'stb':
self.insert_base_data(col_type,tbname,row_num,base_data) tdSql.checkRows((i+1)*tb_num)
elif tb_type == 'stb': for j in range(tb_num):
tdSql.checkRows(i*tb_num) self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data)
for j in range(tb_num): for i in range(row_num):
self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) tdSql.execute(f'delete from {tbname} where ts>={self.ts+i}')
for i in range(row_num): tdSql.execute(f'flush database {dbname}')
tdSql.execute(f'delete from {tbname} where ts<={self.ts+i}') tdSql.execute('reset query cache')
tdSql.execute(f'flush database {dbname}') tdSql.query(f'select {col_name} from {tbname}')
tdSql.execute('reset query cache') if tb_type == 'ntb' or tb_type == 'ctb':
tdSql.query(f'select {col_name} from {tbname}') tdSql.checkRows(i)
if tb_type == 'ntb' or tb_type == 'ctb': self.insert_base_data(col_type,tbname,row_num,base_data)
tdSql.checkRows(row_num-i-1) elif tb_type == 'stb':
self.insert_base_data(col_type,tbname,row_num,base_data) tdSql.checkRows(i*tb_num)
elif tb_type == 'stb': for j in range(tb_num):
tdSql.checkRows((row_num-i-1)*tb_num) self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data)
for j in range(tb_num): for i in range(row_num):
self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) tdSql.execute(f'delete from {tbname} where ts<={self.ts+i}')
for i in range(row_num): tdSql.execute(f'flush database {dbname}')
tdSql.execute(f'delete from {tbname} where ts<{self.ts+i}') tdSql.execute('reset query cache')
tdSql.execute(f'flush database {dbname}') tdSql.query(f'select {col_name} from {tbname}')
tdSql.execute('reset query cache') if tb_type == 'ntb' or tb_type == 'ctb':
tdSql.query(f'select {col_name} from {tbname}') tdSql.checkRows(row_num-i-1)
if tb_type == 'ntb' or tb_type == 'ctb': self.insert_base_data(col_type,tbname,row_num,base_data)
tdSql.checkRows(row_num-i) elif tb_type == 'stb':
self.insert_base_data(col_type,tbname,row_num,base_data) tdSql.checkRows((row_num-i-1)*tb_num)
elif tb_type == 'stb': for j in range(tb_num):
tdSql.checkRows((row_num-i)*tb_num) self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data)
for j in range(tb_num): for i in range(row_num):
self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) tdSql.execute(f'delete from {tbname} where ts<{self.ts+i}')
for i in range(row_num): tdSql.execute(f'flush database {dbname}')
tdSql.execute(f'delete from {tbname} where ts between {self.ts} and {self.ts+i}') tdSql.execute('reset query cache')
tdSql.execute(f'flush database {dbname}') tdSql.query(f'select {col_name} from {tbname}')
tdSql.execute('reset query cache') if tb_type == 'ntb' or tb_type == 'ctb':
tdSql.query(f'select {col_name} from {tbname}') tdSql.checkRows(row_num-i)
if tb_type == 'ntb' or tb_type == 'ctb': self.insert_base_data(col_type,tbname,row_num,base_data)
tdSql.checkRows(row_num - i-1) elif tb_type == 'stb':
self.insert_base_data(col_type,tbname,row_num,base_data) tdSql.checkRows((row_num-i)*tb_num)
elif tb_type == 'stb': for j in range(tb_num):
tdSql.checkRows(tb_num*(row_num - i-1)) self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data)
for j in range(tb_num): for i in range(row_num):
self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) tdSql.execute(f'delete from {tbname} where ts between {self.ts} and {self.ts+i}')
tdSql.execute(f'delete from {tbname} where ts between {self.ts+i+1} and {self.ts}') tdSql.execute(f'flush database {dbname}')
tdSql.query(f'select {col_name} from {tbname}') tdSql.execute('reset query cache')
if tb_type == 'ntb' or tb_type == 'ctb': tdSql.query(f'select {col_name} from {tbname}')
tdSql.checkRows(row_num) if tb_type == 'ntb' or tb_type == 'ctb':
elif tb_type == 'stb': tdSql.checkRows(row_num - i-1)
tdSql.checkRows(tb_num*row_num) self.insert_base_data(col_type,tbname,row_num,base_data)
def delete_error(self,tbname,column_name,column_type,base_data): elif tb_type == 'stb':
for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']: tdSql.checkRows(tb_num*(row_num - i-1))
if 'binary' in column_type.lower(): for j in range(tb_num):
tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''') self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data)
elif 'nchar' in column_type.lower(): tdSql.execute(f'delete from {tbname} where ts between {self.ts+i+1} and {self.ts}')
tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''') tdSql.query(f'select {col_name} from {tbname}')
else: if tb_type == 'ntb' or tb_type == 'ctb':
tdSql.error(f'delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}') tdSql.checkRows(row_num)
elif tb_type == 'stb':
def delete_data_ntb(self): tdSql.checkRows(tb_num*row_num)
tdSql.execute(f'create database if not exists {self.dbname}') def delete_error(self,tbname,column_name,column_type,base_data):
tdSql.execute(f'use {self.dbname}') for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']:
for col_name,col_type in self.column_dict.items(): if 'binary' in column_type.lower():
tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''')
self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data) elif 'nchar' in column_type.lower():
self.delete_one_row(self.ntbname,col_type,col_name,self.base_data,self.rowNum,self.dbname,'ntb') tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''')
self.delete_all_data(self.ntbname,col_type,self.rowNum,self.base_data,self.dbname,'ntb') else:
self.delete_error(self.ntbname,col_name,col_type,self.base_data) tdSql.error(f'delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}')
self.delete_rows(self.dbname,self.ntbname,col_name,col_type,self.base_data,self.rowNum,'ntb')
for func in ['first','last']: def delete_data_ntb(self):
tdSql.query(f'select {func}(*) from {self.ntbname}') tdSql.execute(f'create database if not exists {self.dbname}')
tdSql.execute(f'drop table {self.ntbname}') tdSql.execute(f'use {self.dbname}')
tdSql.execute(f'drop database {self.dbname}') for col_name,col_type in self.column_dict.items():
def delete_data_ctb(self): tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})')
tdSql.execute(f'create database if not exists {self.dbname}') self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data)
tdSql.execute(f'use {self.dbname}') self.delete_one_row(self.ntbname,col_type,col_name,self.base_data,self.rowNum,self.dbname,'ntb')
for col_name,col_type in self.column_dict.items(): self.delete_all_data(self.ntbname,col_type,self.rowNum,self.base_data,self.dbname,'ntb')
tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t1 int)') self.delete_error(self.ntbname,col_name,col_type,self.base_data)
for i in range(self.tbnum): self.delete_rows(self.dbname,self.ntbname,col_name,col_type,self.base_data,self.rowNum,'ntb')
tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags(1)') for func in ['first','last']:
self.insert_base_data(col_type,f'{self.stbname}_{i}',self.rowNum,self.base_data) tdSql.query(f'select {func}(*) from {self.ntbname}')
self.delete_one_row(f'{self.stbname}_{i}',col_type,col_name,self.base_data,self.rowNum,self.dbname,'ctb') tdSql.execute(f'drop table {self.ntbname}')
self.delete_all_data(f'{self.stbname}_{i}',col_type,self.rowNum,self.base_data,self.dbname,'ctb') tdSql.execute(f'drop database {self.dbname}')
self.delete_error(f'{self.stbname}_{i}',col_name,col_type,self.base_data) def delete_data_ctb(self):
self.delete_rows(self.dbname,f'{self.stbname}_{i}',col_name,col_type,self.base_data,self.rowNum,'ctb') tdSql.execute(f'create database if not exists {self.dbname}')
for func in ['first','last']: tdSql.execute(f'use {self.dbname}')
tdSql.query(f'select {func}(*) from {self.stbname}_{i}') for col_name,col_type in self.column_dict.items():
tdSql.execute(f'drop table {self.stbname}') tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t1 int)')
def delete_data_stb(self): for i in range(self.tbnum):
tdSql.execute(f'create database if not exists {self.dbname}') tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags(1)')
tdSql.execute(f'use {self.dbname}') self.insert_base_data(col_type,f'{self.stbname}_{i}',self.rowNum,self.base_data)
for col_name,col_type in self.column_dict.items(): self.delete_one_row(f'{self.stbname}_{i}',col_type,col_name,self.base_data,self.rowNum,self.dbname,'ctb')
tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t1 int)') self.delete_all_data(f'{self.stbname}_{i}',col_type,self.rowNum,self.base_data,self.dbname,'ctb',i+1,self.stbname)
for i in range(self.tbnum): self.delete_error(f'{self.stbname}_{i}',col_name,col_type,self.base_data)
tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags(1)') self.delete_rows(self.dbname,f'{self.stbname}_{i}',col_name,col_type,self.base_data,self.rowNum,'ctb')
self.insert_base_data(col_type,f'{self.stbname}_{i}',self.rowNum,self.base_data) for func in ['first','last']:
self.delete_error(self.stbname,col_name,col_type,self.base_data) tdSql.query(f'select {func}(*) from {self.stbname}_{i}')
self.delete_one_row(self.stbname,col_type,col_name,self.base_data,self.rowNum,self.dbname,'stb',self.tbnum) tdSql.execute(f'drop table {self.stbname}')
self.delete_all_data(self.stbname,col_type,self.rowNum,self.base_data,self.dbname,'stb',self.tbnum) def delete_data_stb(self):
self.delete_rows(self.dbname,self.stbname,col_name,col_type,self.base_data,self.rowNum,'stb',self.tbnum) tdSql.execute(f'create database if not exists {self.dbname}')
for func in ['first','last']: tdSql.execute(f'use {self.dbname}')
tdSql.query(f'select {func}(*) from {self.stbname}') for col_name,col_type in self.column_dict.items():
tdSql.execute(f'drop table {self.stbname}') tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t1 int)')
tdSql.execute(f'drop database {self.dbname}') for i in range(self.tbnum):
def run(self): tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags(1)')
self.delete_data_ntb() self.insert_base_data(col_type,f'{self.stbname}_{i}',self.rowNum,self.base_data)
self.delete_data_ctb() self.delete_error(self.stbname,col_name,col_type,self.base_data)
self.delete_data_stb() self.delete_one_row(self.stbname,col_type,col_name,self.base_data,self.rowNum,self.dbname,'stb',self.tbnum)
tdDnodes.stoptaosd(1) self.delete_all_data(self.stbname,col_type,self.rowNum,self.base_data,self.dbname,'stb',self.tbnum)
tdDnodes.starttaosd(1) self.delete_rows(self.dbname,self.stbname,col_name,col_type,self.base_data,self.rowNum,'stb',self.tbnum)
self.delete_data_ntb() for func in ['first','last']:
def stop(self): tdSql.query(f'select {func}(*) from {self.stbname}')
tdSql.close() tdSql.execute(f'drop table {self.stbname}')
tdLog.success("%s successfully executed" % __file__) tdSql.execute(f'drop database {self.dbname}')
def run(self):
tdCases.addWindows(__file__, TDTestCase()) self.delete_data_ntb()
tdCases.addLinux(__file__, TDTestCase()) self.delete_data_ctb()
self.delete_data_stb()
tdDnodes.stoptaosd(1)
tdDnodes.starttaosd(1)
self.delete_data_ntb()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())