xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into enh/row_optimize

This commit is contained in:
Hongze Cheng 2022-11-20 10:12:38 +08:00
parent fb1f9ce1aa 7e8873a3e0
commit a0dc7859ec
35 changed files with 1058 additions and 609 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/en/07-develop/09-udf.md
View File

@ -205,13 +205,13 @@ Additional functions are defined in `taosudf.h` to make it easier to work with t
To use your user-defined function in TDengine, first compile it to a dynamically linked library (DLL). To use your user-defined function in TDengine, first compile it to a dynamically linked library (DLL).
For example, the sample UDF `add_one.c` can be compiled into a DLL as follows: For example, the sample UDF `bit_and.c` can be compiled into a DLL as follows:
```bash ```bash
gcc -g -O0 -fPIC -shared add_one.c -o add_one.so gcc -g -O0 -fPIC -shared bit_and.c -o libbitand.so
``` ```
The generated DLL file `add_one.so` can now be used to implement your function. Note: GCC 7.5 or later is required. The generated DLL file `libbitand.so` can now be used to implement your function. Note: GCC 7.5 or later is required.
## Manage and Use User-Defined Functions ## Manage and Use User-Defined Functions
After compiling your function into a DLL, you add it to TDengine. For more information, see [User-Defined Functions](../12-taos-sql/26-udf.md). After compiling your function into a DLL, you add it to TDengine. For more information, see [User-Defined Functions](../12-taos-sql/26-udf.md).

4
docs/en/12-taos-sql/26-udf.md
View File

@ -62,7 +62,7 @@ SHOW FUNCTIONS;
The function name specified when creating UDF can be used directly in SQL statements, just like builtin functions. For example: The function name specified when creating UDF can be used directly in SQL statements, just like builtin functions. For example:
```sql ```sql
SELECT X(c1,c2) FROM table/stable; SELECT bit_and(c1,c2) FROM table;
``` ```
The above SQL statement invokes function X for column c1 and c2. You can use query keywords like WHERE with user-defined functions. The above SQL statement invokes function X for column c1 and c2 on table. You can use query keywords like WHERE with user-defined functions.

2
docs/examples/rust/nativeexample/Cargo.toml
View File

@ -10,4 +10,4 @@ chrono = "0.4"
serde = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] }
tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] }
taos = { version = "0.*" } taos = { version = "0.4.8" }

5
docs/examples/rust/nativeexample/examples/stmt_example.rs
View File

@ -12,7 +12,10 @@ async fn main() -> anyhow::Result<()> {
// bind table name and tags // bind table name and tags
stmt.set_tbname_tags( stmt.set_tbname_tags(
"d1001", "d1001",
&[Value::VarChar("California.SanFransico".into()), Value::Int(2)], &[
Value::VarChar("California.SanFransico".into()),
Value::Int(2),
],
)?; )?;
// bind values. // bind values.
let values = vec![ let values = vec![

26
docs/examples/rust/nativeexample/examples/subscribe_demo.rs
View File

@ -50,7 +50,7 @@ async fn main() -> anyhow::Result<()> {
// create super table // create super table
format!("CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` BINARY(24))"), format!("CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` BINARY(24))"),
// create topic for subscription // create topic for subscription
format!("CREATE TOPIC tmq_meters with META AS DATABASE {db}") format!("CREATE TOPIC tmq_meters AS SELECT * FROM `meters`")
]) ])
.await?; .await?;
@ -64,13 +64,9 @@ async fn main() -> anyhow::Result<()> {
let mut consumer = tmq.build()?; let mut consumer = tmq.build()?;
consumer.subscribe(["tmq_meters"]).await?; consumer.subscribe(["tmq_meters"]).await?;
{ consumer
let mut stream = consumer.stream(); .stream()
.try_for_each(|(offset, message)| async {
while let Some((offset, message)) = stream.try_next().await? {
// get information from offset
// the topic
let topic = offset.topic(); let topic = offset.topic();
// the vgroup id, like partition id in kafka. // the vgroup id, like partition id in kafka.
let vgroup_id = offset.vgroup_id(); let vgroup_id = offset.vgroup_id();
@ -78,20 +74,14 @@ async fn main() -> anyhow::Result<()> {
if let Some(data) = message.into_data() { if let Some(data) = message.into_data() {
while let Some(block) = data.fetch_raw_block().await? { while let Some(block) = data.fetch_raw_block().await? {
// one block for one table, get table name if needed
let name = block.table_name();
let records: Vec<Record> = block.deserialize().try_collect()?; let records: Vec<Record> = block.deserialize().try_collect()?;
println!( println!("** read {} records: {:#?}\n", records.len(), records);
"** table: {}, got {} records: {:#?}\n",
name.unwrap(),
records.len(),
records
);
} }
} }
consumer.commit(offset).await?; consumer.commit(offset).await?;
} Ok(())
} })
.await?;
consumer.unsubscribe().await; consumer.unsubscribe().await;

1
docs/examples/rust/restexample/examples/insert_example.rs
View File

@ -5,7 +5,6 @@ async fn main() -> anyhow::Result<()> {
let dsn = "ws://"; let dsn = "ws://";
let taos = TaosBuilder::from_dsn(dsn)?.build()?; let taos = TaosBuilder::from_dsn(dsn)?.build()?;
taos.exec_many([ taos.exec_many([
"DROP DATABASE IF EXISTS power", "DROP DATABASE IF EXISTS power",
"CREATE DATABASE power", "CREATE DATABASE power",

6
docs/zh/07-develop/09-udf.md
View File

@ -205,13 +205,13 @@ typedef struct SUdfInterBuf {
用户定义函数的 C 语言源代码无法直接被 TDengine 系统使用，而是需要先编译为 动态链接库，之后才能载入 TDengine 系统。 用户定义函数的 C 语言源代码无法直接被 TDengine 系统使用，而是需要先编译为 动态链接库，之后才能载入 TDengine 系统。
例如，按照上一章节描述的规则准备好了用户定义函数的源代码 add_one.c，以 Linux 为例可以执行如下指令编译得到动态链接库文件： 例如，按照上一章节描述的规则准备好了用户定义函数的源代码 bit_and.c，以 Linux 为例可以执行如下指令编译得到动态链接库文件：
```bash ```bash
gcc -g -O0 -fPIC -shared add_one.c -o add_one.so gcc -g -O0 -fPIC -shared bit_and.c -o libbitand.so
``` ```
这样就准备好了动态链接库 add_one.so 文件，可以供后文创建 UDF 时使用了。为了保证可靠的系统运行，编译器 GCC 推荐使用 7.5 及以上版本。 这样就准备好了动态链接库 libbitand.so 文件，可以供后文创建 UDF 时使用了。为了保证可靠的系统运行，编译器 GCC 推荐使用 7.5 及以上版本。
## 管理和使用UDF ## 管理和使用UDF
编译好的UDF，还需要将其加入到系统才能被正常的SQL调用。关于如何管理和使用UDF，参见[UDF使用说明](../12-taos-sql/26-udf.md) 编译好的UDF，还需要将其加入到系统才能被正常的SQL调用。关于如何管理和使用UDF，参见[UDF使用说明](../12-taos-sql/26-udf.md)

4
docs/zh/12-taos-sql/26-udf.md
View File

@ -63,7 +63,7 @@ SHOW FUNCTIONS;
在 SQL 指令中，可以直接以在系统中创建 UDF 时赋予的函数名来调用用户定义函数。例如： 在 SQL 指令中，可以直接以在系统中创建 UDF 时赋予的函数名来调用用户定义函数。例如：
```sql ```sql
SELECT X(c1,c2) FROM table/stable; SELECT bit_and(c1,c2) FROM table;
``` ```
表示对名为 c1, c2 的数据列调用名为 X 的用户定义函数。SQL 指令中用户定义函数可以配合 WHERE 等查询特性来使用。 表示对表 table 上名为 c1, c2 的数据列调用名为 bit_and 的用户定义函数。SQL 指令中用户定义函数可以配合 WHERE 等查询特性来使用。

4
include/libs/stream/streamState.h
View File

@ -35,6 +35,7 @@ typedef struct {
TTB* pFuncStateDb; TTB* pFuncStateDb;
TTB* pFillStateDb; // todo refactor TTB* pFillStateDb; // todo refactor
TTB* pSessionStateDb; TTB* pSessionStateDb;
TTB* pParNameDb;
TXN txn; TXN txn;
int32_t number; int32_t number;
} SStreamState; } SStreamState;
@ -99,6 +100,9 @@ int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal);
#if 0 #if 0
char* streamStateSessionDump(SStreamState* pState); char* streamStateSessionDump(SStreamState* pState);
#endif #endif

1
include/util/taoserror.h
View File

@ -352,6 +352,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TDB_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x061A) #define TSDB_CODE_TDB_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x061A)
#define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x061B) #define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x061B)
#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061C) #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061C)
#define TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE TAOS_DEF_ERROR_CODE(0, 0x061D)
// query // query
#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700)

2
packaging/tools/tdengine.iss
View File

@ -63,7 +63,7 @@ Source: {#MyAppSourceDir}{#MyAppTaosdemoExeName}; DestDir: "{app}"; Flags: igNor
[run] [run]
Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\TDengine\\taosd.exe --win_service""" ; Flags: runhidden Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\TDengine\\taosd.exe --win_service""" ; Flags: runhidden
Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\TDengine\\taosadapter.exe --win_service""" ; Flags: runhidden Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\TDengine\\taosadapter.exe""" ; Flags: runhidden
[UninstallRun] [UninstallRun]
RunOnceId: "stoptaosd"; Filename: {sys}\sc.exe; Parameters: "stop taosd" ; Flags: runhidden RunOnceId: "stoptaosd"; Filename: {sys}\sc.exe; Parameters: "stop taosd" ; Flags: runhidden

3
source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
View File

@ -56,6 +56,9 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) {
if (IsReq(pMsg) && pMsg->info.handle != NULL && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (IsReq(pMsg) && pMsg->info.handle != NULL && code != TSDB_CODE_ACTION_IN_PROGRESS) {
if (code != 0 && terrno != 0) code = terrno; if (code != 0 && terrno != 0) code = terrno;
mmSendRsp(pMsg, code); mmSendRsp(pMsg, code);
} else {
rpcFreeCont(pMsg->info.rsp);
pMsg->info.rsp = NULL;
} }
if (code == TSDB_CODE_RPC_REDIRECT) { if (code == TSDB_CODE_RPC_REDIRECT) {

2
source/dnode/mnode/impl/src/mndMain.c
View File

@ -577,9 +577,9 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);
pMsg->info.rsp = rpcMallocCont(contLen); pMsg->info.rsp = rpcMallocCont(contLen);
pMsg->info.hasEpSet = 1;
if (pMsg->info.rsp != NULL) { if (pMsg->info.rsp != NULL) {
tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet); tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet);
pMsg->info.hasEpSet = 1;
pMsg->info.rspLen = contLen; pMsg->info.rspLen = contLen;
terrno = TSDB_CODE_RPC_REDIRECT; terrno = TSDB_CODE_RPC_REDIRECT;
} else { } else {

5
source/dnode/vnode/src/meta/metaTable.c
View File

@ -403,6 +403,11 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe
// validate req // validate req
metaReaderInit(&mr, pMeta, 0); metaReaderInit(&mr, pMeta, 0);
if (metaGetTableEntryByName(&mr, pReq->name) == 0) { if (metaGetTableEntryByName(&mr, pReq->name) == 0) {
if (pReq->type == TSDB_CHILD_TABLE && pReq->ctb.suid != mr.me.ctbEntry.suid) {
terrno = TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE;
metaReaderClear(&mr);
return -1;
}
pReq->uid = mr.me.uid; pReq->uid = mr.me.uid;
if (pReq->type == TSDB_CHILD_TABLE) { if (pReq->type == TSDB_CHILD_TABLE) {
pReq->ctb.suid = mr.me.ctbEntry.suid; pReq->ctb.suid = mr.me.ctbEntry.suid;

142
source/dnode/vnode/src/tq/tqRead.c
View File

@ -398,6 +398,35 @@ bool tqNextDataBlock(STqReader* pReader) {
return false; return false;
} }
int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrapper* pSrc, char* mask) {
int32_t code;
int32_t cnt = 0;
for (int32_t i = 0; i < pSrc->nCols; i++) {
cnt += mask[i];
}
pDst->nCols = cnt;
pDst->pSchema = taosMemoryCalloc(cnt, sizeof(SSchema));
if (pDst->pSchema == NULL) {
return -1;
}
int32_t j = 0;
for (int32_t i = 0; i < pSrc->nCols; i++) {
if (mask[i]) {
pDst->pSchema[j++] = pSrc->pSchema[i];
SColumnInfoData colInfo =
createColumnInfoData(pSrc->pSchema[i].type, pSrc->pSchema[i].bytes, pSrc->pSchema[i].colId);
code = blockDataAppendColInfo(pBlock, &colInfo);
if (code != 0) {
return -1;
}
}
}
return 0;
}
bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) {
while (1) { while (1) {
if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
@ -527,6 +556,119 @@ FAIL:
return -1; return -1;
} }
int32_t tqSplitRetrieveDataBlock(STqReader* pReader, SArray* blocks, SArray* schemas) {
int32_t sversion = htonl(pReader->pBlock->sversion);
if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion ||
pReader->cachedSchemaSuid != pReader->msgIter.suid) {
if (pReader->pSchema) taosMemoryFree(pReader->pSchema);
pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1);
if (pReader->pSchema == NULL) {
tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table",
pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer);
/*ASSERT(0);*/
pReader->cachedSchemaSuid = 0;
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1;
}
if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1);
if (pReader->pSchemaWrapper == NULL) {
tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
pReader->msgIter.uid, pReader->cachedSchemaVer);
/*ASSERT(0);*/
pReader->cachedSchemaSuid = 0;
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1;
}
pReader->cachedSchemaVer = sversion;
pReader->cachedSchemaSuid = pReader->msgIter.suid;
}
STSchema* pTschema = pReader->pSchema;
SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
int32_t colAtMost = pSchemaWrapper->nCols;
int32_t curRow = 0;
char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols);
if (assigned) return -1;
tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter);
STSRowIter iter = {0};
tdSTSRowIterInit(&iter, pTschema);
STSRow* row;
while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) {
bool buildNew = false;
tdSTSRowIterReset(&iter, row);
for (int32_t i = 0; i < colAtMost; i++) {
SCellVal sVal = {0};
if (!tdSTSRowIterFetch(&iter, pSchemaWrapper->pSchema[i].colId, pSchemaWrapper->pSchema[i].type, &sVal)) {
break;
}
if (curRow == 0) {
assigned[i] = sVal.valType != TD_VTYPE_NONE;
buildNew = true;
} else {
bool currentRowAssigned = sVal.valType != TD_VTYPE_NONE;
if (currentRowAssigned != assigned[i]) {
assigned[i] = currentRowAssigned;
buildNew = true;
}
}
}
if (buildNew) {
SSDataBlock block;
SSchemaWrapper sw;
if (tqMaskBlock(&sw, &block, pSchemaWrapper, assigned) < 0) {
goto FAIL;
}
taosArrayPush(blocks, &block);
taosArrayPush(schemas, &sw);
}
SSDataBlock* pBlock = taosArrayGetLast(blocks);
pBlock->info.uid = pReader->msgIter.uid;
pBlock->info.rows = pReader->msgIter.numOfRows;
pBlock->info.version = pReader->pMsg->version;
if (blockDataEnsureCapacity(pBlock, pReader->msgIter.numOfRows - curRow) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto FAIL;
}
tdSTSRowIterInit(&iter, pTschema);
for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); i++) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SCellVal sVal = {0};
if (!tdSTSRowIterFetch(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
break;
}
ASSERT(sVal.valType != TD_VTYPE_NONE);
if (colDataAppend(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) {
goto FAIL;
}
}
curRow++;
}
taosMemoryFree(assigned);
return 0;
FAIL:
taosMemoryFree(assigned);
return -1;
}
void tqReaderSetColIdList(STqReader* pReader, SArray* pColIdList) { pReader->pColIdList = pColIdList; } void tqReaderSetColIdList(STqReader* pReader, SArray* pColIdList) { pReader->pColIdList = pColIdList; }
int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) { int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) {

2
source/dnode/vnode/src/tsdb/tsdbOpen.c
View File

@ -49,7 +49,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
pTsdb->path = (char *)&pTsdb[1]; pTsdb->path = (char *)&pTsdb[1];
snprintf(pTsdb->path, TD_PATH_MAX, "%s%s%s", pVnode->path, TD_DIRSEP, dir); snprintf(pTsdb->path, TD_PATH_MAX, "%s%s%s", pVnode->path, TD_DIRSEP, dir);
taosRealPath(pTsdb->path, NULL, slen); // taosRealPath(pTsdb->path, NULL, slen);
pTsdb->pVnode = pVnode; pTsdb->pVnode = pVnode;
taosThreadRwlockInit(&pTsdb->rwLock, NULL); taosThreadRwlockInit(&pTsdb->rwLock, NULL);
if (!pKeepCfg) { if (!pKeepCfg) {

62
source/libs/executor/inc/executorimpl.h
View File

@ -163,7 +163,7 @@ typedef struct {
SArray* pStopInfo; SArray* pStopInfo;
} STaskStopInfo; } STaskStopInfo;
typedef struct SExecTaskInfo { struct SExecTaskInfo {
STaskIdInfo id; STaskIdInfo id;
uint32_t status; uint32_t status;
STimeWindow window; STimeWindow window;
@ -182,7 +182,7 @@ typedef struct SExecTaskInfo {
struct SOperatorInfo* pRoot; struct SOperatorInfo* pRoot;
SLocalFetch localFetch; SLocalFetch localFetch;
STaskStopInfo stopInfo; STaskStopInfo stopInfo;
} SExecTaskInfo; };
enum { enum {
OP_NOT_OPENED = 0x0, OP_NOT_OPENED = 0x0,
@ -315,37 +315,39 @@ typedef struct STableMetaCacheInfo {
uint64_t cacheHit; uint64_t cacheHit;
} STableMetaCacheInfo; } STableMetaCacheInfo;
typedef struct STableScanInfo { typedef struct STableScanBase {
STsdbReader* dataReader; STsdbReader* dataReader;
SReadHandle readHandle;
SLimitInfo limitInfo;
SFileBlockLoadRecorder readRecorder; SFileBlockLoadRecorder readRecorder;
SQueryTableDataCond cond;
SAggOptrPushDownInfo pdInfo;
SColMatchInfo matchInfo;
SReadHandle readHandle;
SExprSupp pseudoSup;
STableMetaCacheInfo metaCache;
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
int32_t dataBlockLoadFlag;
SLimitInfo limitInfo;
} STableScanBase;
typedef struct STableScanInfo {
STableScanBase base;
SScanInfo scanInfo; SScanInfo scanInfo;
int32_t scanTimes; int32_t scanTimes;
SSDataBlock* pResBlock; SSDataBlock* pResBlock;
SColMatchInfo matchInfo;
SExprSupp pseudoSup;
SQueryTableDataCond cond;
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
int32_t dataBlockLoadFlag;
SSampleExecInfo sample; // sample execution info SSampleExecInfo sample; // sample execution info
int32_t currentGroupId; int32_t currentGroupId;
int32_t currentTable; int32_t currentTable;
int8_t scanMode; int8_t scanMode;
SAggOptrPushDownInfo pdInfo;
int8_t assignBlockUid; int8_t assignBlockUid;
STableMetaCacheInfo metaCache;
} STableScanInfo; } STableScanInfo;
typedef struct STableMergeScanInfo { typedef struct STableMergeScanInfo {
STableListInfo* tableListInfo;
int32_t tableStartIndex; int32_t tableStartIndex;
int32_t tableEndIndex; int32_t tableEndIndex;
bool hasGroupId; bool hasGroupId;
uint64_t groupId; uint64_t groupId;
SArray* queryConds; // array of queryTableDataCond SArray* queryConds; // array of queryTableDataCond
STsdbReader* pReader; STableScanBase base;
SReadHandle readHandle;
int32_t bufPageSize; int32_t bufPageSize;
uint32_t sortBufSize; // max buffer size for in-memory sort uint32_t sortBufSize; // max buffer size for in-memory sort
SArray* pSortInfo; SArray* pSortInfo;
@ -354,25 +356,10 @@ typedef struct STableMergeScanInfo {
int64_t startTs; // sort start time int64_t startTs; // sort start time
SArray* sortSourceParams; SArray* sortSourceParams;
SLimitInfo limitInfo; SLimitInfo limitInfo;
SFileBlockLoadRecorder readRecorder;
int64_t numOfRows; int64_t numOfRows;
SScanInfo scanInfo; SScanInfo scanInfo;
int32_t scanTimes; int32_t scanTimes;
SqlFunctionCtx* pCtx; // which belongs to the direct upstream operator operator query context
SResultRowInfo* pResultRowInfo;
int32_t* rowEntryInfoOffset;
SExprInfo* pExpr;
SSDataBlock* pResBlock; SSDataBlock* pResBlock;
SColMatchInfo matchInfo;
int32_t numOfOutput;
SExprSupp pseudoSup;
SQueryTableDataCond cond;
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
int32_t dataBlockLoadFlag;
// if the upstream is an interval operator, the interval info is also kept here to get the time
// window to check if current data block needs to be loaded.
SInterval interval;
SSampleExecInfo sample; // sample execution info SSampleExecInfo sample; // sample execution info
SSortExecInfo sortExecInfo; SSortExecInfo sortExecInfo;
} STableMergeScanInfo; } STableMergeScanInfo;
@ -414,13 +401,6 @@ enum {
PROJECT_RETRIEVE_DONE = 0x2, PROJECT_RETRIEVE_DONE = 0x2,
}; };
typedef struct SCatchSupporter {
SHashObj* pWindowHashTable; // quick locate the window object for each window
SDiskbasedBuf* pDataBuf; // buffer based on blocked-wised disk file
int32_t keySize;
int64_t* pKeyBuf;
} SCatchSupporter;
typedef struct SStreamAggSupporter { typedef struct SStreamAggSupporter {
int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row
SSDataBlock* pScanBlock; SSDataBlock* pScanBlock;
@ -504,7 +484,6 @@ typedef struct SStreamScanInfo {
STimeWindow updateWin; STimeWindow updateWin;
STimeWindowAggSupp twAggSup; STimeWindowAggSupp twAggSup;
SSDataBlock* pUpdateDataRes; SSDataBlock* pUpdateDataRes;
SHashObj* pGroupIdTbNameMap;
// status for tmq // status for tmq
SNodeList* pGroupTags; SNodeList* pGroupTags;
SNode* pTagCond; SNode* pTagCond;
@ -612,7 +591,6 @@ typedef struct SStreamIntervalOperatorInfo {
SArray* pChildren; SArray* pChildren;
SStreamState* pState; SStreamState* pState;
SWinKey delKey; SWinKey delKey;
SHashObj* pGroupIdTbNameMap; // uint64_t -> char[TSDB_TABLE_NAME_LEN]
} SStreamIntervalOperatorInfo; } SStreamIntervalOperatorInfo;
typedef struct SAggOperatorInfo { typedef struct SAggOperatorInfo {
@ -745,7 +723,6 @@ typedef struct SStreamSessionAggOperatorInfo {
SPhysiNode* pPhyNode; // create new child SPhysiNode* pPhyNode; // create new child
bool isFinal; bool isFinal;
bool ignoreExpiredData; bool ignoreExpiredData;
SHashObj* pGroupIdTbNameMap;
} SStreamSessionAggOperatorInfo; } SStreamSessionAggOperatorInfo;
typedef struct SStreamStateAggOperatorInfo { typedef struct SStreamStateAggOperatorInfo {
@ -761,7 +738,6 @@ typedef struct SStreamStateAggOperatorInfo {
void* pDelIterator; void* pDelIterator;
SArray* pChildren; // cache for children's result; SArray* pChildren; // cache for children's result;
bool ignoreExpiredData; bool ignoreExpiredData;
SHashObj* pGroupIdTbNameMap;
} SStreamStateAggOperatorInfo; } SStreamStateAggOperatorInfo;
typedef struct SStreamPartitionOperatorInfo { typedef struct SStreamPartitionOperatorInfo {
@ -1046,8 +1022,8 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos
SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
SExecTaskInfo* pTaskInfo); SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); SExecTaskInfo* pTaskInfo);
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);

22
source/libs/executor/src/executor.c
View File

@ -1026,8 +1026,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
SStreamScanInfo* pInfo = pOperator->info; SStreamScanInfo* pInfo = pOperator->info;
if (pOffset->type == TMQ_OFFSET__LOG) { if (pOffset->type == TMQ_OFFSET__LOG) {
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTSInfo->dataReader); tsdbReaderClose(pTSInfo->base.dataReader);
pTSInfo->dataReader = NULL; pTSInfo->base.dataReader = NULL;
#if 0 #if 0
if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus) && if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus) &&
pInfo->tqReader->pWalReader->curVersion != pOffset->version) { pInfo->tqReader->pWalReader->curVersion != pOffset->version) {
@ -1079,23 +1079,23 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
// TODO after dropping table, table may not found // TODO after dropping table, table may not found
ASSERT(found); ASSERT(found);
if (pTableScanInfo->dataReader == NULL) { if (pTableScanInfo->base.dataReader == NULL) {
STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0); STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0);
int32_t num = tableListGetSize(pTaskInfo->pTableInfoList); int32_t num = tableListGetSize(pTaskInfo->pTableInfoList);
if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond, pList, num, if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num,
&pTableScanInfo->dataReader, NULL) < 0 || &pTableScanInfo->base.dataReader, NULL) < 0 ||
pTableScanInfo->dataReader == NULL) { pTableScanInfo->base.dataReader == NULL) {
ASSERT(0); ASSERT(0);
} }
} }
STableKeyInfo tki = {.uid = uid}; STableKeyInfo tki = {.uid = uid};
tsdbSetTableList(pTableScanInfo->dataReader, &tki, 1); tsdbSetTableList(pTableScanInfo->base.dataReader, &tki, 1);
int64_t oldSkey = pTableScanInfo->cond.twindows.skey; int64_t oldSkey = pTableScanInfo->base.cond.twindows.skey;
pTableScanInfo->cond.twindows.skey = ts + 1; pTableScanInfo->base.cond.twindows.skey = ts + 1;
tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
pTableScanInfo->cond.twindows.skey = oldSkey; pTableScanInfo->base.cond.twindows.skey = oldSkey;
pTableScanInfo->scanTimes = 0; pTableScanInfo->scanTimes = 0;
qDebug("tsdb reader offset seek to uid %" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid, qDebug("tsdb reader offset seek to uid %" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid,

135
source/libs/executor/src/executorimpl.c
View File

@ -1000,12 +1000,6 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void updateTableQueryInfoForReverseScan(STableQueryInfo* pTableQueryInfo) {
if (pTableQueryInfo == NULL) {
return;
}
}
void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) {
if (status == TASK_NOT_COMPLETED) { if (status == TASK_NOT_COMPLETED) {
pTaskInfo->status = status; pTaskInfo->status = status;
@ -1336,27 +1330,14 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr
pBlock->info.groupId = 0; pBlock->info.groupId = 0;
ASSERT(!pbInfo->mergeResultBlock); ASSERT(!pbInfo->mergeResultBlock);
doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo); doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo);
if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) {
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
char* tbname = taosHashGet(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t)); void* tbname = NULL;
if (tbname != NULL) { if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.groupId, &tbname) < 0) {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
} else {
pBlock->info.parTbName[0] = 0; pBlock->info.parTbName[0] = 0;
}
} else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION ||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
char* tbname = taosHashGet(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t));
if (tbname != NULL) {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
} else { } else {
pBlock->info.parTbName[0] = 0; memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
}
} }
tdbFree(tbname);
} }
void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
@ -1665,55 +1646,6 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t
static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize,
const char* pKey); const char* pKey);
static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int32_t rowIndex) {
size_t size = taosArrayGetSize(groupInfo);
if (size == 0) {
return true;
}
for (int32_t i = 0; i < size; ++i) {
int32_t* index = taosArrayGet(groupInfo, i);
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, *index);
bool isNull = colDataIsNull(pColInfo, rowIndex, pBlock->info.rows, NULL);
if ((isNull && buf[i] != NULL) || (!isNull && buf[i] == NULL)) {
return false;
}
char* pCell = colDataGetData(pColInfo, rowIndex);
if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
if (varDataLen(pCell) != varDataLen(buf[i])) {
return false;
} else {
if (memcmp(varDataVal(pCell), varDataVal(buf[i]), varDataLen(pCell)) != 0) {
return false;
}
}
} else {
if (memcmp(pCell, buf[i], pColInfo->info.bytes) != 0) {
return false;
}
}
}
return 0;
}
static bool saveCurrentTuple(char** rowColData, SArray* pColumnList, SSDataBlock* pBlock, int32_t rowIndex) {
int32_t size = (int32_t)taosArrayGetSize(pColumnList);
for (int32_t i = 0; i < size; ++i) {
int32_t* index = taosArrayGet(pColumnList, i);
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, *index);
char* data = colDataGetData(pColInfo, rowIndex);
memcpy(rowColData[i], data, colDataGetLength(pColInfo, rowIndex));
}
return true;
}
int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag) { int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag) {
// todo add more information about exchange operation // todo add more information about exchange operation
int32_t type = pOperator->operatorType; int32_t type = pOperator->operatorType;
@ -1725,13 +1657,13 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
STableScanInfo* pTableScanInfo = pOperator->info; STableScanInfo* pTableScanInfo = pOperator->info;
*order = pTableScanInfo->cond.order; *order = pTableScanInfo->base.cond.order;
*scanFlag = pTableScanInfo->scanFlag; *scanFlag = pTableScanInfo->base.scanFlag;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN) { } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN) {
STableMergeScanInfo* pTableScanInfo = pOperator->info; STableMergeScanInfo* pTableScanInfo = pOperator->info;
*order = pTableScanInfo->cond.order; *order = pTableScanInfo->base.cond.order;
*scanFlag = pTableScanInfo->scanFlag; *scanFlag = pTableScanInfo->base.scanFlag;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else { } else {
if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) { if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) {
@ -2372,14 +2304,14 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock; pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock;
pInfo->groupId = UINT64_MAX; pInfo->groupId = UINT64_MAX;
setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo, pTaskInfo); setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo,
pOperator->fpSet = pTaskInfo);
createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, NULL); pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, NULL);
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
STableScanInfo* pTableScanInfo = downstream->info; STableScanInfo* pTableScanInfo = downstream->info;
pTableScanInfo->pdInfo.pExprSup = &pOperator->exprSupp; pTableScanInfo->base.pdInfo.pExprSup = &pOperator->exprSupp;
pTableScanInfo->pdInfo.pAggSup = &pInfo->aggSup; pTableScanInfo->base.pdInfo.pAggSup = &pInfo->aggSup;
} }
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
@ -2744,7 +2676,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
} }
STableScanInfo* pScanInfo = pOperator->info; STableScanInfo* pScanInfo = pOperator->info;
pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; pTaskInfo->cost.pRecoder = &pScanInfo->base.readRecorder;
} else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
@ -2762,14 +2694,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
return NULL; return NULL;
} }
pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo); pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo);
if (NULL == pOperator) { if (NULL == pOperator) {
pTaskInfo->code = terrno; pTaskInfo->code = terrno;
return NULL; return NULL;
} }
STableScanInfo* pScanInfo = pOperator->info; STableScanInfo* pScanInfo = pOperator->info;
pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; pTaskInfo->cost.pRecoder = &pScanInfo->base.readRecorder;
} else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) {
pOperator = createExchangeOperatorInfo(pHandle ? pHandle->pMsgCb->clientRpc : NULL, (SExchangePhysiNode*)pPhyNode, pOperator = createExchangeOperatorInfo(pHandle ? pHandle->pMsgCb->clientRpc : NULL, (SExchangePhysiNode*)pPhyNode,
pTaskInfo); pTaskInfo);
@ -3351,13 +3283,13 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat
if (pBlock->info.groupId == 0) { if (pBlock->info.groupId == 0) {
pBlock->info.groupId = pPos->groupId; pBlock->info.groupId = pPos->groupId;
SStreamIntervalOperatorInfo* pInfo = pOperator->info; void* tbname = NULL;
char* tbname = taosHashGet(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t)); if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.groupId, &tbname) < 0) {
if (tbname != NULL) {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
} else {
pBlock->info.parTbName[0] = 0; pBlock->info.parTbName[0] = 0;
} else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
} }
tdbFree(tbname);
} else { } else {
// current value belongs to different group, it can't be packed into one datablock // current value belongs to different group, it can't be packed into one datablock
if (pBlock->info.groupId != pPos->groupId) { if (pBlock->info.groupId != pPos->groupId) {
@ -3443,30 +3375,13 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
if (pBlock->info.groupId == 0) { if (pBlock->info.groupId == 0) {
pBlock->info.groupId = pKey->groupId; pBlock->info.groupId = pKey->groupId;
if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { void* tbname = NULL;
SStreamStateAggOperatorInfo* pInfo = pOperator->info; if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.groupId, &tbname) < 0) {
char* tbname = taosHashGet(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t));
if (tbname != NULL) {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
} else {
pBlock->info.parTbName[0] = 0; pBlock->info.parTbName[0] = 0;
} } else {
} else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION ||
pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
char* tbname = taosHashGet(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t));
if (tbname != NULL) {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
} else {
pBlock->info.parTbName[0] = 0;
} }
} else { tdbFree(tbname);
ASSERT(0);
}
} else { } else {
// current value belongs to different group, it can't be packed into one datablock // current value belongs to different group, it can't be packed into one datablock
if (pBlock->info.groupId != pKey->groupId) { if (pBlock->info.groupId != pKey->groupId) {

421
source/libs/executor/src/scanoperator.c
View File

@ -233,25 +233,25 @@ static SResultRow* getTableGroupOutputBuf(SOperatorInfo* pOperator, uint64_t gro
STableScanInfo* pTableScanInfo = pOperator->info; STableScanInfo* pTableScanInfo = pOperator->info;
SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pTableScanInfo->pdInfo.pAggSup->pResultRowHashTable, buf, SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pTableScanInfo->base.pdInfo.pAggSup->pResultRowHashTable, buf,
GET_RES_WINDOW_KEY_LEN(sizeof(groupId))); GET_RES_WINDOW_KEY_LEN(sizeof(groupId)));
if (p1 == NULL) { if (p1 == NULL) {
return NULL; return NULL;
} }
*pPage = getBufPage(pTableScanInfo->pdInfo.pAggSup->pResultBuf, p1->pageId); *pPage = getBufPage(pTableScanInfo->base.pdInfo.pAggSup->pResultBuf, p1->pageId);
return (SResultRow*)((char*)(*pPage) + p1->offset); return (SResultRow*)((char*)(*pPage) + p1->offset);
} }
static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* pBlockInfo, uint32_t* status) { static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* pBlockInfo, uint32_t* status) {
STableScanInfo* pTableScanInfo = pOperator->info; STableScanInfo* pTableScanInfo = pOperator->info;
if (pTableScanInfo->pdInfo.pExprSup == NULL) { if (pTableScanInfo->base.pdInfo.pExprSup == NULL) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SExprSupp* pSup1 = pTableScanInfo->pdInfo.pExprSup; SExprSupp* pSup1 = pTableScanInfo->base.pdInfo.pExprSup;
SFilePage* pPage = NULL; SFilePage* pPage = NULL;
SResultRow* pRow = getTableGroupOutputBuf(pOperator, pBlockInfo->groupId, &pPage); SResultRow* pRow = getTableGroupOutputBuf(pOperator, pBlockInfo->groupId, &pPage);
@ -264,7 +264,7 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo*
for (int32_t i = 0; i < pSup1->numOfExprs; ++i) { for (int32_t i = 0; i < pSup1->numOfExprs; ++i) {
int32_t functionId = pSup1->pCtx[i].functionId; int32_t functionId = pSup1->pCtx[i].functionId;
SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, pTableScanInfo->pdInfo.pExprSup->rowEntryInfoOffset); SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, pTableScanInfo->base.pdInfo.pExprSup->rowEntryInfoOffset);
int32_t reqStatus = fmFuncDynDataRequired(functionId, pEntry, &pBlockInfo->window); int32_t reqStatus = fmFuncDynDataRequired(functionId, pEntry, &pBlockInfo->window);
if (reqStatus != FUNC_DATA_REQUIRED_NOT_LOAD) { if (reqStatus != FUNC_DATA_REQUIRED_NOT_LOAD) {
@ -274,7 +274,7 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo*
} }
// release buffer pages // release buffer pages
releaseBufPage(pTableScanInfo->pdInfo.pAggSup->pResultBuf, pPage); releaseBufPage(pTableScanInfo->base.pdInfo.pAggSup->pResultBuf, pPage);
if (notLoadBlock) { if (notLoadBlock) {
*status = FUNC_DATA_REQUIRED_NOT_LOAD; *status = FUNC_DATA_REQUIRED_NOT_LOAD;
@ -293,7 +293,7 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA
return keep; return keep;
} }
static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) {
bool allColumnsHaveAgg = true; bool allColumnsHaveAgg = true;
SColumnDataAgg** pColAgg = NULL; SColumnDataAgg** pColAgg = NULL;
@ -330,7 +330,7 @@ static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock,
return true; return true;
} }
static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, static void doSetTagColumnData(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo,
int32_t rows) { int32_t rows) {
if (pTableScanInfo->pseudoSup.numOfExprs > 0) { if (pTableScanInfo->pseudoSup.numOfExprs > 0) {
SExprSupp* pSup = &pTableScanInfo->pseudoSup; SExprSupp* pSup = &pTableScanInfo->pseudoSup;
@ -374,19 +374,16 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo
} }
} }
static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableScanInfo, SSDataBlock* pBlock,
uint32_t* status) { uint32_t* status) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
STableScanInfo* pInfo = pOperator->info;
SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder; SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder;
pCost->totalBlocks += 1; pCost->totalBlocks += 1;
pCost->totalRows += pBlock->info.rows; pCost->totalRows += pBlock->info.rows;
bool loadSMA = false; bool loadSMA = false;
*status = pTableScanInfo->dataBlockLoadFlag;
*status = pInfo->dataBlockLoadFlag;
if (pOperator->exprSupp.pFilterInfo != NULL || if (pOperator->exprSupp.pFilterInfo != NULL ||
overlapWithTimeWindow(&pTableScanInfo->pdInfo.interval, &pBlock->info, pTableScanInfo->cond.order)) { overlapWithTimeWindow(&pTableScanInfo->pdInfo.interval, &pBlock->info, pTableScanInfo->cond.order)) {
(*status) = FUNC_DATA_REQUIRED_DATA_LOAD; (*status) = FUNC_DATA_REQUIRED_DATA_LOAD;
@ -485,14 +482,14 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
} }
} }
applyLimitOffset(&pInfo->limitInfo, pBlock, pTaskInfo, pOperator); applyLimitOffset(&pTableScanInfo->limitInfo, pBlock, pTaskInfo, pOperator);
pCost->totalRows += pBlock->info.rows; pCost->totalRows += pBlock->info.rows;
pInfo->limitInfo.numOfOutputRows = pCost->totalRows; pTableScanInfo->limitInfo.numOfOutputRows = pCost->totalRows;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunctionCtx* pCtx, int32_t numOfOutput) { static void prepareForDescendingScan(STableScanBase* pTableScanInfo, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
SET_REVERSE_SCAN_FLAG(pTableScanInfo); SET_REVERSE_SCAN_FLAG(pTableScanInfo);
switchCtxOrder(pCtx, numOfOutput); switchCtxOrder(pCtx, numOfOutput);
@ -560,7 +557,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
code = metaGetTableEntryByUid(&mr, pBlock->info.uid); code = metaGetTableEntryByUid(&mr, pBlock->info.uid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr); qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid,
tstrerror(terrno), idStr);
} else { } else {
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr); qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr);
} }
@ -583,9 +581,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
code = metaGetTableEntryByUid(&mr, pBlock->info.uid); code = metaGetTableEntryByUid(&mr, pBlock->info.uid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr); qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s",
pBlock->info.uid, tstrerror(terrno), idStr);
} else { } else {
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr); qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno),
idStr);
} }
metaReaderClear(&mr); metaReaderClear(&mr);
return terrno; return terrno;
@ -697,7 +697,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
while (tsdbNextDataBlock(pTableScanInfo->dataReader)) { while (tsdbNextDataBlock(pTableScanInfo->base.dataReader)) {
if (isTaskKilled(pTaskInfo)) { if (isTaskKilled(pTaskInfo)) {
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); T_LONG_JMP(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
} }
@ -712,7 +712,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
SDataBlockInfo* pBInfo = &pBlock->info; SDataBlockInfo* pBInfo = &pBlock->info;
int32_t rows = 0; int32_t rows = 0;
tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &rows, &pBInfo->uid, &pBInfo->window); tsdbRetrieveDataBlockInfo(pTableScanInfo->base.dataReader, &rows, &pBInfo->uid, &pBInfo->window);
blockDataEnsureCapacity(pBlock, rows); // todo remove it latter blockDataEnsureCapacity(pBlock, rows); // todo remove it latter
pBInfo->rows = rows; pBInfo->rows = rows;
@ -721,7 +721,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.uid); pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.uid);
uint32_t status = 0; uint32_t status = 0;
int32_t code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status); int32_t code = loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status);
// int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pOperator->pTaskInfo->env, code); T_LONG_JMP(pOperator->pTaskInfo->env, code);
@ -732,10 +732,10 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
continue; continue;
} }
pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows; pOperator->resultInfo.totalRows = pTableScanInfo->base.readRecorder.totalRows;
pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
pOperator->cost.totalCost = pTableScanInfo->readRecorder.elapsedTime; pOperator->cost.totalCost = pTableScanInfo->base.readRecorder.elapsedTime;
// todo refactor // todo refactor
/*pTableScanInfo->lastStatus.uid = pBlock->info.uid;*/ /*pTableScanInfo->lastStatus.uid = pBlock->info.uid;*/
@ -755,7 +755,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
// The read handle is not initialized yet, since no qualified tables exists // The read handle is not initialized yet, since no qualified tables exists
if (pTableScanInfo->dataReader == NULL || pOperator->status == OP_EXEC_DONE) { if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) {
return NULL; return NULL;
} }
@ -770,19 +770,19 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) {
if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) {
setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
pTableScanInfo->scanFlag = REPEAT_SCAN; pTableScanInfo->base.scanFlag = REPEAT_SCAN;
qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo)); qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo));
// do prepare for the next round table scan operation // do prepare for the next round table scan operation
tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
} }
} }
int32_t total = pTableScanInfo->scanInfo.numOfAsc + pTableScanInfo->scanInfo.numOfDesc; int32_t total = pTableScanInfo->scanInfo.numOfAsc + pTableScanInfo->scanInfo.numOfDesc;
if (pTableScanInfo->scanTimes < total) { if (pTableScanInfo->scanTimes < total) {
if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) { if (pTableScanInfo->base.cond.order == TSDB_ORDER_ASC) {
prepareForDescendingScan(pTableScanInfo, pOperator->exprSupp.pCtx, 0); prepareForDescendingScan(&pTableScanInfo->base, pOperator->exprSupp.pCtx, 0);
tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo));
} }
@ -796,10 +796,10 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) {
if (pTableScanInfo->scanTimes < total) { if (pTableScanInfo->scanTimes < total) {
setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
pTableScanInfo->scanFlag = REPEAT_SCAN; pTableScanInfo->base.scanFlag = REPEAT_SCAN;
qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo)); qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo));
tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
} }
} }
} }
@ -828,11 +828,11 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
} }
STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->currentTable); STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->currentTable);
tsdbSetTableList(pInfo->dataReader, pTableInfo, 1); tsdbSetTableList(pInfo->base.dataReader, pTableInfo, 1);
qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d %s", pTableInfo->uid, numOfTables, qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d %s", pTableInfo->uid, numOfTables,
pInfo->currentTable, pTaskInfo->id.str); pInfo->currentTable, pTaskInfo->id.str);
tsdbReaderReset(pInfo->dataReader, &pInfo->cond); tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond);
pInfo->scanTimes = 0; pInfo->scanTimes = 0;
} }
} else { // scan table group by group sequentially } else { // scan table group by group sequentially
@ -845,10 +845,10 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
int32_t num = 0; int32_t num = 0;
STableKeyInfo* pList = NULL; STableKeyInfo* pList = NULL;
tableListGetGroupList(pTaskInfo->pTableInfoList, pInfo->currentGroupId, &pList, &num); tableListGetGroupList(pTaskInfo->pTableInfoList, pInfo->currentGroupId, &pList, &num);
ASSERT(pInfo->dataReader == NULL); ASSERT(pInfo->base.dataReader == NULL);
int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, pList, num, int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num,
(STsdbReader**)&pInfo->dataReader, GET_TASKID(pTaskInfo)); (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
} }
@ -867,15 +867,15 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
// reset value for the next group data output // reset value for the next group data output
pOperator->status = OP_OPENED; pOperator->status = OP_OPENED;
pInfo->limitInfo.numOfOutputRows = 0; pInfo->base.limitInfo.numOfOutputRows = 0;
pInfo->limitInfo.remainOffset = pInfo->limitInfo.limit.offset; pInfo->base.limitInfo.remainOffset = pInfo->base.limitInfo.limit.offset;
int32_t num = 0; int32_t num = 0;
STableKeyInfo* pList = NULL; STableKeyInfo* pList = NULL;
tableListGetGroupList(pTaskInfo->pTableInfoList, pInfo->currentGroupId, &pList, &num); tableListGetGroupList(pTaskInfo->pTableInfoList, pInfo->currentGroupId, &pList, &num);
tsdbSetTableList(pInfo->dataReader, pList, num); tsdbSetTableList(pInfo->base.dataReader, pList, num);
tsdbReaderReset(pInfo->dataReader, &pInfo->cond); tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond);
pInfo->scanTimes = 0; pInfo->scanTimes = 0;
result = doGroupedTableScan(pOperator); result = doGroupedTableScan(pOperator);
@ -891,7 +891,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) {
SFileBlockLoadRecorder* pRecorder = taosMemoryCalloc(1, sizeof(SFileBlockLoadRecorder)); SFileBlockLoadRecorder* pRecorder = taosMemoryCalloc(1, sizeof(SFileBlockLoadRecorder));
STableScanInfo* pTableScanInfo = pOptr->info; STableScanInfo* pTableScanInfo = pOptr->info;
*pRecorder = pTableScanInfo->readRecorder; *pRecorder = pTableScanInfo->base.readRecorder;
*pOptrExplain = pRecorder; *pOptrExplain = pRecorder;
*len = sizeof(SFileBlockLoadRecorder); *len = sizeof(SFileBlockLoadRecorder);
return 0; return 0;
@ -900,17 +900,17 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr
static void destroyTableScanOperatorInfo(void* param) { static void destroyTableScanOperatorInfo(void* param) {
STableScanInfo* pTableScanInfo = (STableScanInfo*)param; STableScanInfo* pTableScanInfo = (STableScanInfo*)param;
blockDataDestroy(pTableScanInfo->pResBlock); blockDataDestroy(pTableScanInfo->pResBlock);
cleanupQueryTableDataCond(&pTableScanInfo->cond); cleanupQueryTableDataCond(&pTableScanInfo->base.cond);
tsdbReaderClose(pTableScanInfo->dataReader); tsdbReaderClose(pTableScanInfo->base.dataReader);
pTableScanInfo->dataReader = NULL; pTableScanInfo->base.dataReader = NULL;
if (pTableScanInfo->matchInfo.pList != NULL) { if (pTableScanInfo->base.matchInfo.pList != NULL) {
taosArrayDestroy(pTableScanInfo->matchInfo.pList); taosArrayDestroy(pTableScanInfo->base.matchInfo.pList);
} }
taosLRUCacheCleanup(pTableScanInfo->metaCache.pTableMetaEntryCache); taosLRUCacheCleanup(pTableScanInfo->base.metaCache.pTableMetaEntryCache);
cleanupExprSupp(&pTableScanInfo->pseudoSup); cleanupExprSupp(&pTableScanInfo->base.pseudoSup);
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
@ -927,30 +927,32 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
int32_t numOfCols = 0; int32_t numOfCols = 0;
int32_t code = int32_t code =
extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->base.matchInfo);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }
initLimitInfo(pScanNode->node.pLimit, pScanNode->node.pSlimit, &pInfo->limitInfo); initLimitInfo(pScanNode->node.pLimit, pScanNode->node.pSlimit, &pInfo->base.limitInfo);
code = initQueryTableDataCond(&pInfo->cond, pTableScanNode); code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }
if (pScanNode->pScanPseudoCols != NULL) { if (pScanNode->pScanPseudoCols != NULL) {
SExprSupp* pSup = &pInfo->pseudoSup; SExprSupp* pSup = &pInfo->base.pseudoSup;
pSup->pExprInfo = createExprInfo(pScanNode->pScanPseudoCols, NULL, &pSup->numOfExprs); pSup->pExprInfo = createExprInfo(pScanNode->pScanPseudoCols, NULL, &pSup->numOfExprs);
pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset); pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset);
} }
pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]};
pInfo->pdInfo.interval = extractIntervalInfo(pTableScanNode);
pInfo->readHandle = *readHandle; pInfo->base.scanFlag = MAIN_SCAN;
pInfo->base.pdInfo.interval = extractIntervalInfo(pTableScanNode);
pInfo->base.readHandle = *readHandle;
pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.sampleRatio = pTableScanNode->ratio;
pInfo->sample.seed = taosGetTimestampSec(); pInfo->sample.seed = taosGetTimestampSec();
pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired;
initResultSizeInfo(&pOperator->resultInfo, 4096); initResultSizeInfo(&pOperator->resultInfo, 4096);
pInfo->pResBlock = createResDataBlock(pDescNode); pInfo->pResBlock = createResDataBlock(pDescNode);
@ -961,7 +963,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
goto _error; goto _error;
} }
pInfo->scanFlag = MAIN_SCAN;
pInfo->currentGroupId = -1; pInfo->currentGroupId = -1;
pInfo->assignBlockUid = pTableScanNode->assignBlockUid; pInfo->assignBlockUid = pTableScanNode->assignBlockUid;
@ -969,13 +970,13 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
pTaskInfo); pTaskInfo);
pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.numOfExprs = numOfCols;
pInfo->metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5);
if (pInfo->metaCache.pTableMetaEntryCache == NULL) { if (pInfo->base.metaCache.pTableMetaEntryCache == NULL) {
code = terrno; code = terrno;
goto _error; goto _error;
} }
taosLRUCacheSetStrictCapacity(pInfo->metaCache.pTableMetaEntryCache, false); taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false);
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo,
getTableScannerExecInfo); getTableScannerExecInfo);
@ -997,7 +998,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo*
STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
pInfo->dataReader = pReadHandle; pInfo->base.dataReader = pReadHandle;
// pInfo->prevGroupId = -1; // pInfo->prevGroupId = -1;
setOperatorInfo(pOperator, "TableSeqScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, false, OP_NOT_OPENED, setOperatorInfo(pOperator, "TableSeqScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, false, OP_NOT_OPENED,
@ -1204,11 +1205,11 @@ static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t grou
} }
void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) { void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) {
pTableScanInfo->cond.twindows = *pWin; pTableScanInfo->base.cond.twindows = *pWin;
pTableScanInfo->scanTimes = 0; pTableScanInfo->scanTimes = 0;
pTableScanInfo->currentGroupId = -1; pTableScanInfo->currentGroupId = -1;
tsdbReaderClose(pTableScanInfo->dataReader); tsdbReaderClose(pTableScanInfo->base.dataReader);
pTableScanInfo->dataReader = NULL; pTableScanInfo->base.dataReader = NULL;
} }
static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs, static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs,
@ -1216,7 +1217,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
STableKeyInfo tblInfo = {.uid = tbUid, .groupId = 0}; STableKeyInfo tblInfo = {.uid = tbUid, .groupId = 0};
STableScanInfo* pTableScanInfo = pTableScanOp->info; STableScanInfo* pTableScanInfo = pTableScanOp->info;
SQueryTableDataCond cond = pTableScanInfo->cond; SQueryTableDataCond cond = pTableScanInfo->base.cond;
cond.startVersion = -1; cond.startVersion = -1;
cond.endVersion = maxVersion; cond.endVersion = maxVersion;
@ -1228,7 +1229,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
blockDataCleanup(pBlock); blockDataCleanup(pBlock);
STsdbReader* pReader = NULL; STsdbReader* pReader = NULL;
int32_t code = tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &cond, &tblInfo, 1, (STsdbReader**)&pReader, int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, (STsdbReader**)&pReader,
GET_TASKID(pTaskInfo)); GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code; terrno = code;
@ -1247,8 +1248,8 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
blockDataEnsureCapacity(pBlock, rows); blockDataEnsureCapacity(pBlock, rows);
pBlock->info.rows = rows; pBlock->info.rows = rows;
relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true); relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true);
doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, rows); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, rows);
pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->uid); pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->uid);
} }
@ -1375,8 +1376,8 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
*pRowIndex = 0; *pRowIndex = 0;
pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTableScanInfo->dataReader); tsdbReaderClose(pTableScanInfo->base.dataReader);
pTableScanInfo->dataReader = NULL; pTableScanInfo->base.dataReader = NULL;
return NULL; return NULL;
} }
@ -1539,7 +1540,9 @@ static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pS
groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version);
} }
if (pInfo->tbnameCalSup.pExprInfo) { if (pInfo->tbnameCalSup.pExprInfo) {
char* parTbname = taosHashGet(pInfo->pGroupIdTbNameMap, &groupId, sizeof(int64_t)); void* parTbname = NULL;
streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname);
memcpy(varDataVal(tbname), parTbname, TSDB_TABLE_NAME_LEN); memcpy(varDataVal(tbname), parTbname, TSDB_TABLE_NAME_LEN);
varDataSetLen(tbname, strlen(varDataVal(tbname))); varDataSetLen(tbname, strlen(varDataVal(tbname)));
} }
@ -1590,14 +1593,17 @@ static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock*
void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
SExprSupp* pTbNameCalSup = &pInfo->tbnameCalSup; SExprSupp* pTbNameCalSup = &pInfo->tbnameCalSup;
SStreamState* pState = pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState;
if (pTbNameCalSup == NULL || pTbNameCalSup->numOfExprs == 0) return; if (pTbNameCalSup == NULL || pTbNameCalSup->numOfExprs == 0) return;
if (pBlock == NULL || pBlock->info.rows == 0) return; if (pBlock == NULL || pBlock->info.rows == 0) return;
if (pBlock->info.groupId) {
char* tbname = taosHashGet(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t)); void* tbname = NULL;
if (tbname != NULL) { if (streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, pBlock->info.groupId, &tbname) < 0) {
pBlock->info.parTbName[0] = 0;
} else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
} }
} tdbFree(tbname);
SSDataBlock* pSrcBlock = blockCopyOneRow(pBlock, 0); SSDataBlock* pSrcBlock = blockCopyOneRow(pBlock, 0);
ASSERT(pSrcBlock->info.rows == 1); ASSERT(pSrcBlock->info.rows == 1);
@ -1625,9 +1631,8 @@ void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
pBlock->info.parTbName[0] = 0; pBlock->info.parTbName[0] = 0;
} }
if (pBlock->info.groupId) { if (pBlock->info.groupId && pBlock->info.parTbName[0]) {
taosHashPut(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t), pBlock->info.parTbName, streamStatePutParName(pState, pBlock->info.groupId, pBlock->info.parTbName);
TSDB_TABLE_NAME_LEN);
} }
blockDataDestroy(pSrcBlock); blockDataDestroy(pSrcBlock);
@ -1806,8 +1811,8 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
} else { } else {
if (!pTaskInfo->streamInfo.returned) { if (!pTaskInfo->streamInfo.returned) {
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTSInfo->dataReader); tsdbReaderClose(pTSInfo->base.dataReader);
pTSInfo->dataReader = NULL; pTSInfo->base.dataReader = NULL;
tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer); tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer);
qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1);
if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) { if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) {
@ -1973,22 +1978,22 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1 || if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1 ||
pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE2) { pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE2) {
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
memcpy(&pTSInfo->cond, &pTaskInfo->streamInfo.tableCond, sizeof(SQueryTableDataCond)); memcpy(&pTSInfo->base.cond, &pTaskInfo->streamInfo.tableCond, sizeof(SQueryTableDataCond));
if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1) { if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1) {
pTSInfo->cond.startVersion = 0; pTSInfo->base.cond.startVersion = 0;
pTSInfo->cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer1; pTSInfo->base.cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer1;
qDebug("stream recover step 1, from %" PRId64 " to %" PRId64, pTSInfo->cond.startVersion, qDebug("stream recover step 1, from %" PRId64 " to %" PRId64, pTSInfo->base.cond.startVersion,
pTSInfo->cond.endVersion); pTSInfo->base.cond.endVersion);
} else { } else {
pTSInfo->cond.startVersion = pTaskInfo->streamInfo.fillHistoryVer1 + 1; pTSInfo->base.cond.startVersion = pTaskInfo->streamInfo.fillHistoryVer1 + 1;
pTSInfo->cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer2; pTSInfo->base.cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer2;
qDebug("stream recover step 2, from %" PRId64 " to %" PRId64, pTSInfo->cond.startVersion, qDebug("stream recover step 2, from %" PRId64 " to %" PRId64, pTSInfo->base.cond.startVersion,
pTSInfo->cond.endVersion); pTSInfo->base.cond.endVersion);
} }
/*resetTableScanInfo(pTSInfo, pWin);*/ /*resetTableScanInfo(pTSInfo, pWin);*/
tsdbReaderClose(pTSInfo->dataReader); tsdbReaderClose(pTSInfo->base.dataReader);
pTSInfo->dataReader = NULL; pTSInfo->base.dataReader = NULL;
pTSInfo->scanTimes = 0; pTSInfo->scanTimes = 0;
pTSInfo->currentGroupId = -1; pTSInfo->currentGroupId = -1;
@ -2009,11 +2014,11 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
} }
pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE;
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTSInfo->dataReader); tsdbReaderClose(pTSInfo->base.dataReader);
pTSInfo->dataReader = NULL; pTSInfo->base.dataReader = NULL;
pTSInfo->cond.startVersion = -1; pTSInfo->base.cond.startVersion = -1;
pTSInfo->cond.endVersion = -1; pTSInfo->base.cond.endVersion = -1;
return NULL; return NULL;
} }
@ -2030,6 +2035,9 @@ FETCH_NEXT_BLOCK:
int32_t current = pInfo->validBlockIndex++; int32_t current = pInfo->validBlockIndex++;
SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current);
if (pBlock->info.groupId && pBlock->info.parTbName[0]) {
streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.groupId, pBlock->info.parTbName);
}
// TODO move into scan // TODO move into scan
pBlock->info.calWin.skey = INT64_MIN; pBlock->info.calWin.skey = INT64_MIN;
pBlock->info.calWin.ekey = INT64_MAX; pBlock->info.calWin.ekey = INT64_MAX;
@ -2118,8 +2126,8 @@ FETCH_NEXT_BLOCK:
SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
if (pSDB) { if (pSDB) {
STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
uint64_t version = getReaderMaxVersion(pTableScanInfo->dataReader); uint64_t version = getReaderMaxVersion(pTableScanInfo->base.dataReader);
updateInfoSetScanRange(pInfo->pUpdateInfo, &pTableScanInfo->cond.twindows, pInfo->groupId, version); updateInfoSetScanRange(pInfo->pUpdateInfo, &pTableScanInfo->base.cond.twindows, pInfo->groupId, version);
pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA; pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA;
checkUpdateData(pInfo, true, pSDB, false); checkUpdateData(pInfo, true, pSDB, false);
// printDataBlock(pSDB, "stream scan update"); // printDataBlock(pSDB, "stream scan update");
@ -2428,7 +2436,6 @@ static void destroyStreamScanOperatorInfo(void* param) {
} }
cleanupExprSupp(&pStreamScan->tbnameCalSup); cleanupExprSupp(&pStreamScan->tbnameCalSup);
taosHashCleanup(pStreamScan->pGroupIdTbNameMap);
updateInfoDestroy(pStreamScan->pUpdateInfo); updateInfoDestroy(pStreamScan->pUpdateInfo);
blockDataDestroy(pStreamScan->pRes); blockDataDestroy(pStreamScan->pRes);
@ -2486,8 +2493,6 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
if (initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1) != 0) { if (initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1) != 0) {
goto _error; goto _error;
} }
pInfo->pGroupIdTbNameMap =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
} }
if (pTableScanNode->pTags != NULL) { if (pTableScanNode->pTags != NULL) {
@ -2513,7 +2518,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo);
STableScanInfo* pTSInfo = (STableScanInfo*)pTableScanOp->info; STableScanInfo* pTSInfo = (STableScanInfo*)pTableScanOp->info;
if (pHandle->version > 0) { if (pHandle->version > 0) {
pTSInfo->cond.endVersion = pHandle->version; pTSInfo->base.cond.endVersion = pHandle->version;
} }
STableKeyInfo* pList = NULL; STableKeyInfo* pList = NULL;
@ -2522,8 +2527,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
if (pHandle->initTableReader) { if (pHandle->initTableReader) {
pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER; pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
pTSInfo->dataReader = NULL; pTSInfo->base.dataReader = NULL;
code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, pList, num, &pTSInfo->dataReader, NULL); code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, &pTSInfo->base.dataReader, NULL);
if (code != 0) { if (code != 0) {
terrno = code; terrno = code;
destroyTableScanOperatorInfo(pTableScanOp); destroyTableScanOperatorInfo(pTableScanOp);
@ -2559,7 +2564,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
goto _error; goto _error;
} }
taosArrayDestroy(tableIdList); taosArrayDestroy(tableIdList);
memcpy(&pTaskInfo->streamInfo.tableCond, &pTSInfo->cond, sizeof(SQueryTableDataCond)); memcpy(&pTaskInfo->streamInfo.tableCond, &pTSInfo->base.cond, sizeof(SQueryTableDataCond));
} else { } else {
taosArrayDestroy(pColIds); taosArrayDestroy(pColIds);
} }
@ -4374,123 +4379,6 @@ _error:
return NULL; return NULL;
} }
// todo refactor
static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
SSDataBlock* pBlock, uint32_t* status) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
STableMergeScanInfo* pInfo = pOperator->info;
SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder;
pCost->totalBlocks += 1;
pCost->totalRows += pBlock->info.rows;
*status = pInfo->dataBlockLoadFlag;
if (pOperator->exprSupp.pFilterInfo != NULL ||
overlapWithTimeWindow(&pTableScanInfo->interval, &pBlock->info, pTableScanInfo->cond.order)) {
(*status) = FUNC_DATA_REQUIRED_DATA_LOAD;
}
SDataBlockInfo* pBlockInfo = &pBlock->info;
taosMemoryFreeClear(pBlock->pBlockAgg);
if (*status == FUNC_DATA_REQUIRED_FILTEROUT) {
qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
pCost->filterOutBlocks += 1;
return TSDB_CODE_SUCCESS;
} else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
pCost->skipBlocks += 1;
// clear all data in pBlock that are set when handing the previous block
for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i);
pcol->pData = NULL;
}
return TSDB_CODE_SUCCESS;
} else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) {
pCost->loadBlockStatis += 1;
bool allColumnsHaveAgg = true;
SColumnDataAgg** pColAgg = NULL;
if (allColumnsHaveAgg == true) {
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
// todo create this buffer during creating operator
if (pBlock->pBlockAgg == NULL) {
pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES);
}
for (int32_t i = 0; i < numOfCols; ++i) {
SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i);
if (!pColMatchInfo->needOutput) {
continue;
}
pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i];
}
return TSDB_CODE_SUCCESS;
} else { // failed to load the block sma data, data block statistics does not exist, load data block instead
*status = FUNC_DATA_REQUIRED_DATA_LOAD;
}
}
ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD);
// todo filter data block according to the block sma data firstly
#if 0
if (!doFilterByBlockSMA(pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) {
pCost->filterOutBlocks += 1;
qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey,
pBlockInfo->window.ekey, pBlockInfo->rows);
(*status) = FUNC_DATA_REQUIRED_FILTEROUT;
return TSDB_CODE_SUCCESS;
}
#endif
pCost->totalCheckedRows += pBlock->info.rows;
pCost->loadBlocks += 1;
STsdbReader* reader = pTableScanInfo->pReader;
SArray* pCols = tsdbRetrieveDataBlock(reader, NULL);
if (pCols == NULL) {
return terrno;
}
relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
// currently only the tbname pseudo column
SExprSupp* pSup = &pTableScanInfo->pseudoSup;
int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock,
pBlock->info.rows, GET_TASKID(pTaskInfo), NULL);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
if (pOperator->exprSupp.pFilterInfo != NULL) {
int64_t st = taosGetTimestampMs();
doFilter(pBlock, pOperator->exprSupp.pFilterInfo, &pTableScanInfo->matchInfo);
double el = (taosGetTimestampUs() - st) / 1000.0;
pTableScanInfo->readRecorder.filterTime += el;
if (pBlock->info.rows == 0) {
pCost->filterOutBlocks += 1;
qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%.2f ms",
GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, el);
} else {
qDebug("%s data block filter applied, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), el);
}
}
return TSDB_CODE_SUCCESS;
}
static SSDataBlock* getTableDataBlockImpl(void* param) { static SSDataBlock* getTableDataBlockImpl(void* param) {
STableMergeScanSortSourceParam* source = param; STableMergeScanSortSourceParam* source = param;
SOperatorInfo* pOperator = source->pOperator; SOperatorInfo* pOperator = source->pOperator;
@ -4505,15 +4393,15 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
void* p = tableListGetInfo(pInfo->tableListInfo, readIdx + pInfo->tableStartIndex); void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex);
SReadHandle* pHandle = &pInfo->readHandle; SReadHandle* pHandle = &pInfo->base.readHandle;
int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->pReader, GET_TASKID(pTaskInfo)); int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->base.dataReader, GET_TASKID(pTaskInfo));
if (code != 0) { if (code != 0) {
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
} }
STsdbReader* reader = pInfo->pReader; STsdbReader* reader = pInfo->base.dataReader;
while (tsdbNextDataBlock(reader)) { while (tsdbNextDataBlock(reader)) {
if (isTaskKilled(pTaskInfo)) { if (isTaskKilled(pTaskInfo)) {
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); T_LONG_JMP(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
@ -4539,7 +4427,8 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
} }
uint32_t status = 0; uint32_t status = 0;
code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status); loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status);
// code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
} }
@ -4552,15 +4441,15 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.uid); pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.uid);
pOperator->resultInfo.totalRows += pBlock->info.rows; pOperator->resultInfo.totalRows += pBlock->info.rows;
pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
tsdbReaderClose(pInfo->pReader); tsdbReaderClose(pInfo->base.dataReader);
pInfo->pReader = NULL; pInfo->base.dataReader = NULL;
return pBlock; return pBlock;
} }
tsdbReaderClose(pInfo->pReader); tsdbReaderClose(pInfo->base.dataReader);
pInfo->pReader = NULL; pInfo->base.dataReader = NULL;
return NULL; return NULL;
} }
@ -4598,10 +4487,10 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
{ {
size_t numOfTables = tableListGetSize(pInfo->tableListInfo); size_t numOfTables = tableListGetSize(pTaskInfo->pTableInfoList);
int32_t i = pInfo->tableStartIndex + 1; int32_t i = pInfo->tableStartIndex + 1;
for (; i < numOfTables; ++i) { for (; i < numOfTables; ++i) {
STableKeyInfo* tableKeyInfo = tableListGetInfo(pInfo->tableListInfo, i); STableKeyInfo* tableKeyInfo = tableListGetInfo(pTaskInfo->pTableInfoList, i);
if (tableKeyInfo->groupId != pInfo->groupId) { if (tableKeyInfo->groupId != pInfo->groupId) {
break; break;
} }
@ -4612,7 +4501,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
int32_t tableStartIdx = pInfo->tableStartIndex; int32_t tableStartIdx = pInfo->tableStartIndex;
int32_t tableEndIdx = pInfo->tableEndIndex; int32_t tableEndIdx = pInfo->tableEndIndex;
pInfo->pReader = NULL; pInfo->base.dataReader = NULL;
// todo the total available buffer should be determined by total capacity of buffer of this task. // todo the total available buffer should be determined by total capacity of buffer of this task.
// the additional one is reserved for merge result // the additional one is reserved for merge result
@ -4635,7 +4524,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
taosArrayPush(pInfo->sortSourceParams, &param); taosArrayPush(pInfo->sortSourceParams, &param);
SQueryTableDataCond cond; SQueryTableDataCond cond;
dumpSQueryTableCond(&pInfo->cond, &cond); dumpSQueryTableCond(&pInfo->base.cond, &cond);
taosArrayPush(pInfo->queryConds, &cond); taosArrayPush(pInfo->queryConds, &cond);
} }
@ -4675,6 +4564,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
taosArrayClear(pInfo->sortSourceParams); taosArrayClear(pInfo->sortSourceParams);
tsortDestroySortHandle(pInfo->pSortHandle); tsortDestroySortHandle(pInfo->pSortHandle);
pInfo->pSortHandle = NULL;
for (int32_t i = 0; i < taosArrayGetSize(pInfo->queryConds); i++) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->queryConds); i++) {
SQueryTableDataCond* cond = taosArrayGet(pInfo->queryConds, i); SQueryTableDataCond* cond = taosArrayGet(pInfo->queryConds, i);
@ -4725,7 +4615,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
} }
size_t tableListSize = tableListGetSize(pInfo->tableListInfo); size_t tableListSize = tableListGetSize(pTaskInfo->pTableInfoList);
if (!pInfo->hasGroupId) { if (!pInfo->hasGroupId) {
pInfo->hasGroupId = true; pInfo->hasGroupId = true;
@ -4734,7 +4624,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
return NULL; return NULL;
} }
pInfo->tableStartIndex = 0; pInfo->tableStartIndex = 0;
pInfo->groupId = ((STableKeyInfo*)tableListGetInfo(pInfo->tableListInfo, pInfo->tableStartIndex))->groupId; pInfo->groupId = ((STableKeyInfo*)tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->tableStartIndex))->groupId;
startGroupTableMergeScan(pOperator); startGroupTableMergeScan(pOperator);
} }
@ -4753,7 +4643,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
break; break;
} }
pInfo->tableStartIndex = pInfo->tableEndIndex + 1; pInfo->tableStartIndex = pInfo->tableEndIndex + 1;
pInfo->groupId = tableListGetInfo(pInfo->tableListInfo, pInfo->tableStartIndex)->groupId; pInfo->groupId = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->tableStartIndex)->groupId;
startGroupTableMergeScan(pOperator); startGroupTableMergeScan(pOperator);
} }
} }
@ -4763,7 +4653,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
void destroyTableMergeScanOperatorInfo(void* param) { void destroyTableMergeScanOperatorInfo(void* param) {
STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param; STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param;
cleanupQueryTableDataCond(&pTableScanInfo->cond); cleanupQueryTableDataCond(&pTableScanInfo->base.cond);
int32_t numOfTable = taosArrayGetSize(pTableScanInfo->queryConds); int32_t numOfTable = taosArrayGetSize(pTableScanInfo->queryConds);
@ -4773,9 +4663,11 @@ void destroyTableMergeScanOperatorInfo(void* param) {
} }
taosArrayDestroy(pTableScanInfo->sortSourceParams); taosArrayDestroy(pTableScanInfo->sortSourceParams);
tsortDestroySortHandle(pTableScanInfo->pSortHandle);
pTableScanInfo->pSortHandle = NULL;
tsdbReaderClose(pTableScanInfo->pReader); tsdbReaderClose(pTableScanInfo->base.dataReader);
pTableScanInfo->pReader = NULL; pTableScanInfo->base.dataReader = NULL;
for (int i = 0; i < taosArrayGetSize(pTableScanInfo->queryConds); i++) { for (int i = 0; i < taosArrayGetSize(pTableScanInfo->queryConds); i++) {
SQueryTableDataCond* pCond = taosArrayGet(pTableScanInfo->queryConds, i); SQueryTableDataCond* pCond = taosArrayGet(pTableScanInfo->queryConds, i);
@ -4783,17 +4675,20 @@ void destroyTableMergeScanOperatorInfo(void* param) {
} }
taosArrayDestroy(pTableScanInfo->queryConds); taosArrayDestroy(pTableScanInfo->queryConds);
if (pTableScanInfo->matchInfo.pList != NULL) { if (pTableScanInfo->base.matchInfo.pList != NULL) {
taosArrayDestroy(pTableScanInfo->matchInfo.pList); taosArrayDestroy(pTableScanInfo->base.matchInfo.pList);
} }
pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock);
pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock);
taosArrayDestroy(pTableScanInfo->pSortInfo); taosArrayDestroy(pTableScanInfo->pSortInfo);
cleanupExprSupp(&pTableScanInfo->pseudoSup); cleanupExprSupp(&pTableScanInfo->base.pseudoSup);
tsdbReaderClose(pTableScanInfo->base.dataReader);
pTableScanInfo->base.dataReader = NULL;
taosLRUCacheCleanup(pTableScanInfo->base.metaCache.pTableMetaEntryCache);
taosMemoryFreeClear(pTableScanInfo->rowEntryInfoOffset);
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
@ -4802,7 +4697,7 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla
// TODO: merge these two info into one struct // TODO: merge these two info into one struct
STableMergeScanExecInfo* execInfo = taosMemoryCalloc(1, sizeof(STableMergeScanExecInfo)); STableMergeScanExecInfo* execInfo = taosMemoryCalloc(1, sizeof(STableMergeScanExecInfo));
STableMergeScanInfo* pInfo = pOptr->info; STableMergeScanInfo* pInfo = pOptr->info;
execInfo->blockRecorder = pInfo->readRecorder; execInfo->blockRecorder = pInfo->base.readRecorder;
execInfo->sortExecInfo = pInfo->sortExecInfo; execInfo->sortExecInfo = pInfo->sortExecInfo;
*pOptrExplain = execInfo; *pOptrExplain = execInfo;
@ -4811,8 +4706,8 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { SExecTaskInfo* pTaskInfo) {
STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo)); STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
@ -4823,38 +4718,46 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
int32_t numOfCols = 0; int32_t numOfCols = 0;
int32_t code = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, int32_t code = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID,
&pInfo->matchInfo); &pInfo->base.matchInfo);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }
code = initQueryTableDataCond(&pInfo->cond, pTableScanNode); code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(pInfo->matchInfo.pList); taosArrayDestroy(pInfo->base.matchInfo.pList);
goto _error; goto _error;
} }
if (pTableScanNode->scan.pScanPseudoCols != NULL) { if (pTableScanNode->scan.pScanPseudoCols != NULL) {
SExprSupp* pSup = &pInfo->pseudoSup; SExprSupp* pSup = &pInfo->base.pseudoSup;
pSup->pExprInfo = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pSup->numOfExprs); pSup->pExprInfo = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pSup->numOfExprs);
pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset); pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset);
} }
pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]};
pInfo->readHandle = *readHandle; pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5);
pInfo->interval = extractIntervalInfo(pTableScanNode); if (pInfo->base.metaCache.pTableMetaEntryCache == NULL) {
code = terrno;
goto _error;
}
pInfo->base.dataBlockLoadFlag = FUNC_DATA_REQUIRED_DATA_LOAD;
pInfo->base.scanFlag = MAIN_SCAN;
pInfo->base.readHandle = *readHandle;
pInfo->base.limitInfo.limit.limit = -1;
pInfo->base.limitInfo.slimit.limit = -1;
pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.sampleRatio = pTableScanNode->ratio;
pInfo->sample.seed = taosGetTimestampSec(); pInfo->sample.seed = taosGetTimestampSec();
pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired;
code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }
pInfo->tableListInfo = pTableListInfo;
pInfo->scanFlag = MAIN_SCAN;
initResultSizeInfo(&pOperator->resultInfo, 1024); initResultSizeInfo(&pOperator->resultInfo, 1024);
pInfo->pResBlock = createResDataBlock(pDescNode); pInfo->pResBlock = createResDataBlock(pDescNode);
@ -4862,7 +4765,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam)); pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
pInfo->pSortInfo = generateSortByTsInfo(pInfo->matchInfo.pList, pInfo->cond.order); pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order);
pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false); pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false);
initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo); initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo);

81
source/libs/executor/src/timewindowoperator.c
View File

@ -1450,7 +1450,6 @@ static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp
} }
} }
tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter); tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter);
/*taosHashRemove(pInfo->pGroupIdTbNameMap, &pWinKey->groupId, sizeof(int64_t));*/
} }
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1547,7 +1546,8 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin
uint64_t uid = 0; uint64_t uid = 0;
for (int32_t i = *index; i < size; i++) { for (int32_t i = *index; i < size; i++) {
SWinKey* pWin = taosArrayGet(pWins, i); SWinKey* pWin = taosArrayGet(pWins, i);
char* tbname = taosHashGet(pInfo->pGroupIdTbNameMap, &pWin->groupId, sizeof(int64_t)); void* tbname = NULL;
streamStateGetParName(pInfo->pState, pWin->groupId, &tbname);
if (tbname == NULL) { if (tbname == NULL) {
appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL); appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL);
} else { } else {
@ -1555,6 +1555,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin
STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName); appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName);
} }
tdbFree(tbname);
(*index)++; (*index)++;
} }
} }
@ -1617,7 +1618,6 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) {
nodesDestroyNode((SNode*)pInfo->pPhyNode); nodesDestroyNode((SNode*)pInfo->pPhyNode);
colDataDestroy(&pInfo->twAggSup.timeWindowData); colDataDestroy(&pInfo->twAggSup.timeWindowData);
cleanupGroupResInfo(&pInfo->groupResInfo); cleanupGroupResInfo(&pInfo->groupResInfo);
taosHashCleanup(pInfo->pGroupIdTbNameMap);
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
@ -2545,9 +2545,11 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode
pInfo->interval.interval = pInterpPhyNode->interval; pInfo->interval.interval = pInterpPhyNode->interval;
pInfo->current = pInfo->win.skey; pInfo->current = pInfo->win.skey;
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
STableScanInfo* pScanInfo = (STableScanInfo*)downstream->info; STableScanInfo* pScanInfo = (STableScanInfo*)downstream->info;
pScanInfo->cond.twindows = pInfo->win; pScanInfo->base.cond.twindows = pInfo->win;
pScanInfo->cond.type = TIMEWINDOW_RANGE_EXTERNAL; pScanInfo->base.cond.type = TIMEWINDOW_RANGE_EXTERNAL;
}
setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo, setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo,
pTaskInfo); pTaskInfo);
@ -3153,11 +3155,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
} }
printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv"); printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv");
if (pBlock->info.parTbName[0]) {
taosHashPut(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t), &pBlock->info.parTbName,
TSDB_TABLE_NAME_LEN);
}
ASSERT(pBlock->info.type != STREAM_INVERT); ASSERT(pBlock->info.type != STREAM_INVERT);
if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) { if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {
pInfo->binfo.pRes->info.type = pBlock->info.type; pInfo->binfo.pRes->info.type = pBlock->info.type;
@ -3372,9 +3369,6 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
pInfo->delKey.ts = INT64_MAX; pInfo->delKey.ts = INT64_MAX;
pInfo->delKey.groupId = 0; pInfo->delKey.groupId = 0;
pInfo->pGroupIdTbNameMap =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
pOperator->operatorType = pPhyNode->type; pOperator->operatorType = pPhyNode->type;
pOperator->blocking = true; pOperator->blocking = true;
pOperator->status = OP_NOT_OPENED; pOperator->status = OP_NOT_OPENED;
@ -3425,7 +3419,6 @@ void destroyStreamSessionAggOperatorInfo(void* param) {
blockDataDestroy(pInfo->pWinBlock); blockDataDestroy(pInfo->pWinBlock);
blockDataDestroy(pInfo->pUpdateRes); blockDataDestroy(pInfo->pUpdateRes);
tSimpleHashCleanup(pInfo->pStDeleted); tSimpleHashCleanup(pInfo->pStDeleted);
taosHashCleanup(pInfo->pGroupIdTbNameMap);
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
@ -3857,30 +3850,18 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo
SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
colDataAppendNULL(pCalEdCol, pBlock->info.rows); colDataAppendNULL(pCalEdCol, pBlock->info.rows);
SHashObj* pGroupIdTbNameMap = NULL;
if (pOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
pOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION ||
pOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
SStreamSessionAggOperatorInfo* pInfo = pOp->info;
pGroupIdTbNameMap = pInfo->pGroupIdTbNameMap;
} else if (pOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) {
SStreamStateAggOperatorInfo* pInfo = pOp->info;
pGroupIdTbNameMap = pInfo->pGroupIdTbNameMap;
} else {
ASSERT(0);
}
char* tbname = taosHashGet(pGroupIdTbNameMap, &res->groupId, sizeof(int64_t));
SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
void* tbname = NULL;
streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname);
if (tbname == NULL) { if (tbname == NULL) {
/*printf("\n\n no tbname for group id %" PRId64 "%p %p\n\n", res->groupId, pOp->info, pGroupIdTbNameMap);*/
colDataAppendNULL(pTableCol, pBlock->info.rows); colDataAppendNULL(pTableCol, pBlock->info.rows);
} else { } else {
char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
colDataAppend(pTableCol, pBlock->info.rows, (const char*)parTbName, false); colDataAppend(pTableCol, pBlock->info.rows, (const char*)parTbName, false);
/*printf("\n\n get tbname %s group id %" PRId64 "\n\n", tbname, res->groupId);*/
} }
tdbFree(tbname);
pBlock->info.rows += 1; pBlock->info.rows += 1;
} }
if ((*Ite) == NULL) { if ((*Ite) == NULL) {
@ -4053,19 +4034,6 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
} }
printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "final session recv" : "single session recv"); printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "final session recv" : "single session recv");
if (pBlock->info.parTbName[0]) {
taosHashPut(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t), &pBlock->info.parTbName,
TSDB_TABLE_NAME_LEN);
/*printf("\n\n put tbname %s group id %" PRId64 "\n\n into %p %p", pBlock->info.parTbName, pBlock->info.groupId,*/
/*pInfo, pInfo->pGroupIdTbNameMap);*/
}
if (pBlock->info.parTbName[0]) {
taosHashPut(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t), &pBlock->info.parTbName,
TSDB_TABLE_NAME_LEN);
/*printf("\n\n put tbname %s\n\n", pBlock->info.parTbName);*/
}
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) { pBlock->info.type == STREAM_CLEAR) {
SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));
@ -4209,8 +4177,6 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
pInfo->isFinal = false; pInfo->isFinal = false;
pInfo->pPhyNode = pPhyNode; pInfo->pPhyNode = pPhyNode;
pInfo->ignoreExpiredData = pSessionNode->window.igExpired; pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
pInfo->pGroupIdTbNameMap =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,
OP_NOT_OPENED, pInfo, pTaskInfo); OP_NOT_OPENED, pInfo, pTaskInfo);
@ -4285,12 +4251,6 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
} }
printDataBlock(pBlock, "semi session recv"); printDataBlock(pBlock, "semi session recv");
if (pBlock->info.parTbName[0]) {
taosHashPut(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t), &pBlock->info.parTbName,
TSDB_TABLE_NAME_LEN);
/*printf("\n\n put tbname %s\n\n", pBlock->info.parTbName);*/
}
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) { pBlock->info.type == STREAM_CLEAR) {
// gap must be 0 // gap must be 0
@ -4418,7 +4378,6 @@ void destroyStreamStateOperatorInfo(void* param) {
colDataDestroy(&pInfo->twAggSup.timeWindowData); colDataDestroy(&pInfo->twAggSup.timeWindowData);
blockDataDestroy(pInfo->pDelRes); blockDataDestroy(pInfo->pDelRes);
tSimpleHashCleanup(pInfo->pSeDeleted); tSimpleHashCleanup(pInfo->pSeDeleted);
taosHashCleanup(pInfo->pGroupIdTbNameMap);
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
@ -4612,12 +4571,6 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
} }
printDataBlock(pBlock, "single state recv"); printDataBlock(pBlock, "single state recv");
if (pBlock->info.parTbName[0]) {
taosHashPut(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t), &pBlock->info.parTbName,
TSDB_TABLE_NAME_LEN);
/*printf("\n\n put tbname %s\n\n", pBlock->info.parTbName);*/
}
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) { pBlock->info.type == STREAM_CLEAR) {
SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));
@ -4731,9 +4684,6 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->pChildren = NULL; pInfo->pChildren = NULL;
pInfo->ignoreExpiredData = pStateNode->window.igExpired; pInfo->ignoreExpiredData = pStateNode->window.igExpired;
pInfo->pGroupIdTbNameMap =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED,
pInfo, pTaskInfo); pInfo, pTaskInfo);
pOperator->fpSet = pOperator->fpSet =
@ -5384,12 +5334,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
} }
printDataBlock(pBlock, "single interval recv"); printDataBlock(pBlock, "single interval recv");
if (pBlock->info.parTbName[0]) {
taosHashPut(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t), &pBlock->info.parTbName,
TSDB_TABLE_NAME_LEN);
/*printf("\n\n put tbname %s\n\n", pBlock->info.parTbName);*/
}
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) { pBlock->info.type == STREAM_CLEAR) {
doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pUpdatedMap); doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pUpdatedMap);
@ -5547,9 +5491,6 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->delKey.ts = INT64_MAX; pInfo->delKey.ts = INT64_MAX;
pInfo->delKey.groupId = 0; pInfo->delKey.groupId = 0;
pInfo->pGroupIdTbNameMap =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED,
pInfo, pTaskInfo); pInfo, pTaskInfo);
pOperator->fpSet = pOperator->fpSet =

7
source/libs/scalar/src/filter.c
View File

@ -1704,7 +1704,8 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
} }
qDebug("GROUP Num:%u", info->groupNum); qDebug("GROUP Num:%u", info->groupNum);
for (uint32_t i = 0; i < info->groupNum; ++i) { uint32_t maxDbgGrpNum = TMIN(info->groupNum, 1000);
for (uint32_t i = 0; i < maxDbgGrpNum; ++i) {
SFilterGroup *group = &info->groups[i]; SFilterGroup *group = &info->groups[i];
qDebug("Group%d : unit num[%u]", i, group->unitNum); qDebug("Group%d : unit num[%u]", i, group->unitNum);
@ -3920,6 +3921,10 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) {
} else { } else {
SColumnNode *refNode = (SColumnNode *)node->pLeft; SColumnNode *refNode = (SColumnNode *)node->pLeft;
SNodeListNode *listNode = (SNodeListNode *)node->pRight; SNodeListNode *listNode = (SNodeListNode *)node->pRight;
if (LIST_LENGTH(listNode->pNodeList) > 10) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
int32_t type = vectorGetConvertType(refNode->node.resType.type, listNode->dataType.type); int32_t type = vectorGetConvertType(refNode->node.resType.type, listNode->dataType.type);
if (0 != type && type != refNode->node.resType.type) { if (0 != type && type != refNode->node.resType.type) {
stat->scalarMode = true; stat->scalarMode = true;

18
source/libs/stream/src/streamState.c
View File

@ -16,6 +16,7 @@
#include "executor.h" #include "executor.h"
#include "streamInc.h" #include "streamInc.h"
#include "tcommon.h" #include "tcommon.h"
#include "tcompare.h"
#include "ttimer.h" #include "ttimer.h"
// todo refactor // todo refactor
@ -144,6 +145,11 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int
goto _err; goto _err;
} }
if (tdbTbOpen("parname.state.db", sizeof(int64_t), TSDB_TABLE_NAME_LEN, NULL, pState->db, &pState->pParNameDb, 0) <
0) {
goto _err;
}
if (streamStateBegin(pState) < 0) { if (streamStateBegin(pState) < 0) {
goto _err; goto _err;
} }
@ -157,6 +163,7 @@ _err:
tdbTbClose(pState->pFuncStateDb); tdbTbClose(pState->pFuncStateDb);
tdbTbClose(pState->pFillStateDb); tdbTbClose(pState->pFillStateDb);
tdbTbClose(pState->pSessionStateDb); tdbTbClose(pState->pSessionStateDb);
tdbTbClose(pState->pParNameDb);
tdbClose(pState->db); tdbClose(pState->db);
taosMemoryFree(pState); taosMemoryFree(pState);
return NULL; return NULL;
@ -169,6 +176,7 @@ void streamStateClose(SStreamState* pState) {
tdbTbClose(pState->pFuncStateDb); tdbTbClose(pState->pFuncStateDb);
tdbTbClose(pState->pFillStateDb); tdbTbClose(pState->pFillStateDb);
tdbTbClose(pState->pSessionStateDb); tdbTbClose(pState->pSessionStateDb);
tdbTbClose(pState->pParNameDb);
tdbClose(pState->db); tdbClose(pState->db);
taosMemoryFree(pState); taosMemoryFree(pState);
@ -812,6 +820,16 @@ _end:
return res; return res;
} }
int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) {
tdbTbUpsert(pState->pParNameDb, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN, &pState->txn);
return 0;
}
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal) {
int32_t len;
return tdbTbGet(pState->pParNameDb, &groupId, sizeof(int64_t), pVal, &len);
}
#if 0 #if 0
char* streamStateSessionDump(SStreamState* pState) { char* streamStateSessionDump(SStreamState* pState) {
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));

13
source/libs/sync/src/syncMain.c
View File

@ -436,8 +436,15 @@ int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode) {
} }
int32_t ret = 0; int32_t ret = 0;
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { if (pSyncNode->state == TAOS_SYNC_STATE_LEADER && pSyncNode->replicaNum > 1) {
SNodeInfo newLeader = (pSyncNode->peersNodeInfo)[0]; SNodeInfo newLeader = (pSyncNode->peersNodeInfo)[0];
if (pSyncNode->peersNum == 2) {
SyncIndex matchIndex0 = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, &(pSyncNode->peersId[0]));
SyncIndex matchIndex1 = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, &(pSyncNode->peersId[1]));
if (matchIndex1 > matchIndex0) {
newLeader = (pSyncNode->peersNodeInfo)[1];
}
}
ret = syncNodeLeaderTransferTo(pSyncNode, newLeader); ret = syncNodeLeaderTransferTo(pSyncNode, newLeader);
} }
@ -1396,7 +1403,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
// reset sender // reset sender
bool reset = false; bool reset = false;
for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) {
if (syncUtilSameId(&(pSyncNode->replicasId)[i], &oldReplicasId[j])) { if (syncUtilSameId(&(pSyncNode->replicasId)[i], &oldReplicasId[j]) && oldSenders[j] != NULL) {
char host[128]; char host[128];
uint16_t port; uint16_t port;
syncUtilU642Addr((pSyncNode->replicasId)[i].addr, host, sizeof(host), &port); syncUtilU642Addr((pSyncNode->replicasId)[i].addr, host, sizeof(host), &port);
@ -1413,6 +1420,8 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
sNTrace(pSyncNode, "snapshot sender udpate replicaIndex from %d to %d, %s:%d, %p, reset:%d", oldreplicaIndex, sNTrace(pSyncNode, "snapshot sender udpate replicaIndex from %d to %d, %s:%d, %p, reset:%d", oldreplicaIndex,
i, host, port, (pSyncNode->senders)[i], reset); i, host, port, (pSyncNode->senders)[i], reset);
break;
} }
} }
} }

8
source/libs/transport/src/tmsgcb.c
View File

@ -44,7 +44,13 @@ int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) {
return code; return code;
} }
void tmsgSendRsp(SRpcMsg* pMsg) { return (*defaultMsgCb.sendRspFp)(pMsg); } void tmsgSendRsp(SRpcMsg* pMsg) {
#if 1
rpcSendResponse(pMsg);
#else
return (*defaultMsgCb.sendRspFp)(pMsg);
#endif
}
void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) { (*defaultMsgCb.sendRedirectRspFp)(pMsg, pNewEpSet); } void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) { (*defaultMsgCb.sendRedirectRspFp)(pMsg, pNewEpSet); }

14
source/libs/transport/src/transSvr.c
View File

@ -462,8 +462,6 @@ static void uvStartSendResp(SSvrMsg* smsg) {
if (pConn->broken == true) { if (pConn->broken == true) {
// persist by // persist by
destroySmsg(smsg); destroySmsg(smsg);
// transFreeMsg(smsg->msg.pCont);
// taosMemoryFree(smsg);
transUnrefSrvHandle(pConn); transUnrefSrvHandle(pConn);
return; return;
} }
@ -1234,7 +1232,9 @@ int transReleaseSrvHandle(void* handle) {
m->type = Release; m->type = Release;
tTrace("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle); tTrace("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle);
transAsyncSend(pThrd->asyncPool, &m->q); if (0 != transAsyncSend(pThrd->asyncPool, &m->q)) {
destroySmsg(m);
}
transReleaseExHandle(transGetRefMgt(), refId); transReleaseExHandle(transGetRefMgt(), refId);
return 0; return 0;
@ -1269,7 +1269,9 @@ int transSendResponse(const STransMsg* msg) {
STraceId* trace = (STraceId*)&msg->info.traceId; STraceId* trace = (STraceId*)&msg->info.traceId;
tGTrace("conn %p start to send resp (1/2)", exh->handle); tGTrace("conn %p start to send resp (1/2)", exh->handle);
transAsyncSend(pThrd->asyncPool, &m->q); if (0 != transAsyncSend(pThrd->asyncPool, &m->q)) {
destroySmsg(m);
}
transReleaseExHandle(transGetRefMgt(), refId); transReleaseExHandle(transGetRefMgt(), refId);
return 0; return 0;
@ -1303,7 +1305,9 @@ int transRegisterMsg(const STransMsg* msg) {
STrans* pTransInst = pThrd->pTransInst; STrans* pTransInst = pThrd->pTransInst;
tTrace("%s conn %p start to register brokenlink callback", transLabel(pTransInst), exh->handle); tTrace("%s conn %p start to register brokenlink callback", transLabel(pTransInst), exh->handle);
transAsyncSend(pThrd->asyncPool, &m->q); if (0 != transAsyncSend(pThrd->asyncPool, &m->q)) {
destroySmsg(m);
}
transReleaseExHandle(transGetRefMgt(), refId); transReleaseExHandle(transGetRefMgt(), refId);
return 0; return 0;

2
source/os/src/osDir.c
View File

@ -336,6 +336,7 @@ int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) {
#else #else
if (realpath(dirname, tmp) != NULL) { if (realpath(dirname, tmp) != NULL) {
#endif #endif
if (strlen(tmp) < maxlen) {
if (realPath == NULL) { if (realPath == NULL) {
strncpy(dirname, tmp, maxlen); strncpy(dirname, tmp, maxlen);
} else { } else {
@ -343,6 +344,7 @@ int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) {
} }
return 0; return 0;
} }
}
return -1; return -1;
} }

1
source/util/src/terror.c
View File

@ -344,6 +344,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_ALREADY_EXIST, "Stable already exists
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST, "Stable not exists") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST, "Stable not exists")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER, "Table schema is old") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER, "Table schema is old")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE, "Table already exists in other stables")
// query // query
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle")

34
tests/parallel_test/cases.task
View File

@ -11,7 +11,7 @@
,,y,script,./test.sh -f tsim/user/privilege_db.sim ,,y,script,./test.sh -f tsim/user/privilege_db.sim
,,y,script,./test.sh -f tsim/user/privilege_sysinfo.sim ,,y,script,./test.sh -f tsim/user/privilege_sysinfo.sim
,,y,script,./test.sh -f tsim/db/alter_option.sim ,,y,script,./test.sh -f tsim/db/alter_option.sim
#,,y,script,./test.sh -f tsim/db/alter_replica_13.sim ,,y,script,./test.sh -f tsim/db/alter_replica_13.sim
,,y,script,./test.sh -f tsim/db/alter_replica_31.sim ,,y,script,./test.sh -f tsim/db/alter_replica_31.sim
,,y,script,./test.sh -f tsim/db/basic1.sim ,,y,script,./test.sh -f tsim/db/basic1.sim
,,y,script,./test.sh -f tsim/db/basic2.sim ,,y,script,./test.sh -f tsim/db/basic2.sim
@ -48,7 +48,7 @@
,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim ,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim ,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim ,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
,,,script,./test.sh -f tsim/dnode/drop_dnode_force.sim ,,y,script,./test.sh -f tsim/dnode/drop_dnode_force.sim
,,y,script,./test.sh -f tsim/dnode/offline_reason.sim ,,y,script,./test.sh -f tsim/dnode/offline_reason.sim
,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim ,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim
,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim ,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
@ -56,7 +56,7 @@
,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim ,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim
,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim ,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim
,,y,script,./test.sh -f tsim/dnode/vnode_clean.sim ,,y,script,./test.sh -f tsim/dnode/vnode_clean.sim
,,,script,./test.sh -f tsim/dnode/use_dropped_dnode.sim ,,y,script,./test.sh -f tsim/dnode/use_dropped_dnode.sim
,,y,script,./test.sh -f tsim/dnode/split_vgroup_replica1.sim ,,y,script,./test.sh -f tsim/dnode/split_vgroup_replica1.sim
,,y,script,./test.sh -f tsim/dnode/split_vgroup_replica3.sim ,,y,script,./test.sh -f tsim/dnode/split_vgroup_replica3.sim
,,y,script,./test.sh -f tsim/import/basic.sim ,,y,script,./test.sh -f tsim/import/basic.sim
@ -113,7 +113,7 @@
,,y,script,./test.sh -f tsim/parser/first_last.sim ,,y,script,./test.sh -f tsim/parser/first_last.sim
,,y,script,./test.sh -f tsim/parser/fill_stb.sim ,,y,script,./test.sh -f tsim/parser/fill_stb.sim
,,y,script,./test.sh -f tsim/parser/interp.sim ,,y,script,./test.sh -f tsim/parser/interp.sim
#,,y,script,./test.sh -f tsim/parser/limit2.sim ,,y,script,./test.sh -f tsim/parser/limit2.sim
,,y,script,./test.sh -f tsim/parser/fourArithmetic-basic.sim ,,y,script,./test.sh -f tsim/parser/fourArithmetic-basic.sim
,,y,script,./test.sh -f tsim/parser/function.sim ,,y,script,./test.sh -f tsim/parser/function.sim
,,y,script,./test.sh -f tsim/parser/groupby-basic.sim ,,y,script,./test.sh -f tsim/parser/groupby-basic.sim
@ -136,8 +136,8 @@
,,y,script,./test.sh -f tsim/parser/lastrow.sim ,,y,script,./test.sh -f tsim/parser/lastrow.sim
,,y,script,./test.sh -f tsim/parser/lastrow2.sim ,,y,script,./test.sh -f tsim/parser/lastrow2.sim
,,y,script,./test.sh -f tsim/parser/like.sim ,,y,script,./test.sh -f tsim/parser/like.sim
,,,script,./test.sh -f tsim/parser/limit.sim ,,y,script,./test.sh -f tsim/parser/limit.sim
,,,script,./test.sh -f tsim/parser/limit1.sim ,,y,script,./test.sh -f tsim/parser/limit1.sim
,,y,script,./test.sh -f tsim/parser/mixed_blocks.sim ,,y,script,./test.sh -f tsim/parser/mixed_blocks.sim
,,y,script,./test.sh -f tsim/parser/nchar.sim ,,y,script,./test.sh -f tsim/parser/nchar.sim
,,y,script,./test.sh -f tsim/parser/nestquery.sim ,,y,script,./test.sh -f tsim/parser/nestquery.sim
@ -163,7 +163,7 @@
,,y,script,./test.sh -f tsim/parser/timestamp.sim ,,y,script,./test.sh -f tsim/parser/timestamp.sim
,,y,script,./test.sh -f tsim/parser/top_groupby.sim ,,y,script,./test.sh -f tsim/parser/top_groupby.sim
,,y,script,./test.sh -f tsim/parser/topbot.sim ,,y,script,./test.sh -f tsim/parser/topbot.sim
,,,script,./test.sh -f tsim/parser/union.sim ,,y,script,./test.sh -f tsim/parser/union.sim
,,y,script,./test.sh -f tsim/parser/union_sysinfo.sim ,,y,script,./test.sh -f tsim/parser/union_sysinfo.sim
,,y,script,./test.sh -f tsim/parser/where.sim ,,y,script,./test.sh -f tsim/parser/where.sim
,,y,script,./test.sh -f tsim/query/charScalarFunction.sim ,,y,script,./test.sh -f tsim/query/charScalarFunction.sim
@ -176,11 +176,11 @@
,,y,script,./test.sh -f tsim/query/udf.sim ,,y,script,./test.sh -f tsim/query/udf.sim
,,y,script,./test.sh -f tsim/qnode/basic1.sim ,,y,script,./test.sh -f tsim/qnode/basic1.sim
,,y,script,./test.sh -f tsim/snode/basic1.sim ,,y,script,./test.sh -f tsim/snode/basic1.sim
,,,script,./test.sh -f tsim/mnode/basic1.sim ,,y,script,./test.sh -f tsim/mnode/basic1.sim
,,,script,./test.sh -f tsim/mnode/basic2.sim ,,y,script,./test.sh -f tsim/mnode/basic2.sim
,,,script,./test.sh -f tsim/mnode/basic3.sim ,,y,script,./test.sh -f tsim/mnode/basic3.sim
,,,script,./test.sh -f tsim/mnode/basic4.sim ,,y,script,./test.sh -f tsim/mnode/basic4.sim
,,,script,./test.sh -f tsim/mnode/basic5.sim ,,y,script,./test.sh -f tsim/mnode/basic5.sim
,,y,script,./test.sh -f tsim/show/basic.sim ,,y,script,./test.sh -f tsim/show/basic.sim
,,y,script,./test.sh -f tsim/table/autocreate.sim ,,y,script,./test.sh -f tsim/table/autocreate.sim
,,y,script,./test.sh -f tsim/table/basic1.sim ,,y,script,./test.sh -f tsim/table/basic1.sim
@ -278,7 +278,7 @@
,,y,script,./test.sh -f tsim/stable/values.sim ,,y,script,./test.sh -f tsim/stable/values.sim
,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/vnode3.sim
,,y,script,./test.sh -f tsim/stable/metrics_idx.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim
,,,script,./test.sh -f tsim/sma/drop_sma.sim ,,n,script,./test.sh -f tsim/sma/drop_sma.sim
,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim
,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
@ -291,7 +291,7 @@
,,n,script,./test.sh -f tsim/valgrind/checkError7.sim ,,n,script,./test.sh -f tsim/valgrind/checkError7.sim
,,n,script,./test.sh -f tsim/valgrind/checkError8.sim ,,n,script,./test.sh -f tsim/valgrind/checkError8.sim
,,n,script,./test.sh -f tsim/valgrind/checkUdf.sim ,,n,script,./test.sh -f tsim/valgrind/checkUdf.sim
,,,script,./test.sh -f tsim/vnode/replica3_basic.sim ,,y,script,./test.sh -f tsim/vnode/replica3_basic.sim
,,y,script,./test.sh -f tsim/vnode/replica3_repeat.sim ,,y,script,./test.sh -f tsim/vnode/replica3_repeat.sim
,,y,script,./test.sh -f tsim/vnode/replica3_vgroup.sim ,,y,script,./test.sh -f tsim/vnode/replica3_vgroup.sim
,,y,script,./test.sh -f tsim/vnode/replica3_many.sim ,,y,script,./test.sh -f tsim/vnode/replica3_many.sim
@ -424,6 +424,7 @@
,,,system-test,python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py
,,,system-test,python3 ./test.py -f 1-insert/alter_stable.py ,,,system-test,python3 ./test.py -f 1-insert/alter_stable.py
,,,system-test,python3 ./test.py -f 1-insert/alter_table.py ,,,system-test,python3 ./test.py -f 1-insert/alter_table.py
,,,system-test,python3 ./test.py -f 1-insert/boundary.py
,,,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py ,,,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py
,,,system-test,python3 ./test.py -f 1-insert/table_comment.py ,,,system-test,python3 ./test.py -f 1-insert/table_comment.py
,,,system-test,python3 ./test.py -f 1-insert/time_range_wise.py ,,,system-test,python3 ./test.py -f 1-insert/time_range_wise.py
@ -607,6 +608,8 @@
,,,system-test,python3 ./test.py -f 2-query/upper.py -R ,,,system-test,python3 ./test.py -f 2-query/upper.py -R
,,,system-test,python3 ./test.py -f 2-query/varchar.py ,,,system-test,python3 ./test.py -f 2-query/varchar.py
,,,system-test,python3 ./test.py -f 2-query/varchar.py -R ,,,system-test,python3 ./test.py -f 2-query/varchar.py -R
,,,system-test,python3 ./test.py -f 2-query/case_when.py
,,,system-test,python3 ./test.py -f 2-query/case_when.py -R
,,,system-test,python3 ./test.py -f 1-insert/update_data.py ,,,system-test,python3 ./test.py -f 1-insert/update_data.py
,,,system-test,python3 ./test.py -f 1-insert/tb_100w_data_order.py ,,,system-test,python3 ./test.py -f 1-insert/tb_100w_data_order.py
,,,system-test,python3 ./test.py -f 1-insert/delete_stable.py ,,,system-test,python3 ./test.py -f 1-insert/delete_stable.py
@ -813,6 +816,7 @@
,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 2
,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 2
,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 2
,,,system-test,python3 ./test.py -f 2-query/case_when.py -Q 2
,,,system-test,python3 ./test.py -f 2-query/between.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/between.py -Q 3
,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 3
,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 3
@ -906,6 +910,7 @@
,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 3
,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 3
,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 3
,,,system-test,python3 ./test.py -f 2-query/case_when.py -Q 3
,,,system-test,python3 ./test.py -f 2-query/between.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/between.py -Q 4
,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 4
,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 4
@ -998,6 +1003,7 @@
,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 4
,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 4
,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 4
,,,system-test,python3 ./test.py -f 2-query/case_when.py -Q 4
#develop test #develop test
,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py ,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py

11
tests/pytest/util/sql.py
View File

@ -73,8 +73,15 @@ class TDSql:
expectErrNotOccured = True expectErrNotOccured = True
try: try:
self.cursor.execute(sql) self.cursor.execute(sql)
except BaseException: except BaseException as e:
expectErrNotOccured = False expectErrNotOccured = False
caller = inspect.getframeinfo(inspect.stack()[1][0])
self.error_info = repr(e)
# print(error_info)
# self.error_info = error_info[error_info.index('(')+1:-1].split(",")[0].replace("'","")
# self.error_info = (','.join(error_info.split(",")[:-1]).split("(",1)[1:][0]).replace("'","")
# print("!!!!!!!!!!!!!!",self.error_info)
if expectErrNotOccured: if expectErrNotOccured:
caller = inspect.getframeinfo(inspect.stack()[1][0]) caller = inspect.getframeinfo(inspect.stack()[1][0])
tdLog.exit("%s(%d) failed: sql:%s, expect error not occured" % (caller.filename, caller.lineno, sql)) tdLog.exit("%s(%d) failed: sql:%s, expect error not occured" % (caller.filename, caller.lineno, sql))
@ -83,6 +90,8 @@ class TDSql:
self.queryCols = 0 self.queryCols = 0
self.queryResult = None self.queryResult = None
tdLog.info("sql:%s, expect error occured" % (sql)) tdLog.info("sql:%s, expect error occured" % (sql))
return self.error_info
def query(self, sql, row_tag=None,queryTimes=10): def query(self, sql, row_tag=None,queryTimes=10):
self.sql = sql self.sql = sql

4
tests/script/tsim/db/alter_replica_13.sim
View File

@ -138,10 +138,10 @@ while $i < 10
if $data[0][4] != leader then if $data[0][4] != leader then
return -1 return -1
endi endi
if $data[0][6] != follower then if $data[0][6] == leader then
return -1 return -1
endi endi
if $data[0][8] != follower then if $data[0][8] == leader then
return -1 return -1
endi endi
endw endw

34
tests/script/tsim/parser/limit2_query.sim
View File

@ -321,55 +321,41 @@ endi
### [TBASE-350] ### [TBASE-350]
## stb + interval + fill + group by + limit offset ## stb + interval + fill + group by + limit offset
sql select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') limit 2 offset 10 sql select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') limit 2 offset 10
if $rows != 2 then if $rows != 2 then
return -1 return -1
endi endi
#add one more test case
sql select max(c1), last(c8) from lm2_db0.lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(linear) limit 10 offset 4089;"
$limit = 5 $limit = 5
$offset = $rowNum * 2 $offset = $rowNum * 2
$offset = $offset - 2 $offset = $offset - 2
sql select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') order by t1 limit $limit offset $offset sql select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') order by t1 limit $limit offset $offset
if $rows != $tbNum then if $rows != 1 then
return -1 return -1
endi endi
if $data00 != @18-11-25 19:30:00.000@ then if $data00 != 9 then
return -1 return -1
endi endi
if $data01 != 9 then if $data01 != 9 then
return -1 return -1
endi endi
if $data12 != 9 then if $data02 != 9.000000000 then
return -1 return -1
endi endi
if $data23 != 9.000000000 then if $data03 != 9.000000000 then
return -1 return -1
endi endi
if $data34 != 9.000000000 then if $data04 != 1 then
return -1 return -1
endi endi
if $data45 != 1 then if $data05 != binary9 then
return -1 return -1
endi endi
if $data56 != binary9 then if $data06 != nchar9 then
return -1
endi
if $data68 != 6 then
return -1
endi
if $data72 != -2 then
return -1
endi
if $data84 != -2.000000000 then
return -1
endi
if $data98 != 9 then
return -1 return -1
endi endi
#add one more test case #add one more test case
sql select max(c1), last(c8) from lm2_db0.lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(linear) limit 10 offset 4089;" sql select max(c1), last(c8) from lm2_db0.lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(linear) limit 10 offset 4089;"

183
tests/system-test/1-insert/boundary.py Normal file
View File

@ -0,0 +1,183 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import math
from random import randint
from util.log import *
from util.cases import *
from util.sql import *
from util.common import *
from util.sqlset import *
from util.boundary import *
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.boundary = DataBoundary()
self.dbname_length_boundary = self.boundary.DBNAME_MAX_LENGTH
self.tbname_length_boundary = self.boundary.TBNAME_MAX_LENGTH
self.stbname_length_boundary = self.boundary.STBNAME_MAX_LENGTH
self.colname_length_boundary = self.boundary.COL_KEY_MAX_LENGTH
self.tagname_length_boundary = self.boundary.TAG_KEY_MAX_LENGTH
self.username_length_boundary = 23
self.password_length_boundary = 128
def dbname_length_check(self):
dbname_length = randint(1,self.dbname_length_boundary-1)
for dbname in [tdCom.get_long_name(self.dbname_length_boundary),tdCom.get_long_name(dbname_length)]:
tdSql.execute(f'create database if not exists {dbname}')
tdSql.query(f'select name from information_schema.ins_databases where name = "{dbname}"')
tdSql.checkEqual(tdSql.queryResult[0][0],dbname)
tdSql.execute(f'drop database if exists {dbname}')
dbname = tdCom.get_long_name(self.dbname_length_boundary+1)
tdSql.error(f'create database if not exists {dbname}')
if "Invalid identifier name" in tdSql.error_info:
tdLog.info("error info is true!")
else:
tdLog.exit("error info is not true")
def tbname_length_check(self):
tdSql.prepare()
tdSql.execute('use db')
tbname_length = randint(1,self.tbname_length_boundary-1)
tdSql.execute(f'create table stb (ts timestamp,c0 int) tags(t0 int)')
for tbname in [tdCom.get_long_name(self.tbname_length_boundary),tdCom.get_long_name(tbname_length)]:
tdSql.execute(f'create table {tbname} using stb tags(1)')
tdSql.query(f'select table_name from information_schema.ins_tables where table_name = "{tbname}"')
tdSql.checkEqual(tdSql.queryResult[0][0],tbname)
tdSql.execute(f'drop table {tbname}')
tbname = tdCom.get_long_name(self.tbname_length_boundary+1)
tdSql.error(f'create table {tbname} using stb tags(1)')
if "Invalid identifier name" in tdSql.error_info:
tdLog.info("error info is true!")
else:
tdLog.exit("error info is not true")
stbname_length = randint(1,self.stbname_length_boundary-1)
for stbname in [tdCom.get_long_name(self.stbname_length_boundary),tdCom.get_long_name(stbname_length)]:
tdSql.execute(f'create table {stbname} (ts timestamp,c0 int) tags(t0 int)')
tdSql.query(f'select stable_name from information_schema.ins_stables where stable_name = "{stbname}"')
tdSql.checkEqual(tdSql.queryResult[0][0],stbname)
tdSql.execute(f'drop table {stbname}')
stbname = tdCom.get_long_name(self.stbname_length_boundary+1)
tdSql.error(f'create table {stbname} (ts timestamp,c0 int) tags(t0 int)')
print(tdSql.error_info)
if "Invalid identifier name" in tdSql.error_info:
tdLog.info("error info is true!")
else:
tdLog.exit("error info is not true")
tdSql.execute('drop database db')
def colname_length_check(self):
tdSql.prepare()
tdSql.execute('use db')
column_name_length = randint(1,self.colname_length_boundary-1)
for colname in [tdCom.get_long_name(column_name_length),tdCom.get_long_name(self.colname_length_boundary)]:
stbname = tdCom.get_long_name(3)
ntbname = tdCom.get_long_name(4)
tdSql.execute(f'create table {stbname} (ts timestamp,{colname} int) tags(t0 int)')
tdSql.query(f'describe {stbname}')
tdSql.checkEqual(tdSql.queryResult[1][0],colname)
tdSql.execute(f'create table {ntbname} (ts timestamp,{colname} int)')
tdSql.query(f'describe {ntbname}')
tdSql.checkEqual(tdSql.queryResult[1][0],colname)
colname = tdCom.get_long_name(self.colname_length_boundary+1)
tdSql.error(f'create table stb (ts timestamp,{colname} int) tags(t0 int)')
if "Invalid identifier name" in tdSql.error_info:
tdLog.info("error info is true!")
else:
tdLog.exit("error info is not true")
tdSql.execute('drop database db')
def tagname_length_check(self):
tdSql.prepare()
tdSql.execute('use db')
tag_name_length = randint(1,self.tagname_length_boundary-1)
for tagname in (tdCom.get_long_name(tag_name_length),tdCom.get_long_name(self.tagname_length_boundary)):
stbname = tdCom.get_long_name(3)
tdSql.execute(f'create table {stbname} (ts timestamp,c0 int) tags({tagname} int)')
tdSql.query(f'describe {stbname}')
tdSql.checkEqual(tdSql.queryResult[-1][0],tagname)
tagname = tdCom.get_long_name(self.tagname_length_boundary+1)
tdSql.error(f'create table {stbname} (ts timestamp,c0 int) tags({tagname} int)')
if "Invalid identifier name" in tdSql.error_info:
tdLog.info("error info is true!")
else:
tdLog.exit("error info is not true")
tdSql.execute('drop database db')
def username_length_check(self):
username_length = randint(1,self.username_length_boundary-1)
for username in [tdCom.get_long_name(username_length),tdCom.get_long_name(self.username_length_boundary)]:
tdSql.execute(f'create user {username} pass "123"')
tdSql.query('show users')
for user in tdSql.queryResult:
if user[0].lower() != 'root':
tdSql.checkEqual(user[0],username)
tdSql.execute(f'drop user {username}')
username = tdCom.get_long_name(self.username_length_boundary+1)
tdSql.error(f'create user {username} pass "123"')
if "Name or password too long" in tdSql.error_info:
tdLog.info("error info is true!")
else:
tdLog.exit("error info is not true")
def password_length_check(self):
password_length = randint(1,self.password_length_boundary-1)
for password in [tdCom.get_long_name(password_length),tdCom.get_long_name(self.password_length_boundary)]:
username = tdCom.get_long_name(3)
tdSql.execute(f'create user {username} pass "{password}"')
password = tdCom.get_long_name(self.password_length_boundary+1)
tdSql.error(f'create user {username} pass "{password}"')
if "Name or password too long" in tdSql.error_info:
tdLog.info("error info is true!")
else:
tdLog.exit("error info is not true")
def sql_length_check(self):
insert_rows = 1021
tdSql.prepare()
tdSql.execute('use db')
tdSql.execute('create table ntb (ts timestamp,c0 binary(1013))')
values_sql = ''
value = tdCom.get_long_name(1013)
for num in range(insert_rows):
values_sql += f' (now+{num}s,"{value}")'
value = tdCom.get_long_name(65)
values_sql += f"(now-1s,'{value}')"
tdSql.execute(f'insert into ntb values{values_sql}')
tdSql.query('select * from ntb')
tdSql.checkRows(insert_rows+1)
tdSql.execute('create table ntb1 (ts timestamp,c0 binary(1013))')
tdSql.error(f'insert into ntb1 values{values_sql};')
print(tdSql.error_info)
if "SQL statement too long" in tdSql.error_info:
tdLog.info("error info is true!")
else:
tdLog.exit("error info is not true")
tdSql.execute('drop database db')
def run(self):
self.dbname_length_check()
self.tbname_length_check()
self.colname_length_check()
self.tagname_length_check()
self.username_length_check()
self.password_length_check()
self.sql_length_check()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

2
tests/system-test/1-insert/update_data.py
View File

@ -23,7 +23,7 @@ class TDTestCase:
def init(self, conn, logSql, replicaVar=1): def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar) self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__) tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(),logSql) tdSql.init(conn.cursor())
self.setsql = TDSetSql() self.setsql = TDSetSql()
self.dbname = 'db_test' self.dbname = 'db_test'
self.ntbname = 'ntb' self.ntbname = 'ntb'

338
tests/system-test/2-query/case_when.py Executable file
View File

@ -0,0 +1,338 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import random
import os
import time
import taos
import subprocess
from faker import Faker
from util.log import tdLog
from util.cases import tdCases
from util.sql import tdSql
from util.dnodes import tdDnodes
from util.dnodes import *
class TDTestCase:
updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"querySmaOptimize":1}
def init(self, conn, logSql, replicaVar):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
self.testcasePath = os.path.split(__file__)[0]
self.testcaseFilename = os.path.split(__file__)[-1]
os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename))
self.db = "case_when"
def dropandcreateDB_random(self,database,n):
ts = 1630000000000
num_random = 10
fake = Faker('zh_CN')
tdSql.execute('''drop database if exists %s ;''' %database)
tdSql.execute('''create database %s keep 36500 ;'''%(database))
tdSql.execute('''use %s;'''%database)
tdSql.execute('''create stable %s.stable_1 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \
q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) \
tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);'''%database)
tdSql.execute('''create stable %s.stable_2 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , \
q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) \
tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);'''%database)
for i in range(num_random):
tdSql.execute('''create table %s.table_%d \
(ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp ) ;'''%(database,i))
tdSql.execute('''create table %s.stable_1_%d using %s.stable_1 tags('stable_1_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;'''
%(database,i,database,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1),
fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) ,
fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1)))
tdSql.execute('''create table %s.stable_%d_a using %s.stable_2 tags('stable_2_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;'''
%(database,i,database,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1),
fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) ,
fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1)))
tdSql.execute('''create table %s.stable_%d_b using %s.stable_2 tags('stable_2_%d', '%d' , '%d', '%d' , '%d' , 1 , 'binary1.%s' , 'nchar1.%s' , '%f', '%f' ,'%d') ;'''
%(database,i,database,i,fake.random_int(min=-2147483647, max=2147483647, step=1), fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1),
fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) ,
fake.pystr() ,fake.pystr() ,fake.pyfloat(),fake.pyfloat(),fake.random_int(min=-2147483647, max=2147483647, step=1)))
# insert data
for i in range(num_random):
for j in range(n):
tdSql.execute('''insert into %s.stable_1_%d (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts)\
values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d) ;'''
% (database,i,ts + i*1000 + j, fake.random_int(min=-2147483647, max=2147483647, step=1),
fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1),
fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) ,
fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i))
tdSql.execute('''insert into %s.table_%d (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) \
values(%d, %d, %d, %d, %d, %f, %f, 0, 'binary.%s', 'nchar.%s', %d) ;'''
% (database,i,ts + i*1000 + j, fake.random_int(min=-2147483647, max=2147483647, step=1),
fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1),
fake.random_int(min=-32767, max=32767, step=1) , fake.random_int(min=-127, max=127, step=1) ,
fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i ))
tdSql.execute('''insert into %s.stable_%d_a (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts)\
values(%d, %d, %d, %d, %d, %f, %f, 1, 'binary.%s', 'nchar.%s', %d) ;'''
% (database,i,ts + i*1000 + j, fake.random_int(min=0, max=2147483647, step=1),
fake.random_int(min=0, max=9223372036854775807, step=1),
fake.random_int(min=0, max=32767, step=1) , fake.random_int(min=0, max=127, step=1) ,
fake.pyfloat() , fake.pyfloat() , fake.pystr() , fake.pystr() , ts + i))
tdSql.query("select count(*) from %s.stable_1;" %database)
tdSql.checkData(0,0,num_random*n)
tdSql.query("select count(*) from %s.table_0;"%database)
tdSql.checkData(0,0,n)
def users_bug(self,database):
sql1 = "select (case when `q_smallint` >0 then 'many--' when `q_smallint`<0 then 'little' end),q_int,loc from %s.stable_1 where tbname = 'stable_1_1' limit 100;" %database
sql2 = "select (case when `q_smallint` >0 then 'many--' when `q_smallint`<0 then 'little' end),q_int,loc from %s.stable_1_1 limit 100;" %database
self.constant_check(database,sql1,sql2,0)
sql1 = "select (case when `q_smallint` >0 then 'many![;;][][]]' when `q_smallint`<0 then 'little' end),q_int,loc from %s.stable_1 where tbname = 'stable_1_1' limit 100;" %database
sql2 = "select (case when `q_smallint` >0 then 'many![;;][][]]' when `q_smallint`<0 then 'little' end),q_int,loc from %s.stable_1_1 limit 100;" %database
self.constant_check(database,sql1,sql2,0)
sql1 = "select (case when sum(q_smallint)=0 then null else sum(q_smallint) end) from %s.stable_1 where tbname = 'stable_1_1' limit 100;" %database
sql2 = "select (case when sum(q_smallint)=0 then null else sum(q_smallint) end) from %s.stable_1_1 limit 100;" %database
self.constant_check(database,sql1,sql2,0)
sql1 = "select _wstart,avg(q_int),min(q_smallint) from %s.stable_1 where tbname = 'stable_1_1' and ts < now state_window(case when q_smallint <0 then 1 else 0 end);" %database
sql2 = "select _wstart,avg(q_int),min(q_smallint) from %s.stable_1_1 where ts < now state_window(case when q_smallint <0 then 1 else 0 end);" %database
self.constant_check(database,sql1,sql2,0)
self.constant_check(database,sql1,sql2,1)
self.constant_check(database,sql1,sql2,2)
def casewhen_list(self):
a1,a2,a3 = random.randint(-2147483647,2147483647),random.randint(-2147483647,2147483647),random.randint(-2147483647,2147483647)
casewhen_lists = ['first case when %d then %d end last' %(a1,a2) , #'first case when 3 then 4 end last' ,
'first case when 0 then %d end last' %(a1), #'first case when 0 then 4 end last' ,
'first case when null then %d end last' %(a1) , #'first case when null then 4 end last' ,
'first case when 1 then %d+(%d) end last' %(a1,a2) , #'first case when 1 then 4+1 end last' ,
'first case when %d-(%d) then 0 end last' %(a1,a1) , #'first case when 1-1 then 0 end last' ,
'first case when %d+(%d) then 0 end last' %(a1,a1), #'first case when 1+1 then 0 end last' ,
'first case when 1 then %d-(%d)+(%d) end last' %(a1,a1,a2), #'first case when 1 then 1-1+2 end last' ,
'first case when %d > 0 then %d < %d end last' %(a1,a1,a2), #'first case when 1 > 0 then 1 < 2 end last' ,
'first case when %d > %d then %d < %d end last' %(a1,a2,a1,a2), #'first case when 1 > 2 then 1 < 2 end last' ,
'first case when abs(%d) then abs(-(%d)) end last' %(a1,a2) ,#'first case when abs(3) then abs(-1) end last' ,
'first case when abs(%d+(%d)) then abs(-(%d))+abs(%d) end last' %(a1,a2,a1,a2) , #'first case when abs(1+1) then abs(-1)+abs(3) end last' ,
'first case when 0 then %d else %d end last' %(a1,a2), #'first case when 0 then 1 else 3 end last' ,
'first case when 0 then %d when 1 then %d else %d end last' %(a1,a1,a3), #'first case when 0 then 1 when 1 then 0 else 3 end last' ,
'first case when 0 then %d when 1 then %d when 2 then %d end last' %(a1,a1,a3), #'first case when 0 then 1 when 1 then 0 when 2 then 3 end last' ,
'first case when \'a\' then \'b\' when null then 0 end last' , #'first case when \'a\' then \'b\' when null then 0 end last' ,
'first case when \'%d\' then \'b\' when null then %d end last' %(a1,a2), #'first case when \'2\' then \'b\' when null then 0 end last' ,
'first case when \'%d\' then \'b\' else null end last' %(a1), #'first case when \'0\' then \'b\' else null end last',
'first case when \'%d\' then \'b\' else %d end last' %(a1,a2), #'first case when \'0\' then \'b\' else 2 end last',
'first case when sum(%d) then sum(%d)-sum(%d) end last' %(a1,a1,a3), #'first case when sum(2) then sum(2)-sum(1) end last' ,
'first case when sum(%d) then abs(-(%d)) end last' %(a1,a2), #'first case when sum(2) then abs(-2) end last' ,
'first case when q_int then ts end last' ,
'first case when q_int then q_int when q_int + (%d) then q_int + (%d) else q_int is null end last' %(a1,a2) , #'first case when q_int then q_int when q_int + 1 then q_int + 1 else q_int is null end last' ,
'first case when q_int then %d when ts then ts end last' %(a1), #'first case when q_int then 3 when ts then ts end last' ,
'first case when %d then q_int end last' %(a1), #'first case when 3 then q_int end last' ,
'first case when q_int then %d when %d then %d end last' %(a1,a1,a3), #'first case when q_int then 3 when 1 then 2 end last' ,
'first case when sum(q_int) then sum(q_int)-abs(-(%d)) end last' %(a1), #'first case when sum(q_int) then sum(q_int)-abs(-1) end last' ,
'first case when q_int < %d then %d when q_int >= %d then %d else %d end last' %(a1,a2,a1,a2,a3), #'first case when q_int < 3 then 1 when q_int >= 3 then 2 else 3 end last' ,
'first cast(case q_int when q_int then q_int + (%d) else q_int is null end as double) last' %(a1), #'first cast(case q_int when q_int then q_int + 1 else q_int is null end as double) last' ,
'first sum(case q_int when q_int then q_int + (%d) else q_int is null end + (%d)) last' %(a1,a2), #'first sum(case q_int when q_int then q_int + 1 else q_int is null end + 1) last' ,
'first case when q_int is not null then case when q_int <= %d then q_int else q_int * (%d) end else -(%d) end last' %(a1,a1,a3), #'first case when q_int is not null then case when q_int <= 0 then q_int else q_int * 10 end else -1 end last' ,
'first case %d when %d then %d end last' %(a1,a2,a3), # 'first case 3 when 3 then 4 end last' ,
'first case %d when %d then %d end last' %(a1,a2,a3), # 'first case 3 when 1 then 4 end last' ,
'first case %d when %d then %d else %d end last' %(a1,a1,a2,a3), # 'first case 3 when 1 then 4 else 2 end last' ,
'first case %d when null then %d when \'%d\' then %d end last' %(a1,a1,a2,a3) , # 'first case 3 when null then 4 when \'3\' then 1 end last' ,
'first case \'%d\' when null then %d when %d then %d end last' %(a1,a1,a2,a3), # 'first case \'3\' when null then 4 when 3 then 1 end last' ,
'first case null when null then %d when %d then %d end last' %(a1,a2,a3), # 'first case null when null then 4 when 3 then 1 end last' ,
'first case %d.0 when null then %d when \'%d\' then %d end last' %(a1,a1,a2,a3) , # 'first case 3.0 when null then 4 when \'3\' then 1 end last' ,
'first case q_double when \'a\' then %d when \'%d\' then %d end last' %(a1,a2,a3) , # 'first case q_double when \'a\' then 4 when \'0\' then 1 end last' ,
'first case q_double when q_int then q_int when q_int - (%d) then q_int else %d end last' %(a1,a2), # 'first case q_double when q_int then q_int when q_int - 1 then q_int else 99 end last' ,
'first case cast(q_double as int) when %d then q_double when q_int then %d else ts end last' %(a1,a2), #'first case cast(q_double as int) when 0 then q_double when q_int then 11 else ts end last' ,
'first case q_int + (%d) when %d then %d when %d then %d else %d end last' %(a1,a2,a3,a1,a2,a3), #'first case q_int + 1 when 1 then 1 when 2 then 2 else 3 end last' ,
'first case when \'a\' then \'b\' when null then %d end last' %(a1), # 'first case when \'a\' then \'b\' when null then 0 end last' ,
'first case when \'%d\' then \'b\' when null then %d end last' %(a1,a2), # 'first case when \'2\' then \'b\' when null then 0 end last' ,
'first case when %d then \'b\' else null end last' %(a1), # 'first case when 0 then \'b\' else null end last' ,
'first case when %d then \'b\' else %d+abs(%d) end last' %(a1,a2,a3), # 'first case when 0 then \'b\' else 2+abs(-2) end last' ,
'first case when %d then %d end last' %(a1,a2), # 'first case when 3 then 4 end last' ,
'first case when %d then %d end last' %(a1,a2), # 'first case when 0 then 4 end last' ,
'first case when null then %d end last' %(a1), # 'first case when null then 4 end last' ,
'first case when %d then %d+(%d) end last' %(a1,a2,a3), # 'first case when 1 then 4+1 end last' ,
'first case when %d-(%d) then %d end last' %(a1,a2,a3), # 'first case when 1-1 then 0 end last' ,
'first case when %d+(%d) then %d end last' %(a1,a2,a3), # 'first case when 1+1 then 0 end last' ,
'first case when abs(%d) then abs(%d) end last' %(a1,a2), # 'first case when abs(3) then abs(-1) end last' ,
'first case when abs(%d+(%d)) then abs(%d)+abs(%d) end last' %(a1,a2,a3,a1), # 'first case when abs(1+1) then abs(-1)+abs(3) end last' ,
'first case when %d then %d else %d end last' %(a1,a2,a3), # 'first case when 0 then 1 else 3 end last' ,
'first case when %d then %d when %d then %d else %d end last' %(a1,a2,a3,a1,a2), # 'first case when 0 then 1 when 1 then 0 else 3 end last' ,
'first case when %d then %d when %d then %d when %d then %d end last' %(a1,a2,a3,a1,a2,a3), # 'first case when 0 then 1 when 1 then 0 when 2 then 3 end last' ,
'first case %d when %d then %d end last' %(a1,a1,a3), # 'first case 3 when 3 then 4 end last' ,
'first case %d when %d then %d end last' %(a1,a2,a3), # 'first case 3 when 1 then 4 end last' ,
'first case %d when %d then %d else %d end last' %(a1,a2,a3,a1), # 'first case 3 when 1 then 4 else 2 end last' ,
'first case %d when null then %d when \'%d\' then %d end last' %(a1,a2,a1,a3), # 'first case 3 when null then 4 when \'3\' then 1 end last' ,
'first case null when null then %d when %d then %d end last' %(a1,a2,a3), # 'first case null when null then 4 when 3 then 1 end last' ,
'first case %d.0 when null then %d when \'%d\' then %d end last' %(a1,a2,a1,a3), # 'first case 3.0 when null then 4 when \'3\' then 1 end last' ,
'first q_double,case q_double when \'a\' then %d when \'%d\' then %d end last' %(a1,a2,a3), #'first q_double,case q_double when \'a\' then 4 when \'0\' then 1 end last' ,
'first case null when null then %d when %d then %d end last' %(a1,a2,a3), #'first case null when null then 4 when 3 then 1 end last' ,
'first q_double,q_int,case q_double when q_int then q_int when q_int - (%d ) then q_int else %d end last' %(a1,a2), # 'first q_double,q_int,case q_double when q_int then q_int when q_int - 1 then q_int else 99 end last' ,
'first case cast(q_double as int) when %d then q_double when q_int then %d else ts end last' %(a1,a2), # 'first case cast(q_double as int) when 0 then q_double when q_int then 11 else ts end last' ,
'first q_int, case q_int + (%d) when %d then %d when %d then %d else %d end last' %(a1,a1,a1,a2,a2,a3), #'first q_int, case q_int + 1 when 1 then 1 when 2 then 2 else 3 end last' ,
'first distinct loc, case t_int when t_bigint then t_ts else t_smallint + (%d) end last' %(a1), #'first distinct loc, case t_int when t_bigint then t_ts else t_smallint + 100 end last' ,
]
#num = len(casewhen_lists)
casewhen_list = str(random.sample(casewhen_lists,50)).replace("[","").replace("]","").replace("'first","").replace("last'","").replace("\"first","").replace("last\"","")
return casewhen_list
def base_case(self,database):
for i in range(30):
cs = self.casewhen_list().split(',')[i]
sql1 = "select %s from %s.stable_1 where tbname = 'stable_1_1';" % (cs ,database)
sql2 = "select %s from %s.stable_1_1 ;" % (cs ,database)
self.constant_check(database,sql1,sql2,0)
def state_window_list(self):
a1,a2,a3 = random.randint(-2147483647,2147483647),random.randint(-2147483647,2147483647),random.randint(-2147483647,2147483647)
state_window_lists = ['first case when %d then %d end last' %(a1,a2) , #'first case when 3 then 4 end last' ,
'first case when 0 then %d end last' %(a1), #'first case when 0 then 4 end last' ,
'first case when null then %d end last' %(a1) , #'first case when null then 4 end last' ,
'first case when %d-(%d) then 0 end last' %(a1,a1) , #'first case when 1-1 then 0 end last' ,
'first case when %d+(%d) then 0 end last' %(a1,a1), #'first case when 1+1 then 0 end last' ,
'first case when %d > 0 then %d < %d end last' %(a1,a1,a2), #'first case when 1 > 0 then 1 < 2 end last' ,
'first case when %d > %d then %d < %d end last' %(a1,a2,a1,a2), #'first case when 1 > 2 then 1 < 2 end last' ,
'first case when abs(%d) then abs(-(%d)) end last' %(a1,a2) ,#'first case when abs(3) then abs(-1) end last' ,
'first case when 0 then %d else %d end last' %(a1,a2), #'first case when 0 then 1 else 3 end last' ,
'first case when 0 then %d when 1 then %d else %d end last' %(a1,a1,a3), #'first case when 0 then 1 when 1 then 0 else 3 end last' ,
'first case when 0 then %d when 1 then %d when 2 then %d end last' %(a1,a1,a3), #'first case when 0 then 1 when 1 then 0 when 2 then 3 end last' ,
'first case when \'a\' then \'b\' when null then 0 end last' , #'first case when \'a\' then \'b\' when null then 0 end last' ,
'first case when \'%d\' then \'b\' when null then %d end last' %(a1,a2) , #'first case when \'2\' then \'b\' when null then 0 end last' ,
'first case when \'%d\' then \'b\' else null end last' %(a1), #'first case when \'0\' then \'b\' else null end last',
'first case when \'%d\' then \'b\' else %d end last' %(a1,a2), #'first case when \'0\' then \'b\' else 2 end last',
'first case when q_int then q_int when q_int + (%d) then q_int + (%d) else q_int is null end last' %(a1,a2) , #'first case when q_int then q_int when q_int + 1 then q_int + 1 else q_int is null end last' ,
'first case when q_int then %d when ts then ts end last' %(a1), #'first case when q_int then 3 when ts then ts end last' ,
'first case when %d then q_int end last' %(a1), #'first case when 3 then q_int end last' ,
'first case when q_int then %d when %d then %d end last' %(a1,a1,a3), #'first case when q_int then 3 when 1 then 2 end last' ,
'first case when q_int < %d then %d when q_int >= %d then %d else %d end last' %(a1,a2,a1,a2,a3), #'first case when q_int < 3 then 1 when q_int >= 3 then 2 else 3 end last' ,
'first case when q_int is not null then case when q_int <= %d then q_int else q_int * (%d) end else -(%d) end last' %(a1,a1,a3), #'first case when q_int is not null then case when q_int <= 0 then q_int else q_int * 10 end else -1 end last' ,
'first case %d when %d then %d end last' %(a1,a2,a3), # 'first case 3 when 3 then 4 end last' ,
'first case %d when %d then %d end last' %(a1,a2,a3), # 'first case 3 when 1 then 4 end last' ,
'first case %d when %d then %d else %d end last' %(a1,a1,a2,a3), # 'first case 3 when 1 then 4 else 2 end last' ,
'first case %d when null then %d when \'%d\' then %d end last' %(a1,a1,a2,a3) , # 'first case 3 when null then 4 when \'3\' then 1 end last' ,
'first case \'%d\' when null then %d when %d then %d end last' %(a1,a1,a2,a3), # 'first case \'3\' when null then 4 when 3 then 1 end last' ,
'first case null when null then %d when %d then %d end last' %(a1,a2,a3), # 'first case null when null then 4 when 3 then 1 end last' ,
'first case %d.0 when null then %d when \'%d\' then %d end last' %(a1,a1,a2,a3) , # 'first case 3.0 when null then 4 when \'3\' then 1 end last' ,
'first case q_double when \'a\' then %d when \'%d\' then %d end last' %(a1,a2,a3) , # 'first case q_double when \'a\' then 4 when \'0\' then 1 end last' ,
'first case q_double when q_int then q_int when q_int - (%d) then q_int else %d end last' %(a1,a2), # 'first case q_double when q_int then q_int when q_int - 1 then q_int else 99 end last' ,
'first case q_int + (%d) when %d then %d when %d then %d else %d end last' %(a1,a2,a3,a1,a2,a3), #'first case q_int + 1 when 1 then 1 when 2 then 2 else 3 end last' ,
'first case when \'a\' then \'b\' when null then %d end last' %(a1), # 'first case when \'a\' then \'b\' when null then 0 end last' ,
'first case when \'%d\' then \'b\' when null then %d end last' %(a1,a2), # 'first case when \'2\' then \'b\' when null then 0 end last' ,
'first case when %d then \'b\' else null end last' %(a1), # 'first case when 0 then \'b\' else null end last' ,
'first case when %d then \'b\' else %d+abs(%d) end last' %(a1,a2,a3), # 'first case when 0 then \'b\' else 2+abs(-2) end last' ,
'first case when %d then %d end last' %(a1,a2), # 'first case when 3 then 4 end last' ,
'first case when %d then %d end last' %(a1,a2), # 'first case when 0 then 4 end last' ,
'first case when null then %d end last' %(a1), # 'first case when null then 4 end last' ,
#'first case when %d then %d+(%d) end last' %(a1,a2,a3), # 'first case when 1 then 4+1 end last' ,
'first case when %d-(%d) then %d end last' %(a1,a2,a3), # 'first case when 1-1 then 0 end last' ,
'first case when %d+(%d) then %d end last' %(a1,a2,a3), # 'first case when 1+1 then 0 end last' ,
'first case when abs(%d) then abs(%d) end last' %(a1,a2), # 'first case when abs(3) then abs(-1) end last' ,
#'first case when abs(%d+(%d)) then abs(%d)+abs(%d) end last' %(a1,a2,a3,a1), # 'first case when abs(1+1) then abs(-1)+abs(3) end last' ,
'first case when %d then %d else %d end last' %(a1,a2,a3), # 'first case when 0 then 1 else 3 end last' ,
'first case when %d then %d when %d then %d else %d end last' %(a1,a2,a3,a1,a2), # 'first case when 0 then 1 when 1 then 0 else 3 end last' ,
'first case when %d then %d when %d then %d when %d then %d end last' %(a1,a2,a3,a1,a2,a3), # 'first case when 0 then 1 when 1 then 0 when 2 then 3 end last' ,
'first case %d when %d then %d end last' %(a1,a1,a3), # 'first case 3 when 3 then 4 end last' ,
'first case %d when %d then %d end last' %(a1,a2,a3), # 'first case 3 when 1 then 4 end last' ,
'first case %d when %d then %d else %d end last' %(a1,a2,a3,a1), # 'first case 3 when 1 then 4 else 2 end last' ,
'first case %d when null then %d when \'%d\' then %d end last' %(a1,a2,a1,a3), # 'first case 3 when null then 4 when \'3\' then 1 end last' ,
'first case null when null then %d when %d then %d end last' %(a1,a2,a3), # 'first case null when null then 4 when 3 then 1 end last' ,
'first case %d.0 when null then %d when \'%d\' then %d end last' %(a1,a2,a1,a3), # 'first case 3.0 when null then 4 when \'3\' then 1 end last' ,
'first case null when null then %d when %d then %d end last' %(a1,a2,a3), #'first case null when null then 4 when 3 then 1 end last' ,
'first q_int, case q_int + (%d) when %d then %d when %d then %d else %d end last' %(a1,a1,a1,a2,a2,a3), #'first q_int, case q_int + 1 when 1 then 1 when 2 then 2 else 3 end last' ,
]
state_window_list = str(random.sample(state_window_lists,50)).replace("[","").replace("]","").replace("'first","").replace("last'","").replace("\"first","").replace("last\"","")
return state_window_list
def state_window_case(self,database):
for i in range(30):
cs = self.state_window_list().split(',')[i]
sql1 = "select _wstart,avg(q_int),min(q_smallint) from %s.stable_1 where tbname = 'stable_1_1' state_window(%s);" % (database,cs)
sql2 = "select _wstart,avg(q_int),min(q_smallint) from %s.stable_1_1 state_window(%s) ;" % (database,cs)
self.constant_check(database,sql1,sql2,0)
self.constant_check(database,sql1,sql2,1)
self.constant_check(database,sql1,sql2,2)
def constant_check(self,database,sql1,sql2,column):
#column =0 代表0列， column = n代表n-1列
tdLog.info("\n=============sql1:(%s)___sql2:(%s) ====================\n" %(sql1,sql2))
tdSql.query(sql1)
queryRows = len(tdSql.queryResult)
for i in range(queryRows):
tdSql.query(sql1)
sql1_value = tdSql.getData(i,column)
tdSql.execute(" flush database %s;" %database)
tdSql.query(sql2)
sql2_value = tdSql.getData(i,column)
self.value_check(sql1_value,sql2_value)
def value_check(self,base_value,check_value):
if base_value==check_value:
tdLog.info(f"checkEqual success, base_value={base_value},check_value={check_value}")
else :
tdLog.exit(f"checkEqual error, base_value=={base_value},check_value={check_value}")
def run(self):
fake = Faker('zh_CN')
fake_data = fake.random_int(min=-9223372036854775807, max=9223372036854775807, step=1)
fake_float = fake.pyfloat()
fake_str = fake.pystr()
startTime = time.time()
os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename))
self.dropandcreateDB_random("%s" %self.db, 10)
self.users_bug("%s" %self.db)
self.base_case("%s" %self.db)
self.state_window_case("%s" %self.db)
#taos -f sql
print("taos -f sql start!")
taos_cmd1 = "taos -f %s/%s.sql" % (self.testcasePath,self.testcaseFilename)
_ = subprocess.check_output(taos_cmd1, shell=True)
print("taos -f sql over!")
endTime = time.time()
print("total time %ds" % (endTime - startTime))
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())