From ce59e4fff4266715ceec0209407a15760e06aec5 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Tue, 22 Nov 2022 11:45:25 +0800 Subject: [PATCH 01/46] add test case --- tests/parallel_test/cases.task | 2 + tests/system-test/1-insert/drop.py | 128 +++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 tests/system-test/1-insert/drop.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index a9a02d8380..7b06fa5eab 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -616,6 +616,8 @@ ,,,system-test,python3 ./test.py -f 1-insert/delete_childtable.py ,,,system-test,python3 ./test.py -f 1-insert/delete_normaltable.py ,,,system-test,python3 ./test.py -f 1-insert/keep_expired.py +,,,system-test,python3 ./test.py -f 1-insert/drop.py +,,,system-test,python3 ./test.py -f 1-insert/drop.py -N 3 -M 3 -i False -n 3 ,,,system-test,python3 ./test.py -f 2-query/join2.py ,,,system-test,python3 ./test.py -f 2-query/union1.py ,,,system-test,python3 ./test.py -f 2-query/concat2.py diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py new file mode 100644 index 0000000000..d511d1aa7a --- /dev/null +++ b/tests/system-test/1-insert/drop.py @@ -0,0 +1,128 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +import math +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + self.dbname = 'db' + self.ntbname = f"{self.dbname}.ntb" + self.rowNum = 10 + self.tbnum = 20 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': 'binary(20)', + 'col13': 'nchar(20)' + } + def insert_data(self,column_dict,tbname,row_num): + insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) + for i in range(row_num): + insert_list = [] + self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) + def drop_ntb_check(self): + tdSql.execute(f'create database if not exists {self.dbname} replica {self.replicaVar}') + tdSql.execute(f'use {self.dbname}') + tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) + self.insert_data(self.column_dict,self.ntbname,self.rowNum) + for k,v in self.column_dict.items(): + if v.lower() == "timestamp": + tdSql.query(f'select * from {self.ntbname} where {k} = {self.ts}') + tdSql.checkRows(1) + tdSql.execute(f'drop table {self.ntbname}') + tdSql.execute(f'flush database {self.dbname}') + tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) + self.insert_data(self.column_dict,self.ntbname,self.rowNum) + for k,v in self.column_dict.items(): + if v.lower() == "timestamp": + tdSql.query(f'select * from {self.ntbname} where {k} = {self.ts}') + tdSql.checkRows(1) + tdSql.execute(f'drop database {self.dbname}') + + def drop_stb_ctb_check(self): + stbname = f'{self.dbname}.{tdCom.getLongName(5,"letters")}' + tag_dict = { + 't0':'int' + } + tag_values = [ + f'1' + ] + tdSql.execute(f"create database if not exists {self.dbname} replica {self.replicaVar}") + tdSql.execute(f'use {self.dbname}') + tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f"create table {stbname}_{i} using {stbname} tags({tag_values[0]})") + self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum) + for k,v in self.column_dict.items(): + for i in range(self.tbnum): + if v.lower() == "timestamp": + tdSql.query(f'select * from {stbname}_{i} where {k} = {self.ts}') + tdSql.checkRows(1) + tdSql.execute(f'drop table {stbname}_{i}') + tdSql.execute(f'flush database {self.dbname}') + for i in range(self.tbnum): + tdSql.execute(f"create table {stbname}_{i} using {stbname} tags({tag_values[0]})") + self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum) + for k,v in self.column_dict.items(): + for i in range(self.tbnum): + if v.lower() == "timestamp": + tdSql.query(f'select * from {stbname}_{i} where {k} = {self.ts}') + tdSql.checkRows(1) + if v.lower() == "timestamp": + tdSql.query(f'select * from {stbname} where {k} = {self.ts}') + tdSql.checkRows(self.tbnum) + tdSql.execute(f'drop table {stbname}') + tdSql.execute(f'flush database {self.dbname}') + tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f"create table {stbname}_{i} using {stbname} tags({tag_values[0]})") + self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum) + for k,v in self.column_dict.items(): + if v.lower() == "timestamp": + tdSql.query(f'select * from {stbname} where {k} = {self.ts}') + tdSql.checkRows(self.tbnum) + tdSql.execute(f'drop database {self.dbname}') + + def run(self): + self.drop_ntb_check() + self.drop_stb_ctb_check() + pass + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 526f8261525cbeefb053719e4cdcdb3b37717ade Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Wed, 23 Nov 2022 12:01:21 +0800 Subject: [PATCH 02/46] update test case --- tests/system-test/1-insert/alter_table.py | 7 ++++- tests/system-test/1-insert/drop.py | 37 +++++++++++++++++++++-- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/tests/system-test/1-insert/alter_table.py b/tests/system-test/1-insert/alter_table.py index 6a437d4601..f9f20096a7 100644 --- a/tests/system-test/1-insert/alter_table.py +++ b/tests/system-test/1-insert/alter_table.py @@ -87,7 +87,6 @@ class TDTestCase: } def alter_check_ntb(self): - tdSql.prepare() tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) for i in self.values_list: @@ -107,6 +106,7 @@ class TDTestCase: v = f'binary({self.binary_length+1})' v_error = f'binary({self.binary_length-1})' tdSql.error(f'alter table {self.ntbname} modify column {key} {v_error}') + tdSql.error(f'alter table {self.ntbname} set tag {key} = "abcd1"') tdSql.execute(f'alter table {self.ntbname} modify column {key} {v}') tdSql.query(f'describe {self.ntbname}') result = tdCom.getOneRow(1,'VARCHAR') @@ -115,6 +115,7 @@ class TDTestCase: v = f'nchar({self.binary_length+1})' v_error = f'nchar({self.binary_length-1})' tdSql.error(f'alter table {self.ntbname} modify column {key} {v_error}') + tdSql.error(f'alter table {self.ntbname} set tag {key} = "abcd1"') tdSql.execute(f'alter table {self.ntbname} modify column {key} {v}') tdSql.query(f'describe {self.ntbname}') result = tdCom.getOneRow(1,'NCHAR') @@ -122,6 +123,7 @@ class TDTestCase: else: for v in self.column_dict.values(): tdSql.error(f'alter table {self.ntbname} modify column {key} {v}') + tdSql.error(f'alter table {self.ntbname} set tag {key} = "abcd1"') for key,values in self.column_dict.items(): rename_str = f'{tdCom.getLongName(constant.COL_NAME_LENGTH_MAX,"letters")}' tdSql.execute(f'alter table {self.ntbname} rename column {key} {rename_str}') @@ -285,6 +287,7 @@ class TDTestCase: v = f'binary({self.binary_length+1})' v_error = f'binary({self.binary_length-1})' tdSql.error(f'alter table {self.stbname} modify column {key} {v_error}') + tdSql.error(f'alter table {self.stbname} set tag {key} = "abcd1"') tdSql.execute(f'alter table {self.stbname} modify column {key} {v}') tdSql.query(f'describe {self.stbname}') result = tdCom.getOneRow(1,'VARCHAR') @@ -297,6 +300,7 @@ class TDTestCase: v = f'nchar({self.binary_length+1})' v_error = f'nchar({self.binary_length-1})' tdSql.error(f'alter table {self.stbname} modify column {key} {v_error}') + tdSql.error(f'alter table {self.stbname} set tag {key} = "abcd1"') tdSql.execute(f'alter table {self.stbname} modify column {key} {v}') tdSql.query(f'describe {self.stbname}') result = tdCom.getOneRow(1,'NCHAR') @@ -308,6 +312,7 @@ class TDTestCase: else: for v in self.column_dict.values(): tdSql.error(f'alter table {self.stbname} modify column {key} {v}') + tdSql.error(f'alter table {self.stbname} set tag {key} = "abcd1"') for key,values in self.column_dict.items(): rename_str = f'{tdCom.getLongName(constant.COL_NAME_LENGTH_MAX,"letters")}' tdSql.error(f'alter table {self.stbname} rename column {key} {rename_str}') diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py index d511d1aa7a..bbb0b512e9 100644 --- a/tests/system-test/1-insert/drop.py +++ b/tests/system-test/1-insert/drop.py @@ -115,10 +115,41 @@ class TDTestCase: tdSql.query(f'select * from {stbname} where {k} = {self.ts}') tdSql.checkRows(self.tbnum) tdSql.execute(f'drop database {self.dbname}') - + def drop_topic_check(self): + tdSql.execute(f'create database {self.dbname} replica {self.replicaVar}') + tdSql.execute(f'use {self.dbname}') + stbname = tdCom.getLongName(5,"letters") + topic_name = tdCom.getLongName(5,"letters") + tdSql.execute(f'create table {stbname} (ts timestamp,c0 int) tags(t0 int)') + tdSql.execute(f'create topic {topic_name} as select * from {self.dbname}.{stbname}') + tdSql.query(f'select * from information_schema.ins_topics where topic_name = "{topic_name}"') + tdSql.checkEqual(tdSql.queryResult[0][3],f'create topic {topic_name} as select * from {self.dbname}.{stbname}') + tdSql.execute(f'drop topic {topic_name}') + tdSql.execute(f'create topic {topic_name} as select c0 from {self.dbname}.{stbname}') + tdSql.query(f'select * from information_schema.ins_topics where topic_name = "{topic_name}"') + tdSql.checkEqual(tdSql.queryResult[0][3],f'create topic {topic_name} as select c0 from {self.dbname}.{stbname}') + tdSql.execute(f'drop topic {topic_name}') + tdSql.execute(f'drop database {self.dbname}') + + def drop_stream_check(self): + tdSql.execute(f'create database {self.dbname} replica {self.replicaVar}') + tdSql.execute(f'use {self.dbname}') + stbname = tdCom.getLongName(5,"letters") + stream_name = tdCom.getLongName(5,"letters") + tdSql.execute(f'create table {stbname} (ts timestamp,c0 int) tags(t0 int)') + tdSql.execute(f'create stream {stream_name} into stb as select * from {self.dbname}.{stbname} partition by tbname') + tdSql.query(f'select * from information_schema.ins_streams where stream_name = "{stream_name}"') + print(tdSql.queryResult) + tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} into stb as select * from {self.dbname}.{stbname} partition by tbname') + tdSql.execute(f'drop stream {stream_name}') + tdSql.execute(f'create stream {stream_name} into stb as select c0 from {self.dbname}.{stbname} partition by tbname') + tdSql.query(f'select * from information_schema.ins_streams where stream_name = "{stream_name}"') + tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} into stb as select c0 from {stbname} partition by tbname') def run(self): - self.drop_ntb_check() - self.drop_stb_ctb_check() + # self.drop_ntb_check() + # self.drop_stb_ctb_check() + # self.drop_topic_check() + self.drop_stream_check() pass def stop(self): tdSql.close() From 6b53003f94aad0bd5d315fc15b01cfdf701c0fea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 12:18:38 +0800 Subject: [PATCH 03/46] refactor: add some logs. --- source/libs/executor/src/exchangeoperator.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index c858536bb1..3c27c8064a 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -75,6 +75,8 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn } while (1) { + qDebug("prepare wait for ready, %p, %s", pExchangeInfo, GET_TASKID(pTaskInfo)); + tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); @@ -360,7 +362,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId); if (pExchangeInfo == NULL) { - qWarn("failed to acquire exchange operator, since it may have been released"); + qWarn("failed to acquire exchange operator, since it may have been released, %p", pExchangeInfo); taosMemoryFree(pMsg->pData); return TSDB_CODE_SUCCESS; } @@ -368,6 +370,9 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { int32_t index = pWrapper->sourceIndex; SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index); + int32_t v = 0; + sem_getvalue(&pExchangeInfo->ready, &v); + if (code == TSDB_CODE_SUCCESS) { pSourceDataInfo->pRsp = pMsg->pData; @@ -379,19 +384,20 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, - pRsp->numOfRows); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, sem:%d, %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + pRsp->numOfRows, v, pExchangeInfo); } else { taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%s", pSourceDataInfo->taskId, index, tstrerror(code)); + qDebug("%s fetch rsp received, index:%d, error:%s, sem:%d, %p", pSourceDataInfo->taskId, index, tstrerror(code), v, pExchangeInfo); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; - tsem_post(&pExchangeInfo->ready); - taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); + code = tsem_post(&pExchangeInfo->ready); + ASSERT(code == TSDB_CODE_SUCCESS); + taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); return TSDB_CODE_SUCCESS; } @@ -557,6 +563,9 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; pOperator->cost.openCost = taosGetTimestampUs() - startTs; + int32_t value = 0; + sem_getvalue(&pExchangeInfo->ready, &value); + tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); From cf245fa52096edbc64037870968a7b5a4ee4672a Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 23 Nov 2022 12:56:16 +0800 Subject: [PATCH 04/46] enh: last cache optimize --- source/libs/planner/src/planLogicCreater.c | 86 +++++++++++++++++----- source/libs/planner/src/planOptimizer.c | 26 +++++-- 2 files changed, 87 insertions(+), 25 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 89e8a85895..dc28b2ba31 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -36,6 +36,7 @@ static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogi typedef struct SRewriteExprCxt { int32_t errCode; SNodeList* pExprs; + bool* pOutputs; } SRewriteExprCxt; static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol) { @@ -63,14 +64,30 @@ static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol) { } static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { + SRewriteExprCxt* pCxt = (SRewriteExprCxt*)pContext; switch (nodeType(*pNode)) { + case QUERY_NODE_COLUMN: { + if (NULL != pCxt->pOutputs) { + SNode* pExpr; + int32_t index = 0; + FOREACH(pExpr, pCxt->pExprs) { + if (QUERY_NODE_GROUPING_SET == nodeType(pExpr)) { + pExpr = nodesListGetNode(((SGroupingSetNode*)pExpr)->pParameterList, 0); + } + if (nodesEqualNode(pExpr, *pNode)) { + pCxt->pOutputs[index] = true; + break; + } + } + } + break; + } case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_FUNCTION: case QUERY_NODE_CASE_WHEN: { - SRewriteExprCxt* pCxt = (SRewriteExprCxt*)pContext; - SNode* pExpr; - int32_t index = 0; + SNode* pExpr; + int32_t index = 0; FOREACH(pExpr, pCxt->pExprs) { if (QUERY_NODE_GROUPING_SET == nodeType(pExpr)) { pExpr = nodesListGetNode(((SGroupingSetNode*)pExpr)->pParameterList, 0); @@ -89,6 +106,9 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { } nodesDestroyNode(*pNode); *pNode = (SNode*)pCol; + if (NULL != pCxt->pOutputs) { + pCxt->pOutputs[index] = true; + } return DEAL_RES_IGNORE_CHILD; } ++index; @@ -121,7 +141,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { static int32_t rewriteExprForSelect(SNode* pExpr, SSelectStmt* pSelect, ESqlClause clause) { nodesWalkExpr(pExpr, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL}; cxt.errCode = nodesListMakeAppend(&cxt.pExprs, pExpr); if (TSDB_CODE_SUCCESS == cxt.errCode) { nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); @@ -130,23 +150,50 @@ static int32_t rewriteExprForSelect(SNode* pExpr, SSelectStmt* pSelect, ESqlClau return cxt.errCode; } -static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { +static int32_t cloneRewriteExprs(SNodeList* pExprs, bool* pOutputs, SNodeList** pRewriteExpr) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t index = 0; + SNode* pExpr = NULL; + FOREACH(pExpr, pExprs) { + if (pOutputs[index]) { + code = nodesListMakeStrictAppend(pRewriteExpr, nodesCloneNode(pExpr)); + if (TSDB_CODE_SUCCESS != code) { + NODES_DESTORY_LIST(*pRewriteExpr); + break; + } + } + } + return code; +} + +static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause, + SNodeList** pRewriteExpr) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + if (NULL != pRewriteExpr) { + cxt.pOutputs = taosMemoryCalloc(LIST_LENGTH(pExprs), sizeof(bool)); + if (NULL == cxt.pOutputs) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); + if (TSDB_CODE_SUCCESS == cxt.errCode && NULL != pRewriteExpr) { + cxt.errCode = cloneRewriteExprs(pExprs, cxt.pOutputs, pRewriteExpr); + } + taosMemoryFree(cxt.pOutputs); return cxt.errCode; } static int32_t rewriteExpr(SNodeList* pExprs, SNode** pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; nodesRewriteExpr(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; nodesRewriteExprs(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } @@ -311,7 +358,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM); + code = rewriteExprsForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM, NULL); } pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType, pSelect->tagScan); @@ -509,7 +556,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pAggFuncs) { - code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); + code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY, NULL); } if (NULL != pSelect->pGroupByList) { @@ -524,8 +571,9 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } // rewrite the expression in subsequent clauses + SNodeList* pOutputGroupKeys = NULL; if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY); + code = rewriteExprsForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY, &pOutputGroupKeys); } if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) { @@ -536,8 +584,8 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } // set the output - if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pGroupKeys) { - code = createColumnByRewriteExprs(pAgg->pGroupKeys, &pAgg->node.pTargets); + if (TSDB_CODE_SUCCESS == code && NULL != pOutputGroupKeys) { + code = createColumnByRewriteExprs(pOutputGroupKeys, &pAgg->node.pTargets); } if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pAggFuncs) { code = createColumnByRewriteExprs(pAgg->pAggFuncs, &pAgg->node.pTargets); @@ -574,7 +622,7 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt // indefinite rows functions and _select_values functions int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pFuncs); if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pIdfRowsFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT); + code = rewriteExprsForSelect(pIdfRowsFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT, NULL); } // set the output @@ -612,7 +660,7 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p // interp functions and _group_key functions int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, isInterpFunc, &pInterpFunc->pFuncs); if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pInterpFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT); + code = rewriteExprsForSelect(pInterpFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT, NULL); } if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pFill) { @@ -656,7 +704,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_WINDOW, fmIsWindowClauseFunc, &pWindow->pFuncs); if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW); + code = rewriteExprsForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW, NULL); } if (TSDB_CODE_SUCCESS == code) { @@ -854,10 +902,10 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect int32_t code = partFillExprs(pSelect, &pFill->pFillExprs, &pFill->pNotFillExprs); if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pFill->pFillExprs, pSelect, SQL_CLAUSE_FILL); + code = rewriteExprsForSelect(pFill->pFillExprs, pSelect, SQL_CLAUSE_FILL, NULL); } if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pFill->pNotFillExprs, pSelect, SQL_CLAUSE_FILL); + code = rewriteExprsForSelect(pFill->pNotFillExprs, pSelect, SQL_CLAUSE_FILL, NULL); } if (TSDB_CODE_SUCCESS == code) { code = createColumnByRewriteExprs(pFill->pFillExprs, &pFill->node.pTargets); @@ -1066,7 +1114,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_DISTINCT); + code = rewriteExprsForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_DISTINCT, NULL); } // set the output diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 45fa67faef..b8b6e44412 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1476,19 +1476,33 @@ static bool partTagsHasIndefRowsSelectFunc(SNodeList* pFuncs) { return false; } -static int32_t partTagsRewriteGroupTagsToFuncs(SNodeList* pGroupTags, int32_t start, SNodeList* pAggFuncs) { - bool hasIndefRowsSelectFunc = partTagsHasIndefRowsSelectFunc(pAggFuncs); +static bool partTagsNeedOutput(SNode* pExpr, SNodeList* pTargets) { + SNode* pOutput = NULL; + FOREACH(pOutput, pTargets) { + if (QUERY_NODE_COLUMN == nodeType(pExpr)) { + if (nodesEqualNode(pExpr, pOutput)) { + return true; + } + } else if (0 == strcmp(((SExprNode*)pExpr)->aliasName, ((SColumnNode*)pOutput)->colName)) { + return true; + } + } + return false; +} + +static int32_t partTagsRewriteGroupTagsToFuncs(SNodeList* pGroupTags, int32_t start, SAggLogicNode* pAgg) { + bool hasIndefRowsSelectFunc = partTagsHasIndefRowsSelectFunc(pAgg->pAggFuncs); int32_t code = TSDB_CODE_SUCCESS; int32_t index = 0; SNode* pNode = NULL; FOREACH(pNode, pGroupTags) { - if (index++ < start) { + if (index++ < start || !partTagsNeedOutput(pNode, pAgg->node.pTargets)) { continue; } if (hasIndefRowsSelectFunc) { - code = nodesListStrictAppend(pAggFuncs, partTagsCreateWrapperFunc("_select_value", pNode)); + code = nodesListStrictAppend(pAgg->pAggFuncs, partTagsCreateWrapperFunc("_select_value", pNode)); } else { - code = nodesListStrictAppend(pAggFuncs, partTagsCreateWrapperFunc("_group_key", pNode)); + code = nodesListStrictAppend(pAgg->pAggFuncs, partTagsCreateWrapperFunc("_group_key", pNode)); } if (TSDB_CODE_SUCCESS != code) { break; @@ -1541,7 +1555,7 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub } NODES_DESTORY_LIST(pAgg->pGroupKeys); if (TSDB_CODE_SUCCESS == code && start >= 0) { - code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, start, pAgg->pAggFuncs); + code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, start, pAgg); } } if (TSDB_CODE_SUCCESS == code) { From 906fafc578912c9621414153601d8c5f49c0322a Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 23 Nov 2022 12:58:27 +0800 Subject: [PATCH 05/46] enh: last cache optimize --- source/libs/planner/src/planLogicCreater.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index dc28b2ba31..de0feceb8f 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -560,13 +560,9 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } if (NULL != pSelect->pGroupByList) { - if (NULL != pAgg->pGroupKeys) { - code = nodesListStrictAppendList(pAgg->pGroupKeys, nodesCloneList(pSelect->pGroupByList)); - } else { - pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList); - if (NULL == pAgg->pGroupKeys) { - code = TSDB_CODE_OUT_OF_MEMORY; - } + pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList); + if (NULL == pAgg->pGroupKeys) { + code = TSDB_CODE_OUT_OF_MEMORY; } } @@ -587,6 +583,8 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, if (TSDB_CODE_SUCCESS == code && NULL != pOutputGroupKeys) { code = createColumnByRewriteExprs(pOutputGroupKeys, &pAgg->node.pTargets); } + nodesDestroyList(pOutputGroupKeys); + if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pAggFuncs) { code = createColumnByRewriteExprs(pAgg->pAggFuncs, &pAgg->node.pTargets); } From f92c1d4271293f5a1ec2e05b75699f62434aeb55 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 13:48:15 +0800 Subject: [PATCH 06/46] refactor: add some logs. --- source/libs/executor/src/exchangeoperator.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 3c27c8064a..cff0c50644 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -75,7 +75,9 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn } while (1) { - qDebug("prepare wait for ready, %p, %s", pExchangeInfo, GET_TASKID(pTaskInfo)); + int32_t v = 0; + sem_getvalue(&pExchangeInfo->ready, &v); + qDebug("prepare wait for ready, sem:(%d,%p), %p, %s", v, pExchangeInfo->ready.__align, pExchangeInfo, GET_TASKID(pTaskInfo)); tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { @@ -370,7 +372,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { int32_t index = pWrapper->sourceIndex; SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index); - int32_t v = 0; + int32_t v = 0, v1 = 0; sem_getvalue(&pExchangeInfo->ready, &v); if (code == TSDB_CODE_SUCCESS) { @@ -384,12 +386,13 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, sem:%d, %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, - pRsp->numOfRows, v, pExchangeInfo); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, sem:(%d,%p), %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + pRsp->numOfRows, v, pExchangeInfo->ready.__align, pExchangeInfo); } else { taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%s, sem:%d, %p", pSourceDataInfo->taskId, index, tstrerror(code), v, pExchangeInfo); + qDebug("%s fetch rsp received, index:%d, error:%s, sem:(%d,%p), %p", pSourceDataInfo->taskId, index, tstrerror(code), v, + pExchangeInfo->ready.__align, pExchangeInfo); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; @@ -429,9 +432,9 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas return pTaskInfo->code; } - qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, + qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %p, %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, - pSource->execId, sourceIndex, totalSources); + pSource->execId, pExchangeInfo, sourceIndex, totalSources); pMsg->header.vgId = htonl(pSource->addr.nodeId); pMsg->sId = htobe64(pSource->schedId); From c623336b6ed2f758e6be80703eacb7f9bdbe4ea8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 13:59:38 +0800 Subject: [PATCH 07/46] refactor: enable continue query when the buffer of sink node is lower or empty. --- source/libs/executor/src/dataDispatcher.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 3d9757c96f..d4248fc420 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -138,7 +138,9 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, toDataCacheEntry(pDispatcher, pInput, pBuf); taosWriteQitem(pDispatcher->pDataBlocks, pBuf); - *pContinue = (DS_BUF_LOW == updateStatus(pDispatcher) ? true : false); + + int32_t status = updateStatus(pDispatcher); + *pContinue = (status == DS_BUF_LOW || status == DS_BUF_EMPTY); return TSDB_CODE_SUCCESS; } From 852f52687672eb67881c9c90bff4dd178881c869 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Wed, 23 Nov 2022 14:41:30 +0800 Subject: [PATCH 08/46] update --- tests/system-test/1-insert/drop.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py index bbb0b512e9..f8796bcf6a 100644 --- a/tests/system-test/1-insert/drop.py +++ b/tests/system-test/1-insert/drop.py @@ -137,18 +137,20 @@ class TDTestCase: stbname = tdCom.getLongName(5,"letters") stream_name = tdCom.getLongName(5,"letters") tdSql.execute(f'create table {stbname} (ts timestamp,c0 int) tags(t0 int)') + tdSql.execute(f'create table tb using {stbname} tags(1)') tdSql.execute(f'create stream {stream_name} into stb as select * from {self.dbname}.{stbname} partition by tbname') tdSql.query(f'select * from information_schema.ins_streams where stream_name = "{stream_name}"') print(tdSql.queryResult) tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} into stb as select * from {self.dbname}.{stbname} partition by tbname') tdSql.execute(f'drop stream {stream_name}') - tdSql.execute(f'create stream {stream_name} into stb as select c0 from {self.dbname}.{stbname} partition by tbname') + tdSql.execute(f'create stream {stream_name} into stb1 as select * from tb') tdSql.query(f'select * from information_schema.ins_streams where stream_name = "{stream_name}"') - tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} into stb as select c0 from {stbname} partition by tbname') + tdSql.checkEqual(tdSql.queryResult[0][2],f'create stream {stream_name} into stb1 as select * from tb') + tdSql.execute(f'drop database {self.dbname}') def run(self): - # self.drop_ntb_check() - # self.drop_stb_ctb_check() - # self.drop_topic_check() + self.drop_ntb_check() + self.drop_stb_ctb_check() + self.drop_topic_check() self.drop_stream_check() pass def stop(self): From 37735cf861963e4c3f0179cafbfeed7b27212500 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Wed, 23 Nov 2022 15:09:39 +0800 Subject: [PATCH 09/46] remove test case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e5ab69396e..07937c817c 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -424,7 +424,7 @@ ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_stable.py -,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_table.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_table.py ,,,system-test,python3 ./test.py -f 1-insert/boundary.py ,,,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py ,,,system-test,python3 ./test.py -f 1-insert/table_comment.py From 05f84c184e0db320bfff1c946b732a016ad27b3c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 15:36:38 +0800 Subject: [PATCH 10/46] fix(query): fix syntax error. --- source/libs/executor/src/exchangeoperator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index cff0c50644..04caccbf8f 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -77,7 +77,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn while (1) { int32_t v = 0; sem_getvalue(&pExchangeInfo->ready, &v); - qDebug("prepare wait for ready, sem:(%d,%p), %p, %s", v, pExchangeInfo->ready.__align, pExchangeInfo, GET_TASKID(pTaskInfo)); + qDebug("prepare wait for ready, sem:(%d,%p), %p, %s", v, (void*)pExchangeInfo->ready.__align, pExchangeInfo, GET_TASKID(pTaskInfo)); tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { @@ -387,12 +387,12 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { ASSERT(pRsp != NULL); qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, sem:(%d,%p), %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, - pRsp->numOfRows, v, pExchangeInfo->ready.__align, pExchangeInfo); + pRsp->numOfRows, v, (void*)pExchangeInfo->ready.__align, pExchangeInfo); } else { taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; qDebug("%s fetch rsp received, index:%d, error:%s, sem:(%d,%p), %p", pSourceDataInfo->taskId, index, tstrerror(code), v, - pExchangeInfo->ready.__align, pExchangeInfo); + (void*) pExchangeInfo->ready.__align, pExchangeInfo); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; From 0a8fc50024eca59f7d01f80b4c84b208308bd0e8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 15:47:12 +0800 Subject: [PATCH 11/46] test:update test cases. --- source/client/test/clientTests.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 85814305bd..007e0bd931 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -692,7 +692,6 @@ TEST(testCase, insert_test) { taos_free_result(pRes); taos_close(pConn); } -#endif TEST(testCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -752,9 +751,6 @@ TEST(testCase, projection_query_tables) { taos_close(pConn); } - -#if 0 - TEST(testCase, tsbs_perf_test) { TdThread qid[20] = {0}; @@ -764,15 +760,16 @@ TEST(testCase, tsbs_perf_test) { getchar(); } +#endif TEST(testCase, projection_query_stables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); + TAOS_RES* pRes = taos_query(pConn, "use test"); taos_free_result(pRes); - pRes = taos_query(pConn, "select ts from st1"); + pRes = taos_query(pConn, "select * from meters"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); @@ -793,6 +790,7 @@ TEST(testCase, projection_query_stables) { taos_close(pConn); } +#if 0 TEST(testCase, agg_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); From 4d75bd69b255977f9cd46beb8c466125035f314d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 15:49:59 +0800 Subject: [PATCH 12/46] test: disable print result. --- source/client/test/clientTests.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 007e0bd931..78a14d3292 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -782,8 +782,8 @@ TEST(testCase, projection_query_stables) { char str[512] = {0}; while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); } taos_free_result(pRes); From ca1dcf4ea1a2f1d4934b670e7bbec4b09d8b63a7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 15:54:00 +0800 Subject: [PATCH 13/46] test: limit the output --- source/client/test/clientTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 78a14d3292..dcacaf3471 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -769,7 +769,7 @@ TEST(testCase, projection_query_stables) { TAOS_RES* pRes = taos_query(pConn, "use test"); taos_free_result(pRes); - pRes = taos_query(pConn, "select * from meters"); + pRes = taos_query(pConn, "select * from meters limit 30000000"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); From f9e338ba2d25bdb93b4a03b3f1320553804944ab Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Nov 2022 15:54:30 +0800 Subject: [PATCH 14/46] test: limit the output. --- source/client/test/clientTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index dcacaf3471..82202b8820 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -769,7 +769,7 @@ TEST(testCase, projection_query_stables) { TAOS_RES* pRes = taos_query(pConn, "use test"); taos_free_result(pRes); - pRes = taos_query(pConn, "select * from meters limit 30000000"); + pRes = taos_query(pConn, "select * from meters limit 50000000"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); From 2d86f8e91cd1bf659ecba4717ffffaaeb431827d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 23 Nov 2022 16:20:54 +0800 Subject: [PATCH 15/46] enh: group by tbname optimize --- source/libs/parser/src/parTranslater.c | 5 +- source/libs/planner/src/planSpliter.c | 81 +++++++++++++++++++------- 2 files changed, 62 insertions(+), 24 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 54f450e971..bb4aa67767 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7089,9 +7089,10 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, SVAlterTbReq* pReq) { - SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + SSchema* pSchema = getTagSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "Invalid tag name: %s", + pStmt->colName); } pReq->tagName = strdup(pStmt->colName); diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 489046d88e..0cbca8b032 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -292,6 +292,43 @@ static bool stbSplNeedSplitJoin(bool streamQuery, SJoinLogicNode* pJoin) { return true; } +static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + return ((SScanLogicNode*)pNode)->pGroupTags; + } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { + return ((SPartitionLogicNode*)pNode)->pPartitionKeys; + } else { + return NULL; + } +} + +static bool stbSplHasPartTbname(SNodeList* pPartKeys) { + if (NULL == pPartKeys) { + return false; + } + SNode* pPartKey = NULL; + FOREACH(pPartKey, pPartKeys) { + if (QUERY_NODE_GROUPING_SET == nodeType(pPartKey)) { + pPartKey = nodesListGetNode(((SGroupingSetNode*)pPartKey)->pParameterList, 0); + } + if ((QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType) || + (QUERY_NODE_COLUMN == nodeType(pPartKey) && COLUMN_TYPE_TBNAME == ((SColumnNode*)pPartKey)->colType)) { + return true; + } + } + return false; +} + +static bool stbSplIsPartTableAgg(SAggLogicNode* pAgg) { + if (NULL != pAgg->pGroupKeys) { + return stbSplHasPartTbname(pAgg->pGroupKeys); + } + if (1 != LIST_LENGTH(pAgg->node.pChildren)) { + return false; + } + return stbSplHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))); +} + static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: @@ -301,7 +338,9 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { case QUERY_NODE_LOGIC_PLAN_PARTITION: return streamQuery ? false : stbSplIsMultiTbScanChild(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_AGG: - return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); + return (!stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) || + stbSplIsPartTableAgg((SAggLogicNode*)pNode)) && + stbSplHasMultiTbScan(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_WINDOW: return stbSplNeedSplitWindow(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_SORT: @@ -676,27 +715,8 @@ static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) { } } -static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) { - if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { - return ((SScanLogicNode*)pNode)->pGroupTags; - } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { - return ((SPartitionLogicNode*)pNode)->pPartitionKeys; - } else { - return NULL; - } -} - -static bool stbSplIsPartTbanme(SNodeList* pPartKeys) { - if (NULL == pPartKeys || 1 != LIST_LENGTH(pPartKeys)) { - return false; - } - SNode* pPartKey = nodesListGetNode(pPartKeys, 0); - return (QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType) || - (QUERY_NODE_COLUMN == nodeType(pPartKey) && COLUMN_TYPE_TBNAME == ((SColumnNode*)pPartKey)->colType); -} - static bool stbSplIsPartTableWinodw(SWindowLogicNode* pWindow) { - return stbSplIsPartTbanme(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0))); + return stbSplHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0))); } static int32_t stbSplSplitWindowForCrossTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { @@ -797,7 +817,17 @@ static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pO return code; } -static int32_t stbSplSplitAggNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { +static int32_t stbSplSplitAggNodeForPartTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + int32_t code = splCreateExchangeNodeForSubplan(pCxt, pInfo->pSubplan, pInfo->pSplitNode, SUBPLAN_TYPE_MERGE); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, + (SNode*)splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT)); + } + ++(pCxt->groupId); + return code; +} + +static int32_t stbSplSplitAggNodeForCrossTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { SLogicNode* pPartAgg = NULL; int32_t code = stbSplCreatePartAggNode((SAggLogicNode*)pInfo->pSplitNode, &pPartAgg); if (TSDB_CODE_SUCCESS == code) { @@ -812,6 +842,13 @@ static int32_t stbSplSplitAggNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) return code; } +static int32_t stbSplSplitAggNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + if (stbSplIsPartTableAgg((SAggLogicNode*)pInfo->pSplitNode)) { + return stbSplSplitAggNodeForPartTable(pCxt, pInfo); + } + return stbSplSplitAggNodeForCrossTable(pCxt, pInfo); +} + static SNode* stbSplCreateColumnNode(SExprNode* pExpr) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { From f7a058c87c5d9d7c7d5bdf337ff48c8341ac3922 Mon Sep 17 00:00:00 2001 From: Jason-Jia <714897623@qq.com> Date: Wed, 23 Nov 2022 20:34:28 +0800 Subject: [PATCH 16/46] Update alter_database.py --- tests/system-test/1-insert/alter_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index aec18a1cf1..12b8861707 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -53,7 +53,7 @@ class TDTestCase: tdSql.execute('drop database db') def run(self): - tdSql.error('create database db1 vgroups 10 buffer 12289') + self.alter_buffer() self.alter_pages() From f9bf35c2f5d8dfe4f6b69c193cbabeffab490547 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 24 Nov 2022 09:00:57 +0800 Subject: [PATCH 17/46] fix(sync): use local timestamp --- source/libs/sync/src/syncMain.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index d9f6753dac..008e49c20c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2222,8 +2222,10 @@ int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncHeartbeatReply* pMsg = pRpcMsg->pCont; syncLogRecvHeartbeatReply(ths, pMsg, ""); + int64_t tsMs = taosGetTimestampMs(); + // update last reply time, make decision whether the other node is alive or not - syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, pMsg->timeStamp); + syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, tsMs); return 0; } From f49e44c256f106a015de536a3f73ff5399f588ed Mon Sep 17 00:00:00 2001 From: Jason-Jia <714897623@qq.com> Date: Thu, 24 Nov 2022 09:05:42 +0800 Subject: [PATCH 18/46] Update alter_database.py --- tests/system-test/1-insert/alter_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index 12b8861707..1fca294a47 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -18,7 +18,7 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) self.buffer_boundary = [3, 4097, 8193, 12289, 16384] self.buffer_error = [self.buffer_boundary[0] - - 1, self.buffer_boundary[-1]+1, 12289, 256] + 1, self.buffer_boundary[-1]+1, 256] # pages_boundary >= 64 self.pages_boundary = [64, 128, 512] self.pages_error = [self.pages_boundary[0]-1] From ffbb04d40a15ffe92e5d533968bd11df11c42efb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Nov 2022 09:21:26 +0800 Subject: [PATCH 19/46] Revert "enh: row optimize2" --- include/common/tdataformat.h | 103 +- include/common/trow.h | 10 +- source/client/src/clientRawBlockWrite.c | 17 +- source/common/src/tdatablock.c | 1 - source/common/src/tdataformat.c | 1210 +++++++++----------- source/common/src/trow.c | 13 +- source/common/src/ttypes.c | 4 +- source/dnode/vnode/src/inc/tsdb.h | 10 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 19 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 10 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 14 +- source/libs/parser/src/parInsertUtil.c | 2 +- 13 files changed, 674 insertions(+), 743 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 6855287fb2..2eda2f66cc 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -27,17 +27,17 @@ extern "C" { #endif -typedef struct SBuffer SBuffer; -typedef struct SSchema SSchema; -typedef struct STColumn STColumn; -typedef struct STSchema STSchema; -typedef struct SValue SValue; -typedef struct SColVal SColVal; -typedef struct SRow SRow; -typedef struct SRowIter SRowIter; -typedef struct STagVal STagVal; -typedef struct STag STag; -typedef struct SColData SColData; +typedef struct SBuffer SBuffer; +typedef struct SSchema SSchema; +typedef struct STColumn STColumn; +typedef struct STSchema STSchema; +typedef struct SValue SValue; +typedef struct SColVal SColVal; +typedef struct STSRow2 STSRow2; +typedef struct STSRowBuilder STSRowBuilder; +typedef struct STagVal STagVal; +typedef struct STag STag; +typedef struct SColData SColData; #define HAS_NONE ((uint8_t)0x1) #define HAS_NULL ((uint8_t)0x2) @@ -68,10 +68,13 @@ struct SBuffer { void tBufferDestroy(SBuffer *pBuffer); int32_t tBufferInit(SBuffer *pBuffer, int64_t size); int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData); -int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData); // STSchema ================================ -void tDestroyTSchema(STSchema *pTSchema); +int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); +void tTSchemaDestroy(STSchema *pTSchema); + +// SValue ================================ +static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type); // SColVal ================================ #define CV_FLAG_VALUE ((int8_t)0x0) @@ -86,14 +89,26 @@ void tDestroyTSchema(STSchema *pTSchema); #define COL_VAL_IS_NULL(CV) ((CV)->flag == CV_FLAG_NULL) #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE) -// SRow ================================ -int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer); -void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); +// STSRow2 ================================ +#define TSROW_LEN(PROW, V) tGetI32v((uint8_t *)(PROW)->data, (V) ? &(V) : NULL) +#define TSROW_SVER(PROW, V) tGetI32v((PROW)->data + TSROW_LEN(PROW, NULL), (V) ? &(V) : NULL) -// SRowIter ================================ -int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); -void tRowIterClose(SRowIter **ppIter); -SColVal *tRowIterNext(SRowIter *pIter); +int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow); +int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow); +void tTSRowFree(STSRow2 *pRow); +void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); +int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray); +int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); +int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow); + +// STSRowBuilder ================================ +#define tsRowBuilderInit() ((STSRowBuilder){0}) +#define tsRowBuilderClear(B) \ + do { \ + if ((B)->pBuf) { \ + taosMemoryFree((B)->pBuf); \ + } \ + } while (0) // STag ================================ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); @@ -132,17 +147,29 @@ struct STSchema { int32_t numOfCols; int32_t version; int32_t flen; + int32_t vlen; int32_t tlen; STColumn columns[]; }; -struct SRow { - uint8_t flag; - uint8_t rsv; - uint16_t sver; - uint32_t len; - TSKEY ts; - uint8_t data[]; +#define TSROW_HAS_NONE ((uint8_t)0x1) +#define TSROW_HAS_NULL ((uint8_t)0x2U) +#define TSROW_HAS_VAL ((uint8_t)0x4U) +#define TSROW_KV_SMALL ((uint8_t)0x10U) +#define TSROW_KV_MID ((uint8_t)0x20U) +#define TSROW_KV_BIG ((uint8_t)0x40U) +#pragma pack(push, 1) +struct STSRow2 { + TSKEY ts; + uint8_t flags; + uint8_t data[]; +}; +#pragma pack(pop) + +struct STSRowBuilder { + // STSRow2 tsRow; + int32_t szBuf; + uint8_t *pBuf; }; struct SValue { @@ -231,17 +258,37 @@ typedef struct { int32_t nCols; schema_ver_t version; uint16_t flen; + int32_t vlen; int32_t tlen; STColumn *columns; } STSchemaBuilder; +// use 2 bits for bitmap(default: STSRow/sub block) +#define TD_VTYPE_BITS 2 +#define TD_VTYPE_PARTS 4 // PARTITIONS: 1 byte / 2 bits +#define TD_VTYPE_OPTR 3 // OPERATOR: 4 - 1, utilize to get remainder +#define TD_BITMAP_BYTES(cnt) (((cnt) + TD_VTYPE_OPTR) >> 2) + +// use 1 bit for bitmap(super block) +#define TD_VTYPE_BITS_I 1 +#define TD_VTYPE_PARTS_I 8 // PARTITIONS: 1 byte / 1 bit +#define TD_VTYPE_OPTR_I 7 // OPERATOR: 8 - 1, utilize to get remainder +#define TD_BITMAP_BYTES_I(cnt) (((cnt) + TD_VTYPE_OPTR_I) >> 3) + int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); -STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version); +static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { + if (IS_VAR_DATA_TYPE(type)) { + return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); + } else { + memcpy(&pValue->val, p, tDataTypes[type].bytes); + return tDataTypes[type].bytes; + } +} #endif diff --git a/include/common/trow.h b/include/common/trow.h index 6a71a8844e..9d183c8f80 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -55,14 +55,6 @@ typedef struct STSRow { #define TD_ROW_TP 0x0U // default #define TD_ROW_KV 0x01U -#define TD_VTYPE_PARTS 4 // PARTITIONS: 1 byte / 2 bits -#define TD_VTYPE_OPTR 3 // OPERATOR: 4 - 1, utilize to get remainder -#define TD_BITMAP_BYTES(cnt) (((cnt) + TD_VTYPE_OPTR) >> 2) - -#define TD_VTYPE_PARTS_I 8 // PARTITIONS: 1 byte / 1 bit -#define TD_VTYPE_OPTR_I 7 // OPERATOR: 8 - 1, utilize to get remainder -#define TD_BITMAP_BYTES_I(cnt) (((cnt) + TD_VTYPE_OPTR_I) >> 3) - /** * @brief value type * - for data from client input and STSRow in memory, 3 types of value none/null/norm available @@ -252,7 +244,7 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal */ static FORCE_INLINE void *tdGetBitmapAddrTp(STSRow *pRow, uint32_t flen) { // The primary TS key is stored separatedly. - return POINTER_SHIFT(TD_ROW_DATA(pRow), flen); + return POINTER_SHIFT(TD_ROW_DATA(pRow), flen - sizeof(TSKEY)); // return POINTER_SHIFT(pRow->ts, flen); } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 6a6a5d56d7..90a2adb647 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1275,7 +1275,6 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) nVar++; } } - fLen -= sizeof(TSKEY); int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + (int32_t)TD_BITMAP_BYTES(numOfCols - 1); @@ -1334,9 +1333,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) } } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } + offset += TYPE_BYTES[pColumn->type]; } tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); @@ -1506,7 +1503,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { nVar++; } } - fLen -= sizeof(TSKEY); int32_t rows = rspObj.resInfo.numOfRows; int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + @@ -1589,9 +1585,8 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); } } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } + + offset += TYPE_BYTES[pColumn->type]; } tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); @@ -1808,7 +1803,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) nVar++; } } - fLen -= sizeof(TSKEY); int32_t rows = rspObj.resInfo.numOfRows; int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + @@ -1894,9 +1888,8 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); } } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } + + offset += TYPE_BYTES[pColumn->type]; } tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 3021c586a3..fa63543439 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2055,7 +2055,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB isStartKey = true; tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, offset, k); - continue; // offset should keep 0 for next column } else if (colDataIsNull_s(pColInfoData, j)) { tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NULL, NULL, diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 728f669fc8..73b887342c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -43,672 +43,606 @@ int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData) { return code; } -int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData) { - int32_t code = tRealloc(&pBuffer->pBuf, pBuffer->nBuf + nData); - if (code) return code; - - *ppData = pBuffer->pBuf + pBuffer->nBuf; - pBuffer->nBuf += nData; - - return code; -} - // ================================ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); -// SRow ======================================================================== -#define KV_FLG_LIT ((uint8_t)0x10) -#define KV_FLG_MID ((uint8_t)0x20) -#define KV_FLG_BIG ((uint8_t)0x30) - -#define ROW_BIT_NONE ((uint8_t)0x0) -#define ROW_BIT_NULL ((uint8_t)0x1) -#define ROW_BIT_VALUE ((uint8_t)0x2) - #pragma pack(push, 1) typedef struct { - int16_t nCol; - char idx[]; // uint8_t * | uint16_t * | uint32_t * -} SKVIdx; + int16_t nCols; + uint8_t idx[]; +} STSKVRow; #pragma pack(pop) -#define ROW_SET_BITMAP(PB, FLAG, IDX, VAL) \ - do { \ - if (PB) { \ - switch (FLAG) { \ - case (HAS_NULL | HAS_NONE): \ - SET_BIT1(PB, IDX, VAL); \ - break; \ - case (HAS_VALUE | HAS_NONE): \ - SET_BIT1(PB, IDX, (VAL) ? (VAL)-1 : 0); \ - break; \ - case (HAS_VALUE | HAS_NULL): \ - SET_BIT1(PB, IDX, (VAL)-1); \ - break; \ - case (HAS_VALUE | HAS_NULL | HAS_NONE): \ - SET_BIT2(PB, IDX, VAL); \ - break; \ - default: \ - ASSERT(0); \ - break; \ - } \ - } \ +#define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) + +// SValue +static FORCE_INLINE int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { + if (IS_VAR_DATA_TYPE(type)) { + return tPutBinary(p, pValue->pData, pValue->nData); + } else { + if (p) memcpy(p, &pValue->val, tDataTypes[type].bytes); + return tDataTypes[type].bytes; + } +} + +// STSRow2 ======================================================================== +static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) { + if (pb) { + switch (flags & 0xf) { + case TSROW_HAS_NULL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NONE: + if (v) { + SET_BIT1(pb, idx, (uint8_t)1); + } else { + SET_BIT1(pb, idx, (uint8_t)0); + } + break; + case TSROW_HAS_VAL | TSROW_HAS_NULL: + v = v - 1; + SET_BIT1(pb, idx, v); + break; + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + SET_BIT2(pb, idx, v); + break; + + default: + ASSERT(0); + } + } +} +#define SET_IDX(p, i, n, f) \ + do { \ + if ((f)&TSROW_KV_SMALL) { \ + ((uint8_t *)(p))[i] = (n); \ + } else if ((f)&TSROW_KV_MID) { \ + ((uint16_t *)(p))[i] = (n); \ + } else { \ + ((uint32_t *)(p))[i] = (n); \ + } \ } while (0) - -int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { +#if 0 +int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { int32_t code = 0; +#if 0 + STColumn *pTColumn; + SColVal *pColVal; + int32_t nColVal = taosArrayGetSize(pArray); + int32_t iColVal; - ASSERT(taosArrayGetSize(aColVal) > 0); - ASSERT(((SColVal *)aColVal->pData)[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); - ASSERT(((SColVal *)aColVal->pData)[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(nColVal > 0); - // scan --------------- - uint8_t flag = 0; - int32_t iColVal = 1; - const int32_t nColVal = taosArrayGetSize(aColVal); - SColVal *pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - int32_t iTColumn = 1; - STColumn *pTColumn = pTSchema->columns + iTColumn; - int32_t ntp = 0; - int32_t nkv = 0; - int32_t maxIdx = 0; - int32_t nIdx = 0; - while (pTColumn) { - if (pColVal) { - if (pColVal->cid == pTColumn->colId) { - ntp += TYPE_BYTES[pTColumn->type]; - if (COL_VAL_IS_VALUE(pColVal)) { // VALUE - flag |= HAS_VALUE; - maxIdx = nkv; - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - ntp = ntp + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; - nkv = nkv + tPutI16v(NULL, pTColumn->colId) + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; + // try + uint8_t flags = 0; + uint32_t ntv = 0; + uint32_t nkv = 0; + int16_t nTag = 0; + uint32_t maxIdx = 0; + + iColVal = 0; + for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; iColumn++) { + pTColumn = &pTSchema->columns[iColumn]; + if (iColVal < nColVal) { + pColVal = (SColVal *)taosArrayGet(pArray, iColVal); + } else { + pColVal = NULL; + } + + if (iColumn == 0) { + ASSERT(pColVal->cid == pTColumn->colId); + ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID); + + iColVal++; + } else { + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + iColVal++; + + if (COL_VAL_IS_NONE(pColVal)) { + flags |= TSROW_HAS_NONE; + } else if (COL_VAL_IS_NULL(pColVal)) { + flags |= TSROW_HAS_NULL; + maxIdx = nkv; + nTag++; + nkv += tPutI16v(NULL, -pTColumn->colId); } else { - nkv = nkv + tPutI16v(NULL, pTColumn->colId) + pTColumn->bytes; + flags |= TSROW_HAS_VAL; + maxIdx = nkv; + nTag++; + nkv += tPutI16v(NULL, pTColumn->colId); + nkv += tPutValue(NULL, &pColVal->value, pTColumn->type); + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + ntv += tPutValue(NULL, &pColVal->value, pTColumn->type); + } } - nIdx++; - } else if (COL_VAL_IS_NONE(pColVal)) { // NONE - flag |= HAS_NONE; - } else if (COL_VAL_IS_NULL(pColVal)) { // NULL - flag |= HAS_NULL; - maxIdx = nkv; - nkv += tPutI16v(NULL, -pTColumn->colId); - nIdx++; + } else if (pColVal->cid > pTColumn->colId) { + flags |= TSROW_HAS_NONE; } else { ASSERT(0); } - - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - } else if (pColVal->cid > pTColumn->colId) { // NONE - flag |= HAS_NONE; - ntp += TYPE_BYTES[pTColumn->type]; - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + flags |= TSROW_HAS_NONE; } - } else { // NONE - flag |= HAS_NONE; - ntp += TYPE_BYTES[pTColumn->type]; - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } } - // compare --------------- - switch (flag) { - case HAS_NONE: - case HAS_NULL: - ntp = sizeof(SRow); - break; - case HAS_VALUE: - ntp = sizeof(SRow) + ntp; - break; - case (HAS_NULL | HAS_NONE): - ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1); - break; - case (HAS_VALUE | HAS_NONE): - case (HAS_VALUE | HAS_NULL): - ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1) + ntp; - break; - case (HAS_VALUE | HAS_NULL | HAS_NONE): - ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + ntp; - break; - default: - ASSERT(0); - break; - } - if (maxIdx <= UINT8_MAX) { - nkv = sizeof(SRow) + sizeof(SKVIdx) + nIdx + nkv; - flag |= KV_FLG_LIT; - } else if (maxIdx <= UINT16_MAX) { - nkv = sizeof(SRow) + sizeof(SKVIdx) + (nIdx << 1) + nkv; - flag |= KV_FLG_MID; + ASSERT(flags); + + // decide + uint32_t nData = 0; + uint32_t nDataT = 0; + uint32_t nDataK = 0; + if (flags == TSROW_HAS_NONE || flags == TSROW_HAS_NULL) { + nData = 0; } else { - nkv = sizeof(SRow) + sizeof(SKVIdx) + (nIdx << 2) + nkv; - flag |= KV_FLG_BIG; - } - int32_t nRow; - if (nkv < ntp) { - nRow = nkv; - } else { - nRow = ntp; - flag &= ((uint8_t)0x0f); - } - - // alloc -------------- - SRow *pRow = NULL; - code = tBufferReserve(pBuffer, nRow, (void **)&pRow); - if (code) return code; - - // build -------------- - pColVal = (SColVal *)taosArrayGet(aColVal, 0); - - pRow->flag = flag; - pRow->rsv = 0; - pRow->sver = pTSchema->version; - pRow->len = nRow; - memcpy(&pRow->ts, &pColVal->value.val, sizeof(TSKEY)); - - if (flag == HAS_NONE || flag == HAS_NULL) { - goto _exit; - } - - iColVal = 1; - pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - iTColumn = 1; - pTColumn = pTSchema->columns + iTColumn; - if (flag & 0xf0) { // KV - SKVIdx *pIdx = (SKVIdx *)pRow->data; - int32_t iIdx = 0; - int32_t nv = 0; - uint8_t *pv = NULL; - if (flag & KV_FLG_LIT) { - pv = pIdx->idx + nIdx; - } else if (flag & KV_FLG_MID) { - pv = pIdx->idx + (nIdx << 1); - } else { - pv = pIdx->idx + (nIdx << 2); - } - pIdx->nCol = nIdx; - - while (pTColumn) { - if (pColVal) { - if (pColVal->cid == pTColumn->colId) { - if (COL_VAL_IS_VALUE(pColVal)) { - if (flag & KV_FLG_LIT) { - ((uint8_t *)pIdx->idx)[iIdx] = (uint8_t)nv; - } else if (flag & KV_FLG_MID) { - ((uint16_t *)pIdx->idx)[iIdx] = (uint16_t)nv; - } else { - ((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv; - } - iIdx++; - - nv += tPutI16v(pv + nv, pTColumn->colId); - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - nv += tPutU32v(pv + nv, pColVal->value.nData); - memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData); - nv += pColVal->value.nData; - } else { - memcpy(pv + nv, &pColVal->value.val, pTColumn->bytes); - nv += pTColumn->bytes; - } - } else if (COL_VAL_IS_NULL(pColVal)) { - if (flag & KV_FLG_LIT) { - ((uint8_t *)pIdx->idx)[iIdx] = (uint8_t)nv; - } else if (flag & KV_FLG_MID) { - ((uint16_t *)pIdx->idx)[iIdx] = (uint16_t)nv; - } else { - ((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv; - } - iIdx++; - nv += tPutI16v(pv + nv, -pTColumn->colId); - } - - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - } else if (pColVal->cid > pTColumn->colId) { // NONE - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - } - } else { // NONE - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - } - } - } else { // TUPLE - uint8_t *pb = NULL; - uint8_t *pf = NULL; - uint8_t *pv = NULL; - int32_t nv = 0; - - switch (flag) { - case (HAS_NULL | HAS_NONE): - pb = pRow->data; + switch (flags) { + case TSROW_HAS_VAL: + nDataT = pTSchema->flen + ntv; break; - case HAS_VALUE: - pf = pRow->data; - pv = pf + pTSchema->flen; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + nDataT = BIT1_SIZE(pTSchema->numOfCols - 1); break; - case (HAS_VALUE | HAS_NONE): - case (HAS_VALUE | HAS_NULL): - pb = pRow->data; - pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; + case TSROW_HAS_VAL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NULL: + nDataT = BIT1_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv; break; - case (HAS_VALUE | HAS_NULL | HAS_NONE): - pb = pRow->data; - pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - default: - ASSERT(0); - break; - } - - // build impl - while (pTColumn) { - if (pColVal) { - if (pColVal->cid == pTColumn->colId) { - if (COL_VAL_IS_VALUE(pColVal)) { // VALUE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_VALUE); - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - *(int32_t *)(pf + pTColumn->offset) = nv; - nv += tPutU32v(pv + nv, pColVal->value.nData); - if (pColVal->value.nData) { - memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData); - nv += pColVal->value.nData; - } - } else { - memcpy(pf + pTColumn->offset, &pColVal->value.val, TYPE_BYTES[pTColumn->type]); - } - } else if (COL_VAL_IS_NONE(pColVal)) { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); - if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); - } else { // NULL - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NULL); - if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); - } - - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - } else if (pColVal->cid > pTColumn->colId) { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); - if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - } - } else { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); - if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); - pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - } - } - } - -_exit: - return code; -} - -void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { - ASSERT(iCol < pTSchema->numOfCols); - ASSERT(pRow->sver == pTSchema->version); - - STColumn *pTColumn = pTSchema->columns + iCol; - - if (iCol == 0) { - pColVal->cid = pTColumn->colId; - pColVal->type = pTColumn->type; - pColVal->flag = CV_FLAG_VALUE; - memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY)); - return; - } - - if (pRow->flag == HAS_NONE) { - *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - return; - } - - if (pRow->flag == HAS_NULL) { - *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - return; - } - - if (pRow->flag & 0xf0) { // KV Row - SKVIdx *pIdx = (SKVIdx *)pRow->data; - uint8_t *pv = NULL; - if (pRow->flag & KV_FLG_LIT) { - pv = pIdx->idx + pIdx->nCol; - } else if (pRow->flag & KV_FLG_MID) { - pv = pIdx->idx + (pIdx->nCol << 1); - } else { - pv = pIdx->idx + (pIdx->nCol << 2); - } - - int16_t lidx = 0; - int16_t ridx = pIdx->nCol - 1; - while (lidx <= ridx) { - int16_t mid = (lidx + ridx) >> 1; - uint8_t *pData = NULL; - if (pRow->flag & KV_FLG_LIT) { - pData = pv + ((uint8_t *)pIdx->idx)[mid]; - } else if (pRow->flag & KV_FLG_MID) { - pData = pv + ((uint16_t *)pIdx->idx)[mid]; - } else { - pData = pv + ((uint32_t *)pIdx->idx)[mid]; - } - - int16_t cid; - pData += tGetI16v(pData, &cid); - - if (TABS(cid) == pTColumn->colId) { - if (cid < 0) { - *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - } else { - pColVal->cid = pTColumn->colId; - pColVal->type = pTColumn->type; - pColVal->flag = CV_FLAG_VALUE; - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - pData += tGetU32v(pData, &pColVal->value.nData); - if (pColVal->value.nData > 0) { - pColVal->value.pData = pData; - } else { - pColVal->value.pData = NULL; - } - } else { - memcpy(&pColVal->value.val, pData, pTColumn->bytes); - } - } - return; - } else if (TABS(cid) < pTColumn->colId) { - lidx = mid + 1; - } else { - ridx = mid - 1; - } - } - - *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - } else { // Tuple Row - uint8_t *pf = NULL; - uint8_t *pv = NULL; - uint8_t bv = ROW_BIT_VALUE; - - switch (pRow->flag) { - case HAS_VALUE: - pf = pRow->data; - pv = pf + pTSchema->flen; - break; - case (HAS_NULL | HAS_NONE): - bv = GET_BIT1(pRow->data, iCol - 1); - break; - case (HAS_VALUE | HAS_NONE): - bv = GET_BIT1(pRow->data, iCol - 1); - if (bv) bv++; - pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NULL): - bv = GET_BIT1(pRow->data, iCol - 1); - bv++; - pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NULL | HAS_NONE): - bv = GET_BIT2(pRow->data, iCol - 1); - pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + nDataT = BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv; break; default: break; - } - - if (bv == ROW_BIT_NONE) { - *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - return; - } else if (bv == ROW_BIT_NULL) { - *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - return; - } - - pColVal->cid = pTColumn->colId; - pColVal->type = pTColumn->type; - pColVal->flag = CV_FLAG_VALUE; - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); - pData += tGetU32v(pData, &pColVal->value.nData); - if (pColVal->value.nData) { - pColVal->value.pData = pData; - } else { - pColVal->value.pData = NULL; - } - } else { - memcpy(&pColVal->value.val, pv + pTColumn->offset, pTColumn->bytes); - } - } -} - -// SRowIter ======================================== -struct SRowIter { - SRow *pRow; - STSchema *pTSchema; - - int32_t iTColumn; - union { - struct { // kv - int32_t iCol; - SKVIdx *pIdx; - }; - struct { // tuple - uint8_t *pb; - uint8_t *pf; - }; - uint8_t *pv; - }; - - SColVal cv; -}; - -int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { - ASSERT(pRow->sver == pTSchema->version); - - int32_t code = 0; - - SRowIter *pIter = taosMemoryCalloc(1, sizeof(*pIter)); - if (pIter == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _exit; - } - - pIter->pRow = pRow; - pIter->pTSchema = pTSchema; - pIter->iTColumn = 0; - - if (pRow->flag == HAS_NONE || pRow->flag == HAS_NULL) goto _exit; - - if (pRow->flag & 0xf0) { - pIter->iCol = 0; - pIter->pIdx = (SKVIdx *)pRow->data; - if (pRow->flag & KV_FLG_LIT) { - pIter->pv = pIter->pIdx->idx + pIter->pIdx->nCol; - } else if (pRow->flag & KV_FLG_MID) { - pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 1); - } else { - pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); - } - } else { - switch (pRow->flag) { - case (HAS_NULL | HAS_NONE): - pIter->pb = pRow->data; - break; - case HAS_VALUE: - pIter->pf = pRow->data; - pIter->pv = pIter->pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NONE): - case (HAS_VALUE | HAS_NULL): - pIter->pb = pRow->data; - pIter->pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); - pIter->pv = pIter->pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NULL | HAS_NONE): - pIter->pb = pRow->data; - pIter->pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); - pIter->pv = pIter->pf + pTSchema->flen; - break; - default: ASSERT(0); - break; + } + + uint8_t tflags = 0; + if (maxIdx <= UINT8_MAX) { + nDataK = sizeof(STSKVRow) + sizeof(uint8_t) * nTag + nkv; + tflags |= TSROW_KV_SMALL; + } else if (maxIdx <= UINT16_MAX) { + nDataK = sizeof(STSKVRow) + sizeof(uint16_t) * nTag + nkv; + tflags |= TSROW_KV_MID; + } else { + nDataK = sizeof(STSKVRow) + sizeof(uint32_t) * nTag + nkv; + tflags |= TSROW_KV_BIG; + } + + if (nDataT <= nDataK) { + nData = nDataT; + } else { + nData = nDataK; + flags |= tflags; } } -_exit: - if (code) { - *ppIter = NULL; - if (pIter) taosMemoryFree(pIter); - } else { - *ppIter = pIter; - } - return code; -} - -void tRowIterClose(SRowIter **ppIter) { - SRowIter *pIter = *ppIter; - if (pIter) { - taosMemoryFree(pIter); - } - *ppIter = NULL; -} - -SColVal *tRowIterNext(SRowIter *pIter) { - if (pIter->iTColumn >= pIter->pTSchema->numOfCols) { - return NULL; - } - - STColumn *pTColumn = pIter->pTSchema->columns + pIter->iTColumn; - - // timestamp - if (0 == pIter->iTColumn) { - pIter->cv.cid = pTColumn->colId; - pIter->cv.type = pTColumn->type; - pIter->cv.flag = CV_FLAG_VALUE; - memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY)); - goto _exit; - } - - if (pIter->pRow->flag == HAS_NONE) { - pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - goto _exit; - } - - if (pIter->pRow->flag == HAS_NULL) { - pIter->cv = COL_VAL_NULL(pTColumn->type, pTColumn->colId); - goto _exit; - } - - if (pIter->pRow->flag & 0xf0) { // KV - if (pIter->iCol < pIter->pIdx->nCol) { - uint8_t *pData; - - if (pIter->pRow->flag & KV_FLG_LIT) { - pData = pIter->pv + ((uint8_t *)pIter->pIdx->idx)[pIter->iCol]; - } else if (pIter->pRow->flag & KV_FLG_MID) { - pData = pIter->pv + ((uint16_t *)pIter->pIdx->idx)[pIter->iCol]; - } else { - pData = pIter->pv + ((uint32_t *)pIter->pIdx->idx)[pIter->iCol]; - } - - int16_t cid; - pData += tGetI16v(pData, &cid); - - if (TABS(cid) == pTColumn->colId) { - if (cid < 0) { - pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - } else { - pIter->cv.cid = pTColumn->colId; - pIter->cv.type = pTColumn->type; - pIter->cv.flag = CV_FLAG_VALUE; - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - pData += tGetU32v(pData, &pIter->cv.value.nData); - if (pIter->cv.value.nData > 0) { - pIter->cv.value.pData = pData; - } else { - pIter->cv.value.pData = NULL; - } - } else { - memcpy(&pIter->cv.value.val, pData, pTColumn->bytes); - } + // alloc + if (pBuilder) { + // create from a builder + if (nData == 0) { + pBuilder->tsRow.nData = 0; + pBuilder->tsRow.pData = NULL; + } else { + if (pBuilder->szBuf < nData) { + uint8_t *p = taosMemoryRealloc(pBuilder->pBuf, nData); + if (p == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; } - - pIter->iCol++; - goto _exit; - } else if (TABS(cid) > pTColumn->colId) { - pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - goto _exit; - } else { - ASSERT(0); + pBuilder->pBuf = p; + pBuilder->szBuf = nData; } - } else { - pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + + pBuilder->tsRow.nData = nData; + pBuilder->tsRow.pData = pBuilder->pBuf; + } + + *ppRow = &pBuilder->tsRow; + } else { + // create a new one + *ppRow = (STSRow2 *)taosMemoryMalloc(sizeof(STSRow2)); + if (*ppRow == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - } else { // Tuple - uint8_t bv = ROW_BIT_VALUE; - if (pIter->pb) { - switch (pIter->pRow->flag) { - case (HAS_NULL | HAS_NONE): - bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1); + if (nData == 0) { + (*ppRow)->nData = 0; + (*ppRow)->pData = NULL; + } else { + (*ppRow)->nData = nData; + (*ppRow)->pData = taosMemoryMalloc(nData); + if ((*ppRow)->pData == NULL) { + taosMemoryFree(*ppRow); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + } + } + + // build + (*ppRow)->flags = flags; + (*ppRow)->sver = pTSchema->version; + + pColVal = (SColVal *)taosArrayGet(pArray, 0); + (*ppRow)->ts = pColVal->value.ts; + + if ((*ppRow)->pData) { + STSKVRow *pTSKVRow = NULL; + uint8_t *pidx = NULL; + uint8_t *pkv = NULL; + uint8_t *pb = NULL; + uint8_t *pf = NULL; + uint8_t *ptv = NULL; + nkv = 0; + ntv = 0; + iColVal = 1; + + if ((flags & 0xf0) == 0) { + switch (flags & 0xf) { + case TSROW_HAS_VAL: + pf = (*ppRow)->pData; + ptv = pf + pTSchema->flen; break; - case (HAS_VALUE | HAS_NONE): - bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1); - if (bv) bv++; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + pb = (*ppRow)->pData; break; - case (HAS_VALUE | HAS_NULL): - bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1) + 1; + case TSROW_HAS_VAL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NULL: + pb = (*ppRow)->pData; + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + ptv = pf + pTSchema->flen; break; - case (HAS_VALUE | HAS_NULL | HAS_NONE): - bv = GET_BIT2(pIter->pb, pIter->iTColumn - 1); + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + pb = (*ppRow)->pData; + pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); + ptv = pf + pTSchema->flen; break; default: ASSERT(0); break; } - - if (bv == ROW_BIT_NONE) { - pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - goto _exit; - } else if (bv == ROW_BIT_NULL) { - pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - goto _exit; - } - } - - pIter->cv.cid = pTColumn->colId; - pIter->cv.type = pTColumn->type; - pIter->cv.flag = CV_FLAG_VALUE; - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - uint8_t *pData = pIter->pv + *(int32_t *)(pIter->pf + pTColumn->offset); - pData += tGetU32v(pData, &pIter->cv.value.nData); - if (pIter->cv.value.nData > 0) { - pIter->cv.value.pData = pData; - } else { - pIter->cv.value.pData = NULL; - } } else { - memcpy(&pIter->cv.value.val, pIter->pv + pTColumn->offset, pTColumn->bytes); + pTSKVRow = (STSKVRow *)(*ppRow)->pData; + pTSKVRow->nCols = 0; + pidx = pTSKVRow->idx; + if (flags & TSROW_KV_SMALL) { + pkv = pidx + sizeof(uint8_t) * nTag; + } else if (flags & TSROW_KV_MID) { + pkv = pidx + sizeof(uint16_t) * nTag; + } else { + pkv = pidx + sizeof(uint32_t) * nTag; + } } + + for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { + pTColumn = &pTSchema->columns[iColumn]; + if (iColVal < nColVal) { + pColVal = (SColVal *)taosArrayGet(pArray, iColVal); + } else { + pColVal = NULL; + } + + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + iColVal++; + + if (COL_VAL_IS_NONE(pColVal)) { + goto _set_none; + } else if (COL_VAL_IS_NULL(pColVal)) { + goto _set_null; + } else { + goto _set_value; + } + } else if (pColVal->cid > pTColumn->colId) { + goto _set_none; + } else { + ASSERT(0); + } + } else { + goto _set_none; + } + + _set_none: + if ((flags & 0xf0) == 0) { + setBitMap(pb, 0, iColumn - 1, flags); + if (flags & TSROW_HAS_VAL) { // set 0 + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + *(VarDataOffsetT *)(pf + pTColumn->offset) = 0; + } else { + tPutValue(pf + pTColumn->offset, &((SValue){0}), pTColumn->type); + } + } + } + continue; + + _set_null: + if ((flags & 0xf0) == 0) { + setBitMap(pb, 1, iColumn - 1, flags); + if (flags & TSROW_HAS_VAL) { // set 0 + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + *(VarDataOffsetT *)(pf + pTColumn->offset) = 0; + } else { + tPutValue(pf + pTColumn->offset, &((SValue){0}), pTColumn->type); + } + } + } else { + SET_IDX(pidx, pTSKVRow->nCols, nkv, flags); + pTSKVRow->nCols++; + nkv += tPutI16v(pkv + nkv, -pTColumn->colId); + } + continue; + + _set_value: + if ((flags & 0xf0) == 0) { + setBitMap(pb, 2, iColumn - 1, flags); + + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + *(VarDataOffsetT *)(pf + pTColumn->offset) = ntv; + ntv += tPutValue(ptv + ntv, &pColVal->value, pTColumn->type); + } else { + tPutValue(pf + pTColumn->offset, &pColVal->value, pTColumn->type); + } + } else { + SET_IDX(pidx, pTSKVRow->nCols, nkv, flags); + pTSKVRow->nCols++; + nkv += tPutI16v(pkv + nkv, pColVal->cid); + nkv += tPutValue(pkv + nkv, &pColVal->value, pTColumn->type); + } + continue; + } + } + +#endif +_exit: + return code; +} + +int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow) { + int32_t code = 0; + int32_t rLen = 0; + + TSROW_LEN(pRow, rLen); + (*ppRow) = (STSRow2 *)taosMemoryMalloc(rLen); + if (*ppRow == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(*ppRow, pRow, rLen); + +_exit: + return code; +} + +void tTSRowFree(STSRow2 *pRow) { + if (pRow) { + taosMemoryFree(pRow); + } +} + +void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { + uint8_t isTuple = ((pRow->flags & 0xf0) == 0) ? 1 : 0; + STColumn *pTColumn = &pTSchema->columns[iCol]; + uint8_t flags = pRow->flags & (uint8_t)0xf; + SValue value; + + ASSERT(iCol < pTSchema->numOfCols); + ASSERT(flags); + ASSERT(pRow->sver == pTSchema->version); + + if (iCol == 0) { + value.ts = pRow->ts; + goto _return_value; + } + + if (flags == TSROW_HAS_NONE) { + goto _return_none; + } else if (flags == TSROW_HAS_NULL) { + goto _return_null; + } + + ASSERT(pRow->nData && pRow->pData); + + if (isTuple) { + uint8_t *pb = pRow->pData; + uint8_t *pf = NULL; + uint8_t *pv = NULL; + uint8_t *p; + uint8_t b; + + // bit + switch (flags) { + case TSROW_HAS_VAL: + pf = pb; + break; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + b = GET_BIT1(pb, iCol - 1); + if (b == 0) { + goto _return_none; + } else { + goto _return_null; + } + case TSROW_HAS_VAL | TSROW_HAS_NONE: + b = GET_BIT1(pb, iCol - 1); + if (b == 0) { + goto _return_none; + } else { + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + break; + } + case TSROW_HAS_VAL | TSROW_HAS_NULL: + b = GET_BIT1(pb, iCol - 1); + if (b == 0) { + goto _return_null; + } else { + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + break; + } + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + b = GET_BIT2(pb, iCol - 1); + if (b == 0) { + goto _return_none; + } else if (b == 1) { + goto _return_null; + } else { + pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); + break; + } + default: + ASSERT(0); + } + + ASSERT(pf); + + p = pf + pTColumn->offset; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + pv = pf + pTSchema->flen; + p = pv + *(VarDataOffsetT *)p; + } + tGetValue(p, &value, pTColumn->type); + goto _return_value; + } else { + STSKVRow *pRowK = (STSKVRow *)pRow->pData; + int16_t lidx = 0; + int16_t ridx = pRowK->nCols - 1; + uint8_t *p; + int16_t midx; + uint32_t n; + int16_t cid; + + ASSERT(pRowK->nCols > 0); + + if (pRow->flags & TSROW_KV_SMALL) { + p = pRow->pData + sizeof(STSKVRow) + sizeof(uint8_t) * pRowK->nCols; + } else if (pRow->flags & TSROW_KV_MID) { + p = pRow->pData + sizeof(STSKVRow) + sizeof(uint16_t) * pRowK->nCols; + } else if (pRow->flags & TSROW_KV_BIG) { + p = pRow->pData + sizeof(STSKVRow) + sizeof(uint32_t) * pRowK->nCols; + } else { + ASSERT(0); + } + while (lidx <= ridx) { + midx = (lidx + ridx) / 2; + + if (pRow->flags & TSROW_KV_SMALL) { + n = ((uint8_t *)pRowK->idx)[midx]; + } else if (pRow->flags & TSROW_KV_MID) { + n = ((uint16_t *)pRowK->idx)[midx]; + } else { + n = ((uint32_t *)pRowK->idx)[midx]; + } + + n += tGetI16v(p + n, &cid); + + if (TABS(cid) == pTColumn->colId) { + if (cid < 0) { + goto _return_null; + } else { + n += tGetValue(p + n, &value, pTColumn->type); + goto _return_value; + } + + return; + } else if (TABS(cid) > pTColumn->colId) { + ridx = midx - 1; + } else { + lidx = midx + 1; + } + } + + // not found, return NONE + goto _return_none; + } + +_return_none: + *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + return; + +_return_null: + *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + return; + +_return_value: + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, value); + return; +} + +int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray) { + int32_t code = 0; + SColVal cv; + + (*ppArray) = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); + if (*ppArray == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } + for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; iColumn++) { + tTSRowGet(pRow, pTSchema, iColumn, &cv); + taosArrayPush(*ppArray, &cv); + } + + _exit: - pIter->iTColumn++; - return &pIter->cv; + return code; +} +#endif +int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { + int32_t n = 0; + + TSROW_LEN(pRow, n); + if (p) { + memcpy(p, pRow, n); + } + + return n; +} + +int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow) { + int32_t n = 0; + + *ppRow = (STSRow2 *)p; + TSROW_LEN(*ppRow, n); + + return n; } // STSchema ======================================== +int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) { + *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols); + if (*ppTSchema == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + (*ppTSchema)->numOfCols = ncols; + (*ppTSchema)->version = sver; + (*ppTSchema)->flen = 0; + (*ppTSchema)->vlen = 0; + (*ppTSchema)->tlen = 0; + + for (int32_t iCol = 0; iCol < ncols; iCol++) { + SSchema *pColumn = &pSchema[iCol]; + STColumn *pTColumn = &((*ppTSchema)->columns[iCol]); + + pTColumn->colId = pColumn->colId; + pTColumn->type = pColumn->type; + pTColumn->flags = pColumn->flags; + pTColumn->bytes = pColumn->bytes; + pTColumn->offset = (*ppTSchema)->flen; + + // skip first column + if (iCol) { + (*ppTSchema)->flen += TYPE_BYTES[pColumn->type]; + if (IS_VAR_DATA_TYPE(pColumn->type)) { + (*ppTSchema)->vlen += (pColumn->bytes + 5); + } + } + } + + return 0; +} + +void tTSchemaDestroy(STSchema *pTSchema) { + if (pTSchema) taosMemoryFree(pTSchema); +} // STag ======================================== static int tTagValCmprFn(const void *p1, const void *p2) { @@ -1108,6 +1042,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { pBuilder->nCols = 0; pBuilder->tlen = 0; pBuilder->flen = 0; + pBuilder->vlen = 0; pBuilder->version = version; } @@ -1126,21 +1061,24 @@ int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, c pCol->colId = colId; pCol->flags = flags; if (pBuilder->nCols == 0) { - pCol->offset = -1; + pCol->offset = 0; } else { - pCol->offset = pBuilder->flen; - pBuilder->flen += TYPE_BYTES[type]; + STColumn *pTCol = &(pBuilder->columns[pBuilder->nCols - 1]); + pCol->offset = pTCol->offset + TYPE_BYTES[pTCol->type]; } if (IS_VAR_DATA_TYPE(type)) { pCol->bytes = bytes; pBuilder->tlen += (TYPE_BYTES[type] + bytes); + pBuilder->vlen += bytes - sizeof(VarDataLenT); } else { pCol->bytes = TYPE_BYTES[type]; pBuilder->tlen += TYPE_BYTES[type]; + pBuilder->vlen += TYPE_BYTES[type]; } pBuilder->nCols++; + pBuilder->flen += TYPE_BYTES[type]; ASSERT(pCol->offset < pBuilder->flen); @@ -1159,6 +1097,7 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { pSchema->numOfCols = pBuilder->nCols; pSchema->tlen = pBuilder->tlen; pSchema->flen = pBuilder->flen; + pSchema->vlen = pBuilder->vlen; #ifdef TD_SUPPORT_BITMAP pSchema->tlen += (int)TD_BITMAP_BYTES(pSchema->numOfCols); @@ -1171,43 +1110,6 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { #endif -STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { - STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols); - if (pTSchema == NULL) return NULL; - - pTSchema->numOfCols = numOfCols; - pTSchema->version = version; - - // timestamp column - ASSERT(aSchema[0].type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(aSchema[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID); - pTSchema->columns[0].colId = aSchema[0].colId; - pTSchema->columns[0].type = aSchema[0].type; - pTSchema->columns[0].flags = aSchema[0].flags; - pTSchema->columns[0].bytes = aSchema[0].bytes; - pTSchema->columns[0].offset = -1; - - // other columns - for (int32_t iCol = 1; iCol < numOfCols; iCol++) { - SSchema *pSchema = &aSchema[iCol]; - STColumn *pTColumn = &pTSchema->columns[iCol]; - - pTColumn->colId = pSchema->colId; - pTColumn->type = pSchema->type; - pTColumn->flags = pSchema->flags; - pTColumn->bytes = pSchema->bytes; - pTColumn->offset = pTSchema->flen; - - pTSchema->flen += TYPE_BYTES[pTColumn->type]; - } - - return pTSchema; -} - -void tDestroyTSchema(STSchema *pTSchema) { - if (pTSchema) taosMemoryFree(pTSchema); -} - // SColData ======================================== void tColDataDestroy(void *ph) { SColData *pColData = (SColData *)ph; @@ -1594,7 +1496,7 @@ static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SCo } value.pData = pColData->pData + pColData->aOffset[iVal]; } else { - memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes); + tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type); } *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value); } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 52ebd7f879..653cd34b37 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -192,7 +192,7 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl return true; } void *pBitmap = tdGetBitmapAddrTp(pRow, flen); - tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset, colIdx); + tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx); return true; } @@ -217,7 +217,7 @@ bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SC return false; } } - tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset, pVal); + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); ++pIter->colIdx; } else if (TD_IS_KV_ROW(pIter->pRow)) { return tdSTSRowIterGetKvVal(pIter, colId, &pIter->kvIdx, pVal); @@ -244,7 +244,7 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { } if (TD_IS_TP_ROW(pIter->pRow)) { - tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset, pVal); + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); } else if (TD_IS_KV_ROW(pIter->pRow)) { tdSTSRowIterGetKvVal(pIter, pCol->colId, &pIter->kvIdx, pVal); } else { @@ -469,7 +469,7 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell #ifdef TD_SUPPORT_BITMAP colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn); #endif - tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset, colIdx - 1); + tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1); } else if (TD_IS_KV_ROW(pRow)) { SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx), compareKvRowColId, TD_EQ); @@ -757,10 +757,11 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset) { - if (colIdx < 1) { + if ((offset < (int32_t)sizeof(TSKEY)) || (colIdx < 1)) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } + offset -= sizeof(TSKEY); --colIdx; #ifdef TD_SUPPORT_BITMAP @@ -852,7 +853,7 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps); #endif // the primary TS key is stored separatedly - len = TD_ROW_HEAD_LEN + pBuilder->flen + pBuilder->nBitmaps; + len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps; TD_ROW_SET_LEN(pBuilder->pBuf, len); TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); break; diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 7b5d0a8805..8c5d44b8d5 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -61,7 +61,7 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { static float floatMin = -FLT_MAX, floatMax = FLT_MAX; static double doubleMin = -DBL_MAX, doubleMax = DBL_MAX; -FORCE_INLINE void *getDataMin(int32_t type, void *value) { +FORCE_INLINE void *getDataMin(int32_t type, void* value) { switch (type) { case TSDB_DATA_TYPE_FLOAT: *(float *)value = floatMin; @@ -77,7 +77,7 @@ FORCE_INLINE void *getDataMin(int32_t type, void *value) { return value; } -FORCE_INLINE void *getDataMax(int32_t type, void *value) { +FORCE_INLINE void *getDataMax(int32_t type, void* value) { switch (type) { case TSDB_DATA_TYPE_FLOAT: *(float *)value = floatMax; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index d335e84c4c..c6e2842c32 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -56,7 +56,7 @@ typedef struct SDataFWriter SDataFWriter; typedef struct SDataFReader SDataFReader; typedef struct SDelFWriter SDelFWriter; typedef struct SDelFReader SDelFReader; -typedef struct STSDBRowIter STSDBRowIter; +typedef struct SRowIter SRowIter; typedef struct STsdbFS STsdbFS; typedef struct SRowMerger SRowMerger; typedef struct STsdbReadSnap STsdbReadSnap; @@ -111,9 +111,9 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); // int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); int32_t tsdbRowCmprFn(const void *p1, const void *p2); -// STSDBRowIter -void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); -SColVal *tsdbRowIterNext(STSDBRowIter *pIter); +// SRowIter +void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); +SColVal *tRowIterNext(SRowIter *pIter); // SRowMerger int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); @@ -562,7 +562,7 @@ struct SDFileSet { SSttFile *aSttF[TSDB_MAX_STT_TRIGGER]; }; -struct STSDBRowIter { +struct SRowIter { TSDBROW *pRow; STSchema *pTSchema; SColVal colVal; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index cf9ae30ff0..65a46331aa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -341,7 +341,7 @@ int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo pSkmInfo->suid = suid; pSkmInfo->uid = uid; - tDestroyTSchema(pSkmInfo->pTSchema); + tTSchemaDestroy(pSkmInfo->pTSchema); code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema); TSDB_CHECK_CODE(code, lino, _exit); @@ -365,7 +365,7 @@ static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid pCommitter->skmRow.suid = suid; pCommitter->skmRow.uid = uid; - tDestroyTSchema(pCommitter->skmRow.pTSchema); + tTSchemaDestroy(pCommitter->skmRow.pTSchema); code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema); TSDB_CHECK_CODE(code, lino, _exit); @@ -498,7 +498,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { #if 0 ASSERT(pCommitter->minKey <= pCommitter->nextKey && pCommitter->maxKey >= pCommitter->nextKey); #endif - + pCommitter->nextKey = TSKEY_MAX; // Reader @@ -623,8 +623,7 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -667,8 +666,7 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -708,8 +706,7 @@ static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilde _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -922,8 +919,8 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { #else tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); #endif - tDestroyTSchema(pCommitter->skmTable.pTSchema); - tDestroyTSchema(pCommitter->skmRow.pTSchema); + tTSchemaDestroy(pCommitter->skmTable.pTSchema); + tTSchemaDestroy(pCommitter->skmRow.pTSchema); } static int32_t tsdbCommitData(SCommitter *pCommitter) { diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index b46a003638..9f59707ddc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -595,21 +595,21 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS if (pBuilder->bi.minKey > kRow.ts) pBuilder->bi.minKey = kRow.ts; if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts; - STSDBRowIter iter = {0}; - tsdbRowIterInit(&iter, pRow, pTSchema); + SRowIter iter = {0}; + tRowIterInit(&iter, pRow, pTSchema); - SColVal *pColVal = tsdbRowIterNext(&iter); + SColVal *pColVal = tRowIterNext(&iter); for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); while (pColVal && pColVal->cid < pDCBuilder->cid) { - pColVal = tsdbRowIterNext(&iter); + pColVal = tRowIterNext(&iter); } if (pColVal && pColVal->cid == pDCBuilder->cid) { code = tDiskColAddVal(pDCBuilder, pColVal); if (code) return code; - pColVal = tsdbRowIterNext(&iter); + pColVal = tRowIterNext(&iter); } else { code = tDiskColAddVal(pDCBuilder, &COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type)); if (code) return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index f4bdeeb387..ec89bed17a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -555,7 +555,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { } tBlockDataDestroy(&pReader->bData, 1); - tDestroyTSchema(pReader->skmTable.pTSchema); + tTSchemaDestroy(pReader->skmTable.pTSchema); // del if (pReader->pDelFReader) tsdbDelFReaderClose(&pReader->pDelFReader); @@ -1416,7 +1416,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { taosArrayDestroy(pWriter->dReader.aBlockIdx); tBlockDataDestroy(&pWriter->bData, 1); - tDestroyTSchema(pWriter->skmTable.pTSchema); + tTSchemaDestroy(pWriter->skmTable.pTSchema); for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { tFree(pWriter->aBuf[iBuf]); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 5994285647..b0cf09662c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -579,8 +579,8 @@ int32_t tsdbRowCmprFn(const void *p1, const void *p2) { return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2)); } -// STSDBRowIter ====================================================== -void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { +// SRowIter ====================================================== +void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { pIter->pRow = pRow; if (pRow->type == 0) { ASSERT(pTSchema); @@ -594,7 +594,7 @@ void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { } } -SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { +SColVal *tRowIterNext(SRowIter *pIter) { if (pIter->pRow->type == 0) { if (pIter->i < pIter->pTSchema->numOfCols) { tTSRowGetVal(pIter->pRow->pTSRow, pIter->pTSchema, pIter->i, &pIter->colVal); @@ -1084,11 +1084,11 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); + void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); cv.value.nData = varDataLen(pData); cv.value.pData = varDataVal(pData); } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); + memcpy(&cv.value.val, pRow->data + pTColumn->offset - sizeof(TSKEY), pTColumn->bytes); } code = tColDataAppendValue(pColData, &cv); @@ -1106,11 +1106,11 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); + void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); cv.value.nData = varDataLen(pData); cv.value.pData = varDataVal(pData); } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); + memcpy(&cv.value.val, pRow->data + pTColumn->offset - sizeof(TSKEY), pTColumn->bytes); } code = tColDataAppendValue(pColData, &cv); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 5f8120171f..bc09163753 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -139,8 +139,8 @@ void insSetBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_i if (i > 0) { pColList->cols[i].offset = pColList->cols[i - 1].offset + pSchema[i - 1].bytes; pColList->cols[i].toffset = pColList->flen; - pColList->flen += TYPE_BYTES[type]; } + pColList->flen += TYPE_BYTES[type]; switch (type) { case TSDB_DATA_TYPE_BINARY: pColList->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); From 335a572d86681411d80d4d749c29f1efb9eaba23 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 10:12:25 +0800 Subject: [PATCH 20/46] Revert "Revert "enh: row optimize2"" --- include/common/tdataformat.h | 103 +- include/common/trow.h | 10 +- source/client/src/clientRawBlockWrite.c | 17 +- source/common/src/tdatablock.c | 1 + source/common/src/tdataformat.c | 1200 +++++++++++--------- source/common/src/trow.c | 13 +- source/common/src/ttypes.c | 4 +- source/dnode/vnode/src/inc/tsdb.h | 10 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 19 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 10 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 14 +- source/libs/parser/src/parInsertUtil.c | 2 +- 13 files changed, 738 insertions(+), 669 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 2eda2f66cc..6855287fb2 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -27,17 +27,17 @@ extern "C" { #endif -typedef struct SBuffer SBuffer; -typedef struct SSchema SSchema; -typedef struct STColumn STColumn; -typedef struct STSchema STSchema; -typedef struct SValue SValue; -typedef struct SColVal SColVal; -typedef struct STSRow2 STSRow2; -typedef struct STSRowBuilder STSRowBuilder; -typedef struct STagVal STagVal; -typedef struct STag STag; -typedef struct SColData SColData; +typedef struct SBuffer SBuffer; +typedef struct SSchema SSchema; +typedef struct STColumn STColumn; +typedef struct STSchema STSchema; +typedef struct SValue SValue; +typedef struct SColVal SColVal; +typedef struct SRow SRow; +typedef struct SRowIter SRowIter; +typedef struct STagVal STagVal; +typedef struct STag STag; +typedef struct SColData SColData; #define HAS_NONE ((uint8_t)0x1) #define HAS_NULL ((uint8_t)0x2) @@ -68,13 +68,10 @@ struct SBuffer { void tBufferDestroy(SBuffer *pBuffer); int32_t tBufferInit(SBuffer *pBuffer, int64_t size); int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData); +int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData); // STSchema ================================ -int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); -void tTSchemaDestroy(STSchema *pTSchema); - -// SValue ================================ -static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type); +void tDestroyTSchema(STSchema *pTSchema); // SColVal ================================ #define CV_FLAG_VALUE ((int8_t)0x0) @@ -89,26 +86,14 @@ static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type); #define COL_VAL_IS_NULL(CV) ((CV)->flag == CV_FLAG_NULL) #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE) -// STSRow2 ================================ -#define TSROW_LEN(PROW, V) tGetI32v((uint8_t *)(PROW)->data, (V) ? &(V) : NULL) -#define TSROW_SVER(PROW, V) tGetI32v((PROW)->data + TSROW_LEN(PROW, NULL), (V) ? &(V) : NULL) +// SRow ================================ +int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer); +void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); -int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow); -int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow); -void tTSRowFree(STSRow2 *pRow); -void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); -int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray); -int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); -int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow); - -// STSRowBuilder ================================ -#define tsRowBuilderInit() ((STSRowBuilder){0}) -#define tsRowBuilderClear(B) \ - do { \ - if ((B)->pBuf) { \ - taosMemoryFree((B)->pBuf); \ - } \ - } while (0) +// SRowIter ================================ +int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); +void tRowIterClose(SRowIter **ppIter); +SColVal *tRowIterNext(SRowIter *pIter); // STag ================================ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); @@ -147,29 +132,17 @@ struct STSchema { int32_t numOfCols; int32_t version; int32_t flen; - int32_t vlen; int32_t tlen; STColumn columns[]; }; -#define TSROW_HAS_NONE ((uint8_t)0x1) -#define TSROW_HAS_NULL ((uint8_t)0x2U) -#define TSROW_HAS_VAL ((uint8_t)0x4U) -#define TSROW_KV_SMALL ((uint8_t)0x10U) -#define TSROW_KV_MID ((uint8_t)0x20U) -#define TSROW_KV_BIG ((uint8_t)0x40U) -#pragma pack(push, 1) -struct STSRow2 { - TSKEY ts; - uint8_t flags; - uint8_t data[]; -}; -#pragma pack(pop) - -struct STSRowBuilder { - // STSRow2 tsRow; - int32_t szBuf; - uint8_t *pBuf; +struct SRow { + uint8_t flag; + uint8_t rsv; + uint16_t sver; + uint32_t len; + TSKEY ts; + uint8_t data[]; }; struct SValue { @@ -258,37 +231,17 @@ typedef struct { int32_t nCols; schema_ver_t version; uint16_t flen; - int32_t vlen; int32_t tlen; STColumn *columns; } STSchemaBuilder; -// use 2 bits for bitmap(default: STSRow/sub block) -#define TD_VTYPE_BITS 2 -#define TD_VTYPE_PARTS 4 // PARTITIONS: 1 byte / 2 bits -#define TD_VTYPE_OPTR 3 // OPERATOR: 4 - 1, utilize to get remainder -#define TD_BITMAP_BYTES(cnt) (((cnt) + TD_VTYPE_OPTR) >> 2) - -// use 1 bit for bitmap(super block) -#define TD_VTYPE_BITS_I 1 -#define TD_VTYPE_PARTS_I 8 // PARTITIONS: 1 byte / 1 bit -#define TD_VTYPE_OPTR_I 7 // OPERATOR: 8 - 1, utilize to get remainder -#define TD_BITMAP_BYTES_I(cnt) (((cnt) + TD_VTYPE_OPTR_I) >> 3) - int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); -static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { - if (IS_VAR_DATA_TYPE(type)) { - return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); - } else { - memcpy(&pValue->val, p, tDataTypes[type].bytes); - return tDataTypes[type].bytes; - } -} +STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version); #endif diff --git a/include/common/trow.h b/include/common/trow.h index 9d183c8f80..6a71a8844e 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -55,6 +55,14 @@ typedef struct STSRow { #define TD_ROW_TP 0x0U // default #define TD_ROW_KV 0x01U +#define TD_VTYPE_PARTS 4 // PARTITIONS: 1 byte / 2 bits +#define TD_VTYPE_OPTR 3 // OPERATOR: 4 - 1, utilize to get remainder +#define TD_BITMAP_BYTES(cnt) (((cnt) + TD_VTYPE_OPTR) >> 2) + +#define TD_VTYPE_PARTS_I 8 // PARTITIONS: 1 byte / 1 bit +#define TD_VTYPE_OPTR_I 7 // OPERATOR: 8 - 1, utilize to get remainder +#define TD_BITMAP_BYTES_I(cnt) (((cnt) + TD_VTYPE_OPTR_I) >> 3) + /** * @brief value type * - for data from client input and STSRow in memory, 3 types of value none/null/norm available @@ -244,7 +252,7 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal */ static FORCE_INLINE void *tdGetBitmapAddrTp(STSRow *pRow, uint32_t flen) { // The primary TS key is stored separatedly. - return POINTER_SHIFT(TD_ROW_DATA(pRow), flen - sizeof(TSKEY)); + return POINTER_SHIFT(TD_ROW_DATA(pRow), flen); // return POINTER_SHIFT(pRow->ts, flen); } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 90a2adb647..6a6a5d56d7 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1275,6 +1275,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) nVar++; } } + fLen -= sizeof(TSKEY); int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + (int32_t)TD_BITMAP_BYTES(numOfCols - 1); @@ -1333,7 +1334,9 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) } } - offset += TYPE_BYTES[pColumn->type]; + if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + offset += TYPE_BYTES[pColumn->type]; + } } tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); @@ -1503,6 +1506,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { nVar++; } } + fLen -= sizeof(TSKEY); int32_t rows = rspObj.resInfo.numOfRows; int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + @@ -1585,8 +1589,9 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); } } - - offset += TYPE_BYTES[pColumn->type]; + if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + offset += TYPE_BYTES[pColumn->type]; + } } tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); @@ -1803,6 +1808,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) nVar++; } } + fLen -= sizeof(TSKEY); int32_t rows = rspObj.resInfo.numOfRows; int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + @@ -1888,8 +1894,9 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); } } - - offset += TYPE_BYTES[pColumn->type]; + if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + offset += TYPE_BYTES[pColumn->type]; + } } tdSRowEnd(&rb); int32_t rowLen = TD_ROW_LEN(rowData); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index fa63543439..3021c586a3 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2055,6 +2055,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB isStartKey = true; tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, offset, k); + continue; // offset should keep 0 for next column } else if (colDataIsNull_s(pColInfoData, j)) { tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NULL, NULL, diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 73b887342c..728f669fc8 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -43,606 +43,672 @@ int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData) { return code; } +int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData) { + int32_t code = tRealloc(&pBuffer->pBuf, pBuffer->nBuf + nData); + if (code) return code; + + *ppData = pBuffer->pBuf + pBuffer->nBuf; + pBuffer->nBuf += nData; + + return code; +} + // ================================ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); +// SRow ======================================================================== +#define KV_FLG_LIT ((uint8_t)0x10) +#define KV_FLG_MID ((uint8_t)0x20) +#define KV_FLG_BIG ((uint8_t)0x30) + +#define ROW_BIT_NONE ((uint8_t)0x0) +#define ROW_BIT_NULL ((uint8_t)0x1) +#define ROW_BIT_VALUE ((uint8_t)0x2) + #pragma pack(push, 1) typedef struct { - int16_t nCols; - uint8_t idx[]; -} STSKVRow; + int16_t nCol; + char idx[]; // uint8_t * | uint16_t * | uint32_t * +} SKVIdx; #pragma pack(pop) -#define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) - -// SValue -static FORCE_INLINE int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { - if (IS_VAR_DATA_TYPE(type)) { - return tPutBinary(p, pValue->pData, pValue->nData); - } else { - if (p) memcpy(p, &pValue->val, tDataTypes[type].bytes); - return tDataTypes[type].bytes; - } -} - -// STSRow2 ======================================================================== -static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) { - if (pb) { - switch (flags & 0xf) { - case TSROW_HAS_NULL | TSROW_HAS_NONE: - case TSROW_HAS_VAL | TSROW_HAS_NONE: - if (v) { - SET_BIT1(pb, idx, (uint8_t)1); - } else { - SET_BIT1(pb, idx, (uint8_t)0); - } - break; - case TSROW_HAS_VAL | TSROW_HAS_NULL: - v = v - 1; - SET_BIT1(pb, idx, v); - break; - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - SET_BIT2(pb, idx, v); - break; - - default: - ASSERT(0); - } - } -} -#define SET_IDX(p, i, n, f) \ - do { \ - if ((f)&TSROW_KV_SMALL) { \ - ((uint8_t *)(p))[i] = (n); \ - } else if ((f)&TSROW_KV_MID) { \ - ((uint16_t *)(p))[i] = (n); \ - } else { \ - ((uint32_t *)(p))[i] = (n); \ - } \ +#define ROW_SET_BITMAP(PB, FLAG, IDX, VAL) \ + do { \ + if (PB) { \ + switch (FLAG) { \ + case (HAS_NULL | HAS_NONE): \ + SET_BIT1(PB, IDX, VAL); \ + break; \ + case (HAS_VALUE | HAS_NONE): \ + SET_BIT1(PB, IDX, (VAL) ? (VAL)-1 : 0); \ + break; \ + case (HAS_VALUE | HAS_NULL): \ + SET_BIT1(PB, IDX, (VAL)-1); \ + break; \ + case (HAS_VALUE | HAS_NULL | HAS_NONE): \ + SET_BIT2(PB, IDX, VAL); \ + break; \ + default: \ + ASSERT(0); \ + break; \ + } \ + } \ } while (0) -#if 0 -int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { + +int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { int32_t code = 0; -#if 0 - STColumn *pTColumn; - SColVal *pColVal; - int32_t nColVal = taosArrayGetSize(pArray); - int32_t iColVal; - ASSERT(nColVal > 0); + ASSERT(taosArrayGetSize(aColVal) > 0); + ASSERT(((SColVal *)aColVal->pData)[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(((SColVal *)aColVal->pData)[0].type == TSDB_DATA_TYPE_TIMESTAMP); - // try - uint8_t flags = 0; - uint32_t ntv = 0; - uint32_t nkv = 0; - int16_t nTag = 0; - uint32_t maxIdx = 0; - - iColVal = 0; - for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; iColumn++) { - pTColumn = &pTSchema->columns[iColumn]; - if (iColVal < nColVal) { - pColVal = (SColVal *)taosArrayGet(pArray, iColVal); - } else { - pColVal = NULL; - } - - if (iColumn == 0) { - ASSERT(pColVal->cid == pTColumn->colId); - ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID); - - iColVal++; - } else { - if (pColVal) { - if (pColVal->cid == pTColumn->colId) { - iColVal++; - - if (COL_VAL_IS_NONE(pColVal)) { - flags |= TSROW_HAS_NONE; - } else if (COL_VAL_IS_NULL(pColVal)) { - flags |= TSROW_HAS_NULL; - maxIdx = nkv; - nTag++; - nkv += tPutI16v(NULL, -pTColumn->colId); + // scan --------------- + uint8_t flag = 0; + int32_t iColVal = 1; + const int32_t nColVal = taosArrayGetSize(aColVal); + SColVal *pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + int32_t iTColumn = 1; + STColumn *pTColumn = pTSchema->columns + iTColumn; + int32_t ntp = 0; + int32_t nkv = 0; + int32_t maxIdx = 0; + int32_t nIdx = 0; + while (pTColumn) { + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + ntp += TYPE_BYTES[pTColumn->type]; + if (COL_VAL_IS_VALUE(pColVal)) { // VALUE + flag |= HAS_VALUE; + maxIdx = nkv; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + ntp = ntp + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; + nkv = nkv + tPutI16v(NULL, pTColumn->colId) + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; } else { - flags |= TSROW_HAS_VAL; - maxIdx = nkv; - nTag++; - nkv += tPutI16v(NULL, pTColumn->colId); - nkv += tPutValue(NULL, &pColVal->value, pTColumn->type); - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - ntv += tPutValue(NULL, &pColVal->value, pTColumn->type); - } + nkv = nkv + tPutI16v(NULL, pTColumn->colId) + pTColumn->bytes; } - } else if (pColVal->cid > pTColumn->colId) { - flags |= TSROW_HAS_NONE; + nIdx++; + } else if (COL_VAL_IS_NONE(pColVal)) { // NONE + flag |= HAS_NONE; + } else if (COL_VAL_IS_NULL(pColVal)) { // NULL + flag |= HAS_NULL; + maxIdx = nkv; + nkv += tPutI16v(NULL, -pTColumn->colId); + nIdx++; } else { ASSERT(0); } + + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } else if (pColVal->cid > pTColumn->colId) { // NONE + flag |= HAS_NONE; + ntp += TYPE_BYTES[pTColumn->type]; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - flags |= TSROW_HAS_NONE; + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } + } else { // NONE + flag |= HAS_NONE; + ntp += TYPE_BYTES[pTColumn->type]; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } + } + + // compare --------------- + switch (flag) { + case HAS_NONE: + case HAS_NULL: + ntp = sizeof(SRow); + break; + case HAS_VALUE: + ntp = sizeof(SRow) + ntp; + break; + case (HAS_NULL | HAS_NONE): + ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1); + break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1) + ntp; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + ntp; + break; + default: + ASSERT(0); + break; + } + if (maxIdx <= UINT8_MAX) { + nkv = sizeof(SRow) + sizeof(SKVIdx) + nIdx + nkv; + flag |= KV_FLG_LIT; + } else if (maxIdx <= UINT16_MAX) { + nkv = sizeof(SRow) + sizeof(SKVIdx) + (nIdx << 1) + nkv; + flag |= KV_FLG_MID; + } else { + nkv = sizeof(SRow) + sizeof(SKVIdx) + (nIdx << 2) + nkv; + flag |= KV_FLG_BIG; + } + int32_t nRow; + if (nkv < ntp) { + nRow = nkv; + } else { + nRow = ntp; + flag &= ((uint8_t)0x0f); + } + + // alloc -------------- + SRow *pRow = NULL; + code = tBufferReserve(pBuffer, nRow, (void **)&pRow); + if (code) return code; + + // build -------------- + pColVal = (SColVal *)taosArrayGet(aColVal, 0); + + pRow->flag = flag; + pRow->rsv = 0; + pRow->sver = pTSchema->version; + pRow->len = nRow; + memcpy(&pRow->ts, &pColVal->value.val, sizeof(TSKEY)); + + if (flag == HAS_NONE || flag == HAS_NULL) { + goto _exit; + } + + iColVal = 1; + pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + iTColumn = 1; + pTColumn = pTSchema->columns + iTColumn; + if (flag & 0xf0) { // KV + SKVIdx *pIdx = (SKVIdx *)pRow->data; + int32_t iIdx = 0; + int32_t nv = 0; + uint8_t *pv = NULL; + if (flag & KV_FLG_LIT) { + pv = pIdx->idx + nIdx; + } else if (flag & KV_FLG_MID) { + pv = pIdx->idx + (nIdx << 1); + } else { + pv = pIdx->idx + (nIdx << 2); + } + pIdx->nCol = nIdx; + + while (pTColumn) { + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + if (COL_VAL_IS_VALUE(pColVal)) { + if (flag & KV_FLG_LIT) { + ((uint8_t *)pIdx->idx)[iIdx] = (uint8_t)nv; + } else if (flag & KV_FLG_MID) { + ((uint16_t *)pIdx->idx)[iIdx] = (uint16_t)nv; + } else { + ((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv; + } + iIdx++; + + nv += tPutI16v(pv + nv, pTColumn->colId); + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + nv += tPutU32v(pv + nv, pColVal->value.nData); + memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData); + nv += pColVal->value.nData; + } else { + memcpy(pv + nv, &pColVal->value.val, pTColumn->bytes); + nv += pTColumn->bytes; + } + } else if (COL_VAL_IS_NULL(pColVal)) { + if (flag & KV_FLG_LIT) { + ((uint8_t *)pIdx->idx)[iIdx] = (uint8_t)nv; + } else if (flag & KV_FLG_MID) { + ((uint16_t *)pIdx->idx)[iIdx] = (uint16_t)nv; + } else { + ((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv; + } + iIdx++; + nv += tPutI16v(pv + nv, -pTColumn->colId); + } + + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } else if (pColVal->cid > pTColumn->colId) { // NONE + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } else { + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } + } else { // NONE + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } + } + } else { // TUPLE + uint8_t *pb = NULL; + uint8_t *pf = NULL; + uint8_t *pv = NULL; + int32_t nv = 0; + + switch (flag) { + case (HAS_NULL | HAS_NONE): + pb = pRow->data; + break; + case HAS_VALUE: + pf = pRow->data; + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pb = pRow->data; + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pb = pRow->data; + pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + default: + ASSERT(0); + break; + } + + // build impl + while (pTColumn) { + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + if (COL_VAL_IS_VALUE(pColVal)) { // VALUE + ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_VALUE); + + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + *(int32_t *)(pf + pTColumn->offset) = nv; + nv += tPutU32v(pv + nv, pColVal->value.nData); + if (pColVal->value.nData) { + memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData); + nv += pColVal->value.nData; + } + } else { + memcpy(pf + pTColumn->offset, &pColVal->value.val, TYPE_BYTES[pTColumn->type]); + } + } else if (COL_VAL_IS_NONE(pColVal)) { // NONE + ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); + } else { // NULL + ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NULL); + if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); + } + + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } else if (pColVal->cid > pTColumn->colId) { // NONE + ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; + } else { + pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + } + } else { // NONE + ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); + pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } } } - ASSERT(flags); +_exit: + return code; +} - // decide - uint32_t nData = 0; - uint32_t nDataT = 0; - uint32_t nDataK = 0; - if (flags == TSROW_HAS_NONE || flags == TSROW_HAS_NULL) { - nData = 0; - } else { - switch (flags) { - case TSROW_HAS_VAL: - nDataT = pTSchema->flen + ntv; +void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { + ASSERT(iCol < pTSchema->numOfCols); + ASSERT(pRow->sver == pTSchema->version); + + STColumn *pTColumn = pTSchema->columns + iCol; + + if (iCol == 0) { + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY)); + return; + } + + if (pRow->flag == HAS_NONE) { + *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + return; + } + + if (pRow->flag == HAS_NULL) { + *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + return; + } + + if (pRow->flag & 0xf0) { // KV Row + SKVIdx *pIdx = (SKVIdx *)pRow->data; + uint8_t *pv = NULL; + if (pRow->flag & KV_FLG_LIT) { + pv = pIdx->idx + pIdx->nCol; + } else if (pRow->flag & KV_FLG_MID) { + pv = pIdx->idx + (pIdx->nCol << 1); + } else { + pv = pIdx->idx + (pIdx->nCol << 2); + } + + int16_t lidx = 0; + int16_t ridx = pIdx->nCol - 1; + while (lidx <= ridx) { + int16_t mid = (lidx + ridx) >> 1; + uint8_t *pData = NULL; + if (pRow->flag & KV_FLG_LIT) { + pData = pv + ((uint8_t *)pIdx->idx)[mid]; + } else if (pRow->flag & KV_FLG_MID) { + pData = pv + ((uint16_t *)pIdx->idx)[mid]; + } else { + pData = pv + ((uint32_t *)pIdx->idx)[mid]; + } + + int16_t cid; + pData += tGetI16v(pData, &cid); + + if (TABS(cid) == pTColumn->colId) { + if (cid < 0) { + *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + } else { + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + pData += tGetU32v(pData, &pColVal->value.nData); + if (pColVal->value.nData > 0) { + pColVal->value.pData = pData; + } else { + pColVal->value.pData = NULL; + } + } else { + memcpy(&pColVal->value.val, pData, pTColumn->bytes); + } + } + return; + } else if (TABS(cid) < pTColumn->colId) { + lidx = mid + 1; + } else { + ridx = mid - 1; + } + } + + *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + } else { // Tuple Row + uint8_t *pf = NULL; + uint8_t *pv = NULL; + uint8_t bv = ROW_BIT_VALUE; + + switch (pRow->flag) { + case HAS_VALUE: + pf = pRow->data; + pv = pf + pTSchema->flen; break; - case TSROW_HAS_NULL | TSROW_HAS_NONE: - nDataT = BIT1_SIZE(pTSchema->numOfCols - 1); + case (HAS_NULL | HAS_NONE): + bv = GET_BIT1(pRow->data, iCol - 1); break; - case TSROW_HAS_VAL | TSROW_HAS_NONE: - case TSROW_HAS_VAL | TSROW_HAS_NULL: - nDataT = BIT1_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv; + case (HAS_VALUE | HAS_NONE): + bv = GET_BIT1(pRow->data, iCol - 1); + if (bv) bv++; + pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; break; - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - nDataT = BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv; + case (HAS_VALUE | HAS_NULL): + bv = GET_BIT1(pRow->data, iCol - 1); + bv++; + pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + bv = GET_BIT2(pRow->data, iCol - 1); + pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; break; default: break; + } + + if (bv == ROW_BIT_NONE) { + *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + return; + } else if (bv == ROW_BIT_NULL) { + *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + return; + } + + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); + pData += tGetU32v(pData, &pColVal->value.nData); + if (pColVal->value.nData) { + pColVal->value.pData = pData; + } else { + pColVal->value.pData = NULL; + } + } else { + memcpy(&pColVal->value.val, pv + pTColumn->offset, pTColumn->bytes); + } + } +} + +// SRowIter ======================================== +struct SRowIter { + SRow *pRow; + STSchema *pTSchema; + + int32_t iTColumn; + union { + struct { // kv + int32_t iCol; + SKVIdx *pIdx; + }; + struct { // tuple + uint8_t *pb; + uint8_t *pf; + }; + uint8_t *pv; + }; + + SColVal cv; +}; + +int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { + ASSERT(pRow->sver == pTSchema->version); + + int32_t code = 0; + + SRowIter *pIter = taosMemoryCalloc(1, sizeof(*pIter)); + if (pIter == NULL) { + code = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _exit; + } + + pIter->pRow = pRow; + pIter->pTSchema = pTSchema; + pIter->iTColumn = 0; + + if (pRow->flag == HAS_NONE || pRow->flag == HAS_NULL) goto _exit; + + if (pRow->flag & 0xf0) { + pIter->iCol = 0; + pIter->pIdx = (SKVIdx *)pRow->data; + if (pRow->flag & KV_FLG_LIT) { + pIter->pv = pIter->pIdx->idx + pIter->pIdx->nCol; + } else if (pRow->flag & KV_FLG_MID) { + pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 1); + } else { + pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); + } + } else { + switch (pRow->flag) { + case (HAS_NULL | HAS_NONE): + pIter->pb = pRow->data; + break; + case HAS_VALUE: + pIter->pf = pRow->data; + pIter->pv = pIter->pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pIter->pb = pRow->data; + pIter->pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pIter->pv = pIter->pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pIter->pb = pRow->data; + pIter->pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); + pIter->pv = pIter->pf + pTSchema->flen; + break; + default: ASSERT(0); - } - - uint8_t tflags = 0; - if (maxIdx <= UINT8_MAX) { - nDataK = sizeof(STSKVRow) + sizeof(uint8_t) * nTag + nkv; - tflags |= TSROW_KV_SMALL; - } else if (maxIdx <= UINT16_MAX) { - nDataK = sizeof(STSKVRow) + sizeof(uint16_t) * nTag + nkv; - tflags |= TSROW_KV_MID; - } else { - nDataK = sizeof(STSKVRow) + sizeof(uint32_t) * nTag + nkv; - tflags |= TSROW_KV_BIG; - } - - if (nDataT <= nDataK) { - nData = nDataT; - } else { - nData = nDataK; - flags |= tflags; + break; } } - // alloc - if (pBuilder) { - // create from a builder - if (nData == 0) { - pBuilder->tsRow.nData = 0; - pBuilder->tsRow.pData = NULL; - } else { - if (pBuilder->szBuf < nData) { - uint8_t *p = taosMemoryRealloc(pBuilder->pBuf, nData); - if (p == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - pBuilder->pBuf = p; - pBuilder->szBuf = nData; +_exit: + if (code) { + *ppIter = NULL; + if (pIter) taosMemoryFree(pIter); + } else { + *ppIter = pIter; + } + return code; +} + +void tRowIterClose(SRowIter **ppIter) { + SRowIter *pIter = *ppIter; + if (pIter) { + taosMemoryFree(pIter); + } + *ppIter = NULL; +} + +SColVal *tRowIterNext(SRowIter *pIter) { + if (pIter->iTColumn >= pIter->pTSchema->numOfCols) { + return NULL; + } + + STColumn *pTColumn = pIter->pTSchema->columns + pIter->iTColumn; + + // timestamp + if (0 == pIter->iTColumn) { + pIter->cv.cid = pTColumn->colId; + pIter->cv.type = pTColumn->type; + pIter->cv.flag = CV_FLAG_VALUE; + memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY)); + goto _exit; + } + + if (pIter->pRow->flag == HAS_NONE) { + pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + goto _exit; + } + + if (pIter->pRow->flag == HAS_NULL) { + pIter->cv = COL_VAL_NULL(pTColumn->type, pTColumn->colId); + goto _exit; + } + + if (pIter->pRow->flag & 0xf0) { // KV + if (pIter->iCol < pIter->pIdx->nCol) { + uint8_t *pData; + + if (pIter->pRow->flag & KV_FLG_LIT) { + pData = pIter->pv + ((uint8_t *)pIter->pIdx->idx)[pIter->iCol]; + } else if (pIter->pRow->flag & KV_FLG_MID) { + pData = pIter->pv + ((uint16_t *)pIter->pIdx->idx)[pIter->iCol]; + } else { + pData = pIter->pv + ((uint32_t *)pIter->pIdx->idx)[pIter->iCol]; } - pBuilder->tsRow.nData = nData; - pBuilder->tsRow.pData = pBuilder->pBuf; - } + int16_t cid; + pData += tGetI16v(pData, &cid); - *ppRow = &pBuilder->tsRow; - } else { - // create a new one - *ppRow = (STSRow2 *)taosMemoryMalloc(sizeof(STSRow2)); - if (*ppRow == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + if (TABS(cid) == pTColumn->colId) { + if (cid < 0) { + pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + } else { + pIter->cv.cid = pTColumn->colId; + pIter->cv.type = pTColumn->type; + pIter->cv.flag = CV_FLAG_VALUE; + + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + pData += tGetU32v(pData, &pIter->cv.value.nData); + if (pIter->cv.value.nData > 0) { + pIter->cv.value.pData = pData; + } else { + pIter->cv.value.pData = NULL; + } + } else { + memcpy(&pIter->cv.value.val, pData, pTColumn->bytes); + } + } + + pIter->iCol++; + goto _exit; + } else if (TABS(cid) > pTColumn->colId) { + pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + goto _exit; + } else { + ASSERT(0); + } + } else { + pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); goto _exit; } - if (nData == 0) { - (*ppRow)->nData = 0; - (*ppRow)->pData = NULL; - } else { - (*ppRow)->nData = nData; - (*ppRow)->pData = taosMemoryMalloc(nData); - if ((*ppRow)->pData == NULL) { - taosMemoryFree(*ppRow); - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - } - - // build - (*ppRow)->flags = flags; - (*ppRow)->sver = pTSchema->version; - - pColVal = (SColVal *)taosArrayGet(pArray, 0); - (*ppRow)->ts = pColVal->value.ts; - - if ((*ppRow)->pData) { - STSKVRow *pTSKVRow = NULL; - uint8_t *pidx = NULL; - uint8_t *pkv = NULL; - uint8_t *pb = NULL; - uint8_t *pf = NULL; - uint8_t *ptv = NULL; - nkv = 0; - ntv = 0; - iColVal = 1; - - if ((flags & 0xf0) == 0) { - switch (flags & 0xf) { - case TSROW_HAS_VAL: - pf = (*ppRow)->pData; - ptv = pf + pTSchema->flen; + } else { // Tuple + uint8_t bv = ROW_BIT_VALUE; + if (pIter->pb) { + switch (pIter->pRow->flag) { + case (HAS_NULL | HAS_NONE): + bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1); break; - case TSROW_HAS_NULL | TSROW_HAS_NONE: - pb = (*ppRow)->pData; + case (HAS_VALUE | HAS_NONE): + bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1); + if (bv) bv++; break; - case TSROW_HAS_VAL | TSROW_HAS_NONE: - case TSROW_HAS_VAL | TSROW_HAS_NULL: - pb = (*ppRow)->pData; - pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); - ptv = pf + pTSchema->flen; + case (HAS_VALUE | HAS_NULL): + bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1) + 1; break; - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - pb = (*ppRow)->pData; - pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); - ptv = pf + pTSchema->flen; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + bv = GET_BIT2(pIter->pb, pIter->iTColumn - 1); break; default: ASSERT(0); break; } - } else { - pTSKVRow = (STSKVRow *)(*ppRow)->pData; - pTSKVRow->nCols = 0; - pidx = pTSKVRow->idx; - if (flags & TSROW_KV_SMALL) { - pkv = pidx + sizeof(uint8_t) * nTag; - } else if (flags & TSROW_KV_MID) { - pkv = pidx + sizeof(uint16_t) * nTag; - } else { - pkv = pidx + sizeof(uint32_t) * nTag; + + if (bv == ROW_BIT_NONE) { + pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + goto _exit; + } else if (bv == ROW_BIT_NULL) { + pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + goto _exit; } } - for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { - pTColumn = &pTSchema->columns[iColumn]; - if (iColVal < nColVal) { - pColVal = (SColVal *)taosArrayGet(pArray, iColVal); - } else { - pColVal = NULL; - } - - if (pColVal) { - if (pColVal->cid == pTColumn->colId) { - iColVal++; - - if (COL_VAL_IS_NONE(pColVal)) { - goto _set_none; - } else if (COL_VAL_IS_NULL(pColVal)) { - goto _set_null; - } else { - goto _set_value; - } - } else if (pColVal->cid > pTColumn->colId) { - goto _set_none; - } else { - ASSERT(0); - } - } else { - goto _set_none; - } - - _set_none: - if ((flags & 0xf0) == 0) { - setBitMap(pb, 0, iColumn - 1, flags); - if (flags & TSROW_HAS_VAL) { // set 0 - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - *(VarDataOffsetT *)(pf + pTColumn->offset) = 0; - } else { - tPutValue(pf + pTColumn->offset, &((SValue){0}), pTColumn->type); - } - } - } - continue; - - _set_null: - if ((flags & 0xf0) == 0) { - setBitMap(pb, 1, iColumn - 1, flags); - if (flags & TSROW_HAS_VAL) { // set 0 - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - *(VarDataOffsetT *)(pf + pTColumn->offset) = 0; - } else { - tPutValue(pf + pTColumn->offset, &((SValue){0}), pTColumn->type); - } - } - } else { - SET_IDX(pidx, pTSKVRow->nCols, nkv, flags); - pTSKVRow->nCols++; - nkv += tPutI16v(pkv + nkv, -pTColumn->colId); - } - continue; - - _set_value: - if ((flags & 0xf0) == 0) { - setBitMap(pb, 2, iColumn - 1, flags); - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - *(VarDataOffsetT *)(pf + pTColumn->offset) = ntv; - ntv += tPutValue(ptv + ntv, &pColVal->value, pTColumn->type); - } else { - tPutValue(pf + pTColumn->offset, &pColVal->value, pTColumn->type); - } - } else { - SET_IDX(pidx, pTSKVRow->nCols, nkv, flags); - pTSKVRow->nCols++; - nkv += tPutI16v(pkv + nkv, pColVal->cid); - nkv += tPutValue(pkv + nkv, &pColVal->value, pTColumn->type); - } - continue; - } - } - -#endif -_exit: - return code; -} - -int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow) { - int32_t code = 0; - int32_t rLen = 0; - - TSROW_LEN(pRow, rLen); - (*ppRow) = (STSRow2 *)taosMemoryMalloc(rLen); - if (*ppRow == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - memcpy(*ppRow, pRow, rLen); - -_exit: - return code; -} - -void tTSRowFree(STSRow2 *pRow) { - if (pRow) { - taosMemoryFree(pRow); - } -} - -void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { - uint8_t isTuple = ((pRow->flags & 0xf0) == 0) ? 1 : 0; - STColumn *pTColumn = &pTSchema->columns[iCol]; - uint8_t flags = pRow->flags & (uint8_t)0xf; - SValue value; - - ASSERT(iCol < pTSchema->numOfCols); - ASSERT(flags); - ASSERT(pRow->sver == pTSchema->version); - - if (iCol == 0) { - value.ts = pRow->ts; - goto _return_value; - } - - if (flags == TSROW_HAS_NONE) { - goto _return_none; - } else if (flags == TSROW_HAS_NULL) { - goto _return_null; - } - - ASSERT(pRow->nData && pRow->pData); - - if (isTuple) { - uint8_t *pb = pRow->pData; - uint8_t *pf = NULL; - uint8_t *pv = NULL; - uint8_t *p; - uint8_t b; - - // bit - switch (flags) { - case TSROW_HAS_VAL: - pf = pb; - break; - case TSROW_HAS_NULL | TSROW_HAS_NONE: - b = GET_BIT1(pb, iCol - 1); - if (b == 0) { - goto _return_none; - } else { - goto _return_null; - } - case TSROW_HAS_VAL | TSROW_HAS_NONE: - b = GET_BIT1(pb, iCol - 1); - if (b == 0) { - goto _return_none; - } else { - pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); - break; - } - case TSROW_HAS_VAL | TSROW_HAS_NULL: - b = GET_BIT1(pb, iCol - 1); - if (b == 0) { - goto _return_null; - } else { - pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); - break; - } - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - b = GET_BIT2(pb, iCol - 1); - if (b == 0) { - goto _return_none; - } else if (b == 1) { - goto _return_null; - } else { - pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); - break; - } - default: - ASSERT(0); - } - - ASSERT(pf); - - p = pf + pTColumn->offset; + pIter->cv.cid = pTColumn->colId; + pIter->cv.type = pTColumn->type; + pIter->cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - pv = pf + pTSchema->flen; - p = pv + *(VarDataOffsetT *)p; - } - tGetValue(p, &value, pTColumn->type); - goto _return_value; - } else { - STSKVRow *pRowK = (STSKVRow *)pRow->pData; - int16_t lidx = 0; - int16_t ridx = pRowK->nCols - 1; - uint8_t *p; - int16_t midx; - uint32_t n; - int16_t cid; - - ASSERT(pRowK->nCols > 0); - - if (pRow->flags & TSROW_KV_SMALL) { - p = pRow->pData + sizeof(STSKVRow) + sizeof(uint8_t) * pRowK->nCols; - } else if (pRow->flags & TSROW_KV_MID) { - p = pRow->pData + sizeof(STSKVRow) + sizeof(uint16_t) * pRowK->nCols; - } else if (pRow->flags & TSROW_KV_BIG) { - p = pRow->pData + sizeof(STSKVRow) + sizeof(uint32_t) * pRowK->nCols; + uint8_t *pData = pIter->pv + *(int32_t *)(pIter->pf + pTColumn->offset); + pData += tGetU32v(pData, &pIter->cv.value.nData); + if (pIter->cv.value.nData > 0) { + pIter->cv.value.pData = pData; + } else { + pIter->cv.value.pData = NULL; + } } else { - ASSERT(0); + memcpy(&pIter->cv.value.val, pIter->pv + pTColumn->offset, pTColumn->bytes); } - while (lidx <= ridx) { - midx = (lidx + ridx) / 2; - - if (pRow->flags & TSROW_KV_SMALL) { - n = ((uint8_t *)pRowK->idx)[midx]; - } else if (pRow->flags & TSROW_KV_MID) { - n = ((uint16_t *)pRowK->idx)[midx]; - } else { - n = ((uint32_t *)pRowK->idx)[midx]; - } - - n += tGetI16v(p + n, &cid); - - if (TABS(cid) == pTColumn->colId) { - if (cid < 0) { - goto _return_null; - } else { - n += tGetValue(p + n, &value, pTColumn->type); - goto _return_value; - } - - return; - } else if (TABS(cid) > pTColumn->colId) { - ridx = midx - 1; - } else { - lidx = midx + 1; - } - } - - // not found, return NONE - goto _return_none; - } - -_return_none: - *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - return; - -_return_null: - *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - return; - -_return_value: - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, value); - return; -} - -int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray) { - int32_t code = 0; - SColVal cv; - - (*ppArray) = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); - if (*ppArray == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; iColumn++) { - tTSRowGet(pRow, pTSchema, iColumn, &cv); - taosArrayPush(*ppArray, &cv); - } - - _exit: - return code; -} -#endif -int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { - int32_t n = 0; - - TSROW_LEN(pRow, n); - if (p) { - memcpy(p, pRow, n); - } - - return n; -} - -int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow) { - int32_t n = 0; - - *ppRow = (STSRow2 *)p; - TSROW_LEN(*ppRow, n); - - return n; + pIter->iTColumn++; + return &pIter->cv; } // STSchema ======================================== -int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) { - *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols); - if (*ppTSchema == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - (*ppTSchema)->numOfCols = ncols; - (*ppTSchema)->version = sver; - (*ppTSchema)->flen = 0; - (*ppTSchema)->vlen = 0; - (*ppTSchema)->tlen = 0; - - for (int32_t iCol = 0; iCol < ncols; iCol++) { - SSchema *pColumn = &pSchema[iCol]; - STColumn *pTColumn = &((*ppTSchema)->columns[iCol]); - - pTColumn->colId = pColumn->colId; - pTColumn->type = pColumn->type; - pTColumn->flags = pColumn->flags; - pTColumn->bytes = pColumn->bytes; - pTColumn->offset = (*ppTSchema)->flen; - - // skip first column - if (iCol) { - (*ppTSchema)->flen += TYPE_BYTES[pColumn->type]; - if (IS_VAR_DATA_TYPE(pColumn->type)) { - (*ppTSchema)->vlen += (pColumn->bytes + 5); - } - } - } - - return 0; -} - -void tTSchemaDestroy(STSchema *pTSchema) { - if (pTSchema) taosMemoryFree(pTSchema); -} // STag ======================================== static int tTagValCmprFn(const void *p1, const void *p2) { @@ -1042,7 +1108,6 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { pBuilder->nCols = 0; pBuilder->tlen = 0; pBuilder->flen = 0; - pBuilder->vlen = 0; pBuilder->version = version; } @@ -1061,24 +1126,21 @@ int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, c pCol->colId = colId; pCol->flags = flags; if (pBuilder->nCols == 0) { - pCol->offset = 0; + pCol->offset = -1; } else { - STColumn *pTCol = &(pBuilder->columns[pBuilder->nCols - 1]); - pCol->offset = pTCol->offset + TYPE_BYTES[pTCol->type]; + pCol->offset = pBuilder->flen; + pBuilder->flen += TYPE_BYTES[type]; } if (IS_VAR_DATA_TYPE(type)) { pCol->bytes = bytes; pBuilder->tlen += (TYPE_BYTES[type] + bytes); - pBuilder->vlen += bytes - sizeof(VarDataLenT); } else { pCol->bytes = TYPE_BYTES[type]; pBuilder->tlen += TYPE_BYTES[type]; - pBuilder->vlen += TYPE_BYTES[type]; } pBuilder->nCols++; - pBuilder->flen += TYPE_BYTES[type]; ASSERT(pCol->offset < pBuilder->flen); @@ -1097,7 +1159,6 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { pSchema->numOfCols = pBuilder->nCols; pSchema->tlen = pBuilder->tlen; pSchema->flen = pBuilder->flen; - pSchema->vlen = pBuilder->vlen; #ifdef TD_SUPPORT_BITMAP pSchema->tlen += (int)TD_BITMAP_BYTES(pSchema->numOfCols); @@ -1110,6 +1171,43 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { #endif +STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { + STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols); + if (pTSchema == NULL) return NULL; + + pTSchema->numOfCols = numOfCols; + pTSchema->version = version; + + // timestamp column + ASSERT(aSchema[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aSchema[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID); + pTSchema->columns[0].colId = aSchema[0].colId; + pTSchema->columns[0].type = aSchema[0].type; + pTSchema->columns[0].flags = aSchema[0].flags; + pTSchema->columns[0].bytes = aSchema[0].bytes; + pTSchema->columns[0].offset = -1; + + // other columns + for (int32_t iCol = 1; iCol < numOfCols; iCol++) { + SSchema *pSchema = &aSchema[iCol]; + STColumn *pTColumn = &pTSchema->columns[iCol]; + + pTColumn->colId = pSchema->colId; + pTColumn->type = pSchema->type; + pTColumn->flags = pSchema->flags; + pTColumn->bytes = pSchema->bytes; + pTColumn->offset = pTSchema->flen; + + pTSchema->flen += TYPE_BYTES[pTColumn->type]; + } + + return pTSchema; +} + +void tDestroyTSchema(STSchema *pTSchema) { + if (pTSchema) taosMemoryFree(pTSchema); +} + // SColData ======================================== void tColDataDestroy(void *ph) { SColData *pColData = (SColData *)ph; @@ -1496,7 +1594,7 @@ static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SCo } value.pData = pColData->pData + pColData->aOffset[iVal]; } else { - tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type); + memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes); } *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value); } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 653cd34b37..52ebd7f879 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -192,7 +192,7 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl return true; } void *pBitmap = tdGetBitmapAddrTp(pRow, flen); - tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx); + tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset, colIdx); return true; } @@ -217,7 +217,7 @@ bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SC return false; } } - tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset, pVal); ++pIter->colIdx; } else if (TD_IS_KV_ROW(pIter->pRow)) { return tdSTSRowIterGetKvVal(pIter, colId, &pIter->kvIdx, pVal); @@ -244,7 +244,7 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { } if (TD_IS_TP_ROW(pIter->pRow)) { - tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset, pVal); } else if (TD_IS_KV_ROW(pIter->pRow)) { tdSTSRowIterGetKvVal(pIter, pCol->colId, &pIter->kvIdx, pVal); } else { @@ -469,7 +469,7 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell #ifdef TD_SUPPORT_BITMAP colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn); #endif - tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1); + tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset, colIdx - 1); } else if (TD_IS_KV_ROW(pRow)) { SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx), compareKvRowColId, TD_EQ); @@ -757,11 +757,10 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset) { - if ((offset < (int32_t)sizeof(TSKEY)) || (colIdx < 1)) { + if (colIdx < 1) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - offset -= sizeof(TSKEY); --colIdx; #ifdef TD_SUPPORT_BITMAP @@ -853,7 +852,7 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps); #endif // the primary TS key is stored separatedly - len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps; + len = TD_ROW_HEAD_LEN + pBuilder->flen + pBuilder->nBitmaps; TD_ROW_SET_LEN(pBuilder->pBuf, len); TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); break; diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 8c5d44b8d5..7b5d0a8805 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -61,7 +61,7 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { static float floatMin = -FLT_MAX, floatMax = FLT_MAX; static double doubleMin = -DBL_MAX, doubleMax = DBL_MAX; -FORCE_INLINE void *getDataMin(int32_t type, void* value) { +FORCE_INLINE void *getDataMin(int32_t type, void *value) { switch (type) { case TSDB_DATA_TYPE_FLOAT: *(float *)value = floatMin; @@ -77,7 +77,7 @@ FORCE_INLINE void *getDataMin(int32_t type, void* value) { return value; } -FORCE_INLINE void *getDataMax(int32_t type, void* value) { +FORCE_INLINE void *getDataMax(int32_t type, void *value) { switch (type) { case TSDB_DATA_TYPE_FLOAT: *(float *)value = floatMax; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c6e2842c32..d335e84c4c 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -56,7 +56,7 @@ typedef struct SDataFWriter SDataFWriter; typedef struct SDataFReader SDataFReader; typedef struct SDelFWriter SDelFWriter; typedef struct SDelFReader SDelFReader; -typedef struct SRowIter SRowIter; +typedef struct STSDBRowIter STSDBRowIter; typedef struct STsdbFS STsdbFS; typedef struct SRowMerger SRowMerger; typedef struct STsdbReadSnap STsdbReadSnap; @@ -111,9 +111,9 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); // int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); int32_t tsdbRowCmprFn(const void *p1, const void *p2); -// SRowIter -void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); -SColVal *tRowIterNext(SRowIter *pIter); +// STSDBRowIter +void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); +SColVal *tsdbRowIterNext(STSDBRowIter *pIter); // SRowMerger int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); @@ -562,7 +562,7 @@ struct SDFileSet { SSttFile *aSttF[TSDB_MAX_STT_TRIGGER]; }; -struct SRowIter { +struct STSDBRowIter { TSDBROW *pRow; STSchema *pTSchema; SColVal colVal; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 65a46331aa..cf9ae30ff0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -341,7 +341,7 @@ int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo pSkmInfo->suid = suid; pSkmInfo->uid = uid; - tTSchemaDestroy(pSkmInfo->pTSchema); + tDestroyTSchema(pSkmInfo->pTSchema); code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema); TSDB_CHECK_CODE(code, lino, _exit); @@ -365,7 +365,7 @@ static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid pCommitter->skmRow.suid = suid; pCommitter->skmRow.uid = uid; - tTSchemaDestroy(pCommitter->skmRow.pTSchema); + tDestroyTSchema(pCommitter->skmRow.pTSchema); code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema); TSDB_CHECK_CODE(code, lino, _exit); @@ -498,7 +498,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { #if 0 ASSERT(pCommitter->minKey <= pCommitter->nextKey && pCommitter->maxKey >= pCommitter->nextKey); #endif - + pCommitter->nextKey = TSKEY_MAX; // Reader @@ -623,7 +623,8 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -666,7 +667,8 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -706,7 +708,8 @@ static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilde _exit: if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -919,8 +922,8 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { #else tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); #endif - tTSchemaDestroy(pCommitter->skmTable.pTSchema); - tTSchemaDestroy(pCommitter->skmRow.pTSchema); + tDestroyTSchema(pCommitter->skmTable.pTSchema); + tDestroyTSchema(pCommitter->skmRow.pTSchema); } static int32_t tsdbCommitData(SCommitter *pCommitter) { diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 9f59707ddc..b46a003638 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -595,21 +595,21 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS if (pBuilder->bi.minKey > kRow.ts) pBuilder->bi.minKey = kRow.ts; if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts; - SRowIter iter = {0}; - tRowIterInit(&iter, pRow, pTSchema); + STSDBRowIter iter = {0}; + tsdbRowIterInit(&iter, pRow, pTSchema); - SColVal *pColVal = tRowIterNext(&iter); + SColVal *pColVal = tsdbRowIterNext(&iter); for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); while (pColVal && pColVal->cid < pDCBuilder->cid) { - pColVal = tRowIterNext(&iter); + pColVal = tsdbRowIterNext(&iter); } if (pColVal && pColVal->cid == pDCBuilder->cid) { code = tDiskColAddVal(pDCBuilder, pColVal); if (code) return code; - pColVal = tRowIterNext(&iter); + pColVal = tsdbRowIterNext(&iter); } else { code = tDiskColAddVal(pDCBuilder, &COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type)); if (code) return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index ec89bed17a..f4bdeeb387 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -555,7 +555,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { } tBlockDataDestroy(&pReader->bData, 1); - tTSchemaDestroy(pReader->skmTable.pTSchema); + tDestroyTSchema(pReader->skmTable.pTSchema); // del if (pReader->pDelFReader) tsdbDelFReaderClose(&pReader->pDelFReader); @@ -1416,7 +1416,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { taosArrayDestroy(pWriter->dReader.aBlockIdx); tBlockDataDestroy(&pWriter->bData, 1); - tTSchemaDestroy(pWriter->skmTable.pTSchema); + tDestroyTSchema(pWriter->skmTable.pTSchema); for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { tFree(pWriter->aBuf[iBuf]); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index b0cf09662c..5994285647 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -579,8 +579,8 @@ int32_t tsdbRowCmprFn(const void *p1, const void *p2) { return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2)); } -// SRowIter ====================================================== -void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { +// STSDBRowIter ====================================================== +void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { pIter->pRow = pRow; if (pRow->type == 0) { ASSERT(pTSchema); @@ -594,7 +594,7 @@ void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { } } -SColVal *tRowIterNext(SRowIter *pIter) { +SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { if (pIter->pRow->type == 0) { if (pIter->i < pIter->pTSchema->numOfCols) { tTSRowGetVal(pIter->pRow->pTSRow, pIter->pTSchema, pIter->i, &pIter->colVal); @@ -1084,11 +1084,11 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); + void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); cv.value.nData = varDataLen(pData); cv.value.pData = varDataVal(pData); } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset - sizeof(TSKEY), pTColumn->bytes); + memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); } code = tColDataAppendValue(pColData, &cv); @@ -1106,11 +1106,11 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); + void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); cv.value.nData = varDataLen(pData); cv.value.pData = varDataVal(pData); } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset - sizeof(TSKEY), pTColumn->bytes); + memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); } code = tColDataAppendValue(pColData, &cv); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index bc09163753..5f8120171f 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -139,8 +139,8 @@ void insSetBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_i if (i > 0) { pColList->cols[i].offset = pColList->cols[i - 1].offset + pSchema[i - 1].bytes; pColList->cols[i].toffset = pColList->flen; + pColList->flen += TYPE_BYTES[type]; } - pColList->flen += TYPE_BYTES[type]; switch (type) { case TSDB_DATA_TYPE_BINARY: pColList->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); From b27faba7efb06f9789c240be1285e15d4453af8a Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 23 Nov 2022 17:07:45 +0800 Subject: [PATCH 21/46] fix:initialize maxts --- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 15 +++-- tests/script/tsim/stream/state0.sim | 60 +++++++++++++++++++ 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index de2a7b9dac..8db450ad50 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1849,12 +1849,12 @@ FETCH_NEXT_BLOCK: prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; printDataBlock(pDelBlock, "stream scan delete data"); if (pInfo->tqReader) { blockDataDestroy(pDelBlock); } if (pInfo->pDeleteDataRes->info.rows > 0) { + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; return pInfo->pDeleteDataRes; } else { goto FETCH_NEXT_BLOCK; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index dd02ce9cd4..7fd2fcb5b8 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3364,22 +3364,23 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num } } -void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t waterMark, uint16_t type, - int32_t tsColIndex) { +void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, + STimeWindowAggSupp* pTwSup) { if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) { SStreamPartitionOperatorInfo* pScanInfo = downstream->info; pScanInfo->tsColIndex = tsColIndex; } if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - initDownStream(downstream->pDownstream[0], pAggSup, waterMark, type, tsColIndex); + initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup); return; } SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; if (!pScanInfo->pUpdateInfo) { - pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark); + pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark); } + pScanInfo->twAggSup = *pTwSup; } int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, int64_t gap, @@ -4102,8 +4103,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); if (downstream) { - initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, - pInfo->primaryTsIndex); + initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); code = appendDownstream(pOperator, &downstream, 1); } return pOperator; @@ -4606,8 +4606,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, NULL); - initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, - pInfo->primaryTsIndex); + initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim index 87d7d4b7fc..7c922658c9 100644 --- a/tests/script/tsim/stream/state0.sim +++ b/tests/script/tsim/stream/state0.sim @@ -731,5 +731,65 @@ if $data32 != 1 then goto loop9 endi +sql drop stream if exists streams5; +sql drop database if exists test5; +sql create database test5; +sql use test5; +sql create table tb (ts timestamp, a int); +sql insert into tb values (now + 1m , 1 ); +sql create table b (c timestamp, d int, e int , f int, g double); +sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a); +sql insert into b values(1648791213000,NULL,NULL,NULL,NULL); +sql select * from streamt order by c1, c2, c3; + +print data00:$data00 +print data01:$data01 + +sql insert into b values(1648791213000,NULL,NULL,NULL,NULL); +sql select * from streamt order by c1, c2, c3; + +print data00:$data00 +print data01:$data01 + +sql insert into b values(1648791213001,1,2,2,2.0); +sql insert into b values(1648791213002,1,3,3,3.0); +sql insert into tb values(1648791213003,1); + +sql select * from streamt; +print data00:$data00 +print data01:$data01 + +sql delete from b where c >= 1648791213001 and c <= 1648791213002; +sql insert into b values(1648791223003,2,2,3,1.0); insert into b values(1648791223002,2,3,3,3.0); +sql insert into tb values (now + 1m , 1 ); + +sql select * from streamt; +print data00:$data00 +print data01:$data01 + +sql insert into b(c,d) values (now + 6m , 6 ); +sql delete from b where c >= 1648791213001 and c <= 1648791233005;; + +$loop_count = 0 +loop10: + +sleep 200 + +sql select c2 from streamt; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop10 +endi + +if $data00 != 2 then + print =====data00=$data00 + goto loop10 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT From b460974f43d248edceee368e371d043136bbbdca Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 24 Nov 2022 10:32:31 +0800 Subject: [PATCH 22/46] fix(query): fix syntax error. --- source/libs/executor/src/exchangeoperator.c | 27 ++++++++------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 04caccbf8f..2c79b77e16 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -75,11 +75,9 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn } while (1) { - int32_t v = 0; - sem_getvalue(&pExchangeInfo->ready, &v); - qDebug("prepare wait for ready, sem:(%d,%p), %p, %s", v, (void*)pExchangeInfo->ready.__align, pExchangeInfo, GET_TASKID(pTaskInfo)); - + qDebug("prepare wait for ready, %p, %s", pExchangeInfo, GET_TASKID(pTaskInfo)); tsem_wait(&pExchangeInfo->ready); + if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -372,9 +370,6 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { int32_t index = pWrapper->sourceIndex; SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index); - int32_t v = 0, v1 = 0; - sem_getvalue(&pExchangeInfo->ready, &v); - if (code == TSDB_CODE_SUCCESS) { pSourceDataInfo->pRsp = pMsg->pData; @@ -386,22 +381,23 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, sem:(%d,%p), %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, - pRsp->numOfRows, v, (void*)pExchangeInfo->ready.__align, pExchangeInfo); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + pRsp->numOfRows, pExchangeInfo); } else { taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%s, sem:(%d,%p), %p", pSourceDataInfo->taskId, index, tstrerror(code), v, - (void*) pExchangeInfo->ready.__align, pExchangeInfo); + qDebug("%s fetch rsp received, index:%d, error:%s, %p", pSourceDataInfo->taskId, index, tstrerror(code), pExchangeInfo); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; - code = tsem_post(&pExchangeInfo->ready); - ASSERT(code == TSDB_CODE_SUCCESS); + if (code != TSDB_CODE_SUCCESS) { + code = TAOS_SYSTEM_ERROR(code); + qError("failed to invoke post when fetch rsp is ready, code:%s, %p", tstrerror(code), pExchangeInfo); + } taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); - return TSDB_CODE_SUCCESS; + return code; } int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) { @@ -566,9 +562,6 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; pOperator->cost.openCost = taosGetTimestampUs() - startTs; - int32_t value = 0; - sem_getvalue(&pExchangeInfo->ready, &value); - tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); From 7a88018b2a5cebd2a336bd357f179ca595efe5a2 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 24 Nov 2022 10:43:54 +0800 Subject: [PATCH 23/46] enh: group by tbname optimize --- source/libs/planner/src/planLogicCreater.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index de0feceb8f..44eb8478f1 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -167,18 +167,18 @@ static int32_t cloneRewriteExprs(SNodeList* pExprs, bool* pOutputs, SNodeList** } static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause, - SNodeList** pRewriteExpr) { + SNodeList** pRewriteExprs) { nodesWalkExprs(pExprs, doNameExpr, NULL); SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; - if (NULL != pRewriteExpr) { + if (NULL != pRewriteExprs) { cxt.pOutputs = taosMemoryCalloc(LIST_LENGTH(pExprs), sizeof(bool)); if (NULL == cxt.pOutputs) { return TSDB_CODE_OUT_OF_MEMORY; } } nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); - if (TSDB_CODE_SUCCESS == cxt.errCode && NULL != pRewriteExpr) { - cxt.errCode = cloneRewriteExprs(pExprs, cxt.pOutputs, pRewriteExpr); + if (TSDB_CODE_SUCCESS == cxt.errCode && NULL != pRewriteExprs) { + cxt.errCode = cloneRewriteExprs(pExprs, cxt.pOutputs, pRewriteExprs); } taosMemoryFree(cxt.pOutputs); return cxt.errCode; From 73d51870ab2b1c35a034235f2f1022da6d81f84c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 11:10:14 +0800 Subject: [PATCH 24/46] fix: memory leak while schedule stream --- source/dnode/mnode/impl/src/mndScheduler.c | 6 +++++- tests/parallel_test/cases.task | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index e1886511b7..e8428ea470 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -224,7 +224,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { ASSERT(taosArrayGetSize(pStream->tasks) == 1); while (1) { - SVgObj* pVgroup; + SVgObj* pVgroup = NULL; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); if (pIter == NULL) break; if (!mndVgroupInDb(pVgroup, pStream->targetDbUid)) { @@ -258,6 +258,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); ASSERT(pTask->tbSink.pSchemaWrapper); } + sdbRelease(pSdb, pVgroup); } return 0; } @@ -382,6 +383,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { qDestroyQueryPlan(pPlan); return -1; } + sdbRelease(pSdb, pVgroup); } else { if (mndAssignTaskToSnode(pMnode, pInnerTask, plan, pSnode) < 0) { sdbRelease(pSdb, pSnode); @@ -396,6 +398,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { qDestroyQueryPlan(pPlan); return -1; } + sdbRelease(pSdb, pVgroup); } } @@ -459,6 +462,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { pEpInfo->nodeId = pTask->nodeId; pEpInfo->taskId = pTask->taskId; taosArrayPush(pInnerTask->childEpInfo, &pEpInfo); + sdbRelease(pSdb, pVgroup); } } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index a42a8af6ee..a47f088b9b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -436,7 +436,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_param_ttl.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data_muti_rows.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/db_tb_name_check.py -,,,system-test,python3 ./test.py -f 1-insert/database_pre_suf.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/database_pre_suf.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/InsertFuturets.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py From cd46ca0f5a3a5cb9e3fbb80f8a12724bd6d91787 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 24 Nov 2022 11:12:08 +0800 Subject: [PATCH 25/46] fix(shell): add help command and fix crete table filed complete include binary --- tools/shell/inc/shellAuto.h | 3 +++ tools/shell/src/shellAuto.c | 43 +++++++++++++++++++++++------------ tools/shell/src/shellEngine.c | 6 +++++ 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/tools/shell/inc/shellAuto.h b/tools/shell/inc/shellAuto.h index f86090d618..b7bf5fa101 100644 --- a/tools/shell/inc/shellAuto.h +++ b/tools/shell/inc/shellAuto.h @@ -39,4 +39,7 @@ void callbackAutoTab(char* sqlstr, TAOS* pSql, bool usedb); // introduction void printfIntroduction(); +// show all commands help +void showHelp(); + #endif diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 534ecf3c4d..01c8042c0e 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -108,6 +108,7 @@ SWords shellCommands[] = { {"drop topic ;", 0, 0, NULL}, {"drop stream ;", 0, 0, NULL}, {"explain select", 0, 0, NULL}, // 44 append sub sql + {"help;", 0, 0, NULL}, {"grant all on to ;", 0, 0, NULL}, {"grant read on to ;", 0, 0, NULL}, {"grant write on to ;", 0, 0, NULL}, @@ -386,6 +387,8 @@ void showHelp() { drop stream ;\n\ ----- E ----- \n\ explain select clause ...\n\ + ----- H ----- \n\ + help;\n\ ----- I ----- \n\ insert into values(...) ;\n\ insert into using tags(...) values(...) ;\n\ @@ -1478,24 +1481,36 @@ bool matchSelectQuery(TAOS* con, SShellCmd* cmd) { // if is input create fields or tags area, return true bool isCreateFieldsArea(char* p) { - char* left = strrchr(p, '('); - if (left == NULL) { - // like 'create table st' - return false; - } + // put to while, support like create table st(ts timestamp, bin1 binary(16), bin2 + blank + TAB + char* p1 = strdup(p); + bool ret = false; + while (1) { + char* left = strrchr(p1, '('); + if (left == NULL) { + // like 'create table st' + ret = false; + break; + } - char* right = strrchr(p, ')'); - if (right == NULL) { - // like 'create table st( ' - return true; - } + char* right = strrchr(p1, ')'); + if (right == NULL) { + // like 'create table st( ' + ret = true; + break; + } - if (left > right) { - // like 'create table st( ts timestamp, age int) tags(area ' - return true; + if (left > right) { + // like 'create table st( ts timestamp, age int) tags(area ' + ret = true; + break; + } + + // set string end by small for next strrchr search + *left = 0; } + taosMemoryFree(p1); - return false; + return ret; } bool matchCreateTable(TAOS* con, SShellCmd* cmd) { diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 8402a5a589..577021f460 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -134,6 +134,12 @@ int32_t shellRunCommand(char *command, bool recordHistory) { return 0; } + // add help or help; + if(strcmp(command, "help") == 0 || strcmp(command, "help;") == 0) { + showHelp(); + return 0; + } + if (recordHistory) shellRecordCommandToHistory(command); char quote = 0, *cmd = command; From a0aa07b41dabde141d880d40b4ebbb7f7afd6ceb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 11:45:40 +0800 Subject: [PATCH 26/46] test: adjust asan case --- tests/parallel_test/cases.task | 6 +++--- tests/system-test/2-query/smaTest.py | 25 ++++++++++++++----------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 27611a1a17..ee6f0a393c 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -417,7 +417,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_control.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py ,,n,system-test,python3 ./test.py -f 0-others/compatibility.py -#,,,system-test,python3 ./test.py -f 1-insert/alter_database.py +,,,system-test,python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py ,,,system-test,python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py @@ -561,8 +561,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -R -,,,system-test,python3 ./test.py -f 2-query/smaTest.py -,,,system-test,python3 ./test.py -f 2-query/smaTest.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py diff --git a/tests/system-test/2-query/smaTest.py b/tests/system-test/2-query/smaTest.py index 0390bae114..04fb893e75 100644 --- a/tests/system-test/2-query/smaTest.py +++ b/tests/system-test/2-query/smaTest.py @@ -44,8 +44,8 @@ class TDTestCase: def run(self): # insert data dbname = "db" - self.insert_data1(f"{dbname}.t1", self.ts, 1000*10000) - self.insert_data1(f"{dbname}.t4", self.ts, 1000*10000) + self.insert_data1(f"{dbname}.t1", self.ts, 10*10000) + self.insert_data1(f"{dbname}.t4", self.ts, 10*10000) # test base case # self.test_case1() tdLog.debug(" LIMIT test_case1 ............ [OK]") @@ -53,7 +53,6 @@ class TDTestCase: # self.test_case2() tdLog.debug(" LIMIT test_case2 ............ [OK]") - # stop def stop(self): tdSql.close() @@ -77,15 +76,17 @@ class TDTestCase: # insert data1 def insert_data(self, tbname, ts_start, count): - pre_insert = "insert into %s values"%tbname + pre_insert = "insert into %s values" % tbname sql = pre_insert - tdLog.debug("insert table %s rows=%d ..."%(tbname, count)) + tdLog.debug("insert table %s rows=%d ..." % (tbname, count)) for i in range(count): - sql += " (%d,%d)"%(ts_start + i*1000, i ) - if i >0 and i%30000 == 0: + sql += " (%d,%d)" % (ts_start + i*1000, i) + if i > 0 and i % 20000 == 0: + tdLog.info("%d rows inserted" % i) tdSql.execute(sql) sql = pre_insert # end sql + tdLog.info("insert_data end") if sql != pre_insert: tdSql.execute(sql) @@ -93,15 +94,17 @@ class TDTestCase: return def insert_data1(self, tbname, ts_start, count): - pre_insert = "insert into %s values"%tbname + pre_insert = "insert into %s values" % tbname sql = pre_insert - tdLog.debug("insert table %s rows=%d ..."%(tbname, count)) + tdLog.debug("insert table %s rows=%d ..." % (tbname, count)) for i in range(count): - sql += " (%d,%d,%d)"%(ts_start + i*1000, i , i+1) - if i >0 and i%30000 == 0: + sql += " (%d,%d,%d)" % (ts_start + i*1000, i, i+1) + if i > 0 and i % 20000 == 0: + tdLog.info("%d rows inserted" % i) tdSql.execute(sql) sql = pre_insert # end sql + tdLog.info("insert_data1 end") if sql != pre_insert: tdSql.execute(sql) From 862a51e6ee0949f4e04d5bcf06a7238b286dd12d Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 24 Nov 2022 12:03:05 +0800 Subject: [PATCH 27/46] refactor(sync): add timestamp trace log when elect and heartbeat --- source/libs/sync/inc/syncInt.h | 2 +- source/libs/sync/inc/syncMessage.h | 1 + source/libs/sync/src/syncMain.c | 17 ++++++-- source/libs/sync/src/syncReplication.c | 3 ++ source/libs/sync/src/syncUtil.c | 55 +++++++++++++++++++++----- 5 files changed, 64 insertions(+), 14 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 1e72b4eb26..aa8d3bef51 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -232,7 +232,7 @@ int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, S int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode); int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHandle** h); -bool syncNodeHeartbeatTimeout(SSyncNode* pSyncNode); +bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode); // raft state change -------------- void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index acaf499cbb..7ceec29be4 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -124,6 +124,7 @@ typedef struct SyncHeartbeat { SyncIndex commitIndex; SyncTerm privateTerm; SyncTerm minMatchIndex; + int64_t timeStamp; } SyncHeartbeat; typedef struct SyncHeartbeatReply { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 008e49c20c..7dab496a5b 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -640,7 +640,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { } // heartbeat timeout - if (syncNodeHeartbeatTimeout(pSyncNode)) { + if (syncNodeHeartbeatReplyTimeout(pSyncNode)) { terrno = TSDB_CODE_SYN_PROPOSE_NOT_READY; sNError(pSyncNode, "failed to sync propose since hearbeat timeout, type:%s, last:%" PRId64 ", cmt:%" PRId64, TMSG_INFO(pMsg->msgType), syncNodeGetLastIndex(pSyncNode), pSyncNode->commitIndex); @@ -2039,6 +2039,7 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { pSyncMsg->commitIndex = pSyncNode->commitIndex; pSyncMsg->minMatchIndex = syncMinMatchIndex(pSyncNode); pSyncMsg->privateTerm = 0; + pSyncMsg->timeStamp = taosGetTimestampMs(); // send msg syncNodeSendHeartbeat(pSyncNode, &pSyncMsg->destId, &rpcMsg); @@ -2094,7 +2095,7 @@ int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHand return code; } -bool syncNodeHeartbeatTimeout(SSyncNode* pSyncNode) { +bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) { if (pSyncNode->replicaNum == 1) { return false; } @@ -2148,7 +2149,11 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) { int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncHeartbeat* pMsg = pRpcMsg->pCont; - syncLogRecvHeartbeat(ths, pMsg, ""); + + int64_t tsMs = taosGetTimestampMs(); + char buf[128]; + snprintf(buf, sizeof(buf), "recv local time:%" PRId64, tsMs); + syncLogRecvHeartbeat(ths, pMsg, buf); SRpcMsg rpcMsg = {0}; (void)syncBuildHeartbeatReply(&rpcMsg, ths->vgId); @@ -2161,6 +2166,8 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pMsgReply->timeStamp = taosGetTimestampMs(); if (pMsg->term == ths->pRaftStore->currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) { + syncIndexMgrSetRecvTime(ths->pNextIndex, &(pMsg->srcId), tsMs); + syncNodeResetElectTimer(ths); ths->minMatchIndex = pMsg->minMatchIndex; @@ -2220,9 +2227,11 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncHeartbeatReply* pMsg = pRpcMsg->pCont; - syncLogRecvHeartbeatReply(ths, pMsg, ""); int64_t tsMs = taosGetTimestampMs(); + char buf[128]; + snprintf(buf, sizeof(buf), "recv local time:%" PRId64, tsMs); + syncLogRecvHeartbeatReply(ths, pMsg, buf); // update last reply time, make decision whether the other node is alive or not syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, tsMs); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index de5f71e5a9..54c29febe5 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -213,9 +213,11 @@ int32_t syncNodeSendHeartbeat(SSyncNode* pSyncNode, const SRaftId* destId, SRpcM } int32_t syncNodeHeartbeatPeers(SSyncNode* pSyncNode) { + int64_t ts = taosGetTimestampMs(); for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { SRpcMsg rpcMsg = {0}; if (syncBuildHeartbeat(&rpcMsg, pSyncNode->vgId) != 0) { + sError("vgId:%d, build sync-heartbeat error", pSyncNode->vgId); continue; } @@ -226,6 +228,7 @@ int32_t syncNodeHeartbeatPeers(SSyncNode* pSyncNode) { pSyncMsg->commitIndex = pSyncNode->commitIndex; pSyncMsg->minMatchIndex = syncMinMatchIndex(pSyncNode); pSyncMsg->privateTerm = 0; + pSyncMsg->timeStamp = ts; // send msg syncNodeSendHeartbeat(pSyncNode, &pSyncMsg->destId, &rpcMsg); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 4c84786286..1e5a268e97 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "syncUtil.h" +#include "syncIndexMgr.h" #include "syncMessage.h" #include "syncRaftCfg.h" #include "syncRaftStore.h" @@ -175,6 +176,36 @@ void syncCfg2SimpleStr(const SSyncCfg* pCfg, char* buf, int32_t bufLen) { } } +// for leader +static void syncHearbeatReplyTime2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { + int32_t len = 5; + + for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { + int64_t tsMs = syncIndexMgrGetRecvTime(pSyncNode->pMatchIndex, &(pSyncNode->replicasId[i])); + + if (i < pSyncNode->replicaNum - 1) { + len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 ",", i, tsMs); + } else { + len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 "}", i, tsMs); + } + } +} + +// for follower +static void syncHearbeatTime2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { + int32_t len = 4; + + for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { + int64_t tsMs = syncIndexMgrGetRecvTime(pSyncNode->pNextIndex, &(pSyncNode->replicasId[i])); + + if (i < pSyncNode->replicaNum - 1) { + len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 ",", i, tsMs); + } else { + len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 "}", i, tsMs); + } + } +} + static void syncPeerState2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { int32_t len = 1; @@ -221,6 +252,12 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo char peerStr[1024] = "{"; syncPeerState2Str(pNode, peerStr, sizeof(peerStr)); + char hbrTimeStr[256] = "hbr:{"; + syncHearbeatReplyTime2Str(pNode, hbrTimeStr, sizeof(hbrTimeStr)); + + char hbTimeStr[256] = "hb:{"; + syncHearbeatTime2Str(pNode, hbTimeStr, sizeof(hbTimeStr)); + int32_t quorum = syncNodeDynamicQuorum(pNode); char eventLog[512]; // {0}; @@ -243,12 +280,13 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo "%s" ", tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 ", snap-tm:%" PRIu64 ", sby:%d, aq:%d, snaping:%" PRId64 ", r-num:%d, lcfg:%" PRId64 - ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s", + ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s, %s, %s", pNode->vgId, syncStr(pNode->state), eventLog, currentTerm, pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->pRaftCfg->isStandBy, aqItems, pNode->snapshottingIndex, pNode->replicaNum, pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum, - pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr); + pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr, hbTimeStr, + hbrTimeStr); } } @@ -395,9 +433,8 @@ void syncLogSendHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, const syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "send sync-heartbeat to %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", pterm:%" PRId64 - "}, %s", - host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->privateTerm, s); + "send sync-heartbeat to %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 "}, %s", + host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s); } void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, const char* s) { @@ -406,9 +443,9 @@ void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, const syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "recv sync-heartbeat from %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", pterm:%" PRId64 + "recv sync-heartbeat from %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 "}, %s", - host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->privateTerm, s); + host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s); } void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, const char* s) { @@ -416,8 +453,8 @@ void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* p uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "send sync-heartbeat-reply from %s:%d {term:%" PRId64 ", pterm:%" PRId64 "}, %s", host, port, - pMsg->term, pMsg->privateTerm, s); + sNTrace(pSyncNode, "send sync-heartbeat-reply from %s:%d {term:%" PRId64 ", ts:%" PRId64 "}, %s", host, port, + pMsg->term, pMsg->timeStamp, s); } void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, const char* s) { From 932cec70dfb2ef176be2718735fae13ab6430d34 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 12:06:52 +0800 Subject: [PATCH 28/46] test: add asan case --- tests/parallel_test/cases.task | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 27611a1a17..c61cc35609 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -595,8 +595,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -R -,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py @@ -820,7 +820,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 3 @@ -913,7 +913,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 3 From 5d5538a12cfb084abbe7609b22a292330b7bfc16 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 24 Nov 2022 12:36:14 +0800 Subject: [PATCH 29/46] fix(query): fix error in seq fetch data block. --- source/libs/executor/src/exchangeoperator.c | 52 ++++++++++++++------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 2c79b77e16..cf6263148a 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -63,6 +63,7 @@ static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator); static int32_t seqLoadRemoteData(SOperatorInfo* pOperator); static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator); static int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); +static int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDataInfo); static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo) { @@ -112,20 +113,12 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn break; } - SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; - int32_t index = 0; - char* pStart = pRetrieveRsp->data; - while (index++ < pRetrieveRsp->numOfBlocks) { - SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); - code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); - if (code != 0) { - taosMemoryFreeClear(pDataInfo->pRsp); - goto _error; - } - - taosArrayPush(pExchangeInfo->pResultBlockList, &pb); + code = doExtractResultBlocks(pExchangeInfo, pDataInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; } + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, pDataInfo->startTime, pOperator); pDataInfo->totalRows += pRetrieveRsp->numOfRows; @@ -571,10 +564,32 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } +int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDataInfo) { + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + + char* pStart = pRetrieveRsp->data; + int32_t index = 0; + int32_t code = 0; + while (index++ < pRetrieveRsp->numOfBlocks) { + SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); + + code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); + if (code != 0) { + taosMemoryFreeClear(pDataInfo->pRsp); + return code; + } + + taosArrayPush(pExchangeInfo->pResultBlockList, &pb); + } + + return code; +} + int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + int32_t code = 0; size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); int64_t startTs = taosGetTimestampUs(); @@ -614,11 +629,12 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { continue; } + code = doExtractResultBlocks(pExchangeInfo, pDataInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; - - char* pStart = pRetrieveRsp->data; - int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, NULL, &pStart); - if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, @@ -641,6 +657,10 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { taosMemoryFreeClear(pDataInfo->pRsp); return TSDB_CODE_SUCCESS; } + + _error: + pTaskInfo->code = code; + return code; } int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { From 1d2944973de5221409308890a501f08c8bbb2df3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 13:36:05 +0800 Subject: [PATCH 30/46] test: remove case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 4054a0c3eb..64eca5fda6 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -418,7 +418,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py ,,n,system-test,python3 ./test.py -f 0-others/compatibility.py ,,,system-test,python3 ./test.py -f 1-insert/alter_database.py -,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py +,,,system-test,python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py ,,,system-test,python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py ,,,system-test,python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py From 991795ca6ec4b09fe15d295a00080e3996ed4c71 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 13:52:09 +0800 Subject: [PATCH 31/46] test: add asan case --- tests/parallel_test/cases.task | 4 ++-- tests/pytest/util/dnodes.py | 14 ++++++++++---- tests/system-test/test.py | 1 + 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 5ba294853f..9c33a07875 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -617,8 +617,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_childtable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_normaltable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/keep_expired.py -,,,system-test,python3 ./test.py -f 1-insert/drop.py -,,,system-test,python3 ./test.py -f 1-insert/drop.py -N 3 -M 3 -i False -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/drop.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/drop.py -N 3 -M 3 -i False -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 22e6127973..ee09130368 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -312,8 +312,14 @@ class TDDnode: cmd = "mintty -h never %s -c %s" % ( binPath, self.cfgDir) else: - cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( - binPath, self.cfgDir) + if self.asan: + asanDir = "%s/sim/asan/dnode%d.asan" % ( + self.path, self.index) + cmd = "nohup %s -c %s > /dev/null 2> %s & " % ( + binPath, self.cfgDir, asanDir) + else: + cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( + binPath, self.cfgDir) else: valgrindCmdline = "valgrind --log-file=\"%s/../log/valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"%self.cfgDir @@ -748,7 +754,7 @@ class TDDnodes: tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) def StopAllSigint(self): - tdLog.info("stop all dnodes sigint") + tdLog.info("stop all dnodes sigint, asan:%d" % self.asan) if self.asan: tdLog.info("execute script: %s" % self.stopDnodesSigintPath) os.system(self.stopDnodesSigintPath) @@ -756,7 +762,7 @@ class TDDnodes: return def stopAll(self): - tdLog.info("stop all dnodes") + tdLog.info("stop all dnodes, asan:%d" % self.asan) if self.asan: tdLog.info("execute script: %s" % self.stopDnodesPath) os.system(self.stopDnodesPath) diff --git a/tests/system-test/test.py b/tests/system-test/test.py index cf9aba123c..2017aad1ca 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -464,6 +464,7 @@ if __name__ == "__main__": tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) tdDnodes.setValgrind(valgrind) + tdDnodes.setAsan(asan) tdDnodes.stopAll() for dnode in tdDnodes.dnodes: tdDnodes.deploy(dnode.index,{}) From ec75eaee78c34cd44b989e9bf9bdc4daa193ad3d Mon Sep 17 00:00:00 2001 From: Pan YANG Date: Thu, 24 Nov 2022 14:05:00 +0800 Subject: [PATCH 32/46] docs: add StackOverflow link --- docs/en/05-get-started/index.md | 20 +++++++++++++++----- docs/en/05-get-started/stackoverflow.svg | 7 +++++++ 2 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 docs/en/05-get-started/stackoverflow.svg diff --git a/docs/en/05-get-started/index.md b/docs/en/05-get-started/index.md index 251581e98f..12cfa22c69 100644 --- a/docs/en/05-get-started/index.md +++ b/docs/en/05-get-started/index.md @@ -8,6 +8,7 @@ import DiscordSVG from './discord.svg' import TwitterSVG from './twitter.svg' import YouTubeSVG from './youtube.svg' import LinkedInSVG from './linkedin.svg' +import StackOverflowSVG from './stackoverflow.svg' You can install and run TDengine on Linux/Windows/macOS machines as well as Docker containers. You can also deploy TDengine as a managed service with TDengine Cloud. @@ -35,10 +36,19 @@ The TDengine Knowledge Map covers the various knowledge points of TDengine, reve - - - - - + + + + + + + + + + + + + +

Star GitHub

Join Discord

Follow Twitter

Subscribe YouTube

Follow LinkedIn

Star GitHubJoin DiscordFollow TwitterSubscribe YouTubeFollow LinkedInAsk StackOverflow
diff --git a/docs/en/05-get-started/stackoverflow.svg b/docs/en/05-get-started/stackoverflow.svg new file mode 100644 index 0000000000..22b4b64d32 --- /dev/null +++ b/docs/en/05-get-started/stackoverflow.svg @@ -0,0 +1,7 @@ + + + + From fcf46059e9060cddd73dbddf0ab3828fdcad9a5d Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 24 Nov 2022 14:33:27 +0800 Subject: [PATCH 33/46] fix(stream): heap uaf --- source/dnode/mnode/impl/src/mndStb.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index cee0b84672..5e16397b14 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1187,11 +1187,11 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, goto NEXT; } if (pCol->colId > 0 && pCol->colId == colId) { - sdbRelease(pSdb, pTopic); - nodesDestroyNode(pAst); - nodesDestroyList(pNodeList); terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC; mError("topic:%s, check colId:%d conflicted", pTopic->name, pCol->colId); + nodesDestroyNode(pAst); + nodesDestroyList(pNodeList); + sdbRelease(pSdb, pTopic); return -1; } mInfo("topic:%s, check colId:%d passed", pTopic->name, pCol->colId); @@ -1230,11 +1230,11 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName goto NEXT; } if (pCol->colId > 0 && pCol->colId == colId) { - sdbRelease(pSdb, pStream); - nodesDestroyNode(pAst); - nodesDestroyList(pNodeList); terrno = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; mError("stream:%s, check colId:%d conflicted", pStream->name, pCol->colId); + nodesDestroyNode(pAst); + nodesDestroyList(pNodeList); + sdbRelease(pSdb, pStream); return -1; } mInfo("stream:%s, check colId:%d passed", pStream->name, pCol->colId); @@ -1279,11 +1279,11 @@ static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName, goto NEXT; } if ((pCol->colId) > 0 && (pCol->colId == colId)) { - sdbRelease(pSdb, pSma); - nodesDestroyNode(pAst); - nodesDestroyList(pNodeList); terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TSMA; mError("tsma:%s, check colId:%d conflicted", pSma->name, pCol->colId); + nodesDestroyNode(pAst); + nodesDestroyList(pNodeList); + sdbRelease(pSdb, pSma); return -1; } mInfo("tsma:%s, check colId:%d passed", pSma->name, pCol->colId); From e0e663c6fac534b576a86917fbccd745c057e2ca Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 24 Nov 2022 10:13:57 +0800 Subject: [PATCH 34/46] fix:Explicit null dereferenced --- source/libs/executor/src/timewindowoperator.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 7fd2fcb5b8..013b8d39de 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3655,6 +3655,11 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData setSessionWinOutputInfo(pStUpdated, &winInfo); winRows = updateSessionWindowInfo(&winInfo, startTsCols, endTsCols, groupId, rows, i, pAggSup->gap, pAggSup->pResultRows, pStUpdated, pStDeleted); + // coverity scan error + if (!winInfo.pOutputBuf) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { From 1e22adeabc489576b3957c0b58777f3b38ba9d27 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 24 Nov 2022 15:11:24 +0800 Subject: [PATCH 35/46] fix(query): set correct data source flag. --- source/libs/executor/src/exchangeoperator.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 08fef72107..64c2b8c27f 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -299,7 +299,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode SExchangeOpStopInfo stopInfo = {QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, pInfo->self}; qAppendTaskStopInfo(pTaskInfo, &stopInfo); - pInfo->seqLoadData = false; + pInfo->seqLoadData = true; pInfo->pTransporter = pTransporter; setOperatorInfo(pOperator, "ExchangeOperator", QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -614,13 +614,15 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); + pDataInfo->status = EX_SOURCE_DATA_NOT_READY; + doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); if (pDataInfo->code != TSDB_CODE_SUCCESS) { From 70164f9f202a8f48b033892f807e91c93098c0c7 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 24 Nov 2022 15:16:38 +0800 Subject: [PATCH 36/46] enh: group by tbname optimize --- include/libs/nodes/plannodes.h | 2 ++ source/libs/executor/src/exchangeoperator.c | 2 +- source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 7 +++++++ source/libs/nodes/src/nodesMsgFuncs.c | 9 ++++++++- source/libs/parser/src/parTranslater.c | 12 +++++++++++- source/libs/planner/src/planPhysiCreater.c | 1 + source/libs/planner/src/planSpliter.c | 12 ++++++++++++ 8 files changed, 43 insertions(+), 3 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index f942713f5d..6a6fd50f2a 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -171,6 +171,7 @@ typedef struct SExchangeLogicNode { SLogicNode node; int32_t srcStartGroupId; int32_t srcEndGroupId; + bool seqRecvData; } SExchangeLogicNode; typedef struct SMergeLogicNode { @@ -416,6 +417,7 @@ typedef struct SExchangePhysiNode { int32_t srcEndGroupId; bool singleChannel; SNodeList* pSrcEndPoints; // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode + bool seqRecvData; } SExchangePhysiNode; typedef struct SMergePhysiNode { diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 08fef72107..db053ca7b2 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -299,7 +299,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode SExchangeOpStopInfo stopInfo = {QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, pInfo->self}; qAppendTaskStopInfo(pTaskInfo, &stopInfo); - pInfo->seqLoadData = false; + pInfo->seqLoadData = true; pInfo->pTransporter = pTransporter; setOperatorInfo(pOperator, "ExchangeOperator", QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index b9b365fb42..5b3e8ce5a9 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -434,6 +434,7 @@ static int32_t logicExchangeCopy(const SExchangeLogicNode* pSrc, SExchangeLogicN COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); COPY_SCALAR_FIELD(srcStartGroupId); COPY_SCALAR_FIELD(srcEndGroupId); + COPY_SCALAR_FIELD(seqRecvData); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 172c769433..462ac513a5 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1864,6 +1864,7 @@ static int32_t jsonToPhysiAggNode(const SJson* pJson, void* pObj) { static const char* jkExchangePhysiPlanSrcStartGroupId = "SrcStartGroupId"; static const char* jkExchangePhysiPlanSrcEndGroupId = "SrcEndGroupId"; static const char* jkExchangePhysiPlanSrcEndPoints = "SrcEndPoints"; +static const char* jkExchangePhysiPlanSeqRecvData = "SeqRecvData"; static int32_t physiExchangeNodeToJson(const void* pObj, SJson* pJson) { const SExchangePhysiNode* pNode = (const SExchangePhysiNode*)pObj; @@ -1878,6 +1879,9 @@ static int32_t physiExchangeNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkExchangePhysiPlanSrcEndPoints, pNode->pSrcEndPoints); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkExchangePhysiPlanSeqRecvData, pNode->seqRecvData); + } return code; } @@ -1895,6 +1899,9 @@ static int32_t jsonToPhysiExchangeNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkExchangePhysiPlanSrcEndPoints, &pNode->pSrcEndPoints); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkExchangePhysiPlanSeqRecvData, &pNode->seqRecvData); + } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 2879d55167..1e8ff8da1a 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2428,7 +2428,8 @@ enum { PHY_EXCHANGE_CODE_SRC_START_GROUP_ID, PHY_EXCHANGE_CODE_SRC_END_GROUP_ID, PHY_EXCHANGE_CODE_SINGLE_CHANNEL, - PHY_EXCHANGE_CODE_SRC_ENDPOINTS + PHY_EXCHANGE_CODE_SRC_ENDPOINTS, + PHY_EXCHANGE_CODE_SEQ_RECV_DATA }; static int32_t physiExchangeNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -2447,6 +2448,9 @@ static int32_t physiExchangeNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeObj(pEncoder, PHY_EXCHANGE_CODE_SRC_ENDPOINTS, nodeListToMsg, pNode->pSrcEndPoints); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_EXCHANGE_CODE_SEQ_RECV_DATA, pNode->seqRecvData); + } return code; } @@ -2473,6 +2477,9 @@ static int32_t msgToPhysiExchangeNode(STlvDecoder* pDecoder, void* pObj) { case PHY_EXCHANGE_CODE_SRC_ENDPOINTS: code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pSrcEndPoints); break; + case PHY_EXCHANGE_CODE_SEQ_RECV_DATA: + code = tlvDecodeBool(pTlv, &pNode->seqRecvData); + break; default: break; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bb4aa67767..0743b40662 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3685,9 +3685,19 @@ static int32_t translateInsertProject(STranslateContext* pCxt, SInsertStmt* pIns return addOrderByPrimaryKeyToQuery(pCxt, pPrimaryKeyExpr, pInsert->pQuery); } +static int32_t translateInsertTable(STranslateContext* pCxt, SNode* pTable) { + int32_t code = translateFrom(pCxt, pTable); + if (TSDB_CODE_SUCCESS == code && TSDB_CHILD_TABLE != ((SRealTableNode*)pTable)->pMeta->tableType && + TSDB_NORMAL_TABLE != ((SRealTableNode*)pTable)->pMeta->tableType) { + code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, + "insert data into super table is not supported"); + } + return code; +} + static int32_t translateInsert(STranslateContext* pCxt, SInsertStmt* pInsert) { pCxt->pCurrStmt = (SNode*)pInsert; - int32_t code = translateFrom(pCxt, pInsert->pTable); + int32_t code = translateInsertTable(pCxt, pInsert->pTable); if (TSDB_CODE_SUCCESS == code) { code = translateInsertCols(pCxt, pInsert); } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 72f3d995bc..379bfe90c8 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1064,6 +1064,7 @@ static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogic pExchange->srcStartGroupId = pExchangeLogicNode->srcStartGroupId; pExchange->srcEndGroupId = pExchangeLogicNode->srcEndGroupId; + pExchange->seqRecvData = pExchangeLogicNode->seqRecvData; *pPhyNode = (SPhysiNode*)pExchange; return TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 0cbca8b032..bcf4b40e69 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -733,6 +733,17 @@ static int32_t stbSplSplitWindowForCrossTable(SSplitContext* pCxt, SStableSplitI return TSDB_CODE_PLAN_INTERNAL_ERROR; } +static bool stbSplNeedSeqRecvData(SLogicNode* pNode) { + if (NULL == pNode) { + return false; + } + + if (NULL != pNode->pLimit || NULL != pNode->pSlimit) { + return true; + } + return stbSplNeedSeqRecvData(pNode->pParent); +} + static int32_t stbSplSplitWindowForPartTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { if (pCxt->pPlanCxt->streamQuery) { SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT); @@ -748,6 +759,7 @@ static int32_t stbSplSplitWindowForPartTable(SSplitContext* pCxt, SStableSplitIn code = replaceLogicNode(pInfo->pSubplan, pInfo->pSplitNode, (SLogicNode*)pExchange); } if (TSDB_CODE_SUCCESS == code) { + pExchange->seqRecvData = stbSplNeedSeqRecvData((SLogicNode*)pExchange); code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, (SNode*)splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT)); } From 5956d8f4fd992a9e010a196c4326c23ad4d2ebe2 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 24 Nov 2022 15:38:48 +0800 Subject: [PATCH 37/46] enh: group by tbname optimize --- source/libs/executor/src/exchangeoperator.c | 54 +++++++++++---------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 64c2b8c27f..b01a4b7871 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -51,9 +51,9 @@ typedef struct SSourceDataInfo { const char* taskId; } SSourceDataInfo; -static void destroyExchangeOperatorInfo(void* param); -static void freeBlock(void* pParam); -static void freeSourceDataInfo(void* param); +static void destroyExchangeOperatorInfo(void* param); +static void freeBlock(void* pParam); +static void freeSourceDataInfo(void* param); static void* setAllSourcesCompleted(SOperatorInfo* pOperator); static int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code); @@ -62,7 +62,8 @@ static int32_t getCompletedSources(const SArray* pArray); static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator); static int32_t seqLoadRemoteData(SOperatorInfo* pOperator); static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator); -static int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); +static int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, + bool holdDataInBuf); static int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDataInfo); static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, @@ -106,7 +107,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn if (pRsp->numOfRows == 0) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", try next %d/%" PRIzu, + ", totalRows:%" PRIu64 ", try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows, pExchangeInfo->loadInfo.totalRows, i + 1, totalSources); taosMemoryFreeClear(pDataInfo->pRsp); @@ -125,14 +126,14 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn if (pRsp->completed == 1) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 - ", total:%.2f Kb, try next %d/%" PRIzu, + " execId:%d index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 + ", total:%.2f Kb, try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, - pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, - i + 1, totalSources); + pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, i + 1, + totalSources); } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", + " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); } @@ -157,7 +158,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn } } - _error: +_error: pTaskInfo->code = code; } @@ -298,17 +299,19 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode SExchangeOpStopInfo stopInfo = {QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, pInfo->self}; qAppendTaskStopInfo(pTaskInfo, &stopInfo); - - pInfo->seqLoadData = true; + + pInfo->seqLoadData = pExNode->seqRecvData; pInfo->pTransporter = pTransporter; - setOperatorInfo(pOperator, "ExchangeOperator", QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); + setOperatorInfo(pOperator, "ExchangeOperator", QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, false, OP_NOT_OPENED, pInfo, + pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); - pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, destroyExchangeOperatorInfo, NULL); + pOperator->fpSet = + createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, destroyExchangeOperatorInfo, NULL); return pOperator; - _error: +_error: if (pInfo != NULL) { doDestroyExchangeOperatorInfo(pInfo); } @@ -379,7 +382,8 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { } else { taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%s, %p", pSourceDataInfo->taskId, index, tstrerror(code), pExchangeInfo); + qDebug("%s fetch rsp received, index:%d, error:%s, %p", pSourceDataInfo->taskId, index, tstrerror(code), + pExchangeInfo); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; @@ -427,14 +431,14 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas taosMemoryFree(pWrapper); return pTaskInfo->code; } - + void* msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; taosMemoryFree(pWrapper); return pTaskInfo->code; } - + if (tSerializeSResFetchReq(msg, msgSize, &req) < 0) { pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; taosMemoryFree(pWrapper); @@ -524,7 +528,7 @@ void* setAllSourcesCompleted(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); qDebug("%s all %" PRIzu " sources are exhausted, total rows: %" PRIu64 ", %.2f Kb, elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, pLoadInfo->totalElapsed / 1000.0); @@ -574,7 +578,7 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } - + tsem_post(&pExchangeInfo->ready); return TSDB_CODE_SUCCESS; } @@ -636,7 +640,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; if (pRsp->numOfRows == 0) { qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 " try next", + ", totalRows:%" PRIu64 " try next", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1, pDataInfo->totalRows, pLoadInfo->totalRows); @@ -654,7 +658,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, + ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, totalSources); @@ -663,7 +667,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { pExchangeInfo->current += 1; } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 - ", totalBytes:%" PRIu64, + ", totalBytes:%" PRIu64, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize); } @@ -675,7 +679,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } - _error: +_error: pTaskInfo->code = code; return code; } From e43794c36650db61b285bdf140e51576dda3635a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 24 Nov 2022 15:51:48 +0800 Subject: [PATCH 38/46] fix(sync): if msg commit, put it into apply-queue, do not care return code --- source/dnode/vnode/src/vnd/vnodeSync.c | 37 +++++++++----------------- 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 38cb534d7f..3151fd0475 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -298,35 +298,24 @@ static int32_t vnodeSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot) static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { SVnode *pVnode = pFsm->data; - if (pMeta->code == 0) { - SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen}; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); - rpcMsg.info = pMsg->info; - rpcMsg.info.conn.applyIndex = pMeta->index; - rpcMsg.info.conn.applyTerm = pMeta->term; + SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen}; + rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); + memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); + rpcMsg.info = pMsg->info; + rpcMsg.info.conn.applyIndex = pMeta->index; + rpcMsg.info.conn.applyTerm = pMeta->term; - const STraceId *trace = &pMsg->info.traceId; - vGTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", term:%" PRIu64 ", msg-index:%" PRId64 - ", weak:%d, code:%d, state:%d %s, type:%s", - pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, rpcMsg.info.conn.applyIndex, pMeta->isWeak, - pMeta->code, pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType)); + const STraceId *trace = &pMsg->info.traceId; + vGTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", term:%" PRIu64 ", msg-index:%" PRId64 + ", weak:%d, code:%d, state:%d %s, type:%s", + pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, rpcMsg.info.conn.applyIndex, pMeta->isWeak, pMeta->code, + pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType)); - tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); - } else { - SRpcMsg rsp = {.code = pMeta->code, .info = pMsg->info}; - vError("vgId:%d, commit-cb execute error, type:%s, index:%" PRId64 ", error:0x%x %s", pVnode->config.vgId, - TMSG_INFO(pMsg->msgType), pMeta->index, pMeta->code, tstrerror(pMeta->code)); - if (rsp.info.handle != NULL) { - tmsgSendRsp(&rsp); - } - } + tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); } static void vnodeSyncCommitMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { - if (pMeta->isWeak == 0) { - vnodeSyncApplyMsg(pFsm, pMsg, pMeta); - } + vnodeSyncApplyMsg(pFsm, pMsg, pMeta); } static void vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { From 7fba711d40b0d145edcb329fddf2a1f81b67542b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 24 Nov 2022 16:03:43 +0800 Subject: [PATCH 39/46] refactor(sync): add info log --- source/dnode/vnode/src/vnd/vnodeSync.c | 10 +++++----- source/libs/sync/src/syncMain.c | 5 ++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 3151fd0475..6337fbe2bd 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -306,10 +306,10 @@ static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const S rpcMsg.info.conn.applyTerm = pMeta->term; const STraceId *trace = &pMsg->info.traceId; - vGTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", term:%" PRIu64 ", msg-index:%" PRId64 - ", weak:%d, code:%d, state:%d %s, type:%s", - pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, rpcMsg.info.conn.applyIndex, pMeta->isWeak, pMeta->code, - pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType)); + vGInfo("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", term:%" PRIu64 ", msg-index:%" PRId64 + ", weak:%d, code:%d, state:%d %s, type:%s", + pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, rpcMsg.info.conn.applyIndex, pMeta->isWeak, pMeta->code, + pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType)); tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); } @@ -409,7 +409,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm) { static void vnodeBecomeFollower(const SSyncFSM *pFsm) { SVnode *pVnode = pFsm->data; - vDebug("vgId:%d, become follower", pVnode->config.vgId); + vInfo("vgId:%d, become follower", pVnode->config.vgId); taosThreadMutexLock(&pVnode->lock); if (pVnode->blocked) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 7dab496a5b..754a3358fb 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2509,6 +2509,8 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde SRpcMsg rpcMsg = {0}; syncEntry2OriginalRpc(pEntry, &rpcMsg); + sInfo("do commit index:%" PRId64 ", type:%s", i, TMSG_INFO(pEntry->msgType)); + // user commit if ((ths->pFsm->FpCommitCb != NULL) && syncUtilUserCommit(pEntry->originalRpcType)) { bool internalExecute = true; @@ -2516,7 +2518,8 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde internalExecute = false; } - sNTrace(ths, "commit index:%" PRId64 ", internal:%d", i, internalExecute); + sNTrace(ths, "user commit index:%" PRId64 ", internal:%d, type:%s", i, internalExecute, + TMSG_INFO(pEntry->msgType)); // execute fsm in apply thread, or execute outside syncPropose if (internalExecute) { From 9f38bfeb2c4159636db881c0d9cffbb93f94fcb3 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 24 Nov 2022 16:50:58 +0800 Subject: [PATCH 40/46] refactor(sync): modify info log to trace log --- source/dnode/vnode/src/vnd/vnodeSync.c | 8 ++++---- source/libs/sync/src/syncMain.c | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 6337fbe2bd..4324c412f7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -306,10 +306,10 @@ static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const S rpcMsg.info.conn.applyTerm = pMeta->term; const STraceId *trace = &pMsg->info.traceId; - vGInfo("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", term:%" PRIu64 ", msg-index:%" PRId64 - ", weak:%d, code:%d, state:%d %s, type:%s", - pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, rpcMsg.info.conn.applyIndex, pMeta->isWeak, pMeta->code, - pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType)); + vGTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", term:%" PRIu64 ", msg-index:%" PRId64 + ", weak:%d, code:%d, state:%d %s, type:%s", + pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, rpcMsg.info.conn.applyIndex, pMeta->isWeak, pMeta->code, + pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType)); tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 754a3358fb..2aaa13f95d 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2509,7 +2509,7 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde SRpcMsg rpcMsg = {0}; syncEntry2OriginalRpc(pEntry, &rpcMsg); - sInfo("do commit index:%" PRId64 ", type:%s", i, TMSG_INFO(pEntry->msgType)); + sTrace("do commit index:%" PRId64 ", type:%s", i, TMSG_INFO(pEntry->msgType)); // user commit if ((ths->pFsm->FpCommitCb != NULL) && syncUtilUserCommit(pEntry->originalRpcType)) { From 30460b9b0d26cefeb6c7f2abb481b1332a5c19d7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 17:22:24 +0800 Subject: [PATCH 41/46] test: add asan case --- tests/parallel_test/cases.task | 10 +++++----- tests/script/sh/checkAsan.sh | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 079dbd8d02..9dc57d1e93 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -278,7 +278,7 @@ ,,y,script,./test.sh -f tsim/stable/values.sim ,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim -,,,script,./test.sh -f tsim/sma/drop_sma.sim +,,n,script,./test.sh -f tsim/sma/drop_sma.sim ,,y,script,./test.sh -f tsim/sma/sma_leak.sim ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim @@ -428,7 +428,7 @@ ,,n,system-test,python3 ./test.py -f 1-insert/boundary.py ,,n,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_comment.py -,,,system-test,python3 ./test.py -f 1-insert/time_range_wise.py +,,n,system-test,python3 ./test.py -f 1-insert/time_range_wise.py ,,,system-test,python3 ./test.py -f 1-insert/block_wise.py ,,,system-test,python3 ./test.py -f 1-insert/create_retentions.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/mutil_stage.py @@ -632,7 +632,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py -,,,system-test,python3 ./test.py -f 2-query/queryQnode.py +,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,,system-test,python3 ./test.py -f 6-cluster/5dnode1mnode.py ,,,system-test,python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 ,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 @@ -669,7 +669,7 @@ ,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1 ,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py -,,,system-test,python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 ,,,system-test,python3 ./test.py -f 7-tmq/basic5.py ,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb.py ,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb0.py @@ -725,7 +725,7 @@ ,,,system-test,python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py ,,,system-test,python3 ./test.py -f 7-tmq/tmq_taosx.py ,,,system-test,python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py -,,,system-test,python3 ./test.py -f 99-TDcase/TD-19201.py +,,y,system-test,./pytest.sh python3 -f 99-TDcase/TD-19201.py ,,,system-test,python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 ,,,system-test,python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 ,,,system-test,python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 8b478384cf..aa0f5b6b76 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -37,8 +37,9 @@ python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` # TD-20569 # /root/TDengine/source/libs/function/src/builtinsimpl.c:856:29: runtime error: signed integer overflow: 9223372036854775806 + 9223372036854775805 cannot be represented in type 'long int' # /root/TDengine/source/libs/scalar/src/sclvector.c:1075:66: runtime error: signed integer overflow: 9223372034707292160 + 1668838476672 cannot be represented in type 'long int' +# /root/TDengine/source/common/src/tdataformat.c:1876:7: runtime error: signed integer overflow: 8252423483843671206 + 2406154664059062870 cannot be represented in type 'long int' -runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "builtinsimpl.c.*signed integer overflow"| grep -v "sclvector.c.*signed integer overflow" | wc -l` +runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "signed integer overflow" | wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" From 1a2983c0676a46e722a0d5fe3f549e63f8ccc9cf Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 24 Nov 2022 18:07:06 +0800 Subject: [PATCH 42/46] fix(tdb): zero pOlds[i]'s nOverflow and assert it's zero when committing --- source/libs/tdb/src/db/tdbBtree.c | 6 ++---- source/libs/tdb/src/db/tdbPager.c | 32 ++++++++++++++++--------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e3860f85c6..fcf9d6ce3e 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -741,14 +741,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx tdbPageCreate(pOlds[0]->pageSize, &pOldsCopy[i], tdbDefaultMalloc, NULL); tdbBtreeInitPage(pOldsCopy[i], &iarg, 0); tdbPageCopy(pOlds[i], pOldsCopy[i], 0); - } - - for (iNew = 0; iNew < nNews; ++iNew) { - tdbBtreeInitPage(pNews[iNew], &iarg, 0); + pOlds[i]->nOverflow = 0; } iNew = 0; nNewCells = 0; + tdbBtreeInitPage(pNews[iNew], &iarg, 0); for (int iOld = 0; iOld < nOlds; iOld++) { SPage *pPage; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index abbad06515..7264e0b5ff 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -28,12 +28,12 @@ typedef struct { TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct"); struct hashset_st { - size_t nbits; - size_t mask; - size_t capacity; + size_t nbits; + size_t mask; + size_t capacity; size_t *items; - size_t nitems; - double load_factor; + size_t nitems; + double load_factor; }; static const unsigned int prime = 39; @@ -68,11 +68,11 @@ void hashset_destroy(hashset_t set) { } int hashset_add_member(hashset_t set, void *item) { - size_t value = (size_t) item; + size_t value = (size_t)item; size_t h; if (value == 0) { - return -1; + return -1; } for (h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { @@ -103,7 +103,7 @@ int hashset_add(hashset_t set, void *item) { set->nitems = 0; for (size_t i = 0; i < old_capacity; ++i) { - hashset_add_member(set, (void*)old_items[i]); + hashset_add_member(set, (void *)old_items[i]); } tdbOsFree(old_items); } @@ -112,7 +112,7 @@ int hashset_add(hashset_t set, void *item) { } int hashset_remove(hashset_t set, void *item) { - size_t value = (size_t) item; + size_t value = (size_t)item; for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { if (set->items[h] == value) { @@ -126,7 +126,7 @@ int hashset_remove(hashset_t set, void *item) { } int hashset_contains(hashset_t set, void *item) { - size_t value = (size_t) item; + size_t value = (size_t)item; for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { if (set->items[h] == value) { @@ -319,7 +319,8 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { tRBTreePut(&pPager->rbt, (SRBTreeNode *)pPage); // Write page to journal if neccessary - if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize && (pPager->jPageSet == NULL || !hashset_contains(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))))) { + if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize && + (pPager->jPageSet == NULL || !hashset_contains(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) { ret = tdbPagerWritePageToJournal(pPager, pPage); if (ret < 0) { tdbError("failed to write page to journal since %s", tstrerror(terrno)); @@ -327,7 +328,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { } if (pPager->jPageSet) { - hashset_add(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))); + hashset_add(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); } } @@ -372,6 +373,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { SRBTreeNode *pNode = NULL; while ((pNode = tRBTreeIterNext(&iter)) != NULL) { pPage = (SPage *)pNode; + ASSERT(pPage->nOverflow == 0); ret = tdbPagerWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); @@ -391,7 +393,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); if (pPager->jPageSet) { - hashset_remove(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))); + hashset_remove(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); } tdbPCacheRelease(pPager->pCache, pPage, pTxn); } @@ -503,7 +505,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { return -1; } - u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize); + u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize); if (pageBuf == NULL) { return -1; } @@ -560,7 +562,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { pPage->isDirty = 0; tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); - hashset_remove(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))); + hashset_remove(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); tdbPCacheRelease(pPager->pCache, pPage, pTxn); } From ad5e85296f61d58095affa0a9840f7b1322e2416 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 24 Nov 2022 18:35:03 +0800 Subject: [PATCH 43/46] fix: auto creating table failed issue when table already exists --- source/client/src/clientStmt.c | 38 ++- source/libs/parser/src/parInsertSql.c | 7 +- tests/script/api/batchprepare.c | 357 ++++++++++++++++++-------- 3 files changed, 289 insertions(+), 113 deletions(-) diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index c5f49bce89..86c86d52ab 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -152,13 +152,13 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char* tbFName, const char* sTableName) { +int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char* tbFName, const char* sTableName, bool autoCreateTbl) { STscStmt* pStmt = (STscStmt*)stmt; strncpy(pStmt->bInfo.tbFName, tbFName, sizeof(pStmt->bInfo.tbFName) - 1); pStmt->bInfo.tbFName[sizeof(pStmt->bInfo.tbFName) - 1] = 0; - pStmt->bInfo.tbUid = pTableMeta->uid; + pStmt->bInfo.tbUid = autoCreateTbl ? 0 : pTableMeta->uid; pStmt->bInfo.tbSuid = pTableMeta->suid; pStmt->bInfo.tbType = pTableMeta->tableType; pStmt->bInfo.boundTags = tags; @@ -182,7 +182,7 @@ int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char SHashObj* pVgHash, SHashObj* pBlockHash, const char* sTableName) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbFName, sTableName)); + STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbFName, sTableName, autoCreateTbl)); STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash, autoCreateTbl)); pStmt->sql.autoCreateTbl = autoCreateTbl; @@ -623,6 +623,8 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); + pStmt->exec.autoCreateTbl = true; + return TSDB_CODE_SUCCESS; } @@ -771,10 +773,6 @@ int stmtAddBatch(TAOS_STMT* stmt) { int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { tscDebug("stmt start to update tbUid, blockNum: %d", pRsp->nBlocks); - if (pRsp->nBlocks <= 0) { - return TSDB_CODE_SUCCESS; - } - size_t keyLen = 0; STableDataBlocks** pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { @@ -809,8 +807,30 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { pMeta->uid = blkRsp->uid; pStmt->bInfo.tbUid = blkRsp->uid; } else { - tscError("table %s not found in submit rsp", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + tscDebug("table %s not found in submit rsp, will update from catalog", pStmt->bInfo.tbFName); + if (NULL == pStmt->pCatalog) { + STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &pStmt->pCatalog)); + } + + STMT_ERR_RET(stmtCreateRequest(pStmt)); + + STableMeta* pTableMeta = NULL; + SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter, + .requestId = pStmt->exec.pRequest->requestId, + .requestObjRefId = pStmt->exec.pRequest->self, + .mgmtEps = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp)}; + int32_t code = catalogGetTableMeta(pStmt->pCatalog, &conn, &pStmt->bInfo.sname, &pTableMeta); + + taos_free_result(pStmt->exec.pRequest); + pStmt->exec.pRequest = NULL; + + if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { + tscDebug("tb %s not exist", pStmt->bInfo.tbFName); + return TSDB_CODE_SUCCESS; + } + + pMeta->uid = pTableMeta->uid; + pStmt->bInfo.tbUid = pTableMeta->uid; } pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 155fc7f831..9c39954f09 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -918,7 +918,12 @@ static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModif static int32_t getTableDataBlocks(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks** pDataBuf) { if (pCxt->pComCxt->async) { - return insGetDataBlockFromList(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), + uint64_t uid = pStmt->pTableMeta->uid; + if (pStmt->usingTableProcessing) { + pStmt->pTableMeta->uid = 0; + } + + return insGetDataBlockFromList(pStmt->pTableBlockHashObj, &uid, sizeof(pStmt->pTableMeta->uid), TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta, pDataBuf, NULL, &pStmt->createTblReq); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index f39d5e6528..74aa7baf0b 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -115,6 +115,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos); int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos); int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos); int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos); +int insertAUTOTest2(TAOS_STMT *stmt, TAOS *taos); int queryColumnTest(TAOS_STMT *stmt, TAOS *taos); int queryMiscTest(TAOS_STMT *stmt, TAOS *taos); @@ -128,7 +129,7 @@ typedef struct { int32_t colNum; int32_t *colList; // full table column list int32_t testType; - bool autoCreateTbl; + int32_t autoCreateTbl; bool fullCol; int32_t (*runFn)(TAOS_STMT*, TAOS*); int32_t tblNum; @@ -142,45 +143,46 @@ typedef struct { } CaseCfg; CaseCfg gCase[] = { - {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 0, 1, -1}, - {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 0, 1, -1}, + {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, 0, true, insertMBSETest1, 1, 10, 10, 0, 0, 0, 1, -1}, + {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, 0, true, insertMBSETest1, 10, 100, 10, 0, 0, 0, 1, -1}, - {"insert:MBSE1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBSE1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBSE1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBSE1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBSETest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBSE1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBSETest1, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBSE1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBSETest1, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBSE2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBSE2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBSE2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBSE2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBSETest2, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBSE2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBSETest2, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBSE2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBSETest2, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBME1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest1, 10, 10, 2, 2, 0, 0, 1, -1}, // 11 - {"insert:MBME2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest2, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBME2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBME2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBME2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBMETest2, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest2, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest2, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBME3-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBME3-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBME3-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBME3-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBMETest3, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME3-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest3, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME3-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest3, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBME4-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBME4-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBME4-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBME4-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBMETest4, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME4-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest4, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME4-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest4, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MPME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMPMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MPME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MPME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMPMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MPME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMPMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, // 22 - {"insert:AUTO1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, true, true, insertAUTOTest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:AUTO1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 1, true, insertAUTOTest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:AUTO1-TBEXISTS", tListLen(fullColList), fullColList, TTYPE_INSERT, 3, true, insertAUTOTest2, 10, 10, 2, 0, 0, 0, 1, -1}, - {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, - {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, + {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, + {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, -// {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, -// {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, +// {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, +// {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, }; @@ -221,7 +223,7 @@ typedef struct { CaseCtrl gCaseCtrl = { .precision = TIME_PRECISION_MICRO, .bindNullNum = 0, - .printCreateTblSql = false, + .printCreateTblSql = true, .printQuerySql = true, .printStmtSql = true, .printVerbose = false, @@ -230,7 +232,7 @@ CaseCtrl gCaseCtrl = { .numericParam = false, .rowNum = 0, .bindColNum = 0, - .bindTagNum = 0, + .bindTagNum = 14, .bindRowNum = 0, .bindColTypeNum = 0, .bindColTypeList = NULL, @@ -242,8 +244,8 @@ CaseCtrl gCaseCtrl = { .funcIdxList = NULL, .checkParamNum = false, .runTimes = 0, - .caseIdx = -1, - .caseNum = -1, + .caseIdx = 23, + .caseNum = 1, .caseRunIdx = -1, .caseRunNum = -1, }; @@ -1946,6 +1948,73 @@ int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos) { } + +/* [prepare [settbnametag [bind add exec]]] */ +int insertAUTOTest2(TAOS_STMT *stmt, TAOS *taos) { + int32_t loop = 0; + + while (gCurCase->bindTagNum > 0 && gCurCase->bindColNum > 0) { + BindData data = {0}; + prepareInsertData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckIsInsert(stmt, 1); + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + for (int32_t b = 0; b tblNum; ++t) { + if (gCurCase->tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = bpSetTableNameTags(&data, t, buf, stmt); + if (code != 0){ + printf("!!!taos_stmt_set_tbname_tags error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + } + + bpCheckIsInsert(stmt, 1); + + destroyData(&data); + + gCurCase->bindColNum -= 2; + gCurCase->bindTagNum -= 2; + gCurCase->fullCol = false; + loop++; + } + + bpCheckAffectedRows(stmt, loop); + + gExecLoopTimes = loop; + + return 0; +} + + /* select * from table */ int queryColumnTest(TAOS_STMT *stmt, TAOS *taos) { BindData data = {0}; @@ -2243,70 +2312,76 @@ int sql_s_perf1(TAOS *taos) { return 0; } -void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t *colList, bool stable) { +void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t *colList, int32_t tableType) { int32_t blen = 0; - blen = sprintf(buf, "create table %s%d ", (stable ? bpStbPrefix : bpTbPrefix), tblIdx); + blen = sprintf(buf, "create table %s%d ", (1 == tableType ? bpStbPrefix : bpTbPrefix), tblIdx); - blen += sprintf(buf + blen, " ("); - - for (int c = 0; c < colNum; ++c) { - if (c > 0) { - blen += sprintf(buf + blen, ","); - } - - switch (colList[c]) { - case TSDB_DATA_TYPE_BOOL: - blen += sprintf(buf + blen, "booldata bool"); - break; - case TSDB_DATA_TYPE_TINYINT: - blen += sprintf(buf + blen, "tinydata tinyint"); - break; - case TSDB_DATA_TYPE_SMALLINT: - blen += sprintf(buf + blen, "smalldata smallint"); - break; - case TSDB_DATA_TYPE_INT: - blen += sprintf(buf + blen, "intdata int"); - break; - case TSDB_DATA_TYPE_BIGINT: - blen += sprintf(buf + blen, "bigdata bigint"); - break; - case TSDB_DATA_TYPE_FLOAT: - blen += sprintf(buf + blen, "floatdata float"); - break; - case TSDB_DATA_TYPE_DOUBLE: - blen += sprintf(buf + blen, "doubledata double"); - break; - case TSDB_DATA_TYPE_VARCHAR: - blen += sprintf(buf + blen, "binarydata binary(%d)", gVarCharSize); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - blen += sprintf(buf + blen, "ts timestamp"); - break; - case TSDB_DATA_TYPE_NCHAR: - blen += sprintf(buf + blen, "nchardata nchar(%d)", gVarCharSize); - break; - case TSDB_DATA_TYPE_UTINYINT: - blen += sprintf(buf + blen, "utinydata tinyint unsigned"); - break; - case TSDB_DATA_TYPE_USMALLINT: - blen += sprintf(buf + blen, "usmalldata smallint unsigned"); - break; - case TSDB_DATA_TYPE_UINT: - blen += sprintf(buf + blen, "uintdata int unsigned"); - break; - case TSDB_DATA_TYPE_UBIGINT: - blen += sprintf(buf + blen, "ubigdata bigint unsigned"); - break; - default: - printf("invalid col type:%d", colList[c]); - exit(1); - } + if (tableType == 3) { + blen += sprintf(buf + blen, "using %s%d", bpStbPrefix, bpDefaultStbId); } - blen += sprintf(buf + blen, ")"); + if (tableType == 0 || tableType == 1) { + blen += sprintf(buf + blen, " ("); + + for (int c = 0; c < colNum; ++c) { + if (c > 0) { + blen += sprintf(buf + blen, ","); + } + + switch (colList[c]) { + case TSDB_DATA_TYPE_BOOL: + blen += sprintf(buf + blen, "booldata bool"); + break; + case TSDB_DATA_TYPE_TINYINT: + blen += sprintf(buf + blen, "tinydata tinyint"); + break; + case TSDB_DATA_TYPE_SMALLINT: + blen += sprintf(buf + blen, "smalldata smallint"); + break; + case TSDB_DATA_TYPE_INT: + blen += sprintf(buf + blen, "intdata int"); + break; + case TSDB_DATA_TYPE_BIGINT: + blen += sprintf(buf + blen, "bigdata bigint"); + break; + case TSDB_DATA_TYPE_FLOAT: + blen += sprintf(buf + blen, "floatdata float"); + break; + case TSDB_DATA_TYPE_DOUBLE: + blen += sprintf(buf + blen, "doubledata double"); + break; + case TSDB_DATA_TYPE_VARCHAR: + blen += sprintf(buf + blen, "binarydata binary(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + blen += sprintf(buf + blen, "ts timestamp"); + break; + case TSDB_DATA_TYPE_NCHAR: + blen += sprintf(buf + blen, "nchardata nchar(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_UTINYINT: + blen += sprintf(buf + blen, "utinydata tinyint unsigned"); + break; + case TSDB_DATA_TYPE_USMALLINT: + blen += sprintf(buf + blen, "usmalldata smallint unsigned"); + break; + case TSDB_DATA_TYPE_UINT: + blen += sprintf(buf + blen, "uintdata int unsigned"); + break; + case TSDB_DATA_TYPE_UBIGINT: + blen += sprintf(buf + blen, "ubigdata bigint unsigned"); + break; + default: + printf("invalid col type:%d", colList[c]); + exit(1); + } + } - if (stable) { - blen += sprintf(buf + blen, "tags ("); + blen += sprintf(buf + blen, ")"); + } + + if (1 == tableType) { + blen += sprintf(buf + blen, " tags ("); for (int c = 0; c < colNum; ++c) { if (c > 0) { blen += sprintf(buf + blen, ","); @@ -2363,6 +2438,64 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t * blen += sprintf(buf + blen, ")"); } + if (3 == tableType) { + blen += sprintf(buf + blen, " tags ("); + for (int c = 0; c < colNum; ++c) { + if (c > 0) { + blen += sprintf(buf + blen, ","); + } + switch (colList[c]) { + case TSDB_DATA_TYPE_BOOL: + blen += sprintf(buf + blen, "%s", rand() % 2 ? "true": "false"); + break; + case TSDB_DATA_TYPE_TINYINT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + case TSDB_DATA_TYPE_SMALLINT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + case TSDB_DATA_TYPE_INT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + case TSDB_DATA_TYPE_BIGINT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + case TSDB_DATA_TYPE_FLOAT: + blen += sprintf(buf + blen, "%f", rand() % 128); + break; + case TSDB_DATA_TYPE_DOUBLE: + blen += sprintf(buf + blen, "%f", rand() % 128); + break; + case TSDB_DATA_TYPE_VARCHAR: + blen += sprintf(buf + blen, "'var%d'", rand() % 128); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + blen += sprintf(buf + blen, "%lld", bpTs); + break; + case TSDB_DATA_TYPE_NCHAR: + blen += sprintf(buf + blen, "'nch%d'", rand() % 128); + break; + case TSDB_DATA_TYPE_UTINYINT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + case TSDB_DATA_TYPE_USMALLINT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + case TSDB_DATA_TYPE_UINT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + case TSDB_DATA_TYPE_UBIGINT: + blen += sprintf(buf + blen, "%d", rand() % 128); + break; + default: + printf("invalid col type:%d", colList[c]); + exit(1); + } + } + + blen += sprintf(buf + blen, ")"); + } + if (gCaseCtrl.printCreateTblSql) { printf("\tCreate Table SQL:%s\n", buf); } @@ -2421,11 +2554,11 @@ void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int prepareStb) { result = taos_query(taos, "use demo"); taos_free_result(result); - if (!prepareStb) { + if (0 == prepareStb) { // create table for (int i = 0 ; i < 10; i++) { char buf[1024]; - generateCreateTableSQL(buf, i, colNum, colList, false); + generateCreateTableSQL(buf, i, colNum, colList, 0); result = taos_query(taos, buf); code = taos_errno(result); if (code != 0) { @@ -2436,17 +2569,35 @@ void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int prepareStb) { taos_free_result(result); } } else { - char buf[1024]; - generateCreateTableSQL(buf, bpDefaultStbId, colNum, colList, true); - - result = taos_query(taos, buf); - code = taos_errno(result); - if (code != 0) { - printf("!!!failed to create table, reason:%s\n", taos_errstr(result)); + if (1 == prepareStb || 3 == prepareStb) { + char buf[1024]; + generateCreateTableSQL(buf, bpDefaultStbId, colNum, colList, 1); + + result = taos_query(taos, buf); + code = taos_errno(result); + if (code != 0) { + printf("!!!failed to create table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } taos_free_result(result); - exit(1); } - taos_free_result(result); + + + if (3 == prepareStb) { + for (int i = 0 ; i < 10; i++) { + char buf[1024]; + generateCreateTableSQL(buf, i, colNum, colList, 3); + result = taos_query(taos, buf); + code = taos_errno(result); + if (code != 0) { + printf("!!!failed to create table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + taos_free_result(result); + } + } } } @@ -2486,7 +2637,7 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) { if (gCaseCtrl.autoCreateTbl) { if (gCurCase->testType == TTYPE_INSERT && gCurCase->tblNum > 1) { - gCurCase->autoCreateTbl = true; + gCurCase->autoCreateTbl = 1; if (gCurCase->bindTagNum <= 0) { gCurCase->bindTagNum = gCurCase->colNum; } From f570d61d48a84f6d26170763a93ffe1ce92315ae Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 24 Nov 2022 19:24:44 +0800 Subject: [PATCH 44/46] fix(meta): return invalid message if cannot getting value from uididx --- source/dnode/vnode/src/meta/metaTable.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 8250d68ae1..841999ef9f 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -981,6 +981,11 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA /* get stbEntry*/ tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal); + if (!pVal) { + terrno = TSDB_CODE_INVALID_MSG; + goto _err; + } + tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = ((SUidIdxVal *)pVal)[0].version}), sizeof(STbDbKey), (void **)&stbEntry.pBuf, &nVal); tdbFree(pVal); From 44eaa279894f5425b7b046e8c8af899c9fcdf83a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 24 Nov 2022 20:54:34 +0800 Subject: [PATCH 45/46] fix invalid packet --- source/libs/transport/src/transCli.c | 21 ++++++++++++--------- source/libs/transport/src/transComm.c | 9 ++++----- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 275048e94a..55bfb57a82 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -794,15 +794,18 @@ void cliSend(SCliConn* pConn) { int msgLen = transMsgLenFromCont(pMsg->contLen); STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); - pHead->ahandle = pCtx != NULL ? (uint64_t)pCtx->ahandle : 0; - pHead->noResp = REQUEST_NO_RESP(pMsg) ? 1 : 0; - pHead->persist = REQUEST_PERSIS_HANDLE(pMsg) ? 1 : 0; - pHead->msgType = pMsg->msgType; - pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); - pHead->release = REQUEST_RELEASE_HANDLE(pCliMsg) ? 1 : 0; - memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user)); - pHead->traceId = pMsg->info.traceId; - pHead->magicNum = htonl(TRANS_MAGIC_NUM); + if (pHead->comp == 0) { + pHead->ahandle = pCtx != NULL ? (uint64_t)pCtx->ahandle : 0; + pHead->noResp = REQUEST_NO_RESP(pMsg) ? 1 : 0; + pHead->persist = REQUEST_PERSIS_HANDLE(pMsg) ? 1 : 0; + pHead->msgType = pMsg->msgType; + pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + pHead->release = REQUEST_RELEASE_HANDLE(pCliMsg) ? 1 : 0; + memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user)); + pHead->traceId = pMsg->info.traceId; + pHead->magicNum = htonl(TRANS_MAGIC_NUM); + } + if (pHead->persist == 1) { CONN_SET_PERSIST_BY_APP(pConn); } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 7710abcaa1..2759fb5aeb 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -60,21 +60,20 @@ int32_t transDecompressMsg(char** msg, int32_t len) { STransMsgHead* pHead = (STransMsgHead*)(*msg); if (pHead->comp == 0) return 0; - char* pCont = transContFromHead(pHead); + char* pCont = transContFromHead(pHead); + STransCompMsg* pComp = (STransCompMsg*)pCont; int32_t oriLen = htonl(pComp->contLen); char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead)); STransMsgHead* pNewHead = (STransMsgHead*)buf; - - int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), pNewHead->content, - len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen); + int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), pNewHead->content, + len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen); memcpy((char*)pNewHead, (char*)pHead, sizeof(STransMsgHead)); pNewHead->msgLen = htonl(oriLen + sizeof(STransMsgHead)); taosMemoryFree(pHead); - *msg = buf; if (decompLen != oriLen) { return -1; From 6cabe54bf2cffb03963c737d539eb7e585143858 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Nov 2022 21:01:06 +0800 Subject: [PATCH 46/46] test: add asan case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 9dc57d1e93..19dcec442a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -725,7 +725,7 @@ ,,,system-test,python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py ,,,system-test,python3 ./test.py -f 7-tmq/tmq_taosx.py ,,,system-test,python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py -,,y,system-test,./pytest.sh python3 -f 99-TDcase/TD-19201.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py ,,,system-test,python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 ,,,system-test,python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 ,,,system-test,python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3