From 665caf144685587caeed1a1c463993faf31cd28f Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 7 Jul 2022 18:16:21 +0800 Subject: [PATCH 01/38] test:add teatcase of tsbs quering --- .../1-insert/test_stmt_insert_query_ex.py | 262 ------------------ tests/system-test/2-query/tsbsQuery.py | 74 +++++ 2 files changed, 74 insertions(+), 262 deletions(-) delete mode 100644 tests/system-test/1-insert/test_stmt_insert_query_ex.py create mode 100644 tests/system-test/2-query/tsbsQuery.py diff --git a/tests/system-test/1-insert/test_stmt_insert_query_ex.py b/tests/system-test/1-insert/test_stmt_insert_query_ex.py deleted file mode 100644 index 2a64c09ff4..0000000000 --- a/tests/system-test/1-insert/test_stmt_insert_query_ex.py +++ /dev/null @@ -1,262 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import os -import threading as thd -import multiprocessing as mp -from numpy.lib.function_base import insert -import taos -from taos import * -from util.log import * -from util.cases import * -from util.sql import * -import numpy as np -import datetime as dt -from datetime import datetime -from ctypes import * -import time -# constant define -WAITS = 5 # wait seconds - -class TDTestCase: - # - # --------------- main frame ------------------- - def caseDescription(self): - ''' - limit and offset keyword function test cases; - case1: limit offset base function test - case2: offset return valid - ''' - return - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files or "taosd.exe" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - return buildPath - - # init - def init(self, conn, logSql): - tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) - # tdSql.prepare() - # self.create_tables(); - self.ts = 1500000000000 - - # stop - def stop(self): - tdSql.close() - tdLog.success("%s successfully executed" % __file__) - - - # --------------- case ------------------- - - - def newcon(self,host,cfg): - user = "root" - password = "taosdata" - port =6030 - con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) - print(con) - return con - - def test_stmt_set_tbname_tag(self,conn): - dbname = "stmt_tag" - - try: - conn.execute("drop database if exists %s" % dbname) - conn.execute("create database if not exists %s PRECISION 'us' " % dbname) - conn.select_db(dbname) - conn.execute("create table if not exists log(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\ - bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \ - ff float, dd double, bb binary(100), nn nchar(100), tt timestamp , vc varchar(100)) tags (t1 timestamp, t2 bool,\ - t3 tinyint, t4 tinyint, t5 smallint, t6 int, t7 bigint, t8 tinyint unsigned, t9 smallint unsigned, \ - t10 int unsigned, t11 bigint unsigned, t12 float, t13 double, t14 binary(100), t15 nchar(100), t16 timestamp)") - - stmt = conn.statement("insert into ? using log tags (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) \ - values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") - tags = new_bind_params(16) - tags[0].timestamp(1626861392589123, PrecisionEnum.Microseconds) - tags[1].bool(True) - tags[2].bool(False) - tags[3].tinyint(2) - tags[4].smallint(3) - tags[5].int(4) - tags[6].bigint(5) - tags[7].tinyint_unsigned(6) - tags[8].smallint_unsigned(7) - tags[9].int_unsigned(8) - tags[10].bigint_unsigned(9) - tags[11].float(10.1) - tags[12].double(10.11) - tags[13].binary("hello") - tags[14].nchar("stmt") - tags[15].timestamp(1626861392589, PrecisionEnum.Milliseconds) - stmt.set_tbname_tags("tb1", tags) - params = new_multi_binds(17) - params[0].timestamp((1626861392589111, 1626861392590111, 1626861392591111)) - params[1].bool((True, None, False)) - params[2].tinyint([-128, -128, None]) # -128 is tinyint null - params[3].tinyint([0, 127, None]) - params[4].smallint([3, None, 2]) - params[5].int([3, 4, None]) - params[6].bigint([3, 4, None]) - params[7].tinyint_unsigned([3, 4, None]) - params[8].smallint_unsigned([3, 4, None]) - params[9].int_unsigned([3, 4, None]) - params[10].bigint_unsigned([3, 4, 5]) - params[11].float([3, None, 1]) - params[12].double([3, None, 1.2]) - params[13].binary(["abc", "dddafadfadfadfadfa", None]) - params[14].nchar(["涛思数据", None, "a long string with 中文字符"]) - params[15].timestamp([None, None, 1626861392591]) - params[16].binary(["涛思数据16", None, "a long string with 中文-字符"]) - - stmt.bind_param_batch(params) - stmt.execute() - - assert stmt.affected_rows == 3 - - #query all - querystmt1=conn.statement("select * from log where bu < ?") - queryparam1=new_bind_params(1) - print(type(queryparam1)) - queryparam1[0].int(10) - querystmt1.bind_param(queryparam1) - querystmt1.execute() - result1=querystmt1.use_result() - rows1=result1.fetch_all() - print(rows1[0]) - print(rows1[1]) - print(rows1[2]) - assert str(rows1[0][0]) == "2021-07-21 17:56:32.589111" - assert rows1[0][10] == 3 - assert rows1[1][10] == 4 - - #query: Numeric Functions - querystmt2=conn.statement("select abs(?) from log where bu < ?") - queryparam2=new_bind_params(2) - print(type(queryparam2)) - queryparam2[0].int(5) - queryparam2[1].int(5) - querystmt2.bind_param(queryparam2) - querystmt2.execute() - result2=querystmt2.use_result() - rows2=result2.fetch_all() - print("2",rows2) - assert rows2[0][0] == 5 - assert rows2[1][0] == 5 - - - #query: Numeric Functions and escapes - - querystmt3=conn.statement("select abs(?) from log where nn= 'a? long string with 中文字符' ") - queryparam3=new_bind_params(1) - print(type(queryparam3)) - queryparam3[0].int(5) - querystmt3.bind_param(queryparam3) - querystmt3.execute() - result3=querystmt3.use_result() - rows3=result3.fetch_all() - print("3",rows3) - assert rows3 == [] - - #query: string Functions - - querystmt9=conn.statement("select CHAR_LENGTH(?) from log ") - queryparam9=new_bind_params(1) - print(type(queryparam9)) - queryparam9[0].binary('中文字符') - querystmt9.bind_param(queryparam9) - querystmt9.execute() - result9=querystmt9.use_result() - rows9=result9.fetch_all() - print("9",rows9) - assert rows9[0][0] == 12, 'fourth case is failed' - assert rows9[1][0] == 12, 'fourth case is failed' - - #query: conversion Functions - - querystmt4=conn.statement("select cast( ? as bigint) from log ") - queryparam4=new_bind_params(1) - print(type(queryparam4)) - queryparam4[0].binary('1232a') - querystmt4.bind_param(queryparam4) - querystmt4.execute() - result4=querystmt4.use_result() - rows4=result4.fetch_all() - print("5",rows4) - assert rows4[0][0] == 1232 - assert rows4[1][0] == 1232 - - querystmt4=conn.statement("select cast( ? as binary(10)) from log ") - queryparam4=new_bind_params(1) - print(type(queryparam4)) - queryparam4[0].int(123) - querystmt4.bind_param(queryparam4) - querystmt4.execute() - result4=querystmt4.use_result() - rows4=result4.fetch_all() - print("6",rows4) - assert rows4[0][0] == '123' - assert rows4[1][0] == '123' - - # #query: datatime Functions - - # querystmt4=conn.statement(" select timediff('2021-07-21 17:56:32.590111',?,1s) from log ") - # queryparam4=new_bind_params(1) - # print(type(queryparam4)) - # queryparam4[0].timestamp(1626861392591111) - # querystmt4.bind_param(queryparam4) - # querystmt4.execute() - # result4=querystmt4.use_result() - # rows4=result4.fetch_all() - # print("7",rows4) - # assert rows4[0][0] == 1, 'seventh case is failed' - # assert rows4[1][0] == 1, 'seventh case is failed' - - - - # conn.execute("drop database if exists %s" % dbname) - conn.close() - - except Exception as err: - # conn.execute("drop database if exists %s" % dbname) - conn.close() - raise err - - def run(self): - buildPath = self.getBuildPath() - config = buildPath+ "../sim/dnode1/cfg/" - host="localhost" - connectstmt=self.newcon(host,config) - self.test_stmt_set_tbname_tag(connectstmt) - - return - - -# add case with filename -# -tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py new file mode 100644 index 0000000000..d24e5ea283 --- /dev/null +++ b/tests/system-test/2-query/tsbsQuery.py @@ -0,0 +1,74 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def prepareData(self): + database="db_tsbs" + ts=1451606400000 + tdSql.execute(f"create database {database};") + tdSql.execute(f"use {database} ") + tdSql.execute(''' + create table readings (ts timestamp,latitude double,longitude double,elevation double,velocity double,heading double,grade double,fuel_consumption double,load_capacity double,fuel_capacity double,nominal_fuel_consumption double) tags (name binary(30),fleet binary(30),driver binary(30),model binary(30),device_version binary(30)); + ''') + tdSql.execute(''' + create table diagnostics (ts timestamp,fuel_state double,current_load double,status bigint,load_capacity double,fuel_capacity double,nominal_fuel_consumption double) tags (name binary(30),fleet binary(30),driver binary(30),model binary(30),device_version binary(30)) ; + ''') + + for i in range(10): + tdSql.execute(f"create table rct{i} using readings (name,fleet,driver,model,device_version) tags ('truck_{i}','South{i}','Trish{i}','H-{i}','v2.3')") + tdSql.execute(f"create table dct{i} using diagnostics (name,fleet,driver,model,device_version) tags ('truck_{i}','South{i}','Trish{i}','H-{i}','v2.3')") + for j in range(10): + for i in range(10): + tdSql.execute( + f"insert into rct{j} values ( {ts+i*10000}, {80+i}, {90+i}, {85+i}, {30+i*10}, {1.2*i}, {221+i*2}, {20+i*0.2}, {1500+i*20}, {150+i*2},{5+i} )" + ) + tdSql.execute( + f"insert into dct{j} values ( {ts+i*10000}, {1+i*0.1},{1400+i*15}, {1+i},{1500+i*20}, {150+i*2},{5+i} )" + ) + + # def check_avg(self ,origin_query , check_query): + # avg_result = tdSql.getResult(origin_query) + # origin_result = tdSql.getResult(check_query) + + # check_status = True + # for row_index , row in enumerate(avg_result): + # for col_index , elem in enumerate(row): + # if avg_result[row_index][col_index] != origin_result[row_index][col_index]: + # check_status = False + # if not check_status: + # tdLog.notice("avg function value has not as expected , sql is \"%s\" "%origin_query ) + # sys.exit(1) + # else: + # tdLog.info("avg value check pass , it work as expected ,sql is \"%s\" "%check_query ) + + + def tsbsIotQuery(self): + tdSql.execute("use db_tsbs") + tdSql.query(" SELECT avg(velocity) as mean_velocity ,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet interval(10m); ") + tdSql.checkRows(1) + + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdLog.printNoPrefix("==========step1:create database and table,insert data ==============") + self.tsbsIotQuery() + self.tsbsIotQuery() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 10f19678985d77ea58cc441a6f2b2e7e21871122 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 8 Jul 2022 15:41:33 +0800 Subject: [PATCH 02/38] fix: fix crash issue --- source/libs/qworker/src/qworker.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 3f8d62b1aa..a4e412ae4a 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -518,7 +518,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex ctx->explain = explain; ctx->queryType = qwMsg->msgType; - QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); + //QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); code = qStringToSubplan(qwMsg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { From df06d22cb3eaf9021526994b83f4636780daea2e Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 8 Jul 2022 17:35:11 +0800 Subject: [PATCH 03/38] fix: some problem of parser and planner --- include/libs/scalar/scalar.h | 2 +- source/common/src/systable.c | 2 +- source/dnode/mnode/impl/src/mndSma.c | 11 +- source/libs/parser/inc/sql.y | 2 +- source/libs/parser/src/parCalcConst.c | 6 +- source/libs/parser/src/parTranslater.c | 147 +---- source/libs/parser/src/sql.c | 834 ++++++++++++------------ source/libs/planner/src/planOptimizer.c | 4 +- source/libs/scalar/src/scalar.c | 149 +++-- 9 files changed, 552 insertions(+), 605 deletions(-) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index aaebffa118..2a1a75c431 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -25,7 +25,7 @@ extern "C" { typedef struct SFilterInfo SFilterInfo; -int32_t scalarGetOperatorResultType(SDataType left, SDataType right, EOperatorType op, SDataType* pRes); +int32_t scalarGetOperatorResultType(SOperatorNode* pOp); /* pNode will be freed in API; diff --git a/source/common/src/systable.c b/source/common/src/systable.c index b8844390d2..ba8a8e1220 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -252,7 +252,7 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)}, {TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema)}, {TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)}, - {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)}, + // {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)}, {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)}, {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)}, {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)}, diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index da5b8cb48e..388441b1dc 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -1131,14 +1131,17 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc SSmaObj *pSma = NULL; int32_t cols = 0; - SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); - if (pDb == NULL) return 0; + SDbObj *pDb = NULL; + if (strlen(pShow->db) > 0) { + pDb = mndAcquireDb(pMnode, pShow->db); + if (pDb == NULL) return 0; + } while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_SMA, pShow->pIter, (void **)&pSma); if (pShow->pIter == NULL) break; - if (pSma->dbUid != pDb->uid) { + if (NULL != pDb && pSma->dbUid != pDb->uid) { sdbRelease(pSdb, pSma); continue; } @@ -1151,7 +1154,7 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc STR_TO_VARSTR(n1, (char *)tNameGetTableName(&smaName)); char n2[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(n2, (char *)mndGetDbStr(pDb->name)); + STR_TO_VARSTR(n2, (char *)mndGetDbStr(pSma->db)); SName stbName = {0}; tNameFromString(&stbName, pSma->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index fd79eaa9b7..4bcfcbf5c0 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -769,7 +769,7 @@ compare_op(A) ::= CONTAINS. in_op(A) ::= IN. { A = OP_TYPE_IN; } in_op(A) ::= NOT IN. { A = OP_TYPE_NOT_IN; } -in_predicate_value(A) ::= NK_LP(C) expression_list(B) NK_RP(D). { A = createRawExprNodeExt(pCxt, &C, &D, createNodeListNode(pCxt, B)); } +in_predicate_value(A) ::= NK_LP(C) literal_list(B) NK_RP(D). { A = createRawExprNodeExt(pCxt, &C, &D, createNodeListNode(pCxt, B)); } /************************************************ boolean_value_expression ********************************************/ boolean_value_expression(A) ::= boolean_primary(B). { A = B; } diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 6c670b3f01..4dff42592a 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -195,8 +195,8 @@ static int32_t calcConstProject(SNode* pProject, SNode** pNew) { return code; } -static bool isUselessCol(bool hasSelectValFunc, SExprNode* pProj) { - if (hasSelectValFunc && QUERY_NODE_FUNCTION == nodeType(pProj) && fmIsSelectFunc(((SFunctionNode*)pProj)->funcId)) { +static bool isUselessCol(SExprNode* pProj) { + if (QUERY_NODE_FUNCTION == nodeType(pProj) && !fmIsScalarFunc(((SFunctionNode*)pProj)->funcId)) { return false; } return NULL == ((SExprNode*)pProj)->pAssociation; @@ -218,7 +218,7 @@ static SNode* createConstantValue() { static int32_t calcConstProjections(SCalcConstContext* pCxt, SSelectStmt* pSelect, bool subquery) { SNode* pProj = NULL; WHERE_EACH(pProj, pSelect->pProjectionList) { - if (subquery && isUselessCol(pSelect->hasSelectValFunc, (SExprNode*)pProj)) { + if (subquery && isUselessCol((SExprNode*)pProj)) { ERASE_NODE(pSelect->pProjectionList); continue; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 97a263f86f..4ffcd34d70 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -977,131 +977,11 @@ static bool isMultiResFunc(SNode* pNode) { return (QUERY_NODE_COLUMN == nodeType(pParam) ? 0 == strcmp(((SColumnNode*)pParam)->colName, "*") : false); } -static int32_t rewriteNegativeOperator(SNode** pOp) { - SNode* pRes = NULL; - int32_t code = scalarCalculateConstants(*pOp, &pRes); - if (TSDB_CODE_SUCCESS == code) { - *pOp = pRes; - } - return code; -} - -static EDealRes translateUnaryOperator(STranslateContext* pCxt, SOperatorNode** pOpRef) { - SOperatorNode* pOp = *pOpRef; - if (OP_TYPE_MINUS == pOp->opType) { - if (!IS_MATHABLE_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); - } - pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; - - pCxt->errCode = rewriteNegativeOperator((SNode**)pOpRef); - } else { - pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - } - return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; -} - -static EDealRes translateArithmeticOperator(STranslateContext* pCxt, SOperatorNode* pOp) { - SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; - SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; - if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); - } - if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) || - (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && (IS_VAR_DATA_TYPE(rdt.type) || IS_FLOAT_TYPE(rdt.type))) || - (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && (IS_VAR_DATA_TYPE(ldt.type) || IS_FLOAT_TYPE(ldt.type)))) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); - } - - if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && IS_INTEGER_TYPE(rdt.type)) || - (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && IS_INTEGER_TYPE(ldt.type)) || - (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_BOOL == rdt.type) || - (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && TSDB_DATA_TYPE_BOOL == ldt.type)) { - pOp->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; - } else { - pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; - } - return DEAL_RES_CONTINUE; -} - static bool dataTypeEqual(const SDataType* l, const SDataType* r) { return (l->type == r->type && l->bytes == r->bytes && l->precision == r->precision && l->scale == r->scale); } -static EDealRes translateComparisonOperator(STranslateContext* pCxt, SOperatorNode* pOp) { - SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; - SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; - if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); - } - if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) { - SNodeListNode* pRight = (SNodeListNode*)pOp->pRight; - bool first = true; - SDataType targetDt = {0}; - SNode* pNode = NULL; - FOREACH(pNode, pRight->pNodeList) { - SDataType dt = ((SExprNode*)pNode)->resType; - if (first) { - targetDt = dt; - if (targetDt.type != TSDB_DATA_TYPE_NULL) { - first = false; - } - } else if (dt.type != targetDt.type && dt.type != TSDB_DATA_TYPE_NULL) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pNode)->aliasName); - } else if (dt.bytes > targetDt.bytes) { - targetDt.bytes = dt.bytes; - } - } - pRight->dataType = targetDt; - } - if (nodesIsRegularOp(pOp)) { - if (!IS_VAR_DATA_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); - } - if (QUERY_NODE_VALUE != nodeType(pOp->pRight) || - ((!IS_STR_DATA_TYPE(((SExprNode*)(pOp->pRight))->resType.type)) && - (((SExprNode*)(pOp->pRight))->resType.type != TSDB_DATA_TYPE_NULL))) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); - } - } - pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - return DEAL_RES_CONTINUE; -} - -static EDealRes translateJsonOperator(STranslateContext* pCxt, SOperatorNode* pOp) { - SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; - SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; - if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); - } - if (pOp->opType == OP_TYPE_JSON_GET_VALUE) { - pOp->node.resType.type = TSDB_DATA_TYPE_JSON; - } else if (pOp->opType == OP_TYPE_JSON_CONTAINS) { - pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; - } - pOp->node.resType.bytes = tDataTypes[pOp->node.resType.type].bytes; - return DEAL_RES_CONTINUE; -} - -static EDealRes translateBitwiseOperator(STranslateContext* pCxt, SOperatorNode* pOp) { - SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; - SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; - if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); - } - pOp->node.resType.type = TSDB_DATA_TYPE_BIGINT; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - return DEAL_RES_CONTINUE; -} - -static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode** pOpRef) { - SOperatorNode* pOp = *pOpRef; - +static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { if (isMultiResFunc(pOp->pLeft)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); } @@ -1109,17 +989,10 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode** pOpRe return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } - if (nodesIsUnaryOp(pOp)) { - return translateUnaryOperator(pCxt, pOpRef); - } else if (nodesIsArithmeticOp(pOp)) { - return translateArithmeticOperator(pCxt, pOp); - } else if (nodesIsComparisonOp(pOp)) { - return translateComparisonOperator(pCxt, pOp); - } else if (nodesIsJsonOp(pOp)) { - return translateJsonOperator(pCxt, pOp); - } else if (nodesIsBitwiseOp(pOp)) { - return translateBitwiseOperator(pCxt, pOp); + if (TSDB_CODE_SUCCESS != scalarGetOperatorResultType(pOp)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pOp->node.aliasName); } + return DEAL_RES_CONTINUE; } @@ -1485,7 +1358,7 @@ static EDealRes doTranslateExpr(SNode** pNode, void* pContext) { case QUERY_NODE_VALUE: return translateValue(pCxt, (SValueNode*)*pNode); case QUERY_NODE_OPERATOR: - return translateOperator(pCxt, (SOperatorNode**)pNode); + return translateOperator(pCxt, (SOperatorNode*)*pNode); case QUERY_NODE_FUNCTION: return translateFunction(pCxt, (SFunctionNode**)pNode); case QUERY_NODE_LOGIC_CONDITION: @@ -3352,7 +3225,7 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs if (NULL == pFuncs) { if (NULL != pDbCfg->pRetensions) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TABLE_OPTION, - "To create a super table in a database with the retensions parameter configured, " + "To create a super table in databases configured with the 'RETENTIONS' option, " "the 'ROLLUP' option must be present"); } return TSDB_CODE_SUCCESS; @@ -3563,10 +3436,12 @@ static int32_t checkTableWatermarkOption(STranslateContext* pCxt, STableOptions* } static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, bool createStable) { - int32_t code = TSDB_CODE_SUCCESS; SDbCfgInfo dbCfg = {0}; - if (createStable) { - code = getDBCfg(pCxt, pStmt->dbName, &dbCfg); + int32_t code = getDBCfg(pCxt, pStmt->dbName, &dbCfg); + if (TSDB_CODE_SUCCESS == code && !createStable && NULL != dbCfg.pRetensions) { + code = generateSyntaxErrMsgExt( + &pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TABLE_OPTION, + "Only super table creation is supported in databases configured with the 'RETENTIONS' option"); } if (TSDB_CODE_SUCCESS == code) { code = checkTableMaxDelayOption(pCxt, pStmt->pOptions, createStable, &dbCfg); diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 124d1b2270..029dae0311 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -216,244 +216,252 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2360) +#define YY_ACTTAB_COUNT (2436) static const YYACTIONTYPE yy_action[] = { /* 0 */ 430, 1920, 431, 1491, 438, 71, 431, 1491, 514, 62, - /* 10 */ 1645, 1693, 40, 38, 1919, 550, 323, 325, 1917, 1690, - /* 20 */ 337, 86, 1254, 1457, 34, 33, 1643, 1589, 41, 39, - /* 30 */ 37, 36, 35, 1329, 119, 1252, 1777, 41, 39, 37, - /* 40 */ 36, 35, 1585, 122, 103, 1734, 550, 102, 101, 100, - /* 50 */ 99, 98, 97, 96, 95, 94, 1324, 526, 550, 23, + /* 10 */ 1645, 1693, 39, 37, 1919, 549, 323, 325, 1917, 1690, + /* 20 */ 337, 86, 1254, 1457, 33, 32, 1643, 1589, 40, 38, + /* 30 */ 36, 35, 34, 1329, 119, 1252, 1777, 40, 38, 36, + /* 40 */ 35, 34, 1585, 122, 103, 1734, 549, 102, 101, 100, + /* 50 */ 99, 98, 97, 96, 95, 94, 1324, 526, 549, 22, /* 60 */ 14, 1016, 342, 1015, 1795, 1638, 1640, 1260, 113, 142, - /* 70 */ 1795, 1468, 578, 79, 122, 468, 466, 1747, 543, 577, - /* 80 */ 40, 38, 1392, 120, 1, 1593, 122, 61, 337, 61, - /* 90 */ 1254, 1017, 358, 555, 1586, 555, 379, 552, 155, 1862, + /* 70 */ 1795, 1468, 578, 79, 122, 468, 466, 1747, 542, 577, + /* 80 */ 39, 37, 1392, 120, 1, 1593, 122, 61, 337, 61, + /* 90 */ 1254, 1017, 358, 554, 1586, 554, 379, 551, 155, 1862, /* 100 */ 1863, 1329, 1867, 1252, 1808, 1479, 659, 1521, 89, 1778, - /* 110 */ 580, 1780, 1781, 576, 120, 571, 542, 553, 1854, 553, + /* 110 */ 580, 1780, 1781, 576, 120, 571, 541, 552, 1854, 552, /* 120 */ 1331, 1332, 304, 1850, 1324, 1478, 120, 526, 14, 249, - /* 130 */ 1862, 549, 1456, 548, 1920, 1260, 1920, 1920, 164, 31, + /* 130 */ 1862, 548, 1456, 547, 1920, 1260, 1920, 1920, 164, 30, /* 140 */ 259, 156, 1862, 1863, 1777, 1867, 1747, 161, 447, 161, /* 150 */ 159, 1917, 2, 1917, 1917, 1593, 112, 111, 110, 109, /* 160 */ 108, 107, 106, 105, 104, 1255, 1747, 1253, 1279, 483, /* 170 */ 482, 481, 1795, 1920, 659, 480, 220, 221, 118, 477, - /* 180 */ 578, 44, 476, 475, 474, 1747, 160, 577, 1331, 1332, + /* 180 */ 578, 43, 476, 475, 474, 1747, 160, 577, 1331, 1332, /* 190 */ 1917, 429, 1258, 1259, 433, 1307, 1308, 1310, 1311, 1312, /* 200 */ 1313, 1314, 573, 569, 1322, 1323, 1325, 1326, 1327, 1328, - /* 210 */ 1330, 1333, 1808, 1413, 1278, 539, 90, 1778, 580, 1780, - /* 220 */ 1781, 576, 1279, 571, 162, 1454, 1854, 162, 553, 162, - /* 230 */ 330, 1850, 1933, 1255, 314, 1253, 61, 303, 140, 143, - /* 240 */ 516, 1888, 999, 1549, 34, 33, 1645, 1596, 41, 39, - /* 250 */ 37, 36, 35, 341, 536, 1411, 1412, 1414, 1415, 1477, + /* 210 */ 1330, 1333, 1808, 1413, 1278, 538, 90, 1778, 580, 1780, + /* 220 */ 1781, 576, 1279, 571, 162, 1454, 1854, 162, 552, 162, + /* 230 */ 328, 1850, 1933, 1255, 314, 1253, 61, 303, 140, 143, + /* 240 */ 516, 1888, 999, 1549, 33, 32, 1645, 1596, 40, 38, + /* 250 */ 36, 35, 34, 341, 535, 1411, 1412, 1414, 1415, 1477, /* 260 */ 1258, 1259, 1643, 1307, 1308, 1310, 1311, 1312, 1313, 1314, /* 270 */ 573, 569, 1322, 1323, 1325, 1326, 1327, 1328, 1330, 1333, - /* 280 */ 40, 38, 1003, 1004, 315, 349, 313, 312, 337, 470, - /* 290 */ 1254, 1777, 43, 472, 545, 540, 305, 1353, 11, 10, + /* 280 */ 39, 37, 1003, 1004, 315, 349, 313, 312, 337, 470, + /* 290 */ 1254, 1777, 42, 472, 544, 539, 305, 1353, 11, 10, /* 300 */ 1747, 1329, 103, 1252, 153, 102, 101, 100, 99, 98, /* 310 */ 97, 96, 95, 94, 385, 471, 378, 1632, 377, 1795, - /* 320 */ 1358, 1292, 1151, 1152, 1324, 1278, 1920, 554, 14, 1351, - /* 330 */ 1639, 1640, 1747, 437, 577, 1260, 433, 40, 38, 159, + /* 320 */ 1358, 1292, 1151, 1152, 1324, 1278, 1920, 553, 14, 1351, + /* 330 */ 1639, 1640, 1747, 437, 577, 1260, 433, 39, 37, 159, /* 340 */ 483, 482, 481, 1917, 1920, 337, 480, 1254, 1518, 118, /* 350 */ 477, 301, 2, 476, 475, 474, 605, 1918, 1329, 1808, - /* 360 */ 1252, 1917, 28, 90, 1778, 580, 1780, 1781, 576, 435, - /* 370 */ 571, 71, 370, 1854, 659, 1276, 162, 330, 1850, 154, + /* 360 */ 1252, 1917, 27, 90, 1778, 580, 1780, 1781, 576, 435, + /* 370 */ 571, 71, 370, 1854, 659, 1276, 162, 328, 1850, 154, /* 380 */ 1396, 1324, 1365, 1352, 117, 1016, 1278, 1015, 1331, 1332, - /* 390 */ 1309, 158, 1260, 1588, 372, 368, 544, 34, 33, 1880, - /* 400 */ 1571, 41, 39, 37, 36, 35, 1357, 479, 478, 8, + /* 390 */ 1309, 158, 1260, 1588, 372, 368, 543, 33, 32, 1880, + /* 400 */ 1571, 40, 38, 36, 35, 34, 1357, 479, 478, 8, /* 410 */ 636, 635, 634, 633, 345, 1017, 632, 631, 630, 123, /* 420 */ 625, 624, 623, 622, 621, 620, 619, 618, 133, 614, - /* 430 */ 324, 659, 162, 1255, 562, 1253, 34, 33, 140, 613, - /* 440 */ 41, 39, 37, 36, 35, 1331, 1332, 1595, 30, 335, + /* 430 */ 324, 659, 162, 1255, 561, 1253, 33, 32, 140, 613, + /* 440 */ 40, 38, 36, 35, 34, 1331, 1332, 1595, 29, 335, /* 450 */ 1346, 1347, 1348, 1349, 1350, 1354, 1355, 1356, 1280, 447, /* 460 */ 1258, 1259, 1476, 1307, 1308, 1310, 1311, 1312, 1313, 1314, /* 470 */ 573, 569, 1322, 1323, 1325, 1326, 1327, 1328, 1330, 1333, - /* 480 */ 34, 33, 214, 526, 41, 39, 37, 36, 35, 170, - /* 490 */ 1255, 207, 1253, 219, 165, 1281, 340, 37, 36, 35, - /* 500 */ 59, 34, 33, 1747, 140, 41, 39, 37, 36, 35, + /* 480 */ 33, 32, 214, 526, 40, 38, 36, 35, 34, 170, + /* 490 */ 1255, 207, 1253, 219, 165, 1281, 340, 36, 35, 34, + /* 500 */ 59, 33, 32, 1747, 140, 40, 38, 36, 35, 34, /* 510 */ 61, 1593, 75, 1595, 629, 627, 69, 1258, 1259, 68, /* 520 */ 1307, 1308, 1310, 1311, 1312, 1313, 1314, 573, 569, 1322, - /* 530 */ 1323, 1325, 1326, 1327, 1328, 1330, 1333, 40, 38, 1334, + /* 530 */ 1323, 1325, 1326, 1327, 1328, 1330, 1333, 39, 37, 1334, /* 540 */ 1689, 139, 298, 1475, 343, 337, 1777, 1254, 1277, 162, /* 550 */ 73, 303, 140, 305, 516, 1234, 1235, 1423, 1329, 418, /* 560 */ 1252, 1595, 1309, 1109, 602, 601, 600, 1113, 599, 1115, /* 570 */ 1116, 598, 1118, 595, 1795, 1124, 592, 1126, 1127, 589, /* 580 */ 586, 1324, 578, 488, 1747, 1449, 1351, 1747, 526, 577, - /* 590 */ 251, 472, 1260, 1474, 40, 38, 1683, 1254, 498, 383, - /* 600 */ 45, 4, 337, 555, 1254, 174, 173, 172, 1260, 9, + /* 590 */ 251, 472, 1260, 1474, 39, 37, 1683, 1254, 498, 383, + /* 600 */ 44, 4, 337, 554, 1254, 174, 173, 172, 1260, 9, /* 610 */ 1252, 1071, 206, 471, 1808, 1329, 1593, 1252, 89, 1778, /* 620 */ 580, 1780, 1781, 576, 497, 571, 491, 231, 1854, 1473, - /* 630 */ 485, 659, 304, 1850, 1747, 205, 563, 495, 1324, 493, - /* 640 */ 1352, 27, 1260, 1073, 1920, 1331, 1332, 34, 33, 1260, - /* 650 */ 162, 41, 39, 37, 36, 35, 1569, 159, 1869, 1472, - /* 660 */ 1471, 1917, 56, 1357, 611, 55, 9, 1869, 559, 526, - /* 670 */ 1747, 34, 33, 1645, 1448, 41, 39, 37, 36, 35, + /* 630 */ 485, 659, 304, 1850, 1747, 205, 562, 495, 1324, 493, + /* 640 */ 1352, 26, 1260, 1073, 1920, 1331, 1332, 33, 32, 1260, + /* 650 */ 162, 40, 38, 36, 35, 34, 1569, 159, 1869, 1472, + /* 660 */ 1471, 1917, 56, 1357, 611, 55, 9, 1869, 558, 526, + /* 670 */ 1747, 33, 32, 1645, 1448, 40, 38, 36, 35, 34, /* 680 */ 384, 659, 1866, 131, 130, 608, 607, 606, 659, 1644, /* 690 */ 1255, 1865, 1253, 1003, 1004, 1869, 513, 1593, 1470, 7, - /* 700 */ 1747, 1747, 1331, 1332, 1570, 30, 335, 1346, 1347, 1348, + /* 700 */ 1747, 1747, 1331, 1332, 1570, 29, 335, 1346, 1347, 1348, /* 710 */ 1349, 1350, 1354, 1355, 1356, 613, 1582, 1258, 1259, 1864, /* 720 */ 1307, 1308, 1310, 1311, 1312, 1313, 1314, 573, 569, 1322, - /* 730 */ 1323, 1325, 1326, 1327, 1328, 1330, 1333, 1920, 557, 1747, - /* 740 */ 1255, 1688, 1253, 298, 617, 29, 1565, 1255, 1403, 1253, - /* 750 */ 159, 34, 33, 514, 1917, 41, 39, 37, 36, 35, - /* 760 */ 347, 1874, 1385, 1263, 1691, 34, 33, 1258, 1259, 41, - /* 770 */ 39, 37, 36, 35, 1258, 1259, 213, 1307, 1308, 1310, + /* 730 */ 1323, 1325, 1326, 1327, 1328, 1330, 1333, 1920, 556, 1747, + /* 740 */ 1255, 1688, 1253, 298, 617, 28, 1565, 1255, 1403, 1253, + /* 750 */ 159, 33, 32, 514, 1917, 40, 38, 36, 35, 34, + /* 760 */ 347, 1874, 1385, 616, 1691, 33, 32, 1258, 1259, 40, + /* 770 */ 38, 36, 35, 34, 1258, 1259, 213, 1307, 1308, 1310, /* 780 */ 1311, 1312, 1313, 1314, 573, 569, 1322, 1323, 1325, 1326, - /* 790 */ 1327, 1328, 1330, 1333, 40, 38, 300, 1578, 1276, 1467, + /* 790 */ 1327, 1328, 1330, 1333, 39, 37, 300, 1578, 1276, 1467, /* 800 */ 611, 1920, 337, 1777, 1254, 411, 74, 609, 423, 1466, /* 810 */ 1636, 1508, 1389, 526, 159, 1329, 499, 1252, 1917, 131, /* 820 */ 130, 608, 607, 606, 388, 396, 1465, 424, 1464, 398, /* 830 */ 1339, 1795, 610, 484, 271, 1636, 1278, 1623, 1324, 578, - /* 840 */ 1747, 1593, 1292, 1568, 1747, 526, 577, 34, 33, 1260, - /* 850 */ 1747, 41, 39, 37, 36, 35, 113, 1920, 550, 52, - /* 860 */ 510, 389, 198, 473, 1262, 196, 2, 1747, 560, 1747, - /* 870 */ 159, 1808, 616, 1593, 1917, 90, 1778, 580, 1780, 1781, - /* 880 */ 576, 1463, 571, 1385, 1462, 1854, 122, 1461, 659, 330, - /* 890 */ 1850, 1933, 1460, 1459, 200, 1266, 1497, 199, 567, 1580, + /* 840 */ 1747, 1593, 1292, 1568, 1747, 526, 577, 33, 32, 1260, + /* 850 */ 1747, 40, 38, 36, 35, 34, 113, 1920, 549, 52, + /* 860 */ 510, 389, 198, 473, 1263, 196, 2, 1747, 559, 1747, + /* 870 */ 159, 1808, 1580, 1593, 1917, 90, 1778, 580, 1780, 1781, + /* 880 */ 576, 1463, 571, 1385, 1462, 1854, 122, 1461, 659, 328, + /* 890 */ 1850, 1933, 1460, 1459, 200, 11, 10, 199, 1262, 1576, /* 900 */ 1911, 422, 1331, 1332, 417, 416, 415, 414, 413, 410, /* 910 */ 409, 408, 407, 406, 402, 401, 400, 399, 393, 392, /* 920 */ 391, 390, 1747, 387, 386, 1747, 120, 526, 1747, 526, - /* 930 */ 628, 141, 1576, 1747, 1747, 526, 277, 526, 403, 611, - /* 940 */ 404, 157, 1862, 1863, 654, 1867, 446, 1255, 1590, 1253, - /* 950 */ 275, 58, 11, 10, 57, 1593, 506, 1593, 131, 130, - /* 960 */ 608, 607, 606, 1593, 210, 1593, 1777, 1451, 1452, 572, - /* 970 */ 176, 426, 373, 42, 1258, 1259, 550, 1307, 1308, 1310, + /* 930 */ 628, 141, 1042, 1747, 1747, 526, 277, 526, 403, 611, + /* 940 */ 404, 157, 1862, 1863, 567, 1867, 446, 1255, 1590, 1253, + /* 950 */ 275, 58, 1451, 1452, 57, 1593, 506, 1593, 131, 130, + /* 960 */ 608, 607, 606, 1593, 1043, 1593, 1777, 210, 572, 604, + /* 970 */ 176, 426, 373, 41, 1258, 1259, 549, 1307, 1308, 1310, /* 980 */ 1311, 1312, 1313, 1314, 573, 569, 1322, 1323, 1325, 1326, - /* 990 */ 1327, 1328, 1330, 1333, 1795, 526, 1265, 61, 202, 526, - /* 1000 */ 526, 201, 554, 526, 122, 204, 1722, 1747, 203, 577, - /* 1010 */ 507, 511, 1309, 1584, 524, 1202, 218, 1469, 1764, 604, - /* 1020 */ 1550, 526, 254, 1593, 1761, 555, 465, 1593, 1593, 1761, - /* 1030 */ 1343, 1593, 525, 1388, 1808, 88, 526, 1777, 90, 1778, + /* 990 */ 1327, 1328, 1330, 1333, 1795, 526, 1266, 61, 202, 526, + /* 1000 */ 526, 201, 553, 526, 122, 204, 1722, 1747, 203, 577, + /* 1010 */ 507, 511, 1309, 1584, 524, 1202, 218, 1469, 1764, 1550, + /* 1020 */ 254, 526, 500, 1593, 1761, 554, 465, 1593, 1593, 1761, + /* 1030 */ 1265, 1593, 525, 1388, 1808, 88, 526, 1777, 90, 1778, /* 1040 */ 580, 1780, 1781, 576, 120, 571, 1503, 260, 1854, 1593, - /* 1050 */ 1757, 1763, 330, 1850, 154, 1757, 1763, 326, 222, 249, - /* 1060 */ 1862, 549, 571, 548, 1593, 1795, 1920, 571, 486, 1501, - /* 1070 */ 66, 65, 382, 578, 1881, 169, 537, 243, 1747, 159, - /* 1080 */ 577, 376, 1767, 1917, 125, 328, 327, 526, 128, 1042, - /* 1090 */ 129, 489, 1796, 50, 299, 1268, 235, 366, 344, 364, - /* 1100 */ 360, 356, 166, 351, 348, 1808, 1329, 1765, 1261, 90, - /* 1110 */ 1778, 580, 1780, 1781, 576, 1593, 571, 42, 1761, 1854, - /* 1120 */ 1769, 1043, 42, 330, 1850, 1933, 519, 346, 42, 1324, - /* 1130 */ 228, 584, 1102, 1777, 1873, 1410, 85, 162, 238, 128, - /* 1140 */ 1260, 129, 114, 128, 1757, 1763, 82, 500, 1492, 1633, - /* 1150 */ 551, 1884, 253, 256, 258, 248, 571, 3, 53, 1359, - /* 1160 */ 80, 1795, 5, 350, 1315, 1276, 353, 357, 310, 578, - /* 1170 */ 270, 1071, 1218, 1130, 1747, 311, 577, 405, 267, 535, - /* 1180 */ 1685, 1134, 171, 1141, 1139, 132, 412, 420, 419, 421, - /* 1190 */ 555, 425, 427, 1282, 1777, 1285, 1284, 428, 436, 439, - /* 1200 */ 179, 1808, 440, 181, 441, 284, 1778, 580, 1780, 1781, - /* 1210 */ 576, 1286, 571, 442, 184, 186, 444, 1283, 188, 70, - /* 1220 */ 445, 448, 1795, 191, 467, 469, 1583, 195, 1579, 197, - /* 1230 */ 578, 1920, 134, 135, 93, 1747, 302, 577, 1269, 268, - /* 1240 */ 1264, 208, 1581, 1577, 161, 136, 137, 1777, 1917, 211, - /* 1250 */ 1727, 555, 502, 501, 508, 505, 512, 534, 1281, 215, - /* 1260 */ 515, 1726, 1808, 520, 1695, 1272, 284, 1778, 580, 1780, - /* 1270 */ 1781, 576, 320, 571, 517, 1795, 569, 1322, 1323, 1325, - /* 1280 */ 1326, 1327, 1328, 578, 126, 322, 521, 127, 1747, 224, - /* 1290 */ 577, 522, 1920, 269, 226, 78, 1594, 530, 1885, 1777, - /* 1300 */ 538, 6, 233, 532, 533, 159, 237, 547, 329, 1917, - /* 1310 */ 531, 541, 1385, 528, 1895, 1808, 529, 247, 121, 91, - /* 1320 */ 1778, 580, 1780, 1781, 576, 1894, 571, 1795, 244, 1854, - /* 1330 */ 246, 1876, 148, 1853, 1850, 578, 1280, 564, 1870, 561, - /* 1340 */ 1747, 331, 577, 242, 19, 245, 1835, 582, 272, 1637, - /* 1350 */ 1566, 252, 1777, 655, 1916, 51, 147, 263, 1741, 558, - /* 1360 */ 1936, 656, 658, 285, 63, 255, 1777, 1808, 276, 565, - /* 1370 */ 257, 91, 1778, 580, 1780, 1781, 576, 1740, 571, 274, - /* 1380 */ 1795, 1854, 1739, 295, 294, 566, 1850, 64, 575, 1738, - /* 1390 */ 352, 1735, 354, 1747, 1795, 577, 355, 1246, 1247, 167, - /* 1400 */ 359, 1733, 578, 361, 362, 363, 1732, 1747, 365, 577, - /* 1410 */ 1731, 1730, 369, 367, 1729, 371, 1712, 168, 374, 1777, - /* 1420 */ 1808, 375, 1221, 1220, 292, 1778, 580, 1780, 1781, 576, - /* 1430 */ 574, 571, 568, 1826, 1808, 1706, 1705, 1777, 144, 1778, - /* 1440 */ 580, 1780, 1781, 576, 380, 571, 381, 1795, 1704, 1703, - /* 1450 */ 1190, 1678, 1677, 1676, 67, 578, 1675, 1674, 1673, 1672, - /* 1460 */ 1747, 1671, 577, 394, 395, 1795, 1670, 397, 1669, 1668, - /* 1470 */ 321, 1667, 1666, 578, 1665, 1664, 1663, 1662, 1747, 1661, - /* 1480 */ 577, 1660, 556, 1934, 1659, 1658, 1657, 1808, 1656, 1777, - /* 1490 */ 1655, 91, 1778, 580, 1780, 1781, 576, 124, 571, 1654, - /* 1500 */ 1653, 1854, 1652, 1651, 1650, 1808, 1851, 1649, 1648, 293, - /* 1510 */ 1778, 580, 1780, 1781, 576, 1192, 571, 1795, 1647, 1646, - /* 1520 */ 1522, 175, 527, 1520, 1488, 578, 152, 1006, 115, 177, - /* 1530 */ 1747, 1777, 577, 1487, 178, 1005, 116, 432, 434, 1720, - /* 1540 */ 1714, 1777, 1702, 185, 183, 1701, 1687, 1572, 1519, 1517, - /* 1550 */ 449, 451, 1515, 455, 1513, 459, 1035, 1808, 450, 1795, - /* 1560 */ 453, 293, 1778, 580, 1780, 1781, 576, 578, 571, 1795, - /* 1570 */ 457, 1511, 1747, 454, 577, 458, 461, 578, 462, 1500, - /* 1580 */ 1499, 1484, 1747, 463, 577, 1574, 1145, 1144, 1573, 194, - /* 1590 */ 1070, 1777, 1069, 1068, 49, 1067, 626, 1509, 1064, 1808, - /* 1600 */ 316, 1063, 628, 288, 1778, 580, 1780, 1781, 576, 1808, - /* 1610 */ 571, 662, 1062, 144, 1778, 580, 1780, 1781, 576, 1795, - /* 1620 */ 571, 1061, 1504, 317, 334, 266, 490, 578, 1502, 318, - /* 1630 */ 1483, 487, 1747, 1777, 577, 492, 1482, 494, 1481, 151, - /* 1640 */ 496, 546, 1719, 92, 652, 648, 644, 640, 264, 1713, - /* 1650 */ 1228, 1777, 1700, 138, 503, 1698, 1699, 1697, 1935, 1808, - /* 1660 */ 1696, 1795, 54, 293, 1778, 580, 1780, 1781, 576, 575, - /* 1670 */ 571, 212, 217, 15, 1747, 87, 577, 1694, 229, 1795, - /* 1680 */ 223, 504, 319, 76, 336, 225, 509, 578, 518, 1238, - /* 1690 */ 1686, 227, 1747, 77, 577, 16, 230, 82, 1425, 1437, - /* 1700 */ 24, 1808, 42, 1777, 232, 292, 1778, 580, 1780, 1781, - /* 1710 */ 576, 523, 571, 236, 1827, 1270, 234, 48, 1407, 1808, - /* 1720 */ 1409, 145, 240, 293, 1778, 580, 1780, 1781, 576, 239, - /* 1730 */ 571, 1795, 25, 241, 1767, 193, 338, 26, 1402, 578, - /* 1740 */ 47, 250, 81, 216, 1747, 17, 577, 1382, 1381, 146, - /* 1750 */ 1766, 149, 1442, 1777, 464, 460, 456, 452, 192, 46, - /* 1760 */ 18, 1431, 13, 1777, 1226, 1436, 209, 332, 1441, 1440, - /* 1770 */ 333, 1808, 10, 1811, 20, 293, 1778, 580, 1780, 1781, - /* 1780 */ 576, 1795, 571, 1319, 570, 72, 1344, 32, 190, 578, - /* 1790 */ 150, 1795, 1317, 1316, 1747, 12, 577, 21, 163, 578, - /* 1800 */ 1300, 581, 579, 22, 1747, 1131, 577, 583, 339, 585, - /* 1810 */ 587, 1128, 588, 1108, 1125, 590, 593, 596, 1123, 1777, - /* 1820 */ 591, 1808, 1122, 1119, 594, 278, 1778, 580, 1780, 1781, - /* 1830 */ 576, 1808, 571, 1117, 597, 279, 1778, 580, 1780, 1781, - /* 1840 */ 576, 603, 571, 83, 84, 1140, 60, 1795, 261, 1121, - /* 1850 */ 189, 182, 1136, 187, 1120, 578, 1033, 443, 612, 1058, - /* 1860 */ 1747, 1777, 577, 1077, 262, 615, 1051, 1056, 1055, 1054, - /* 1870 */ 1053, 1777, 1052, 1050, 1049, 1074, 180, 1072, 1046, 1516, - /* 1880 */ 1045, 1044, 1041, 1777, 1040, 1039, 1038, 1808, 637, 1795, - /* 1890 */ 1514, 280, 1778, 580, 1780, 1781, 576, 578, 571, 1795, - /* 1900 */ 639, 638, 1747, 641, 577, 642, 643, 578, 1512, 645, - /* 1910 */ 646, 1795, 1747, 1510, 577, 647, 649, 650, 1498, 578, - /* 1920 */ 653, 651, 996, 1480, 1747, 1777, 577, 265, 657, 1808, - /* 1930 */ 1455, 1256, 273, 287, 1778, 580, 1780, 1781, 576, 1808, - /* 1940 */ 571, 660, 1777, 289, 1778, 580, 1780, 1781, 576, 661, - /* 1950 */ 571, 1808, 1455, 1795, 1455, 281, 1778, 580, 1780, 1781, - /* 1960 */ 576, 578, 571, 1455, 1455, 1455, 1747, 1455, 577, 1455, - /* 1970 */ 1795, 1455, 1455, 1455, 1455, 1455, 1455, 1455, 578, 1455, - /* 1980 */ 1455, 1455, 1455, 1747, 1777, 577, 1455, 1455, 1455, 1455, - /* 1990 */ 1455, 1455, 1455, 1808, 1777, 1455, 1455, 290, 1778, 580, - /* 2000 */ 1780, 1781, 576, 1455, 571, 1455, 1455, 1455, 1455, 1455, - /* 2010 */ 1808, 1455, 1795, 1455, 282, 1778, 580, 1780, 1781, 576, - /* 2020 */ 578, 571, 1795, 1455, 1455, 1747, 1455, 577, 1455, 1455, - /* 2030 */ 578, 1455, 1455, 1455, 1455, 1747, 1777, 577, 1455, 1455, - /* 2040 */ 1455, 1455, 1455, 1455, 1455, 1455, 1777, 1455, 1455, 1455, - /* 2050 */ 1455, 1455, 1808, 1455, 1455, 1455, 291, 1778, 580, 1780, - /* 2060 */ 1781, 576, 1808, 571, 1795, 1455, 283, 1778, 580, 1780, - /* 2070 */ 1781, 576, 578, 571, 1795, 1455, 1455, 1747, 1455, 577, - /* 2080 */ 1455, 1455, 578, 1455, 1455, 1455, 1455, 1747, 1777, 577, - /* 2090 */ 1455, 1455, 1455, 1455, 1455, 1455, 1455, 1455, 1777, 1455, - /* 2100 */ 1455, 1455, 1455, 1455, 1808, 1455, 1455, 1455, 296, 1778, - /* 2110 */ 580, 1780, 1781, 576, 1808, 571, 1795, 1455, 297, 1778, - /* 2120 */ 580, 1780, 1781, 576, 578, 571, 1795, 1455, 1455, 1747, - /* 2130 */ 1455, 577, 1455, 1455, 578, 1455, 1455, 1455, 1455, 1747, - /* 2140 */ 1455, 577, 1455, 1455, 1455, 1455, 1455, 1455, 1455, 1455, - /* 2150 */ 1777, 1455, 1455, 1455, 1455, 1455, 1808, 1455, 1455, 1455, - /* 2160 */ 1789, 1778, 580, 1780, 1781, 576, 1808, 571, 1455, 1777, - /* 2170 */ 1788, 1778, 580, 1780, 1781, 576, 1455, 571, 1795, 1455, - /* 2180 */ 1455, 1455, 1455, 1455, 1455, 1455, 578, 1455, 1455, 1455, - /* 2190 */ 1455, 1747, 1777, 577, 1455, 1455, 1455, 1795, 1455, 1455, - /* 2200 */ 1455, 1455, 1455, 1455, 1455, 578, 1455, 1455, 1455, 1455, - /* 2210 */ 1747, 1777, 577, 1455, 1455, 1455, 1455, 1455, 1808, 1455, - /* 2220 */ 1795, 1455, 1787, 1778, 580, 1780, 1781, 576, 578, 571, - /* 2230 */ 1455, 1455, 1455, 1747, 1455, 577, 1455, 1808, 1455, 1795, - /* 2240 */ 1455, 308, 1778, 580, 1780, 1781, 576, 578, 571, 1455, - /* 2250 */ 1455, 1455, 1747, 1455, 577, 1455, 1455, 1455, 1455, 1455, - /* 2260 */ 1808, 1455, 1455, 1455, 307, 1778, 580, 1780, 1781, 576, - /* 2270 */ 1777, 571, 1455, 1455, 1455, 1455, 1455, 1455, 1455, 1808, - /* 2280 */ 1777, 1455, 1455, 309, 1778, 580, 1780, 1781, 576, 1455, - /* 2290 */ 571, 1455, 1455, 1455, 1455, 1455, 1455, 1455, 1795, 1455, - /* 2300 */ 1455, 1455, 1455, 1455, 1455, 1455, 578, 1455, 1795, 1455, - /* 2310 */ 1455, 1747, 1455, 577, 1455, 1455, 578, 1455, 1455, 1455, - /* 2320 */ 1455, 1747, 1455, 577, 1455, 1455, 1455, 1455, 1455, 1455, - /* 2330 */ 1455, 1455, 1455, 1455, 1455, 1455, 1455, 1455, 1808, 1455, - /* 2340 */ 1455, 1455, 306, 1778, 580, 1780, 1781, 576, 1808, 571, - /* 2350 */ 1455, 1455, 286, 1778, 580, 1780, 1781, 576, 1455, 571, + /* 1050 */ 1757, 1763, 328, 1850, 154, 1757, 1763, 326, 222, 249, + /* 1060 */ 1862, 548, 571, 547, 1593, 1795, 1920, 571, 486, 1501, + /* 1070 */ 66, 65, 382, 578, 1881, 169, 1343, 85, 1747, 159, + /* 1080 */ 577, 376, 536, 1917, 1767, 334, 333, 82, 125, 243, + /* 1090 */ 128, 489, 1796, 129, 299, 1268, 50, 366, 1492, 364, + /* 1100 */ 360, 356, 166, 351, 348, 1808, 1329, 1764, 1261, 90, + /* 1110 */ 1778, 580, 1780, 1781, 576, 1765, 571, 235, 1761, 1854, + /* 1120 */ 526, 1633, 1769, 328, 1850, 1933, 1761, 1497, 41, 1324, + /* 1130 */ 519, 344, 228, 1777, 1873, 1102, 346, 162, 1410, 50, + /* 1140 */ 1260, 41, 1884, 584, 1757, 1763, 332, 128, 1593, 129, + /* 1150 */ 114, 128, 1757, 1763, 550, 248, 571, 253, 256, 238, + /* 1160 */ 258, 1795, 80, 53, 571, 5, 3, 350, 1276, 578, + /* 1170 */ 270, 353, 357, 310, 1747, 654, 577, 1071, 311, 566, + /* 1180 */ 1218, 1359, 267, 1315, 405, 1130, 1685, 171, 412, 1134, + /* 1190 */ 554, 1141, 1139, 132, 1777, 420, 419, 425, 421, 427, + /* 1200 */ 428, 1808, 1282, 436, 1285, 284, 1778, 580, 1780, 1781, + /* 1210 */ 576, 439, 571, 1284, 179, 440, 181, 441, 1286, 184, + /* 1220 */ 442, 444, 1795, 186, 1283, 188, 445, 70, 448, 191, + /* 1230 */ 578, 1920, 268, 467, 469, 1747, 1583, 577, 1269, 302, + /* 1240 */ 1264, 93, 195, 501, 161, 208, 1579, 1777, 1917, 197, + /* 1250 */ 134, 554, 135, 1581, 1577, 136, 137, 1727, 211, 502, + /* 1260 */ 508, 512, 1808, 534, 215, 1272, 284, 1778, 580, 1780, + /* 1270 */ 1781, 576, 505, 571, 226, 1795, 569, 1322, 1323, 1325, + /* 1280 */ 1326, 1327, 1328, 578, 126, 515, 1726, 1695, 1747, 520, + /* 1290 */ 577, 127, 1920, 521, 320, 224, 517, 322, 522, 1777, + /* 1300 */ 269, 78, 1594, 1281, 530, 159, 532, 1885, 537, 1917, + /* 1310 */ 233, 533, 237, 1895, 1894, 1808, 327, 6, 540, 91, + /* 1320 */ 1778, 580, 1780, 1781, 576, 546, 571, 1795, 531, 1854, + /* 1330 */ 1876, 529, 528, 1853, 1850, 578, 1385, 247, 148, 1280, + /* 1340 */ 1747, 563, 577, 246, 244, 121, 242, 48, 1870, 329, + /* 1350 */ 582, 272, 1777, 245, 560, 1637, 1566, 263, 655, 656, + /* 1360 */ 658, 51, 285, 147, 1741, 1835, 1777, 1808, 295, 276, + /* 1370 */ 252, 91, 1778, 580, 1780, 1781, 576, 63, 571, 274, + /* 1380 */ 1795, 1854, 1916, 557, 1740, 565, 1850, 564, 575, 294, + /* 1390 */ 1739, 1936, 255, 1747, 1795, 577, 257, 64, 1738, 352, + /* 1400 */ 1735, 354, 578, 355, 1246, 1247, 167, 1747, 359, 577, + /* 1410 */ 1733, 361, 362, 363, 1732, 365, 1731, 367, 1730, 1777, + /* 1420 */ 1808, 369, 1729, 371, 292, 1778, 580, 1780, 1781, 576, + /* 1430 */ 574, 571, 568, 1826, 1808, 1712, 168, 1777, 144, 1778, + /* 1440 */ 580, 1780, 1781, 576, 375, 571, 374, 1795, 1221, 1706, + /* 1450 */ 1220, 1705, 380, 381, 1704, 578, 1703, 1190, 1678, 1677, + /* 1460 */ 1747, 1676, 577, 1675, 67, 1795, 1674, 1673, 1672, 1671, + /* 1470 */ 321, 394, 395, 578, 1670, 397, 1669, 1668, 1747, 1667, + /* 1480 */ 577, 1666, 555, 1934, 1665, 1664, 1663, 1808, 1662, 1777, + /* 1490 */ 1661, 91, 1778, 580, 1780, 1781, 576, 1660, 571, 1659, + /* 1500 */ 1658, 1854, 1657, 1656, 124, 1808, 1851, 1655, 1654, 293, + /* 1510 */ 1778, 580, 1780, 1781, 576, 1653, 571, 1795, 1192, 1649, + /* 1520 */ 1648, 1647, 527, 1652, 1651, 578, 1650, 1646, 1522, 175, + /* 1530 */ 1747, 1777, 577, 1520, 1488, 177, 1006, 115, 152, 1005, + /* 1540 */ 1487, 1777, 1720, 1714, 1702, 178, 1701, 432, 116, 183, + /* 1550 */ 434, 185, 1687, 1572, 1519, 1517, 451, 1808, 450, 1795, + /* 1560 */ 1515, 293, 1778, 580, 1780, 1781, 576, 578, 571, 1795, + /* 1570 */ 449, 453, 1747, 1035, 577, 1513, 454, 578, 455, 458, + /* 1580 */ 457, 1511, 1747, 461, 577, 462, 459, 1500, 463, 1499, + /* 1590 */ 1484, 1777, 1574, 1145, 1144, 1573, 1070, 1069, 1068, 1808, + /* 1600 */ 1067, 1064, 626, 288, 1778, 580, 1780, 1781, 576, 1808, + /* 1610 */ 571, 628, 1063, 144, 1778, 580, 1780, 1781, 576, 1795, + /* 1620 */ 571, 1062, 194, 49, 1509, 316, 1061, 575, 1504, 317, + /* 1630 */ 1502, 487, 1747, 318, 577, 1483, 490, 492, 1482, 494, + /* 1640 */ 1481, 545, 496, 1777, 1719, 92, 1713, 1228, 503, 138, + /* 1650 */ 1700, 1698, 1699, 1697, 1696, 15, 1694, 223, 1935, 1808, + /* 1660 */ 1777, 217, 212, 292, 1778, 580, 1780, 1781, 576, 1238, + /* 1670 */ 571, 1795, 1827, 1686, 82, 225, 336, 76, 77, 578, + /* 1680 */ 230, 16, 41, 23, 1747, 47, 577, 241, 1795, 54, + /* 1690 */ 1425, 234, 232, 338, 1407, 509, 578, 236, 240, 227, + /* 1700 */ 1409, 1747, 504, 577, 1402, 145, 239, 518, 319, 1767, + /* 1710 */ 25, 1808, 1382, 662, 1381, 293, 1778, 580, 1780, 1781, + /* 1720 */ 576, 24, 571, 250, 46, 1766, 18, 266, 1808, 81, + /* 1730 */ 149, 1437, 293, 1778, 580, 1780, 1781, 576, 45, 571, + /* 1740 */ 1436, 151, 17, 1442, 1431, 193, 652, 648, 644, 640, + /* 1750 */ 264, 1777, 330, 1441, 1440, 331, 10, 1270, 19, 146, + /* 1760 */ 1811, 1300, 1344, 1777, 464, 460, 456, 452, 192, 150, + /* 1770 */ 1319, 570, 163, 1317, 31, 1777, 1316, 87, 579, 1795, + /* 1780 */ 229, 12, 20, 21, 581, 1131, 583, 578, 339, 1128, + /* 1790 */ 585, 1795, 1747, 587, 577, 72, 1125, 13, 190, 578, + /* 1800 */ 588, 590, 593, 1795, 1747, 591, 577, 1119, 594, 596, + /* 1810 */ 1117, 578, 597, 523, 1108, 1123, 1747, 1122, 577, 1808, + /* 1820 */ 1140, 1121, 1120, 278, 1778, 580, 1780, 1781, 576, 83, + /* 1830 */ 571, 1808, 603, 84, 60, 279, 1778, 580, 1780, 1781, + /* 1840 */ 576, 261, 571, 1808, 1136, 216, 612, 280, 1778, 580, + /* 1850 */ 1780, 1781, 576, 1058, 571, 1033, 1077, 1777, 615, 262, + /* 1860 */ 189, 182, 1056, 187, 1055, 1054, 1226, 443, 209, 1053, + /* 1870 */ 1052, 1777, 1051, 1050, 1049, 1074, 1516, 1072, 1046, 1045, + /* 1880 */ 1044, 1041, 1040, 1039, 1038, 1795, 180, 637, 1514, 1512, + /* 1890 */ 639, 641, 643, 578, 645, 1510, 649, 647, 1747, 1795, + /* 1900 */ 577, 638, 642, 651, 646, 1498, 653, 578, 996, 650, + /* 1910 */ 1480, 657, 1747, 265, 577, 1256, 273, 660, 661, 1455, + /* 1920 */ 1455, 1777, 1455, 1455, 1455, 1808, 1455, 1455, 1455, 287, + /* 1930 */ 1778, 580, 1780, 1781, 576, 1455, 571, 1777, 1455, 1808, + /* 1940 */ 1455, 1455, 1455, 289, 1778, 580, 1780, 1781, 576, 1795, + /* 1950 */ 571, 1455, 1455, 1455, 1455, 1455, 1455, 578, 1455, 1455, + /* 1960 */ 1455, 1455, 1747, 1455, 577, 1795, 1455, 1455, 1455, 1455, + /* 1970 */ 1455, 1455, 1455, 578, 1455, 1455, 1455, 1455, 1747, 1455, + /* 1980 */ 577, 1455, 1455, 1455, 1455, 1455, 1455, 1455, 1455, 1808, + /* 1990 */ 1777, 1455, 1455, 281, 1778, 580, 1780, 1781, 576, 1455, + /* 2000 */ 571, 1455, 1455, 1455, 1777, 1808, 1455, 1455, 1455, 290, + /* 2010 */ 1778, 580, 1780, 1781, 576, 1455, 571, 1455, 1795, 1455, + /* 2020 */ 1455, 1455, 1455, 1455, 1455, 1455, 578, 1455, 1455, 1455, + /* 2030 */ 1455, 1747, 1795, 577, 1455, 1455, 1455, 1455, 1455, 1455, + /* 2040 */ 578, 1455, 1455, 1455, 1455, 1747, 1455, 577, 1455, 1455, + /* 2050 */ 1455, 1455, 1455, 1455, 1455, 1455, 1777, 1455, 1808, 1455, + /* 2060 */ 1455, 1455, 282, 1778, 580, 1780, 1781, 576, 1455, 571, + /* 2070 */ 1455, 1455, 1808, 1455, 1777, 1455, 291, 1778, 580, 1780, + /* 2080 */ 1781, 576, 1455, 571, 1795, 1455, 1455, 1455, 1455, 1455, + /* 2090 */ 1455, 1455, 578, 1455, 1455, 1455, 1455, 1747, 1777, 577, + /* 2100 */ 1455, 1455, 1795, 1455, 1455, 1455, 1455, 1455, 1455, 1455, + /* 2110 */ 578, 1455, 1455, 1455, 1455, 1747, 1455, 577, 1455, 1455, + /* 2120 */ 1455, 1455, 1455, 1455, 1808, 1455, 1795, 1455, 283, 1778, + /* 2130 */ 580, 1780, 1781, 576, 578, 571, 1455, 1455, 1455, 1747, + /* 2140 */ 1455, 577, 1808, 1455, 1455, 1455, 296, 1778, 580, 1780, + /* 2150 */ 1781, 576, 1455, 571, 1455, 1455, 1777, 1455, 1455, 1455, + /* 2160 */ 1455, 1455, 1455, 1455, 1455, 1455, 1808, 1455, 1455, 1455, + /* 2170 */ 297, 1778, 580, 1780, 1781, 576, 1455, 571, 1455, 1777, + /* 2180 */ 1455, 1455, 1455, 1455, 1795, 1455, 1455, 1455, 1455, 1455, + /* 2190 */ 1455, 1455, 578, 1455, 1455, 1455, 1455, 1747, 1455, 577, + /* 2200 */ 1455, 1455, 1455, 1455, 1455, 1455, 1455, 1795, 1455, 1455, + /* 2210 */ 1455, 1455, 1455, 1455, 1455, 578, 1455, 1455, 1455, 1455, + /* 2220 */ 1747, 1455, 577, 1455, 1808, 1455, 1455, 1455, 1789, 1778, + /* 2230 */ 580, 1780, 1781, 576, 1455, 571, 1777, 1455, 1455, 1455, + /* 2240 */ 1455, 1455, 1455, 1455, 1455, 1455, 1455, 1808, 1777, 1455, + /* 2250 */ 1455, 1788, 1778, 580, 1780, 1781, 576, 1455, 571, 1455, + /* 2260 */ 1455, 1455, 1777, 1455, 1795, 1455, 1455, 1455, 1455, 1455, + /* 2270 */ 1455, 1455, 578, 1455, 1455, 1455, 1795, 1747, 1455, 577, + /* 2280 */ 1455, 1455, 1455, 1455, 578, 1455, 1455, 1455, 1455, 1747, + /* 2290 */ 1795, 577, 1455, 1455, 1455, 1455, 1455, 1455, 578, 1455, + /* 2300 */ 1455, 1455, 1455, 1747, 1808, 577, 1455, 1455, 1787, 1778, + /* 2310 */ 580, 1780, 1781, 576, 1777, 571, 1808, 1455, 1455, 1455, + /* 2320 */ 308, 1778, 580, 1780, 1781, 576, 1455, 571, 1455, 1455, + /* 2330 */ 1808, 1455, 1777, 1455, 307, 1778, 580, 1780, 1781, 576, + /* 2340 */ 1455, 571, 1795, 1455, 1455, 1455, 1455, 1455, 1455, 1455, + /* 2350 */ 578, 1455, 1455, 1455, 1455, 1747, 1777, 577, 1455, 1455, + /* 2360 */ 1795, 1455, 1455, 1455, 1455, 1455, 1455, 1455, 578, 1455, + /* 2370 */ 1455, 1455, 1455, 1747, 1455, 577, 1455, 1455, 1455, 1455, + /* 2380 */ 1455, 1455, 1808, 1455, 1795, 1455, 309, 1778, 580, 1780, + /* 2390 */ 1781, 576, 578, 571, 1455, 1455, 1455, 1747, 1455, 577, + /* 2400 */ 1808, 1455, 1455, 1455, 306, 1778, 580, 1780, 1781, 576, + /* 2410 */ 1455, 571, 1455, 1455, 1455, 1455, 1455, 1455, 1455, 1455, + /* 2420 */ 1455, 1455, 1455, 1455, 1808, 1455, 1455, 1455, 286, 1778, + /* 2430 */ 580, 1780, 1781, 576, 1455, 571, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 261, 355, 263, 264, 261, 269, 263, 264, 300, 4, @@ -532,7 +540,7 @@ static const YYCODETYPE yy_lookahead[] = { /* 730 */ 205, 206, 207, 208, 209, 210, 211, 355, 241, 298, /* 740 */ 165, 310, 167, 312, 273, 2, 275, 165, 85, 167, /* 750 */ 368, 8, 9, 300, 372, 12, 13, 14, 15, 16, - /* 760 */ 314, 222, 223, 35, 311, 8, 9, 192, 193, 12, + /* 760 */ 314, 222, 223, 67, 311, 8, 9, 192, 193, 12, /* 770 */ 13, 14, 15, 16, 192, 193, 56, 195, 196, 197, /* 780 */ 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, /* 790 */ 208, 209, 210, 211, 12, 13, 18, 286, 20, 257, @@ -543,165 +551,173 @@ static const YYCODETYPE yy_lookahead[] = { /* 840 */ 298, 293, 85, 0, 298, 265, 300, 8, 9, 67, /* 850 */ 298, 12, 13, 14, 15, 16, 276, 355, 265, 152, /* 860 */ 153, 83, 88, 283, 35, 91, 84, 298, 243, 298, - /* 870 */ 368, 325, 67, 293, 372, 329, 330, 331, 332, 333, + /* 870 */ 368, 325, 286, 293, 372, 329, 330, 331, 332, 333, /* 880 */ 334, 257, 336, 223, 257, 339, 293, 257, 106, 343, - /* 890 */ 344, 345, 257, 257, 88, 167, 0, 91, 60, 286, + /* 890 */ 344, 345, 257, 257, 88, 1, 2, 91, 35, 286, /* 900 */ 354, 123, 120, 121, 126, 127, 128, 129, 130, 131, /* 910 */ 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, /* 920 */ 142, 143, 298, 145, 146, 298, 333, 265, 298, 265, - /* 930 */ 43, 18, 286, 298, 298, 265, 23, 265, 276, 96, - /* 940 */ 276, 348, 349, 350, 48, 352, 276, 165, 276, 167, - /* 950 */ 37, 38, 1, 2, 41, 293, 318, 293, 115, 116, - /* 960 */ 117, 118, 119, 293, 286, 293, 257, 120, 121, 286, + /* 930 */ 43, 18, 35, 298, 298, 265, 23, 265, 276, 96, + /* 940 */ 276, 348, 349, 350, 60, 352, 276, 165, 276, 167, + /* 950 */ 37, 38, 120, 121, 41, 293, 318, 293, 115, 116, + /* 960 */ 117, 118, 119, 293, 67, 293, 257, 286, 286, 286, /* 970 */ 57, 58, 85, 43, 192, 193, 265, 195, 196, 197, /* 980 */ 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, /* 990 */ 208, 209, 210, 211, 285, 265, 167, 84, 88, 265, /* 1000 */ 265, 91, 293, 265, 293, 88, 276, 298, 91, 300, - /* 1010 */ 276, 276, 196, 287, 276, 85, 43, 258, 287, 286, - /* 1020 */ 274, 265, 375, 293, 298, 314, 266, 293, 293, 298, - /* 1030 */ 192, 293, 276, 225, 325, 122, 265, 257, 329, 330, + /* 1010 */ 276, 276, 196, 287, 276, 85, 43, 258, 287, 274, + /* 1020 */ 375, 265, 321, 293, 298, 314, 266, 293, 293, 298, + /* 1030 */ 167, 293, 276, 225, 325, 122, 265, 257, 329, 330, /* 1040 */ 331, 332, 333, 334, 333, 336, 0, 276, 339, 293, /* 1050 */ 324, 325, 343, 344, 345, 324, 325, 326, 85, 348, /* 1060 */ 349, 350, 336, 352, 293, 285, 355, 336, 22, 0, - /* 1070 */ 157, 158, 159, 293, 365, 162, 366, 362, 298, 368, - /* 1080 */ 300, 168, 46, 372, 43, 12, 13, 265, 43, 35, - /* 1090 */ 43, 22, 285, 43, 181, 22, 43, 184, 276, 186, + /* 1070 */ 157, 158, 159, 293, 365, 162, 192, 84, 298, 368, + /* 1080 */ 300, 168, 366, 372, 46, 12, 13, 94, 43, 362, + /* 1090 */ 43, 22, 285, 43, 181, 22, 43, 184, 264, 186, /* 1100 */ 187, 188, 189, 190, 191, 325, 33, 287, 35, 329, - /* 1110 */ 330, 331, 332, 333, 334, 293, 336, 43, 298, 339, - /* 1120 */ 84, 67, 43, 343, 344, 345, 85, 266, 43, 56, - /* 1130 */ 85, 43, 85, 257, 354, 85, 84, 224, 85, 43, - /* 1140 */ 67, 43, 43, 43, 324, 325, 94, 321, 264, 297, - /* 1150 */ 353, 328, 369, 369, 369, 346, 336, 356, 289, 85, - /* 1160 */ 84, 285, 226, 323, 85, 20, 265, 47, 322, 293, - /* 1170 */ 85, 35, 163, 85, 298, 271, 300, 265, 316, 106, - /* 1180 */ 265, 85, 42, 85, 85, 85, 305, 147, 303, 303, - /* 1190 */ 314, 265, 265, 20, 257, 20, 20, 259, 259, 320, - /* 1200 */ 269, 325, 300, 269, 313, 329, 330, 331, 332, 333, - /* 1210 */ 334, 20, 336, 315, 269, 269, 313, 20, 269, 269, - /* 1220 */ 306, 265, 285, 269, 259, 285, 285, 285, 285, 285, - /* 1230 */ 293, 355, 285, 285, 265, 298, 259, 300, 165, 320, - /* 1240 */ 167, 267, 285, 285, 368, 285, 285, 257, 372, 267, - /* 1250 */ 298, 314, 319, 173, 265, 300, 265, 231, 20, 267, - /* 1260 */ 298, 298, 325, 149, 298, 192, 329, 330, 331, 332, - /* 1270 */ 333, 334, 313, 336, 298, 285, 203, 204, 205, 206, - /* 1280 */ 207, 208, 209, 293, 309, 298, 307, 309, 298, 293, - /* 1290 */ 300, 306, 355, 281, 267, 267, 293, 298, 328, 257, - /* 1300 */ 232, 238, 309, 298, 298, 368, 309, 156, 298, 372, - /* 1310 */ 240, 298, 223, 227, 361, 325, 239, 323, 293, 329, - /* 1320 */ 330, 331, 332, 333, 334, 361, 336, 285, 360, 339, - /* 1330 */ 358, 364, 361, 343, 344, 293, 20, 244, 327, 242, - /* 1340 */ 298, 247, 300, 363, 84, 359, 342, 289, 265, 298, - /* 1350 */ 275, 370, 257, 36, 371, 317, 312, 267, 0, 371, - /* 1360 */ 376, 260, 259, 279, 175, 370, 257, 325, 255, 371, - /* 1370 */ 370, 329, 330, 331, 332, 333, 334, 0, 336, 268, - /* 1380 */ 285, 339, 0, 279, 279, 343, 344, 42, 293, 0, - /* 1390 */ 75, 0, 35, 298, 285, 300, 185, 35, 35, 35, - /* 1400 */ 185, 0, 293, 35, 35, 185, 0, 298, 185, 300, - /* 1410 */ 0, 0, 22, 35, 0, 35, 0, 84, 170, 257, - /* 1420 */ 325, 169, 167, 165, 329, 330, 331, 332, 333, 334, - /* 1430 */ 335, 336, 337, 338, 325, 0, 0, 257, 329, 330, - /* 1440 */ 331, 332, 333, 334, 161, 336, 160, 285, 0, 0, - /* 1450 */ 46, 0, 0, 0, 144, 293, 0, 0, 0, 0, - /* 1460 */ 298, 0, 300, 139, 35, 285, 0, 139, 0, 0, - /* 1470 */ 290, 0, 0, 293, 0, 0, 0, 0, 298, 0, + /* 1110 */ 330, 331, 332, 333, 334, 287, 336, 43, 298, 339, + /* 1120 */ 265, 297, 84, 343, 344, 345, 298, 0, 43, 56, + /* 1130 */ 85, 276, 85, 257, 354, 85, 266, 224, 85, 43, + /* 1140 */ 67, 43, 328, 43, 324, 325, 326, 43, 293, 43, + /* 1150 */ 43, 43, 324, 325, 353, 346, 336, 369, 369, 85, + /* 1160 */ 369, 285, 84, 289, 336, 226, 356, 323, 20, 293, + /* 1170 */ 85, 265, 47, 322, 298, 48, 300, 35, 271, 106, + /* 1180 */ 163, 85, 316, 85, 265, 85, 265, 42, 305, 85, + /* 1190 */ 314, 85, 85, 85, 257, 147, 303, 265, 303, 265, + /* 1200 */ 259, 325, 20, 259, 20, 329, 330, 331, 332, 333, + /* 1210 */ 334, 320, 336, 20, 269, 300, 269, 313, 20, 269, + /* 1220 */ 315, 313, 285, 269, 20, 269, 306, 269, 265, 269, + /* 1230 */ 293, 355, 320, 259, 285, 298, 285, 300, 165, 259, + /* 1240 */ 167, 265, 285, 173, 368, 267, 285, 257, 372, 285, + /* 1250 */ 285, 314, 285, 285, 285, 285, 285, 298, 267, 319, + /* 1260 */ 265, 265, 325, 231, 267, 192, 329, 330, 331, 332, + /* 1270 */ 333, 334, 300, 336, 267, 285, 203, 204, 205, 206, + /* 1280 */ 207, 208, 209, 293, 309, 298, 298, 298, 298, 149, + /* 1290 */ 300, 309, 355, 307, 313, 293, 298, 298, 306, 257, + /* 1300 */ 281, 267, 293, 20, 298, 368, 298, 328, 232, 372, + /* 1310 */ 309, 298, 309, 361, 361, 325, 298, 238, 298, 329, + /* 1320 */ 330, 331, 332, 333, 334, 156, 336, 285, 240, 339, + /* 1330 */ 364, 239, 227, 343, 344, 293, 223, 323, 361, 20, + /* 1340 */ 298, 244, 300, 358, 360, 293, 363, 84, 327, 247, + /* 1350 */ 289, 265, 257, 359, 242, 298, 275, 267, 36, 260, + /* 1360 */ 259, 317, 279, 312, 0, 342, 257, 325, 279, 255, + /* 1370 */ 370, 329, 330, 331, 332, 333, 334, 175, 336, 268, + /* 1380 */ 285, 339, 371, 371, 0, 343, 344, 371, 293, 279, + /* 1390 */ 0, 376, 370, 298, 285, 300, 370, 42, 0, 75, + /* 1400 */ 0, 35, 293, 185, 35, 35, 35, 298, 185, 300, + /* 1410 */ 0, 35, 35, 185, 0, 185, 0, 35, 0, 257, + /* 1420 */ 325, 22, 0, 35, 329, 330, 331, 332, 333, 334, + /* 1430 */ 335, 336, 337, 338, 325, 0, 84, 257, 329, 330, + /* 1440 */ 331, 332, 333, 334, 169, 336, 170, 285, 167, 0, + /* 1450 */ 165, 0, 161, 160, 0, 293, 0, 46, 0, 0, + /* 1460 */ 298, 0, 300, 0, 144, 285, 0, 0, 0, 0, + /* 1470 */ 290, 139, 35, 293, 0, 139, 0, 0, 298, 0, /* 1480 */ 300, 0, 373, 374, 0, 0, 0, 325, 0, 257, - /* 1490 */ 0, 329, 330, 331, 332, 333, 334, 42, 336, 0, - /* 1500 */ 0, 339, 0, 0, 0, 325, 344, 0, 0, 329, - /* 1510 */ 330, 331, 332, 333, 334, 22, 336, 285, 0, 0, - /* 1520 */ 0, 56, 290, 0, 0, 293, 43, 14, 39, 42, - /* 1530 */ 298, 257, 300, 0, 40, 14, 39, 46, 46, 0, - /* 1540 */ 0, 257, 0, 156, 39, 0, 0, 0, 0, 0, - /* 1550 */ 35, 39, 0, 39, 0, 39, 61, 325, 47, 285, - /* 1560 */ 35, 329, 330, 331, 332, 333, 334, 293, 336, 285, - /* 1570 */ 35, 0, 298, 47, 300, 47, 35, 293, 47, 0, - /* 1580 */ 0, 0, 298, 39, 300, 0, 35, 22, 0, 91, - /* 1590 */ 35, 257, 35, 35, 93, 35, 43, 0, 35, 325, - /* 1600 */ 22, 35, 43, 329, 330, 331, 332, 333, 334, 325, - /* 1610 */ 336, 19, 35, 329, 330, 331, 332, 333, 334, 285, - /* 1620 */ 336, 35, 0, 22, 290, 33, 35, 293, 0, 22, - /* 1630 */ 0, 49, 298, 257, 300, 35, 0, 35, 0, 47, - /* 1640 */ 22, 367, 0, 20, 52, 53, 54, 55, 56, 0, - /* 1650 */ 35, 257, 0, 171, 22, 0, 0, 0, 374, 325, - /* 1660 */ 0, 285, 152, 329, 330, 331, 332, 333, 334, 293, - /* 1670 */ 336, 149, 85, 84, 298, 83, 300, 0, 86, 285, - /* 1680 */ 84, 152, 152, 84, 290, 39, 154, 293, 150, 180, - /* 1690 */ 0, 148, 298, 84, 300, 228, 46, 94, 85, 35, - /* 1700 */ 84, 325, 43, 257, 84, 329, 330, 331, 332, 333, - /* 1710 */ 334, 119, 336, 84, 338, 22, 85, 43, 85, 325, - /* 1720 */ 85, 84, 43, 329, 330, 331, 332, 333, 334, 84, - /* 1730 */ 336, 285, 84, 46, 46, 33, 290, 43, 85, 293, - /* 1740 */ 43, 46, 84, 151, 298, 228, 300, 85, 85, 47, - /* 1750 */ 46, 46, 85, 257, 52, 53, 54, 55, 56, 222, - /* 1760 */ 43, 85, 228, 257, 172, 35, 174, 35, 35, 35, - /* 1770 */ 35, 325, 2, 84, 43, 329, 330, 331, 332, 333, - /* 1780 */ 334, 285, 336, 85, 84, 83, 192, 84, 86, 293, - /* 1790 */ 46, 285, 85, 85, 298, 84, 300, 84, 46, 293, - /* 1800 */ 22, 95, 194, 84, 298, 85, 300, 35, 35, 84, - /* 1810 */ 35, 85, 84, 22, 85, 35, 35, 35, 108, 257, - /* 1820 */ 84, 325, 108, 85, 84, 329, 330, 331, 332, 333, - /* 1830 */ 334, 325, 336, 85, 84, 329, 330, 331, 332, 333, - /* 1840 */ 334, 96, 336, 84, 84, 35, 84, 285, 43, 108, - /* 1850 */ 148, 149, 22, 151, 108, 293, 61, 155, 60, 35, - /* 1860 */ 298, 257, 300, 67, 43, 82, 22, 35, 35, 35, - /* 1870 */ 35, 257, 35, 35, 35, 67, 174, 35, 35, 0, - /* 1880 */ 35, 35, 35, 257, 35, 35, 35, 325, 35, 285, - /* 1890 */ 0, 329, 330, 331, 332, 333, 334, 293, 336, 285, - /* 1900 */ 39, 47, 298, 35, 300, 47, 39, 293, 0, 35, - /* 1910 */ 47, 285, 298, 0, 300, 39, 35, 47, 0, 293, - /* 1920 */ 35, 39, 35, 0, 298, 257, 300, 22, 21, 325, - /* 1930 */ 377, 22, 22, 329, 330, 331, 332, 333, 334, 325, - /* 1940 */ 336, 21, 257, 329, 330, 331, 332, 333, 334, 20, - /* 1950 */ 336, 325, 377, 285, 377, 329, 330, 331, 332, 333, - /* 1960 */ 334, 293, 336, 377, 377, 377, 298, 377, 300, 377, - /* 1970 */ 285, 377, 377, 377, 377, 377, 377, 377, 293, 377, - /* 1980 */ 377, 377, 377, 298, 257, 300, 377, 377, 377, 377, - /* 1990 */ 377, 377, 377, 325, 257, 377, 377, 329, 330, 331, - /* 2000 */ 332, 333, 334, 377, 336, 377, 377, 377, 377, 377, - /* 2010 */ 325, 377, 285, 377, 329, 330, 331, 332, 333, 334, - /* 2020 */ 293, 336, 285, 377, 377, 298, 377, 300, 377, 377, - /* 2030 */ 293, 377, 377, 377, 377, 298, 257, 300, 377, 377, - /* 2040 */ 377, 377, 377, 377, 377, 377, 257, 377, 377, 377, - /* 2050 */ 377, 377, 325, 377, 377, 377, 329, 330, 331, 332, - /* 2060 */ 333, 334, 325, 336, 285, 377, 329, 330, 331, 332, - /* 2070 */ 333, 334, 293, 336, 285, 377, 377, 298, 377, 300, - /* 2080 */ 377, 377, 293, 377, 377, 377, 377, 298, 257, 300, - /* 2090 */ 377, 377, 377, 377, 377, 377, 377, 377, 257, 377, - /* 2100 */ 377, 377, 377, 377, 325, 377, 377, 377, 329, 330, - /* 2110 */ 331, 332, 333, 334, 325, 336, 285, 377, 329, 330, - /* 2120 */ 331, 332, 333, 334, 293, 336, 285, 377, 377, 298, - /* 2130 */ 377, 300, 377, 377, 293, 377, 377, 377, 377, 298, - /* 2140 */ 377, 300, 377, 377, 377, 377, 377, 377, 377, 377, - /* 2150 */ 257, 377, 377, 377, 377, 377, 325, 377, 377, 377, - /* 2160 */ 329, 330, 331, 332, 333, 334, 325, 336, 377, 257, - /* 2170 */ 329, 330, 331, 332, 333, 334, 377, 336, 285, 377, - /* 2180 */ 377, 377, 377, 377, 377, 377, 293, 377, 377, 377, - /* 2190 */ 377, 298, 257, 300, 377, 377, 377, 285, 377, 377, - /* 2200 */ 377, 377, 377, 377, 377, 293, 377, 377, 377, 377, - /* 2210 */ 298, 257, 300, 377, 377, 377, 377, 377, 325, 377, - /* 2220 */ 285, 377, 329, 330, 331, 332, 333, 334, 293, 336, - /* 2230 */ 377, 377, 377, 298, 377, 300, 377, 325, 377, 285, - /* 2240 */ 377, 329, 330, 331, 332, 333, 334, 293, 336, 377, - /* 2250 */ 377, 377, 298, 377, 300, 377, 377, 377, 377, 377, - /* 2260 */ 325, 377, 377, 377, 329, 330, 331, 332, 333, 334, - /* 2270 */ 257, 336, 377, 377, 377, 377, 377, 377, 377, 325, - /* 2280 */ 257, 377, 377, 329, 330, 331, 332, 333, 334, 377, - /* 2290 */ 336, 377, 377, 377, 377, 377, 377, 377, 285, 377, - /* 2300 */ 377, 377, 377, 377, 377, 377, 293, 377, 285, 377, - /* 2310 */ 377, 298, 377, 300, 377, 377, 293, 377, 377, 377, - /* 2320 */ 377, 298, 377, 300, 377, 377, 377, 377, 377, 377, - /* 2330 */ 377, 377, 377, 377, 377, 377, 377, 377, 325, 377, - /* 2340 */ 377, 377, 329, 330, 331, 332, 333, 334, 325, 336, - /* 2350 */ 377, 377, 329, 330, 331, 332, 333, 334, 377, 336, + /* 1490 */ 0, 329, 330, 331, 332, 333, 334, 0, 336, 0, + /* 1500 */ 0, 339, 0, 0, 42, 325, 344, 0, 0, 329, + /* 1510 */ 330, 331, 332, 333, 334, 0, 336, 285, 22, 0, + /* 1520 */ 0, 0, 290, 0, 0, 293, 0, 0, 0, 56, + /* 1530 */ 298, 257, 300, 0, 0, 42, 14, 39, 43, 14, + /* 1540 */ 0, 257, 0, 0, 0, 40, 0, 46, 39, 39, + /* 1550 */ 46, 156, 0, 0, 0, 0, 39, 325, 47, 285, + /* 1560 */ 0, 329, 330, 331, 332, 333, 334, 293, 336, 285, + /* 1570 */ 35, 35, 298, 61, 300, 0, 47, 293, 39, 47, + /* 1580 */ 35, 0, 298, 35, 300, 47, 39, 0, 39, 0, + /* 1590 */ 0, 257, 0, 35, 22, 0, 35, 35, 35, 325, + /* 1600 */ 35, 35, 43, 329, 330, 331, 332, 333, 334, 325, + /* 1610 */ 336, 43, 35, 329, 330, 331, 332, 333, 334, 285, + /* 1620 */ 336, 35, 91, 93, 0, 22, 35, 293, 0, 22, + /* 1630 */ 0, 49, 298, 22, 300, 0, 35, 35, 0, 35, + /* 1640 */ 0, 367, 22, 257, 0, 20, 0, 35, 22, 171, + /* 1650 */ 0, 0, 0, 0, 0, 84, 0, 84, 374, 325, + /* 1660 */ 257, 85, 149, 329, 330, 331, 332, 333, 334, 180, + /* 1670 */ 336, 285, 338, 0, 94, 39, 290, 84, 84, 293, + /* 1680 */ 46, 228, 43, 84, 298, 43, 300, 46, 285, 152, + /* 1690 */ 85, 85, 84, 290, 85, 154, 293, 84, 43, 148, + /* 1700 */ 85, 298, 152, 300, 85, 84, 84, 150, 152, 46, + /* 1710 */ 43, 325, 85, 19, 85, 329, 330, 331, 332, 333, + /* 1720 */ 334, 84, 336, 46, 43, 46, 43, 33, 325, 84, + /* 1730 */ 46, 35, 329, 330, 331, 332, 333, 334, 222, 336, + /* 1740 */ 35, 47, 228, 85, 85, 33, 52, 53, 54, 55, + /* 1750 */ 56, 257, 35, 35, 35, 35, 2, 22, 43, 47, + /* 1760 */ 84, 22, 192, 257, 52, 53, 54, 55, 56, 46, + /* 1770 */ 85, 84, 46, 85, 84, 257, 85, 83, 194, 285, + /* 1780 */ 86, 84, 84, 84, 95, 85, 35, 293, 35, 85, + /* 1790 */ 84, 285, 298, 35, 300, 83, 85, 228, 86, 293, + /* 1800 */ 84, 35, 35, 285, 298, 84, 300, 85, 84, 35, + /* 1810 */ 85, 293, 84, 119, 22, 108, 298, 108, 300, 325, + /* 1820 */ 35, 108, 108, 329, 330, 331, 332, 333, 334, 84, + /* 1830 */ 336, 325, 96, 84, 84, 329, 330, 331, 332, 333, + /* 1840 */ 334, 43, 336, 325, 22, 151, 60, 329, 330, 331, + /* 1850 */ 332, 333, 334, 35, 336, 61, 67, 257, 82, 43, + /* 1860 */ 148, 149, 35, 151, 35, 35, 172, 155, 174, 35, + /* 1870 */ 35, 257, 22, 35, 35, 67, 0, 35, 35, 35, + /* 1880 */ 35, 35, 35, 35, 35, 285, 174, 35, 0, 0, + /* 1890 */ 39, 35, 39, 293, 35, 0, 35, 39, 298, 285, + /* 1900 */ 300, 47, 47, 39, 47, 0, 35, 293, 35, 47, + /* 1910 */ 0, 21, 298, 22, 300, 22, 22, 21, 20, 377, + /* 1920 */ 377, 257, 377, 377, 377, 325, 377, 377, 377, 329, + /* 1930 */ 330, 331, 332, 333, 334, 377, 336, 257, 377, 325, + /* 1940 */ 377, 377, 377, 329, 330, 331, 332, 333, 334, 285, + /* 1950 */ 336, 377, 377, 377, 377, 377, 377, 293, 377, 377, + /* 1960 */ 377, 377, 298, 377, 300, 285, 377, 377, 377, 377, + /* 1970 */ 377, 377, 377, 293, 377, 377, 377, 377, 298, 377, + /* 1980 */ 300, 377, 377, 377, 377, 377, 377, 377, 377, 325, + /* 1990 */ 257, 377, 377, 329, 330, 331, 332, 333, 334, 377, + /* 2000 */ 336, 377, 377, 377, 257, 325, 377, 377, 377, 329, + /* 2010 */ 330, 331, 332, 333, 334, 377, 336, 377, 285, 377, + /* 2020 */ 377, 377, 377, 377, 377, 377, 293, 377, 377, 377, + /* 2030 */ 377, 298, 285, 300, 377, 377, 377, 377, 377, 377, + /* 2040 */ 293, 377, 377, 377, 377, 298, 377, 300, 377, 377, + /* 2050 */ 377, 377, 377, 377, 377, 377, 257, 377, 325, 377, + /* 2060 */ 377, 377, 329, 330, 331, 332, 333, 334, 377, 336, + /* 2070 */ 377, 377, 325, 377, 257, 377, 329, 330, 331, 332, + /* 2080 */ 333, 334, 377, 336, 285, 377, 377, 377, 377, 377, + /* 2090 */ 377, 377, 293, 377, 377, 377, 377, 298, 257, 300, + /* 2100 */ 377, 377, 285, 377, 377, 377, 377, 377, 377, 377, + /* 2110 */ 293, 377, 377, 377, 377, 298, 377, 300, 377, 377, + /* 2120 */ 377, 377, 377, 377, 325, 377, 285, 377, 329, 330, + /* 2130 */ 331, 332, 333, 334, 293, 336, 377, 377, 377, 298, + /* 2140 */ 377, 300, 325, 377, 377, 377, 329, 330, 331, 332, + /* 2150 */ 333, 334, 377, 336, 377, 377, 257, 377, 377, 377, + /* 2160 */ 377, 377, 377, 377, 377, 377, 325, 377, 377, 377, + /* 2170 */ 329, 330, 331, 332, 333, 334, 377, 336, 377, 257, + /* 2180 */ 377, 377, 377, 377, 285, 377, 377, 377, 377, 377, + /* 2190 */ 377, 377, 293, 377, 377, 377, 377, 298, 377, 300, + /* 2200 */ 377, 377, 377, 377, 377, 377, 377, 285, 377, 377, + /* 2210 */ 377, 377, 377, 377, 377, 293, 377, 377, 377, 377, + /* 2220 */ 298, 377, 300, 377, 325, 377, 377, 377, 329, 330, + /* 2230 */ 331, 332, 333, 334, 377, 336, 257, 377, 377, 377, + /* 2240 */ 377, 377, 377, 377, 377, 377, 377, 325, 257, 377, + /* 2250 */ 377, 329, 330, 331, 332, 333, 334, 377, 336, 377, + /* 2260 */ 377, 377, 257, 377, 285, 377, 377, 377, 377, 377, + /* 2270 */ 377, 377, 293, 377, 377, 377, 285, 298, 377, 300, + /* 2280 */ 377, 377, 377, 377, 293, 377, 377, 377, 377, 298, + /* 2290 */ 285, 300, 377, 377, 377, 377, 377, 377, 293, 377, + /* 2300 */ 377, 377, 377, 298, 325, 300, 377, 377, 329, 330, + /* 2310 */ 331, 332, 333, 334, 257, 336, 325, 377, 377, 377, + /* 2320 */ 329, 330, 331, 332, 333, 334, 377, 336, 377, 377, + /* 2330 */ 325, 377, 257, 377, 329, 330, 331, 332, 333, 334, + /* 2340 */ 377, 336, 285, 377, 377, 377, 377, 377, 377, 377, + /* 2350 */ 293, 377, 377, 377, 377, 298, 257, 300, 377, 377, + /* 2360 */ 285, 377, 377, 377, 377, 377, 377, 377, 293, 377, + /* 2370 */ 377, 377, 377, 298, 377, 300, 377, 377, 377, 377, + /* 2380 */ 377, 377, 325, 377, 285, 377, 329, 330, 331, 332, + /* 2390 */ 333, 334, 293, 336, 377, 377, 377, 298, 377, 300, + /* 2400 */ 325, 377, 377, 377, 329, 330, 331, 332, 333, 334, + /* 2410 */ 377, 336, 377, 377, 377, 377, 377, 377, 377, 377, + /* 2420 */ 377, 377, 377, 377, 325, 377, 377, 377, 329, 330, + /* 2430 */ 331, 332, 333, 334, 377, 336, }; #define YY_SHIFT_COUNT (662) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1929) +#define YY_SHIFT_MAX (1910) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 913, 0, 0, 68, 68, 268, 268, 268, 325, 325, /* 10 */ 268, 268, 525, 582, 782, 582, 582, 582, 582, 582, /* 20 */ 582, 582, 582, 582, 582, 582, 582, 582, 582, 582, /* 30 */ 582, 582, 582, 582, 582, 582, 582, 582, 582, 582, - /* 40 */ 582, 582, 582, 208, 208, 97, 97, 97, 1073, 1073, + /* 40 */ 582, 582, 208, 208, 97, 97, 97, 1073, 1073, 1073, /* 50 */ 1073, 152, 426, 3, 3, 99, 99, 238, 238, 5, /* 60 */ 202, 3, 3, 99, 99, 99, 99, 99, 99, 99, /* 70 */ 99, 99, 89, 99, 99, 99, 148, 305, 99, 99, @@ -710,93 +726,93 @@ static const unsigned short int yy_shift_ofst[] = { /* 100 */ 575, 575, 575, 575, 575, 575, 575, 575, 575, 575, /* 110 */ 575, 575, 575, 197, 202, 355, 355, 400, 576, 656, /* 120 */ 438, 438, 438, 576, 528, 148, 11, 11, 305, 305, - /* 130 */ 541, 541, 261, 805, 466, 466, 466, 466, 466, 466, - /* 140 */ 466, 1592, 23, 107, 428, 21, 41, 375, 65, 366, + /* 130 */ 541, 541, 261, 696, 466, 466, 466, 466, 466, 466, + /* 140 */ 466, 1694, 23, 107, 428, 21, 41, 375, 65, 366, /* 150 */ 816, 365, 649, 495, 475, 539, 660, 539, 558, 497, - /* 160 */ 497, 497, 808, 194, 1076, 936, 1145, 1120, 1136, 1009, - /* 170 */ 1145, 1145, 1140, 1040, 1040, 1145, 1145, 1173, 1173, 1175, - /* 180 */ 89, 148, 89, 1176, 1191, 89, 1176, 89, 1197, 89, - /* 190 */ 89, 1145, 89, 1173, 305, 305, 305, 305, 305, 305, - /* 200 */ 305, 305, 305, 305, 305, 1145, 1173, 541, 1175, 380, - /* 210 */ 1080, 148, 380, 1145, 1145, 1176, 380, 1026, 541, 541, - /* 220 */ 541, 541, 1026, 541, 1114, 528, 1197, 380, 261, 380, - /* 230 */ 528, 1238, 541, 1068, 1026, 541, 541, 1068, 1026, 541, - /* 240 */ 541, 305, 1063, 1151, 1068, 1070, 1077, 1086, 936, 1089, - /* 250 */ 528, 1316, 1093, 1097, 1094, 1093, 1097, 1093, 1097, 1260, - /* 260 */ 1076, 541, 805, 1145, 380, 1317, 1173, 2360, 2360, 2360, - /* 270 */ 2360, 2360, 2360, 2360, 348, 1702, 132, 579, 472, 16, + /* 160 */ 497, 497, 808, 194, 1078, 939, 1148, 1125, 1142, 1017, + /* 170 */ 1148, 1148, 1145, 1048, 1048, 1148, 1148, 1182, 1182, 1184, + /* 180 */ 89, 148, 89, 1193, 1198, 89, 1193, 89, 1204, 89, + /* 190 */ 89, 1148, 89, 1182, 305, 305, 305, 305, 305, 305, + /* 200 */ 305, 305, 305, 305, 305, 1148, 1182, 541, 1184, 380, + /* 210 */ 1070, 148, 380, 1148, 1148, 1193, 380, 1032, 541, 541, + /* 220 */ 541, 541, 1032, 541, 1140, 528, 1204, 380, 261, 380, + /* 230 */ 528, 1283, 541, 1076, 1032, 541, 541, 1076, 1032, 541, + /* 240 */ 541, 305, 1079, 1169, 1076, 1088, 1092, 1105, 939, 1113, + /* 250 */ 528, 1319, 1097, 1112, 1102, 1097, 1112, 1097, 1112, 1263, + /* 260 */ 1078, 541, 696, 1148, 380, 1322, 1182, 2436, 2436, 2436, + /* 270 */ 2436, 2436, 2436, 2436, 348, 1712, 132, 579, 472, 16, /* 280 */ 663, 639, 743, 389, 757, 704, 839, 839, 839, 839, /* 290 */ 839, 839, 839, 839, 843, 568, 25, 25, 61, 212, /* 300 */ 433, 481, 603, 378, 297, 150, 483, 483, 483, 483, /* 310 */ 45, 887, 774, 806, 910, 917, 811, 1046, 1069, 720, - /* 320 */ 707, 930, 973, 1041, 1045, 1047, 1050, 728, 829, 1053, - /* 330 */ 951, 847, 625, 391, 1074, 838, 1079, 1036, 1085, 1088, - /* 340 */ 1096, 1098, 1099, 1100, 1052, 1054, 896, 1358, 1189, 1377, - /* 350 */ 1382, 1345, 1389, 1315, 1391, 1357, 1211, 1362, 1363, 1364, - /* 360 */ 1215, 1401, 1368, 1369, 1220, 1406, 1223, 1410, 1378, 1411, - /* 370 */ 1390, 1414, 1380, 1416, 1333, 1248, 1252, 1255, 1258, 1435, - /* 380 */ 1436, 1283, 1286, 1448, 1449, 1404, 1451, 1452, 1453, 1310, - /* 390 */ 1456, 1457, 1458, 1459, 1461, 1324, 1429, 1466, 1328, 1468, - /* 400 */ 1469, 1471, 1472, 1474, 1475, 1476, 1477, 1479, 1481, 1484, - /* 410 */ 1485, 1486, 1488, 1455, 1490, 1499, 1500, 1502, 1503, 1504, - /* 420 */ 1493, 1507, 1508, 1518, 1519, 1520, 1465, 1523, 1524, 1487, - /* 430 */ 1489, 1483, 1513, 1491, 1521, 1492, 1533, 1494, 1497, 1539, - /* 440 */ 1540, 1542, 1505, 1387, 1545, 1546, 1547, 1495, 1548, 1549, - /* 450 */ 1515, 1511, 1512, 1552, 1525, 1526, 1514, 1554, 1535, 1528, - /* 460 */ 1516, 1571, 1541, 1531, 1544, 1579, 1580, 1581, 1585, 1501, - /* 470 */ 1498, 1551, 1565, 1588, 1555, 1557, 1558, 1560, 1553, 1559, - /* 480 */ 1563, 1566, 1577, 1586, 1597, 1578, 1622, 1601, 1582, 1628, - /* 490 */ 1607, 1591, 1630, 1600, 1636, 1602, 1638, 1618, 1623, 1642, - /* 500 */ 1510, 1615, 1649, 1482, 1632, 1529, 1522, 1652, 1655, 1530, - /* 510 */ 1532, 1656, 1657, 1660, 1589, 1587, 1509, 1677, 1596, 1538, - /* 520 */ 1599, 1690, 1646, 1543, 1609, 1603, 1650, 1659, 1467, 1616, - /* 530 */ 1613, 1620, 1631, 1633, 1629, 1693, 1674, 1635, 1637, 1645, - /* 540 */ 1648, 1653, 1679, 1687, 1688, 1658, 1694, 1517, 1662, 1663, - /* 550 */ 1695, 1537, 1697, 1704, 1705, 1667, 1717, 1534, 1676, 1664, - /* 560 */ 1730, 1732, 1733, 1734, 1735, 1676, 1770, 1594, 1731, 1689, - /* 570 */ 1698, 1700, 1707, 1703, 1708, 1744, 1711, 1713, 1752, 1778, - /* 580 */ 1608, 1719, 1706, 1720, 1772, 1773, 1725, 1726, 1775, 1728, - /* 590 */ 1729, 1780, 1736, 1738, 1781, 1740, 1748, 1782, 1750, 1710, - /* 600 */ 1714, 1741, 1746, 1791, 1745, 1759, 1760, 1810, 1762, 1805, - /* 610 */ 1805, 1830, 1795, 1798, 1824, 1796, 1783, 1821, 1832, 1833, - /* 620 */ 1834, 1835, 1837, 1844, 1838, 1839, 1808, 1553, 1842, 1559, - /* 630 */ 1843, 1845, 1846, 1847, 1849, 1850, 1851, 1879, 1853, 1854, - /* 640 */ 1861, 1890, 1868, 1858, 1867, 1908, 1874, 1863, 1876, 1913, - /* 650 */ 1881, 1870, 1882, 1918, 1885, 1887, 1923, 1905, 1907, 1909, - /* 660 */ 1910, 1920, 1929, + /* 320 */ 707, 930, 973, 1045, 1047, 1050, 1053, 1074, 894, 832, + /* 330 */ 625, 391, 1096, 829, 863, 884, 1098, 1038, 1085, 1100, + /* 340 */ 1104, 1106, 1107, 1108, 993, 897, 1127, 1364, 1202, 1384, + /* 350 */ 1390, 1355, 1398, 1324, 1400, 1366, 1218, 1369, 1370, 1371, + /* 360 */ 1223, 1410, 1376, 1377, 1228, 1414, 1230, 1416, 1382, 1418, + /* 370 */ 1399, 1422, 1388, 1435, 1352, 1276, 1275, 1281, 1285, 1449, + /* 380 */ 1451, 1291, 1293, 1454, 1456, 1411, 1458, 1459, 1461, 1320, + /* 390 */ 1463, 1466, 1467, 1468, 1469, 1332, 1437, 1474, 1336, 1476, + /* 400 */ 1477, 1479, 1481, 1484, 1485, 1486, 1488, 1490, 1497, 1499, + /* 410 */ 1500, 1502, 1503, 1462, 1507, 1508, 1515, 1523, 1524, 1526, + /* 420 */ 1496, 1519, 1520, 1521, 1527, 1528, 1473, 1533, 1534, 1493, + /* 430 */ 1498, 1495, 1522, 1501, 1525, 1504, 1540, 1505, 1509, 1542, + /* 440 */ 1543, 1544, 1510, 1395, 1546, 1552, 1553, 1512, 1554, 1555, + /* 450 */ 1535, 1511, 1517, 1560, 1536, 1529, 1539, 1575, 1545, 1532, + /* 460 */ 1547, 1581, 1548, 1538, 1549, 1587, 1589, 1590, 1592, 1530, + /* 470 */ 1531, 1558, 1572, 1595, 1561, 1562, 1563, 1565, 1559, 1568, + /* 480 */ 1566, 1577, 1586, 1591, 1624, 1603, 1628, 1607, 1582, 1630, + /* 490 */ 1611, 1601, 1635, 1602, 1638, 1604, 1640, 1620, 1625, 1644, + /* 500 */ 1537, 1612, 1646, 1478, 1626, 1550, 1513, 1650, 1651, 1556, + /* 510 */ 1541, 1652, 1653, 1654, 1571, 1576, 1489, 1656, 1573, 1557, + /* 520 */ 1593, 1673, 1636, 1551, 1594, 1580, 1634, 1639, 1453, 1599, + /* 530 */ 1605, 1608, 1606, 1609, 1613, 1642, 1615, 1621, 1622, 1637, + /* 540 */ 1619, 1655, 1641, 1663, 1645, 1667, 1514, 1627, 1629, 1677, + /* 550 */ 1516, 1681, 1679, 1684, 1658, 1683, 1569, 1659, 1696, 1705, + /* 560 */ 1717, 1718, 1719, 1720, 1659, 1754, 1735, 1570, 1715, 1676, + /* 570 */ 1685, 1687, 1688, 1690, 1691, 1723, 1697, 1698, 1726, 1739, + /* 580 */ 1584, 1699, 1689, 1700, 1751, 1753, 1706, 1704, 1758, 1716, + /* 590 */ 1711, 1766, 1721, 1722, 1767, 1724, 1725, 1774, 1728, 1707, + /* 600 */ 1709, 1713, 1714, 1792, 1736, 1745, 1749, 1785, 1750, 1798, + /* 610 */ 1798, 1822, 1794, 1786, 1818, 1789, 1776, 1816, 1827, 1829, + /* 620 */ 1830, 1834, 1835, 1850, 1838, 1839, 1808, 1559, 1842, 1568, + /* 630 */ 1843, 1844, 1845, 1846, 1847, 1848, 1849, 1876, 1852, 1854, + /* 640 */ 1851, 1888, 1856, 1855, 1853, 1889, 1859, 1857, 1858, 1895, + /* 650 */ 1861, 1862, 1864, 1905, 1871, 1873, 1910, 1891, 1890, 1893, + /* 660 */ 1894, 1896, 1898, }; #define YY_REDUCE_COUNT (273) #define YY_REDUCE_MIN (-354) -#define YY_REDUCE_MAX (2023) +#define YY_REDUCE_MAX (2099) static const short yy_reduce_ofst[] = { /* 0 */ -29, -221, 289, 34, 709, -113, 546, 780, 876, 937, /* 10 */ 990, 1042, 1095, 1109, 1162, 1180, 1232, 1274, 1284, 1334, - /* 20 */ 1376, 1394, 1446, 1496, 1506, 1562, 1604, 1614, 1626, 1668, - /* 30 */ 1685, 1727, 1737, 1779, 1789, 1831, 1841, 1893, 1912, 1935, - /* 40 */ 1954, 2013, 2023, -219, 711, -250, -207, 593, 731, 726, - /* 50 */ 820, -218, 382, 446, 502, -208, 580, -261, -257, -354, + /* 20 */ 1386, 1403, 1494, 1506, 1518, 1600, 1614, 1664, 1680, 1733, + /* 30 */ 1747, 1799, 1817, 1841, 1899, 1922, 1979, 1991, 2005, 2057, + /* 40 */ 2075, 2099, -219, 711, -250, -207, 593, 731, 820, 726, + /* 50 */ 828, -218, 382, 446, 502, -208, 580, -261, -257, -354, /* 60 */ -234, -182, -11, -138, 218, 323, 404, 548, 662, 664, /* 70 */ 670, 672, 102, 730, 734, 735, -292, 153, 738, 756, - /* 80 */ -275, -215, 771, 219, -39, 267, 822, -246, 49, -201, + /* 80 */ -275, -215, 771, 219, -39, 267, 855, -246, 49, -201, /* 90 */ -201, -201, -187, -31, -152, -132, 2, 205, 286, 336, /* 100 */ 372, 402, 403, 441, 542, 552, 569, 571, 624, 627, /* 110 */ 630, 635, 636, 20, 31, -71, 71, -264, 136, -194, /* 120 */ 331, 340, 368, 243, 303, 453, 230, 431, -47, 388, - /* 130 */ 512, 537, 556, 471, 430, 511, 613, 646, 678, 683, - /* 140 */ 733, 638, 759, 746, 647, 710, 760, 826, 715, 807, - /* 150 */ 807, 861, 884, 852, 823, 797, 797, 797, 809, 783, - /* 160 */ 784, 785, 801, 807, 869, 840, 901, 846, 904, 862, - /* 170 */ 912, 915, 881, 885, 886, 926, 927, 938, 939, 879, - /* 180 */ 931, 902, 934, 891, 898, 945, 903, 946, 914, 949, - /* 190 */ 950, 956, 954, 965, 940, 941, 942, 943, 944, 947, - /* 200 */ 948, 957, 958, 960, 961, 969, 977, 952, 919, 974, - /* 210 */ 933, 955, 982, 989, 991, 959, 992, 975, 962, 963, - /* 220 */ 966, 976, 978, 987, 979, 996, 985, 1027, 1012, 1028, - /* 230 */ 1003, 970, 999, 953, 993, 1005, 1006, 964, 997, 1010, - /* 240 */ 1013, 807, 967, 980, 971, 968, 986, 972, 994, 797, - /* 250 */ 1025, 1011, 983, 981, 984, 988, 995, 998, 1000, 1004, - /* 260 */ 1058, 1051, 1075, 1083, 1090, 1101, 1103, 1038, 1044, 1084, - /* 270 */ 1104, 1105, 1111, 1113, + /* 130 */ 512, 537, 556, 471, 430, 511, 586, 613, 681, 682, + /* 140 */ 683, 638, 759, 745, 645, 716, 760, 701, 727, 807, + /* 150 */ 807, 870, 834, 824, 814, 801, 801, 801, 809, 788, + /* 160 */ 789, 791, 810, 807, 874, 844, 906, 851, 907, 866, + /* 170 */ 919, 921, 883, 893, 895, 932, 934, 941, 944, 891, + /* 180 */ 945, 915, 947, 904, 905, 950, 908, 954, 920, 956, + /* 190 */ 958, 963, 960, 974, 949, 951, 957, 961, 964, 965, + /* 200 */ 967, 968, 969, 970, 971, 976, 980, 959, 912, 978, + /* 210 */ 940, 972, 991, 995, 996, 981, 997, 975, 987, 988, + /* 220 */ 989, 998, 982, 999, 986, 1002, 992, 1007, 1019, 1034, + /* 230 */ 1009, 979, 1006, 952, 1001, 1008, 1013, 953, 1003, 1018, + /* 240 */ 1020, 807, 966, 983, 977, 984, 994, 985, 1014, 801, + /* 250 */ 1052, 1021, 1011, 1000, 1015, 1012, 1022, 1016, 1026, 1023, + /* 260 */ 1061, 1057, 1081, 1086, 1090, 1099, 1101, 1044, 1051, 1083, + /* 270 */ 1089, 1110, 1111, 1114, }; static const YYACTIONTYPE yy_default[] = { /* 0 */ 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, @@ -831,8 +847,8 @@ static const YYACTIONTYPE yy_default[] = { /* 290 */ 1831, 1830, 1828, 1793, 1453, 1587, 1792, 1791, 1453, 1453, /* 300 */ 1453, 1453, 1453, 1453, 1453, 1453, 1785, 1786, 1784, 1783, /* 310 */ 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, - /* 320 */ 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, - /* 330 */ 1857, 1453, 1925, 1929, 1453, 1453, 1453, 1768, 1453, 1453, + /* 320 */ 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1857, 1453, + /* 330 */ 1925, 1929, 1453, 1453, 1453, 1453, 1453, 1768, 1453, 1453, /* 340 */ 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, /* 350 */ 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, /* 360 */ 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, @@ -853,9 +869,9 @@ static const YYACTIONTYPE yy_default[] = { /* 510 */ 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1727, 1453, 1453, /* 520 */ 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1890, 1453, 1453, /* 530 */ 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, - /* 540 */ 1453, 1453, 1453, 1453, 1768, 1453, 1907, 1453, 1867, 1863, - /* 550 */ 1453, 1453, 1859, 1767, 1453, 1453, 1923, 1453, 1453, 1453, - /* 560 */ 1453, 1453, 1453, 1453, 1453, 1453, 1852, 1453, 1825, 1810, + /* 540 */ 1453, 1453, 1453, 1768, 1453, 1907, 1453, 1867, 1863, 1453, + /* 550 */ 1453, 1859, 1767, 1453, 1453, 1923, 1453, 1453, 1453, 1453, + /* 560 */ 1453, 1453, 1453, 1453, 1453, 1852, 1453, 1453, 1825, 1810, /* 570 */ 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, /* 580 */ 1779, 1453, 1453, 1453, 1453, 1453, 1619, 1453, 1453, 1453, /* 590 */ 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1604, @@ -2003,7 +2019,7 @@ static const char *const yyRuleName[] = { /* 390 */ "compare_op ::= CONTAINS", /* 391 */ "in_op ::= IN", /* 392 */ "in_op ::= NOT IN", - /* 393 */ "in_predicate_value ::= NK_LP expression_list NK_RP", + /* 393 */ "in_predicate_value ::= NK_LP literal_list NK_RP", /* 394 */ "boolean_value_expression ::= boolean_primary", /* 395 */ "boolean_value_expression ::= NOT boolean_primary", /* 396 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", @@ -3089,7 +3105,7 @@ static const struct { { 340, -1 }, /* (390) compare_op ::= CONTAINS */ { 341, -1 }, /* (391) in_op ::= IN */ { 341, -2 }, /* (392) in_op ::= NOT IN */ - { 342, -3 }, /* (393) in_predicate_value ::= NK_LP expression_list NK_RP */ + { 342, -3 }, /* (393) in_predicate_value ::= NK_LP literal_list NK_RP */ { 343, -1 }, /* (394) boolean_value_expression ::= boolean_primary */ { 343, -2 }, /* (395) boolean_value_expression ::= NOT boolean_primary */ { 343, -3 }, /* (396) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ @@ -4477,7 +4493,7 @@ static YYACTIONTYPE yy_reduce( case 392: /* in_op ::= NOT IN */ { yymsp[-1].minor.yy428 = OP_TYPE_NOT_IN; } break; - case 393: /* in_predicate_value ::= NK_LP expression_list NK_RP */ + case 393: /* in_predicate_value ::= NK_LP literal_list NK_RP */ { yylhsminor.yy652 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy210)); } yymsp[-2].minor.yy652 = yylhsminor.yy652; break; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 67b79d5b1c..fb3c420a19 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -896,7 +896,7 @@ static int32_t pushDownCondOptTrivialPushDown(SOptimizeContext* pCxt, SLogicNode return TSDB_CODE_SUCCESS; } SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pLogicNode->pChildren, 0); - int32_t code = pushDownCondOptPushCondToChild(pCxt, pChild, &pLogicNode->pConditions); + int32_t code = pushDownCondOptPushCondToChild(pCxt, pChild, &pLogicNode->pConditions); if (TSDB_CODE_SUCCESS == code) { OPTIMIZE_FLAG_SET_MASK(pLogicNode->optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE); pCxt->optimized = true; @@ -1183,7 +1183,7 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo if (smaFuncIndex < 0) { break; } else { - code = nodesListMakeStrictAppend(&pCols, smaIndexOptCreateSmaCol(pFunc, tableId, smaFuncIndex + 2)); + code = nodesListMakeStrictAppend(&pCols, smaIndexOptCreateSmaCol(pFunc, tableId, smaFuncIndex + 1)); if (TSDB_CODE_SUCCESS != code) { break; } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index cbb1089d61..e1aeaf219b 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1036,71 +1036,124 @@ _return: return code; } -int32_t scalarGetOperatorResultType(SDataType left, SDataType right, EOperatorType op, SDataType* pRes) { - switch (op) { +static int32_t getMinusOperatorResultType(SOperatorNode* pOp) { + if (!IS_MATHABLE_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + return TSDB_CODE_SUCCESS; +} + +static int32_t getArithmeticOperatorResultType(SOperatorNode* pOp) { + SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; + SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; + if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) || + (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && (IS_VAR_DATA_TYPE(rdt.type) || IS_FLOAT_TYPE(rdt.type))) || + (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && (IS_VAR_DATA_TYPE(ldt.type) || IS_FLOAT_TYPE(ldt.type)))) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && IS_INTEGER_TYPE(rdt.type)) || + (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && IS_INTEGER_TYPE(ldt.type)) || + (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_BOOL == rdt.type) || + (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && TSDB_DATA_TYPE_BOOL == ldt.type)) { + pOp->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; + } else { + pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t getComparisonOperatorResultType(SOperatorNode* pOp) { + SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; + if (NULL != pOp->pRight) { + SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; + if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) { + rdt = ldt; + } else if (nodesIsRegularOp(pOp)) { + if (!IS_VAR_DATA_TYPE(ldt.type) || QUERY_NODE_VALUE != nodeType(pOp->pRight) || + (!IS_STR_DATA_TYPE(rdt.type) && (rdt.type != TSDB_DATA_TYPE_NULL))) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + } + pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + return TSDB_CODE_SUCCESS; +} + +static int32_t getJsonOperatorResultType(SOperatorNode* pOp) { + SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; + SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; + if (TSDB_DATA_TYPE_JSON != ldt.type || !IS_STR_DATA_TYPE(rdt.type)) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + if (pOp->opType == OP_TYPE_JSON_GET_VALUE) { + pOp->node.resType.type = TSDB_DATA_TYPE_JSON; + } else if (pOp->opType == OP_TYPE_JSON_CONTAINS) { + pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; + } + pOp->node.resType.bytes = tDataTypes[pOp->node.resType.type].bytes; + return TSDB_CODE_SUCCESS; +} + +static int32_t getBitwiseOperatorResultType(SOperatorNode* pOp) { + pOp->node.resType.type = TSDB_DATA_TYPE_BIGINT; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + return TSDB_CODE_SUCCESS; +} + +int32_t scalarGetOperatorResultType(SOperatorNode* pOp) { + if (TSDB_DATA_TYPE_BLOB == ((SExprNode*)(pOp->pLeft))->resType.type || + (NULL != pOp->pRight && TSDB_DATA_TYPE_BLOB == ((SExprNode*)(pOp->pRight))->resType.type)) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + switch (pOp->opType) { case OP_TYPE_ADD: - if (left.type == TSDB_DATA_TYPE_TIMESTAMP && right.type == TSDB_DATA_TYPE_TIMESTAMP) { - qError("invalid op %d, left type:%d, right type:%d", op, left.type, right.type); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - if ((left.type == TSDB_DATA_TYPE_TIMESTAMP && (IS_INTEGER_TYPE(right.type) || right.type == TSDB_DATA_TYPE_BOOL)) || - (right.type == TSDB_DATA_TYPE_TIMESTAMP && (IS_INTEGER_TYPE(left.type) || left.type == TSDB_DATA_TYPE_BOOL))) { - pRes->type = TSDB_DATA_TYPE_TIMESTAMP; - return TSDB_CODE_SUCCESS; - } - pRes->type = TSDB_DATA_TYPE_DOUBLE; - return TSDB_CODE_SUCCESS; case OP_TYPE_SUB: - if ((left.type == TSDB_DATA_TYPE_TIMESTAMP && right.type == TSDB_DATA_TYPE_BIGINT) || - (right.type == TSDB_DATA_TYPE_TIMESTAMP && left.type == TSDB_DATA_TYPE_BIGINT)) { - pRes->type = TSDB_DATA_TYPE_TIMESTAMP; - return TSDB_CODE_SUCCESS; - } - pRes->type = TSDB_DATA_TYPE_DOUBLE; - return TSDB_CODE_SUCCESS; case OP_TYPE_MULTI: - if (left.type == TSDB_DATA_TYPE_TIMESTAMP && right.type == TSDB_DATA_TYPE_TIMESTAMP) { - qError("invalid op %d, left type:%d, right type:%d", op, left.type, right.type); - return TSDB_CODE_TSC_INVALID_OPERATION; - } case OP_TYPE_DIV: - if (left.type == TSDB_DATA_TYPE_TIMESTAMP && right.type == TSDB_DATA_TYPE_TIMESTAMP) { - qError("invalid op %d, left type:%d, right type:%d", op, left.type, right.type); - return TSDB_CODE_TSC_INVALID_OPERATION; - } case OP_TYPE_REM: + return getArithmeticOperatorResultType(pOp); case OP_TYPE_MINUS: - pRes->type = TSDB_DATA_TYPE_DOUBLE; - return TSDB_CODE_SUCCESS; + return getMinusOperatorResultType(pOp); + case OP_TYPE_ASSIGN: + pOp->node.resType = ((SExprNode*)(pOp->pLeft))->resType; + break; + case OP_TYPE_BIT_AND: + case OP_TYPE_BIT_OR: + return getBitwiseOperatorResultType(pOp); case OP_TYPE_GREATER_THAN: case OP_TYPE_GREATER_EQUAL: case OP_TYPE_LOWER_THAN: case OP_TYPE_LOWER_EQUAL: case OP_TYPE_EQUAL: case OP_TYPE_NOT_EQUAL: - case OP_TYPE_IN: - case OP_TYPE_NOT_IN: + case OP_TYPE_IS_NULL: + case OP_TYPE_IS_NOT_NULL: + case OP_TYPE_IS_TRUE: + case OP_TYPE_IS_FALSE: + case OP_TYPE_IS_UNKNOWN: + case OP_TYPE_IS_NOT_TRUE: + case OP_TYPE_IS_NOT_FALSE: + case OP_TYPE_IS_NOT_UNKNOWN: case OP_TYPE_LIKE: case OP_TYPE_NOT_LIKE: case OP_TYPE_MATCH: case OP_TYPE_NMATCH: - case OP_TYPE_IS_NULL: - case OP_TYPE_IS_NOT_NULL: - case OP_TYPE_IS_TRUE: - case OP_TYPE_JSON_CONTAINS: - pRes->type = TSDB_DATA_TYPE_BOOL; - return TSDB_CODE_SUCCESS; - case OP_TYPE_BIT_AND: - case OP_TYPE_BIT_OR: - pRes->type = TSDB_DATA_TYPE_BIGINT; - return TSDB_CODE_SUCCESS; + case OP_TYPE_IN: + case OP_TYPE_NOT_IN: + return getComparisonOperatorResultType(pOp); case OP_TYPE_JSON_GET_VALUE: - pRes->type = TSDB_DATA_TYPE_JSON; - return TSDB_CODE_SUCCESS; + case OP_TYPE_JSON_CONTAINS: + return getJsonOperatorResultType(pOp); default: - ASSERT(0); - return TSDB_CODE_APP_ERROR; + break; } + + return TSDB_CODE_SUCCESS; } - - From 17f5f3ee820f5cafbe75736d8b5162a39a2c7f1d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 8 Jul 2022 17:51:03 +0800 Subject: [PATCH 04/38] fix: fix server_status() issue --- source/client/src/clientHb.c | 9 ++++----- source/libs/scalar/src/scalar.c | 34 ++++++++++++++++++++++++++------- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 48fa2d7938..96f0eb8e2b 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -264,11 +264,6 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { static int32_t emptyRspNum = 0; - if (code != 0) { - taosMemoryFreeClear(param); - return -1; - } - char *key = (char *)param; SClientHbBatchRsp pRsp = {0}; tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp); @@ -288,6 +283,10 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { taosMemoryFreeClear(param); + if (code != 0) { + (*pInst)->onlineDnodes = 0; + } + if (rspNum) { tscDebug("hb got %d rsp, %d empty rsp received before", rspNum, atomic_val_compare_exchange_32(&emptyRspNum, emptyRspNum, 0)); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index cbb1089d61..f4ba59b721 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -570,7 +570,7 @@ _return: SCL_RET(code); } -EDealRes sclRewriteBasedOnOptr(SNode** pNode, SScalarCtx *ctx, EOperatorType opType) { +EDealRes sclRewriteNullInOptr(SNode** pNode, SScalarCtx *ctx, EOperatorType opType) { if (opType <= OP_TYPE_CALC_MAX) { SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { @@ -602,6 +602,24 @@ EDealRes sclRewriteBasedOnOptr(SNode** pNode, SScalarCtx *ctx, EOperatorType opT return DEAL_RES_CONTINUE; } +EDealRes sclAggFuncWalker(SNode* pNode, void* pContext) { + if (QUERY_NODE_FUNCTION == nodeType(pNode)) { + SFunctionNode* pFunc = (SFunctionNode*)pNode; + *(bool*)pContext = fmIsAggFunc(pFunc->funcId); + if (*(bool*)pContext) { + return DEAL_RES_END; + } + } + + return DEAL_RES_CONTINUE; +} + + +bool sclContainsAggFuncNode(SNode* pNode) { + bool aggFunc = false; + nodesWalkExpr(pNode, sclAggFuncWalker, (void *)&aggFunc); + return aggFunc; +} EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { SOperatorNode *node = (SOperatorNode *)*pNode; @@ -609,8 +627,9 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { if (node->pLeft && (QUERY_NODE_VALUE == nodeType(node->pLeft))) { SValueNode *valueNode = (SValueNode *)node->pLeft; - if (SCL_IS_NULL_VALUE_NODE(valueNode) && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) { - return sclRewriteBasedOnOptr(pNode, ctx, node->opType); + if (SCL_IS_NULL_VALUE_NODE(valueNode) && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL) + && (!sclContainsAggFuncNode(node->pRight))) { + return sclRewriteNullInOptr(pNode, ctx, node->opType); } if (IS_STR_DATA_TYPE(valueNode->node.resType.type) && node->pRight && nodesIsExprNode(node->pRight) @@ -625,8 +644,9 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { if (node->pRight && (QUERY_NODE_VALUE == nodeType(node->pRight))) { SValueNode *valueNode = (SValueNode *)node->pRight; - if (SCL_IS_NULL_VALUE_NODE(valueNode) && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL)) { - return sclRewriteBasedOnOptr(pNode, ctx, node->opType); + if (SCL_IS_NULL_VALUE_NODE(valueNode) && (node->opType != OP_TYPE_IS_NULL && node->opType != OP_TYPE_IS_NOT_NULL) + && (!sclContainsAggFuncNode(node->pLeft))) { + return sclRewriteNullInOptr(pNode, ctx, node->opType); } if (IS_STR_DATA_TYPE(valueNode->node.resType.type) && node->pLeft && nodesIsExprNode(node->pLeft) @@ -648,7 +668,7 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { ERASE_NODE(listNode->pNodeList); continue; } else { //OP_TYPE_NOT_IN - return sclRewriteBasedOnOptr(pNode, ctx, node->opType); + return sclRewriteNullInOptr(pNode, ctx, node->opType); } } @@ -656,7 +676,7 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { } if (listNode->pNodeList->length <= 0) { - return sclRewriteBasedOnOptr(pNode, ctx, node->opType); + return sclRewriteNullInOptr(pNode, ctx, node->opType); } } From 2cf01e0782b078939bddfd68583b15e1e94970ba Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 8 Jul 2022 20:50:11 +0800 Subject: [PATCH 05/38] add test case --- source/dnode/vnode/src/tq/tqMeta.c | 2 +- source/libs/index/inc/indexFstCommon.h | 1 + source/libs/index/inc/indexFstDfa.h | 6 +- source/libs/index/inc/indexFstRegex.h | 1 + source/libs/index/src/indexCache.c | 4 +- source/libs/index/src/indexFst.c | 1 - source/libs/index/src/indexFstCommon.c | 1 + source/libs/index/src/indexFstDfa.c | 14 +++-- source/libs/index/src/indexFstRegex.c | 8 ++- source/libs/index/test/CMakeLists.txt | 24 ++++++++ source/libs/index/test/fstUtilUT.cc | 63 ++++++++++++++++++++ source/libs/index/test/utilUT.cc | 9 +++ source/util/src/ttrace.c | 79 -------------------------- 13 files changed, 119 insertions(+), 94 deletions(-) create mode 100644 source/libs/index/test/fstUtilUT.cc delete mode 100644 source/util/src/ttrace.c diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index d46f12cf77..a9ebcfd4c3 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -87,7 +87,7 @@ int32_t tqMetaOpen(STQ* pTq) { .reader = handle.execHandle.pExecReader[i], .meta = pTq->pVnode->pMeta, .pMsgCb = &pTq->pVnode->msgCb, - .vnode = pTq->pVnode, + .vnode = pTq->pVnode, }; handle.execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(handle.execHandle.execCol.qmsg, &reader); ASSERT(handle.execHandle.execCol.task[i]); diff --git a/source/libs/index/inc/indexFstCommon.h b/source/libs/index/inc/indexFstCommon.h index 8335e437fb..e15df5dc34 100644 --- a/source/libs/index/inc/indexFstCommon.h +++ b/source/libs/index/inc/indexFstCommon.h @@ -4,6 +4,7 @@ #include "tutil.h" extern const uint8_t COMMON_INPUTS[]; extern const char COMMON_INPUTS_INV[]; +extern const int32_t COMMON_INPUTS_LEN; #ifdef __cplusplus extern "C" { diff --git a/source/libs/index/inc/indexFstDfa.h b/source/libs/index/inc/indexFstDfa.h index f6c220bcb7..9ca10897fd 100644 --- a/source/libs/index/inc/indexFstDfa.h +++ b/source/libs/index/inc/indexFstDfa.h @@ -29,16 +29,16 @@ extern "C" { typedef struct FstDfa FstDfa; typedef struct { - SArray * insts; + SArray *insts; uint32_t next[256]; bool isMatch; -} State; +} DfaState; /* * dfa builder related func **/ typedef struct FstDfaBuilder { - FstDfa * dfa; + FstDfa *dfa; SHashObj *cache; } FstDfaBuilder; diff --git a/source/libs/index/inc/indexFstRegex.h b/source/libs/index/inc/indexFstRegex.h index 2bf9c9b791..2814b5dc16 100644 --- a/source/libs/index/inc/indexFstRegex.h +++ b/source/libs/index/inc/indexFstRegex.h @@ -65,6 +65,7 @@ typedef struct { } FstRegex; FstRegex *regexCreate(const char *str); +void regexDestroy(FstRegex *regex); uint32_t regexAutomStart(FstRegex *regex); bool regexAutomIsMatch(FstRegex *regex, uint32_t state); diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 040e8ed830..05ce418037 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -22,7 +22,7 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later #define MEM_TERM_LIMIT 10 * 10000 -#define MEM_THRESHOLD 64 * 1024 +#define MEM_THRESHOLD 512 * 1024 #define MEM_SIGNAL_QUIT MEM_THRESHOLD * 20 #define MEM_ESTIMATE_RADIO 1.5 @@ -204,7 +204,6 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr if (0 == strcmp(c->colVal, pCt->colVal)) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid) - // taosArrayPush(result, &c->uid); *s = kTypeValue; } else if (c->operaType == DEL_VALUE) { INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid) @@ -309,7 +308,6 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR if (cond == MATCH) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid) - // taosArrayPush(result, &c->uid); *s = kTypeValue; } else if (c->operaType == DEL_VALUE) { INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid) diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index 81ac4c9d40..c4b83f8a07 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -1307,7 +1307,6 @@ FStmStRslt* stmStNextWith(FStmSt* sws, StreamCallback callback) { taosArrayPush(sws->inp, &(trn.inp)); if (FST_NODE_IS_FINAL(nextNode)) { - // void *eofState = sws->aut->acceptEof(nextState); void* eofState = automFuncs[aut->type].acceptEof(aut, nextState); if (eofState != NULL) { isMatch = automFuncs[aut->type].isMatch(aut, eofState); diff --git a/source/libs/index/src/indexFstCommon.c b/source/libs/index/src/indexFstCommon.c index 902e68ce09..0b20157009 100644 --- a/source/libs/index/src/indexFstCommon.c +++ b/source/libs/index/src/indexFstCommon.c @@ -294,3 +294,4 @@ const char COMMON_INPUTS_INV[] = { '\xee', '\xef', '\xf0', '\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', '\xf8', '\xf9', '\xfa', '\xfb', '\xfc', '\xfd', '\xfe', '\xff', }; +const int32_t COMMON_INPUTS_LEN = sizeof(COMMON_INPUTS) / sizeof(COMMON_INPUTS[0]); diff --git a/source/libs/index/src/indexFstDfa.c b/source/libs/index/src/indexFstDfa.c index 3011f124c9..b820f16a2a 100644 --- a/source/libs/index/src/indexFstDfa.c +++ b/source/libs/index/src/indexFstDfa.c @@ -41,7 +41,7 @@ FstDfaBuilder *dfaBuilderCreate(SArray *insts) { return NULL; } - SArray *states = taosArrayInit(4, sizeof(State)); + SArray *states = taosArrayInit(4, sizeof(DfaState)); builder->dfa = dfaCreate(insts, states); builder->cache = taosHashInit( @@ -71,7 +71,7 @@ FstDfa *dfaBuilder(FstDfaBuilder *builder) { dfaAdd(builder->dfa, cur, 0); - SArray * states = taosArrayInit(0, sizeof(uint32_t)); + SArray *states = taosArrayInit(0, sizeof(uint32_t)); uint32_t result; if (dfaBuilderCachedState(builder, cur, &result)) { taosArrayPush(states, &result); @@ -98,10 +98,12 @@ FstDfa *dfaBuilder(FstDfaBuilder *builder) { return builder->dfa; } +FstDfa *dfaBuilderBuild(FstDfaBuilder *builer) { return NULL; } + bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t byte, uint32_t *result) { sparSetClear(cur); - State *t = taosArrayGet(builder->dfa->states, state); + DfaState *t = taosArrayGet(builder->dfa->states, state); for (int i = 0; i < taosArrayGetSize(t->insts); i++) { uint32_t ip = *(int32_t *)taosArrayGet(t->insts, i); sparSetAdd(cur, ip); @@ -144,7 +146,7 @@ bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t * *result = *v; taosArrayDestroy(tinsts); } else { - State st; + DfaState st; st.insts = tinsts; st.isMatch = isMatch; taosArrayPush(builder->dfa->states, &st); @@ -169,14 +171,14 @@ bool dfaIsMatch(FstDfa *dfa, uint32_t si) { if (dfa->states == NULL || si < taosArrayGetSize(dfa->states)) { return false; } - State *st = taosArrayGet(dfa->states, si); + DfaState *st = taosArrayGet(dfa->states, si); return st != NULL ? st->isMatch : false; } bool dfaAccept(FstDfa *dfa, uint32_t si, uint8_t byte, uint32_t *result) { if (dfa->states == NULL || si < taosArrayGetSize(dfa->states)) { return false; } - State *st = taosArrayGet(dfa->states, si); + DfaState *st = taosArrayGet(dfa->states, si); *result = st->next[byte]; return true; } diff --git a/source/libs/index/src/indexFstRegex.c b/source/libs/index/src/indexFstRegex.c index 33eeae802e..37cb58996f 100644 --- a/source/libs/index/src/indexFstRegex.c +++ b/source/libs/index/src/indexFstRegex.c @@ -23,7 +23,7 @@ FstRegex *regexCreate(const char *str) { return NULL; } int32_t sz = (int32_t)strlen(str); - char * orig = taosMemoryCalloc(1, sz); + char *orig = taosMemoryCalloc(1, sz); memcpy(orig, str, sz); regex->orig = orig; @@ -36,6 +36,12 @@ FstRegex *regexCreate(const char *str) { return regex; } +void regexDestroy(FstRegex *regex) { + if (regex == NULL) return; + taosMemoryFree(regex->orig); + taosMemoryFree(regex); +} + uint32_t regexAutomStart(FstRegex *regex) { ///// no nothing return 0; diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index 040460ae5c..10becdff40 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -3,6 +3,7 @@ add_executable(idxFstTest "") add_executable(idxFstUT "") add_executable(idxUtilUT "") add_executable(idxJsonUT "") +add_executable(idxFstUtilUT "") target_sources(idxTest PRIVATE @@ -26,6 +27,11 @@ target_sources(idxJsonUT PRIVATE "jsonUT.cc" ) +target_sources(idxFstUtilUT + PRIVATE + "fstUtilUT.cc" +) + target_include_directories (idxTest PUBLIC "${TD_SOURCE_DIR}/include/libs/index" @@ -54,6 +60,12 @@ target_include_directories (idxJsonUT "${TD_SOURCE_DIR}/include/libs/index" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories (idxFstUtilUT + PUBLIC + "${TD_SOURCE_DIR}/include/libs/index" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + target_link_libraries (idxTest os util @@ -91,6 +103,13 @@ target_link_libraries (idxJsonUT gtest_main index ) +target_link_libraries (idxFstUtilUT + os + util + common + gtest_main + index +) add_test( NAME idxtest @@ -108,3 +127,8 @@ add_test( NAME idxFstUT COMMAND idxFstUT ) +add_test( + NAME idxFstUtilUT + COMMAND idxFstUtilUT + +) diff --git a/source/libs/index/test/fstUtilUT.cc b/source/libs/index/test/fstUtilUT.cc new file mode 100644 index 0000000000..3a20661484 --- /dev/null +++ b/source/libs/index/test/fstUtilUT.cc @@ -0,0 +1,63 @@ + +#include +#include +#include +#include +#include +#include +#include "index.h" +#include "indexCache.h" +#include "indexFst.h" +#include "indexFstDfa.h" +#include "indexFstRegex.h" +#include "indexFstSparse.h" +#include "indexFstUtil.h" +#include "indexInt.h" +#include "indexTfile.h" +#include "tglobal.h" +#include "tlog.h" +#include "tskiplist.h" +#include "tutil.h" +class FstUtilEnv : public ::testing::Test { + protected: + virtual void SetUp() { + SArray *inst = taosArrayInit(4, sizeof(char)); + builder = dfaBuilderCreate(inst); + } + virtual void TearDown() { dfaBuilderDestroy(builder); } + + FstDfaBuilder *builder; +}; + +class FstRegexEnv : public ::testing::Test { + protected: + virtual void SetUp() { regex = regexCreate("test"); } + virtual void TearDown() { regexDestroy(regex); } + FstRegex *regex; +}; + +class FstSparseSetEnv : public ::testing::Test { + protected: + virtual void SetUp() { set = sparSetCreate(256); } + virtual void TearDown() { sparSetDestroy(set); } + FstSparseSet *set; +}; + +// test FstDfaBuilder +TEST_F(FstUtilEnv, test1) {} +TEST_F(FstUtilEnv, test2) {} +TEST_F(FstUtilEnv, test3) {} +TEST_F(FstUtilEnv, test4) {} + +// test FstRegex + +TEST_F(FstRegexEnv, test1) {} +TEST_F(FstRegexEnv, test2) {} +TEST_F(FstRegexEnv, test3) {} +TEST_F(FstRegexEnv, test4) {} + +// test FstSparseSet +TEST_F(FstSparseSetEnv, test1) {} +TEST_F(FstSparseSetEnv, test2) {} +TEST_F(FstSparseSetEnv, test3) {} +TEST_F(FstSparseSetEnv, test4) {} diff --git a/source/libs/index/test/utilUT.cc b/source/libs/index/test/utilUT.cc index ab5128cd3e..323a6b4afa 100644 --- a/source/libs/index/test/utilUT.cc +++ b/source/libs/index/test/utilUT.cc @@ -8,6 +8,7 @@ #include "indexCache.h" #include "indexComm.h" #include "indexFst.h" +#include "indexFstCommon.h" #include "indexFstUtil.h" #include "indexInt.h" #include "indexTfile.h" @@ -356,3 +357,11 @@ TEST_F(UtilEnv, TempResultExcept) { idxTRsltMergeTo(relt, f); EXPECT_EQ(taosArrayGetSize(f), 1); } + +TEST_F(UtilEnv, testDictComm) { + int32_t count = COMMON_INPUTS_LEN; + for (int i = 0; i < 256; i++) { + uint8_t v = COMMON_INPUTS_INV[i]; + EXPECT_EQ(COMMON_INPUTS[v], i); + } +} diff --git a/source/util/src/ttrace.c b/source/util/src/ttrace.c deleted file mode 100644 index f183fd58fd..0000000000 --- a/source/util/src/ttrace.c +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "ttrace.h" -#include "taos.h" -#include "thash.h" -#include "tuuid.h" - -// clang-format off -//static TdThreadOnce init = PTHREAD_ONCE_INIT; -//static void * ids = NULL; -//static TdThreadMutex mtx; -// -//void traceInit() { -// ids = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); -// taosThreadMutexInit(&mtx, NULL); -//} -//void traceCreateEnv() { -// taosThreadOnce(&init, traceInit); -//} -//void traceDestroyEnv() { -// taosThreadMutexDestroy(&mtx); -// taosHashCleanup(ids); -//} -// -//STraceId traceInitId(STraceSubId *h, STraceSubId *l) { -// STraceId id = *h; -// id = ((id << 32) & 0xFFFFFFFF) | ((*l) & 0xFFFFFFFF); -// return id; -//} -//void traceId2Str(STraceId *id, char *buf) { -// int32_t f = (*id >> 32) & 0xFFFFFFFF; -// int32_t s = (*id) & 0xFFFFFFFF; -// sprintf(buf, "%d:%d", f, s); -//} -// -//void traceSetSubId(STraceId *id, STraceSubId *subId) { -// int32_t parent = ((*id >> 32) & 0xFFFFFFFF); -// taosThreadMutexLock(&mtx); -// taosHashPut(ids, subId, sizeof(*subId), &parent, sizeof(parent)); -// taosThreadMutexUnlock(&mtx); -//} -// -//STraceSubId traceGetParentId(STraceId *id) { -// int32_t parent = ((*id >> 32) & 0xFFFFFFFF); -// taosThreadMutexLock(&mtx); -// STraceSubId *p = taosHashGet(ids, (void *)&parent, sizeof(parent)); -// parent = *p; -// taosThreadMutexUnlock(&mtx); -// -// return parent; -//} -// -//STraceSubId traceGenSubId() { -// return tGenIdPI32(); -//} -//void traceSetRootId(STraceId *traceid, int64_t rootId) { -// traceId->rootId = rootId; -//} -//int64_t traceGetRootId(STraceId *traceId); -// -//void traceSetMsgId(STraceId *traceid, int64_t msgId); -//int64_t traceGetMsgId(STraceId *traceid); -// -//char *trace2Str(STraceId *id); -// -//void traceSetSubId(STraceId *id, int32_t *subId); -// clang-format on From c8573045f63eff0aa3fb77c2ff8656e67bde2694 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 8 Jul 2022 20:59:37 +0800 Subject: [PATCH 06/38] add test case --- source/libs/index/test/fstUtilUT.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/source/libs/index/test/fstUtilUT.cc b/source/libs/index/test/fstUtilUT.cc index 3a20661484..2c29758756 100644 --- a/source/libs/index/test/fstUtilUT.cc +++ b/source/libs/index/test/fstUtilUT.cc @@ -38,8 +38,15 @@ class FstRegexEnv : public ::testing::Test { class FstSparseSetEnv : public ::testing::Test { protected: - virtual void SetUp() { set = sparSetCreate(256); } - virtual void TearDown() { sparSetDestroy(set); } + virtual void SetUp() { set = sparSetCreate(256); } + virtual void TearDown() { + // tear down + sparSetDestroy(set); + } + void ReBuild(int32_t sz) { + sparSetDestroy(set); + set = sparSetCreate(sz); + } FstSparseSet *set; }; From f99a61a4644e3fb08ad6e531400f068db70dff4e Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 8 Jul 2022 21:05:19 +0800 Subject: [PATCH 07/38] fix: some problem of parser and planner --- include/util/taoserror.h | 1 + source/libs/parser/src/parAstParser.c | 3 + source/libs/parser/src/parTranslater.c | 110 +++++++++++------- source/libs/planner/src/planLogicCreater.c | 7 +- source/libs/planner/test/planIntervalTest.cpp | 2 + source/libs/scalar/src/scalar.c | 14 +-- tests/script/tsim/show/basic.sim | 4 +- .../2-query/query_cols_tags_and_or.py | 20 ++-- 8 files changed, 93 insertions(+), 68 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 41d5910625..05d1bd8ae3 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -582,6 +582,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_INTERP_CLAUSE TAOS_DEF_ERROR_CODE(0, 0x265D) #define TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN TAOS_DEF_ERROR_CODE(0, 0x265E) #define TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE TAOS_DEF_ERROR_CODE(0, 0x265F) +#define TSDB_CODE_PAR_INVALID_SMA_INDEX TAOS_DEF_ERROR_CODE(0, 0x265C) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index f38def0b1d..2060c3da4c 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -270,6 +270,9 @@ static int32_t collectMetaKeyFromCreateIndex(SCollectMetaKeyCxt* pCxt, SCreateIn if (TSDB_CODE_SUCCESS == code) { code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code) { + code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pCxt->pMetaCache); + } } return code; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 4ffcd34d70..46015e5bb5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -962,6 +962,10 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { return translateValueImpl(pCxt, pVal, dt, false); } +static int32_t doTranslateValue(STranslateContext* pCxt, SValueNode* pVal) { + return DEAL_RES_ERROR == translateValue(pCxt, pVal) ? pCxt->errCode : TSDB_CODE_SUCCESS; +} + static bool isMultiResFunc(SNode* pNode) { if (NULL == pNode) { return false; @@ -1430,6 +1434,24 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR; } +static EDealRes rewriteExprToGroupKeyFunc(STranslateContext* pCxt, SNode** pNode) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + strcpy(pFunc->functionName, "_group_key"); + strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName); + pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + *pNode = (SNode*)pFunc; + pCxt->errCode = fmGetFuncInfo(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len); + } + + return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); +} + static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { SCheckExprForGroupByCxt* pCxt = (SCheckExprForGroupByCxt*)pContext; if (!nodesIsExprNode(*pNode) || isAliasColumn(*pNode)) { @@ -1455,7 +1477,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { SNode* pPartKey = NULL; FOREACH(pPartKey, ((SSelectStmt*)pCxt->pTranslateCxt->pCurrStmt)->pPartitionByList) { if (nodesEqualNode(pPartKey, *pNode)) { - return DEAL_RES_IGNORE_CHILD; + return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode); } } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { @@ -1515,35 +1537,30 @@ static int32_t rewriteColsToSelectValFunc(STranslateContext* pCxt, SSelectStmt* typedef struct CheckAggColCoexistCxt { STranslateContext* pTranslateCxt; - bool existAggFunc; + bool existVectorFunc; bool existCol; - bool existIndefiniteRowsFunc; int32_t selectFuncNum; - bool existOtherAggFunc; + bool existOtherVectorFunc; } CheckAggColCoexistCxt; -static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { +static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext; - if (isSelectFunc(pNode)) { + if (isSelectFunc(*pNode)) { ++(pCxt->selectFuncNum); - } else if (isAggFunc(pNode)) { - pCxt->existOtherAggFunc = true; + } else if (isAggFunc(*pNode)) { + pCxt->existOtherVectorFunc = true; } - if (isAggFunc(pNode)) { - pCxt->existAggFunc = true; - return DEAL_RES_IGNORE_CHILD; - } - if (isIndefiniteRowsFunc(pNode)) { - pCxt->existIndefiniteRowsFunc = true; + if (isVectorFunc(*pNode)) { + pCxt->existVectorFunc = true; return DEAL_RES_IGNORE_CHILD; } SNode* pPartKey = NULL; FOREACH(pPartKey, ((SSelectStmt*)pCxt->pTranslateCxt->pCurrStmt)->pPartitionByList) { - if (nodesEqualNode(pPartKey, pNode)) { - return DEAL_RES_IGNORE_CHILD; + if (nodesEqualNode(pPartKey, *pNode)) { + return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode); } } - if (isScanPseudoColumnFunc(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { + if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { pCxt->existCol = true; } return DEAL_RES_CONTINUE; @@ -1554,24 +1571,20 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) return TSDB_CODE_SUCCESS; } CheckAggColCoexistCxt cxt = {.pTranslateCxt = pCxt, - .existAggFunc = false, + .existVectorFunc = false, .existCol = false, - .existIndefiniteRowsFunc = false, .selectFuncNum = 0, - .existOtherAggFunc = false}; - nodesWalkExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); + .existOtherVectorFunc = false}; + nodesRewriteExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); if (!pSelect->isDistinct) { - nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); + nodesRewriteExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); } - if (1 == cxt.selectFuncNum && !cxt.existOtherAggFunc) { + if (1 == cxt.selectFuncNum && !cxt.existOtherVectorFunc) { return rewriteColsToSelectValFunc(pCxt, pSelect); } - if ((cxt.selectFuncNum > 1 || cxt.existAggFunc || NULL != pSelect->pWindow) && cxt.existCol) { + if ((cxt.selectFuncNum > 1 || cxt.existVectorFunc || NULL != pSelect->pWindow) && cxt.existCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SINGLE_GROUP); } - if (cxt.existIndefiniteRowsFunc && cxt.existCol) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); - } return TSDB_CODE_SUCCESS; } @@ -4047,29 +4060,42 @@ static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStm return code; } -static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { - if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pInterval) || - (NULL != pStmt->pOptions->pOffset && - DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pOffset)) || - (NULL != pStmt->pOptions->pSliding && - DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pSliding))) { - return pCxt->errCode; +static int32_t checkCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { + SDbCfgInfo dbCfg = {0}; + int32_t code = getDBCfg(pCxt, pCxt->pParseCxt->db, &dbCfg); + if (TSDB_CODE_SUCCESS == code && NULL != dbCfg.pRetensions) { + code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_SMA_INDEX, + "Tables configured with the 'ROLLUP' option do not support creating sma index"); + } + if (TSDB_CODE_SUCCESS == code) { + code = doTranslateValue(pCxt, (SValueNode*)pStmt->pOptions->pInterval); + } + if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pOptions->pOffset) { + code = doTranslateValue(pCxt, (SValueNode*)pStmt->pOptions->pOffset); + } + if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pOptions->pSliding) { + code = doTranslateValue(pCxt, (SValueNode*)pStmt->pOptions->pSliding); } - if (NULL != pStmt->pOptions->pStreamOptions) { + if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pOptions->pStreamOptions) { SStreamOptions* pStreamOpt = (SStreamOptions*)pStmt->pOptions->pStreamOptions; - if (NULL != pStreamOpt->pWatermark && - (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStreamOpt->pWatermark))) { - return pCxt->errCode; + if (NULL != pStreamOpt->pWatermark) { + code = doTranslateValue(pCxt, (SValueNode*)pStreamOpt->pWatermark); } - - if (NULL != pStreamOpt->pDelay && (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStreamOpt->pDelay))) { - return pCxt->errCode; + if (TSDB_CODE_SUCCESS == code && NULL != pStreamOpt->pDelay) { + code = doTranslateValue(pCxt, (SValueNode*)pStreamOpt->pDelay); } } + return code; +} + +static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { SMCreateSmaReq createSmaReq = {0}; - int32_t code = buildCreateSmaReq(pCxt, pStmt, &createSmaReq); + int32_t code = checkCreateSmaIndex(pCxt, pStmt); + if (TSDB_CODE_SUCCESS == code) { + code = buildCreateSmaReq(pCxt, pStmt, &createSmaReq); + } if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_CREATE_SMA, (FSerializeFunc)tSerializeSMCreateSmaReq, &createSmaReq); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 703395b0d5..cb38e1fc18 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -485,10 +485,6 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); } - if (NULL != pSelect->pPartitionByList) { - code = createGroupKeysFromPartKeys(pSelect->pPartitionByList, &pAgg->pGroupKeys); - } - if (NULL != pSelect->pGroupByList) { if (NULL != pAgg->pGroupKeys) { code = nodesListStrictAppendList(pAgg->pGroupKeys, nodesCloneList(pSelect->pGroupByList)); @@ -845,8 +841,7 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel } static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { - if (NULL == pSelect->pPartitionByList || (pSelect->hasAggFuncs && NULL == pSelect->pWindow) || - NULL != pSelect->pGroupByList) { + if (NULL == pSelect->pPartitionByList) { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/test/planIntervalTest.cpp b/source/libs/planner/test/planIntervalTest.cpp index 10ef09adb9..73fa898bf8 100644 --- a/source/libs/planner/test/planIntervalTest.cpp +++ b/source/libs/planner/test/planIntervalTest.cpp @@ -62,4 +62,6 @@ TEST_F(PlanIntervalTest, stable) { run("SELECT _WSTARTTS, COUNT(*) FROM st1 INTERVAL(10s)"); run("SELECT _WSTARTTS, COUNT(*) FROM st1 PARTITION BY TBNAME INTERVAL(10s)"); + + run("SELECT TBNAME, COUNT(*) FROM st1 PARTITION BY TBNAME INTERVAL(10s)"); } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index e1aeaf219b..91a24986aa 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1069,15 +1069,13 @@ static int32_t getArithmeticOperatorResultType(SOperatorNode* pOp) { static int32_t getComparisonOperatorResultType(SOperatorNode* pOp) { SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; - if (NULL != pOp->pRight) { + if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) { + ((SExprNode*)(pOp->pRight))->resType = ldt; + } else if (nodesIsRegularOp(pOp)) { SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; - if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) { - rdt = ldt; - } else if (nodesIsRegularOp(pOp)) { - if (!IS_VAR_DATA_TYPE(ldt.type) || QUERY_NODE_VALUE != nodeType(pOp->pRight) || - (!IS_STR_DATA_TYPE(rdt.type) && (rdt.type != TSDB_DATA_TYPE_NULL))) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } + if (!IS_VAR_DATA_TYPE(ldt.type) || QUERY_NODE_VALUE != nodeType(pOp->pRight) || + (!IS_STR_DATA_TYPE(rdt.type) && (rdt.type != TSDB_DATA_TYPE_NULL))) { + return TSDB_CODE_TSC_INVALID_OPERATION; } } pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index 4d646f39e3..d0643bc7f2 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -99,7 +99,7 @@ if $rows != 1 then endi #sql select * from information_schema.`streams` sql select * from information_schema.user_tables -if $rows != 31 then +if $rows != 30 then return -1 endi #sql select * from information_schema.user_table_distributed @@ -197,7 +197,7 @@ if $rows != 1 then endi #sql select * from performance_schema.`streams` sql select * from information_schema.user_tables -if $rows != 31 then +if $rows != 30 then return -1 endi #sql select * from information_schema.user_table_distributed diff --git a/tests/system-test/2-query/query_cols_tags_and_or.py b/tests/system-test/2-query/query_cols_tags_and_or.py index f8a44f735f..c9df6f61bb 100644 --- a/tests/system-test/2-query/query_cols_tags_and_or.py +++ b/tests/system-test/2-query/query_cols_tags_and_or.py @@ -1066,7 +1066,7 @@ class TDTestCase: tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False # in query_sql = f'select {select_elm} from {tb_name} where c5 in (1, 6.6)' - tdSql.error(query_sql) + tdSql.query(query_sql) # not in query_sql = f'select {select_elm} from {tb_name} where c5 not in (2, 3)' tdSql.query(query_sql) @@ -1074,10 +1074,10 @@ class TDTestCase: tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False # and query_sql = f'select {select_elm} from {tb_name} where c5 > 0 and c5 >= 1 and c5 < 7 and c5 <= 6.6 and c5 != 2 and c5 <> 2 and c5 = 6.6 and c5 is not null and c5 between 2 and 6.6 and c5 not between 1 and 2 and c5 in (2,6.6) and c5 not in (1,2)' - tdSql.error(query_sql) + tdSql.query(query_sql) # or query_sql = f'select {select_elm} from {tb_name} where c5 > 6 or c5 >= 6.6 or c5 < 1 or c5 <= 0 or c5 != 1.1 or c5 <> 1.1 or c5 = 5 or c5 is null or c5 between 4 and 5 or c5 not between 1 and 3 or c5 in (4,5) or c5 not in (1.1,3)' - tdSql.error(query_sql) + tdSql.query(query_sql) # and or query_sql = f'select {select_elm} from {tb_name} where c5 > 0 and c5 >= 1 or c5 < 5 and c5 <= 6.6 and c5 != 2 and c5 <> 2 and c5 = 4 or c5 is not null and c5 between 2 and 4 and c5 not between 1 and 2 and c5 in (2,4) and c5 not in (1,2)' tdSql.query(query_sql) @@ -1145,7 +1145,7 @@ class TDTestCase: tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False # in query_sql = f'select {select_elm} from {tb_name} where c6 in (1, 7.7)' - tdSql.error(query_sql) + tdSql.query(query_sql) # not in query_sql = f'select {select_elm} from {tb_name} where c6 not in (2, 3)' tdSql.query(query_sql) @@ -1153,10 +1153,10 @@ class TDTestCase: tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False # and query_sql = f'select {select_elm} from {tb_name} where c6 > 0 and c6 >= 1 and c6 < 8 and c6 <= 7.7 and c6 != 2 and c6 <> 2 and c6 = 7.7 and c6 is not null and c6 between 2 and 7.7 and c6 not between 1 and 2 and c6 in (2,7.7) and c6 not in (1,2)' - tdSql.error(query_sql) + tdSql.query(query_sql) # or query_sql = f'select {select_elm} from {tb_name} where c6 > 7 or c6 >= 7.7 or c6 < 1 or c6 <= 0 or c6 != 1.1 or c6 <> 1.1 or c6 = 5 or c6 is null or c6 between 4 and 5 or c6 not between 1 and 3 or c6 in (4,5) or c6 not in (1.1,3)' - tdSql.error(query_sql) + tdSql.query(query_sql) # and or query_sql = f'select {select_elm} from {tb_name} where c6 > 0 and c6 >= 1 or c6 < 5 and c6 <= 7.7 and c6 != 2 and c6 <> 2 and c6 = 4 or c6 is not null and c6 between 2 and 4 and c6 not between 1 and 2 and c6 in (2,4) and c6 not in (1,2)' tdSql.query(query_sql) @@ -1398,7 +1398,7 @@ class TDTestCase: tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False # in query_sql = f'select {select_elm} from {tb_name} where c9 in ("binar", false)' - tdSql.error(query_sql) + tdSql.query(query_sql) # # not in query_sql = f'select {select_elm} from {tb_name} where c9 not in (true)' tdSql.query(query_sql) @@ -1407,13 +1407,13 @@ class TDTestCase: # # and query_sql = f'select {select_elm} from {tb_name} where c9 = true and c9 != "false" and c9 <> "binary" and c9 is not null and c9 in ("binary", true) and c9 not in ("binary")' - tdSql.error(query_sql) + tdSql.query(query_sql) # # or query_sql = f'select {select_elm} from {tb_name} where c9 = true or c9 != "false" or c9 <> "binary" or c9 = "true" or c9 is not null or c9 in ("binary", true) or c9 not in ("binary")' - tdSql.error(query_sql) + tdSql.query(query_sql) # # and or query_sql = f'select {select_elm} from {tb_name} where c9 = true and c9 != "false" or c9 <> "binary" or c9 = "true" and c9 is not null or c9 in ("binary", true) or c9 not in ("binary")' - tdSql.error(query_sql) + tdSql.query(query_sql) query_sql = f'select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13 from {tb_name} where c9 > "binary" and c9 >= "binary8" or c9 < "binary9" and c9 <= "binary" and c9 != 2 and c9 <> 2 and c9 = 4 or c9 is not null and c9 between 2 and 4 and c9 not between 1 and 2 and c9 in (2,4) and c9 not in (1,2)' tdSql.query(query_sql) tdSql.checkRows(9) From c3d702fe7cca73060590f6344481ad9cda855cef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 10 Jul 2022 09:40:35 +0800 Subject: [PATCH 08/38] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 134 ++++++------------ 3 files changed, 46 insertions(+), 92 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 82387119b6..72eddcfc51 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -902,7 +902,7 @@ int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result); int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* length); STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, - int32_t precision, int32_t order); + int32_t order); int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index cdac46c948..471d510385 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -938,7 +938,7 @@ static bool prepareDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t t setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex); (*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, *pRowIndex, gap, NULL); } else { - win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, pInfo->interval.precision, TSDB_ORDER_ASC); + win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, TSDB_ORDER_ASC); setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex); (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, *pRowIndex, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 991bbe525d..987882706a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -93,99 +93,55 @@ static STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, return save; } +static STimeWindow doCalculateTimeWindow(int64_t ts, SInterval* pInterval) { + STimeWindow w = {0}; + + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + w.skey = taosTimeTruncate(ts, pInterval, pInterval->precision); + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + } else { + int64_t st = w.skey; + + if (st > ts) { + st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + int64_t et = st + pInterval->interval - 1; + if (et < ts) { + st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + w.skey = st; + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + } + + return w; +} + // todo do refactor // get the correct time window according to the handled timestamp STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, - int32_t precision, int32_t order) { + int32_t order) { STimeWindow w = {0}; - if (pResultRowInfo->cur.pageId == -1) { // the first window, from the previous stored value - getInitialStartTimeWindow(pInterval, precision, ts, &w, true); - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + getInitialStartTimeWindow(pInterval, pInterval->precision, ts, &w, (order == TSDB_ORDER_ASC)); + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; return w; } w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win; - if (pInterval->interval == pInterval->sliding) { - if (w.skey > ts || w.ekey < ts) { - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - w.skey = taosTimeTruncate(ts, pInterval, precision); - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - int64_t st = w.skey; - - if (st > ts) { - st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - } - - int64_t et = st + pInterval->interval - 1; - if (et < ts) { - st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - } - - w.skey = st; - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } - } - } else { // it is an sliding window query, in which sliding value is not equalled to - // interval value, and we need to find the first qualified time window for asc/desc traverse respectively. - if (order == TSDB_ORDER_ASC) { - if (w.skey <= ts && w.ekey >= ts) { - // ts is resident in current time window, but we need to find the first - //qualified time window that cover this timestamp - w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order); - } else { - // todo refactor: - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - w.skey = taosTimeTruncate(ts, pInterval, precision); - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - int64_t st = w.skey; - - if (st > ts) { - st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - } - - int64_t et = st + pInterval->interval - 1; - if (et < ts) { - st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - } - - w.skey = st; - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - - w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order); - } - } - } else { - if (w.skey <= ts && w.ekey >= ts) { - w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order); - } else { - // todo refactor: - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - w.skey = taosTimeTruncate(ts, pInterval, precision); - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - int64_t st = w.skey; - - if (st > ts) { - st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - } - - int64_t et = st + pInterval->interval - 1; - if (et < ts) { - st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - } - - w.skey = st; - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - - w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order); - } - } - } + // in case of typical time window, we can calculate time window directly. + if (w.skey > ts || w.ekey < ts) { + w = doCalculateTimeWindow(ts, pInterval); } + + if (pInterval->interval != pInterval->sliding) { + // it is an sliding window query, in which sliding value is not equalled to + // interval value, and we need to find the first qualified time window. + w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order); + } + return w; } @@ -930,8 +886,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); SResultRow* pResult = NULL; - STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, - pInfo->interval.precision, pInfo->order); + STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->order); int32_t ret = TSDB_CODE_SUCCESS; if (!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) { ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, @@ -1390,7 +1345,7 @@ void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, for (int32_t i = 0; i < pBlock->info.rows; i++) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, pInterval->precision, TSDB_ORDER_ASC); + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, TSDB_ORDER_ASC); doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]); if (pUpWins) { SWinRes winRes = {.ts = win.skey, .groupId = groupIds[i]}; @@ -1412,7 +1367,7 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* for (int32_t i = 0; i < pBlock->info.rows; i += step) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, TSDB_ORDER_ASC); + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, TSDB_ORDER_ASC); step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); uint64_t winGpId = pGpDatas ? pGpDatas[i] : pBlock->info.groupId; bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), winGpId, numOfOutput); @@ -1452,7 +1407,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, TSDB_ORDER_ASC); + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, TSDB_ORDER_ASC); SWinRes winRe = { .ts = win.skey, .groupId = groupId, @@ -2547,8 +2502,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); - STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, - pInfo->interval.precision, pInfo->order); + STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->order); while (1) { bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); if (pInfo->ignoreExpiredData && isClosed) { @@ -4751,7 +4705,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* SResultRow* pResult = NULL; STimeWindow win = getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval, - iaInfo->interval.precision, iaInfo->order); + iaInfo->order); int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, From 40380ef9c1862f472288be1df19eae181dfbd19a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 10 Jul 2022 10:15:27 +0800 Subject: [PATCH 09/38] refactor: do internal refactor. --- include/common/tcommon.h | 4 +- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 8 +- source/libs/executor/inc/executorimpl.h | 7 -- source/libs/executor/src/executil.c | 94 +++++++++++++++++-- source/libs/executor/src/executorimpl.c | 16 ++-- source/libs/executor/src/scanoperator.c | 73 ++++---------- source/libs/executor/src/timewindowoperator.c | 88 +---------------- 8 files changed, 121 insertions(+), 171 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 28d771bbbd..614e7c9974 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -155,8 +155,8 @@ typedef struct SQueryTableDataCond { int32_t numOfCols; SColumnInfo* colList; int32_t type; // data block load type: - int32_t numOfTWindows; - STimeWindow* twindows; +// int32_t numOfTWindows; + STimeWindow twindows; int64_t startVersion; int64_t endVersion; } SQueryTableDataCond; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index ff29305b74..8ffa569865 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -133,7 +133,7 @@ bool tsdbNextDataBlock(STsdbReader *pReader); void tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo); int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave); SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); -int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond, int32_t tWinIdx); +int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); void *tsdbGetIdx(SMeta *pMeta); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9e01468fc7..f650480ff2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -356,14 +356,14 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd initReaderStatus(&pReader->status); pReader->pTsdb = - getTsdbByRetentions(pVnode, pCond->twindows[0].skey, pVnode->config.tsdbCfg.retentions, idstr, &level); + getTsdbByRetentions(pVnode, pCond->twindows.skey, pVnode->config.tsdbCfg.retentions, idstr, &level); pReader->suid = pCond->suid; pReader->order = pCond->order; pReader->capacity = 4096; pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL; pReader->verRange = getQueryVerRange(pVnode, pCond, level); pReader->type = pCond->type; - pReader->window = updateQueryTimeWindow(pVnode->pTsdb, pCond->twindows); + pReader->window = updateQueryTimeWindow(pVnode->pTsdb, &pCond->twindows); ASSERT(pCond->numOfCols > 0); @@ -2954,7 +2954,7 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { return pReader->pResBlock->pDataBlock; } -int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_t tWinIdx) { +int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { if (isEmptyQueryTimeWindow(&pReader->window)) { return TSDB_CODE_SUCCESS; } @@ -2964,7 +2964,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_ pReader->status.loadFromFile = true; pReader->status.pTableIter = NULL; - pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows[tWinIdx]); + pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); // allocate buffer in order to load data blocks from file memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg)); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 72eddcfc51..c0983e29cc 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -278,9 +278,6 @@ typedef struct STableScanInfo { SScanInfo scanInfo; int32_t scanTimes; SNode* pFilterNode; // filter info, which is push down by optimizer - SqlFunctionCtx* pCtx; // which belongs to the direct upstream operator operator query context,todo: remove this by using SExprSup - int32_t* rowEntryInfoOffset; // todo: remove this by using SExprSup - SExprInfo* pExpr;// todo: remove this by using SExprSup SSDataBlock* pResBlock; SArray* pColMatchInfo; @@ -289,14 +286,10 @@ typedef struct STableScanInfo { int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t dataBlockLoadFlag; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded. - SSampleExecInfo sample; // sample execution info - int32_t curTWinIdx; int32_t currentGroupId; int32_t currentTable; - uint64_t queryId; // todo remove it - uint64_t taskId; // todo remove it struct { uint64_t uid; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index a9cbb89eec..ef16192614 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include "function.h" #include "functionMgt.h" #include "index.h" @@ -769,12 +770,9 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi // pCond->twindow = pTableScanNode->scanRange; // TODO: get it from stable scan node - pCond->numOfTWindows = 1; - pCond->twindows = taosMemoryCalloc(pCond->numOfTWindows, sizeof(STimeWindow)); - pCond->twindows[0] = pTableScanNode->scanRange; - pCond->suid = pTableScanNode->scan.suid; - - pCond->type = BLOCK_LOAD_OFFSET_ORDER; + pCond->twindows = pTableScanNode->scanRange; + pCond->suid = pTableScanNode->scan.suid; + pCond->type = BLOCK_LOAD_OFFSET_ORDER; pCond->startVersion = -1; pCond->endVersion = -1; // pCond->type = pTableScanNode->scanFlag; @@ -826,3 +824,87 @@ int32_t convertFillType(int32_t mode) { return type; } + +static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) { + if (ascQuery) { + getAlignQueryTimeWindow(pInterval, pInterval->precision, ts, w); + } else { + // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + getAlignQueryTimeWindow(pInterval, pInterval->precision, ts, w); + + int64_t key = w->skey; + while (key < ts) { // moving towards end + key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + if (key >= ts) { + break; + } + + w->skey = key; + } + } +} + +static STimeWindow doCalculateTimeWindow(int64_t ts, SInterval* pInterval) { + STimeWindow w = {0}; + + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + w.skey = taosTimeTruncate(ts, pInterval, pInterval->precision); + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + } else { + int64_t st = w.skey; + + if (st > ts) { + st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + int64_t et = st + pInterval->interval - 1; + if (et < ts) { + st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + w.skey = st; + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + } + + return w; +} + +static STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order) { + int32_t factor = (order == TSDB_ORDER_ASC)? -1:1; + + STimeWindow win = *pWindow; + STimeWindow save = win; + while(win.skey <= ts && win.ekey >= ts) { + save = win; + win.skey = taosTimeAdd(win.skey, factor * pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + win.ekey = taosTimeAdd(win.ekey, factor * pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + } + + return save; +} + +// get the correct time window according to the handled timestamp +STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, + int32_t order) { + STimeWindow w = {0}; + if (pResultRowInfo->cur.pageId == -1) { // the first window, from the previous stored value + getInitialStartTimeWindow(pInterval, ts, &w, (order == TSDB_ORDER_ASC)); + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + return w; + } + + w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win; + + // in case of typical time window, we can calculate time window directly. + if (w.skey > ts || w.ekey < ts) { + w = doCalculateTimeWindow(ts, pInterval); + } + + if (pInterval->interval != pInterval->sliding) { + // it is an sliding window query, in which sliding value is not equalled to + // interval value, and we need to find the first qualified time window. + w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order); + } + + return w; +} \ No newline at end of file diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 359b342f93..b8ba72aa8a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1038,6 +1038,7 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI #endif static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { +#if 0 SqlFunctionCtx* pCtx = pTableScanInfo->pCtx; uint32_t status = BLK_DATA_NOT_LOAD; @@ -1059,6 +1060,8 @@ static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSData } return status; +#endif + return 0; } int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, @@ -2887,12 +2890,11 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) { ASSERT(found); tsdbSetTableId(pInfo->dataReader, uid); - int64_t oldSkey = pInfo->cond.twindows[0].skey; - pInfo->cond.twindows[0].skey = ts + 1; - tsdbReaderReset(pInfo->dataReader, &pInfo->cond, 0); - pInfo->cond.twindows[0].skey = oldSkey; + int64_t oldSkey = pInfo->cond.twindows.skey; + pInfo->cond.twindows.skey = ts + 1; + tsdbReaderReset(pInfo->dataReader, &pInfo->cond); + pInfo->cond.twindows.skey = oldSkey; pInfo->scanTimes = 0; - pInfo->curTWinIdx = 0; qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts, pInfo->currentTable, tableSz); @@ -4347,9 +4349,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo cond.colList->type = TSDB_DATA_TYPE_TIMESTAMP; cond.colList->bytes = sizeof(TSKEY); - cond.numOfTWindows = 1; - cond.twindows = taosMemoryCalloc(1, sizeof(STimeWindow)); - cond.twindows[0] = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; + cond.twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; cond.suid = pBlockNode->suid; cond.type = BLOCK_LOAD_OFFSET_ORDER; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 471d510385..db45ddb801 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -293,13 +293,8 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction // setupQueryRangeForReverseScan(pTableScanInfo); pTableScanInfo->cond.order = TSDB_ORDER_DESC; - for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) { - STimeWindow* pTWindow = &pTableScanInfo->cond.twindows[i]; - TSWAP(pTWindow->skey, pTWindow->ekey); - } - - SQueryTableDataCond* pCond = &pTableScanInfo->cond; - taosqsort(pCond->twindows, pCond->numOfTWindows, sizeof(STimeWindow), pCond, compareTimeWindow); + STimeWindow* pTWindow = &pTableScanInfo->cond.twindows; + TSWAP(pTWindow->skey, pTWindow->ekey); } int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, @@ -446,16 +441,10 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { // do the ascending order traverse in the first place. while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { - while (pTableScanInfo->curTWinIdx < pTableScanInfo->cond.numOfTWindows) { - SSDataBlock* p = doTableScanImpl(pOperator); - if (p != NULL) { - ASSERT(p->info.uid != 0); - return p; - } - pTableScanInfo->curTWinIdx += 1; - if (pTableScanInfo->curTWinIdx < pTableScanInfo->cond.numOfTWindows) { - tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, pTableScanInfo->curTWinIdx); - } + SSDataBlock* p = doTableScanImpl(pOperator); + if (p != NULL) { + ASSERT(p->info.uid != 0); + return p; } pTableScanInfo->scanTimes += 1; @@ -464,40 +453,25 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; qDebug("%s start to repeat ascending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); - for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) { - STimeWindow* pWin = &pTableScanInfo->cond.twindows[i]; - qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); - } + // do prepare for the next round table scan operation - tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); - pTableScanInfo->curTWinIdx = 0; + tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); } } int32_t total = pTableScanInfo->scanInfo.numOfAsc + pTableScanInfo->scanInfo.numOfDesc; if (pTableScanInfo->scanTimes < total) { if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) { - prepareForDescendingScan(pTableScanInfo, pTableScanInfo->pCtx, 0); - tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); - pTableScanInfo->curTWinIdx = 0; + prepareForDescendingScan(pTableScanInfo, pOperator->exprSupp.pCtx, 0); + tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); } qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); - for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) { - STimeWindow* pWin = &pTableScanInfo->cond.twindows[i]; - qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); - } while (pTableScanInfo->scanTimes < total) { - while (pTableScanInfo->curTWinIdx < pTableScanInfo->cond.numOfTWindows) { - SSDataBlock* p = doTableScanImpl(pOperator); - if (p != NULL) { - return p; - } - pTableScanInfo->curTWinIdx += 1; - if (pTableScanInfo->curTWinIdx < pTableScanInfo->cond.numOfTWindows) { - tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, pTableScanInfo->curTWinIdx); - } + SSDataBlock* p = doTableScanImpl(pOperator); + if (p != NULL) { + return p; } pTableScanInfo->scanTimes += 1; @@ -508,12 +482,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { qDebug("%s start to repeat descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); - for (int32_t i = 0; i < pTableScanInfo->cond.numOfTWindows; ++i) { - STimeWindow* pWin = &pTableScanInfo->cond.twindows[i]; - qDebug("%s qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); - } - tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); - pTableScanInfo->curTWinIdx = 0; + tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); } } } @@ -540,9 +509,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, pInfo->currentTable); tsdbSetTableId(pInfo->dataReader, pTableInfo->uid); - tsdbReaderReset(pInfo->dataReader, &pInfo->cond, 0); + tsdbReaderReset(pInfo->dataReader, &pInfo->cond); pInfo->scanTimes = 0; - pInfo->curTWinIdx = 0; } } @@ -574,8 +542,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); // tsdbSetTableList(pInfo->dataReader, tableList); - tsdbReaderReset(pInfo->dataReader, &pInfo->cond, 0); - pInfo->curTWinIdx = 0; + tsdbReaderReset(pInfo->dataReader, &pInfo->cond); pInfo->scanTimes = 0; result = doTableScanGroup(pOperator); @@ -646,7 +613,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->pFilterNode = pTableScanNode->scan.node.pConditions; pInfo->scanFlag = MAIN_SCAN; pInfo->pColMatchInfo = pColList; - pInfo->curTWinIdx = 0; pInfo->currentGroupId = -1; pOperator->name = "TableScanOperator"; // for debug purpose @@ -875,12 +841,7 @@ static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t grou } void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) { - pTableScanInfo->cond.twindows[0] = *pWin; - pTableScanInfo->curTWinIdx = 0; - // tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); - // if (!pTableScanInfo->dataReader) { - // return false; - // } + pTableScanInfo->cond.twindows = *pWin; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 987882706a..ebe645df4e 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -59,92 +59,6 @@ static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOper static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; } -static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, - bool ascQuery) { - if (ascQuery) { - getAlignQueryTimeWindow(pInterval, precision, ts, w); - } else { - // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - getAlignQueryTimeWindow(pInterval, precision, ts, w); - - int64_t key = w->skey; - while (key < ts) { // moving towards end - key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision); - if (key >= ts) { - break; - } - - w->skey = key; - } - } -} - -static STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order) { - int32_t factor = (order == TSDB_ORDER_ASC)? -1:1; - - STimeWindow win = *pWindow; - STimeWindow save = win; - while(win.skey <= ts && win.ekey >= ts) { - save = win; - win.skey = taosTimeAdd(win.skey, factor * pInterval->sliding, pInterval->slidingUnit, pInterval->precision); - win.ekey = taosTimeAdd(win.ekey, factor * pInterval->sliding, pInterval->slidingUnit, pInterval->precision); - } - - return save; -} - -static STimeWindow doCalculateTimeWindow(int64_t ts, SInterval* pInterval) { - STimeWindow w = {0}; - - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - w.skey = taosTimeTruncate(ts, pInterval, pInterval->precision); - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; - } else { - int64_t st = w.skey; - - if (st > ts) { - st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - } - - int64_t et = st + pInterval->interval - 1; - if (et < ts) { - st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - } - - w.skey = st; - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; - } - - return w; -} - -// todo do refactor -// get the correct time window according to the handled timestamp -STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, - int32_t order) { - STimeWindow w = {0}; - if (pResultRowInfo->cur.pageId == -1) { // the first window, from the previous stored value - getInitialStartTimeWindow(pInterval, pInterval->precision, ts, &w, (order == TSDB_ORDER_ASC)); - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; - return w; - } - - w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win; - - // in case of typical time window, we can calculate time window directly. - if (w.skey > ts || w.ekey < ts) { - w = doCalculateTimeWindow(ts, pInterval); - } - - if (pInterval->interval != pInterval->sliding) { - // it is an sliding window query, in which sliding value is not equalled to - // interval value, and we need to find the first qualified time window. - w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order); - } - - return w; -} - static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, @@ -4679,7 +4593,7 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t continue; } STimeWindow* prevWin = &prevGrpWin->window; - if ((ascScan && newWin->skey > prevWin->ekey || (!ascScan) && newWin->skey < prevWin->ekey)) { + if ((ascScan && newWin->skey > prevWin->ekey) || ((!ascScan) && newWin->skey < prevWin->ekey)) { finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock); tdListPopNode(miaInfo->groupIntervals, listNode); } From 3ee9d160d9346479146e168a8b6fa82160de5eda Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 10 Jul 2022 11:02:54 +0800 Subject: [PATCH 10/38] fix(query): set correct stddev merge input rows. --- source/libs/executor/inc/executorimpl.h | 1 - source/libs/executor/src/executorimpl.c | 12 ------------ source/libs/executor/src/timewindowoperator.c | 2 ++ source/libs/function/src/builtinsimpl.c | 13 +++++-------- 4 files changed, 7 insertions(+), 21 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index c0983e29cc..612677634f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -911,7 +911,6 @@ int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted); bool functionNeedToExecute(SqlFunctionCtx* pCtx); -int32_t compareTimeWindow(const void* p1, const void* p2, const void* param); int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 842f220c6e..b209684388 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4546,18 +4546,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return pOptr; } -int32_t compareTimeWindow(const void* p1, const void* p2, const void* param) { - const SQueryTableDataCond* pCond = param; - const STimeWindow* pWin1 = p1; - const STimeWindow* pWin2 = p2; - if (pCond->order == TSDB_ORDER_ASC) { - return pWin1->skey - pWin2->skey; - } else if (pCond->order == TSDB_ORDER_DESC) { - return pWin2->skey - pWin1->skey; - } - return 0; -} - SArray* extractColumnInfo(SNodeList* pNodeList) { size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn)); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 2be8b4e579..fb1169b3d3 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1156,11 +1156,13 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doFilter(pInfo->pCondition, pBlock); + bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { doSetOperatorCompleted(pOperator); break; } + if (pBlock->info.rows > 0) { break; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index d8fcca30c0..f95ec78c76 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1863,8 +1863,6 @@ static void stddevTransferInfo(SStddevRes* pInput, SStddevRes* pOutput) { } pOutput->count += pInput->count; - - return; } int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx) { @@ -1874,14 +1872,13 @@ int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx) { SStddevRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - int32_t start = pInput->startRowIndex; - char* data = colDataGetData(pCol, start); - SStddevRes* pInputInfo = (SStddevRes*)varDataVal(data); - - stddevTransferInfo(pInputInfo, pInfo); + for(int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) { + char* data = colDataGetData(pCol, i); + SStddevRes* pInputInfo = (SStddevRes*)varDataVal(data); + stddevTransferInfo(pInputInfo, pInfo); + } SET_VAL(GET_RES_INFO(pCtx), 1, 1); - return TSDB_CODE_SUCCESS; } From f52038dbfc0d99b333f28ac9a990d9245542729a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 10 Jul 2022 11:35:24 +0800 Subject: [PATCH 11/38] add test case --- source/libs/index/inc/indexFstSparse.h | 13 +++--- source/libs/index/src/indexFstDfa.c | 15 ++++--- source/libs/index/src/indexFstSparse.c | 44 +++++++++++++++------ source/libs/index/test/fstUtilUT.cc | 55 ++++++++++++++++++++++---- 4 files changed, 97 insertions(+), 30 deletions(-) diff --git a/source/libs/index/inc/indexFstSparse.h b/source/libs/index/inc/indexFstSparse.h index 665fb2ba5c..bd704fb427 100644 --- a/source/libs/index/inc/indexFstSparse.h +++ b/source/libs/index/inc/indexFstSparse.h @@ -23,17 +23,18 @@ extern "C" { #endif typedef struct FstSparseSet { - uint32_t *dense; - uint32_t *sparse; - int32_t size; + int32_t *dense; + int32_t *sparse; + int32_t size; + int32_t cap; } FstSparseSet; FstSparseSet *sparSetCreate(int32_t sz); void sparSetDestroy(FstSparseSet *s); uint32_t sparSetLen(FstSparseSet *ss); -uint32_t sparSetAdd(FstSparseSet *ss, uint32_t ip); -uint32_t sparSetGet(FstSparseSet *ss, uint32_t i); -bool sparSetContains(FstSparseSet *ss, uint32_t ip); +bool sparSetAdd(FstSparseSet *ss, int32_t ip, int32_t *val); +bool sparSetGet(FstSparseSet *ss, int32_t i, int32_t *val); +bool sparSetContains(FstSparseSet *ss, int32_t ip); void sparSetClear(FstSparseSet *ss); #ifdef __cplusplus diff --git a/source/libs/index/src/indexFstDfa.c b/source/libs/index/src/indexFstDfa.c index b820f16a2a..275580ebdc 100644 --- a/source/libs/index/src/indexFstDfa.c +++ b/source/libs/index/src/indexFstDfa.c @@ -105,8 +105,9 @@ bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet sparSetClear(cur); DfaState *t = taosArrayGet(builder->dfa->states, state); for (int i = 0; i < taosArrayGetSize(t->insts); i++) { - uint32_t ip = *(int32_t *)taosArrayGet(t->insts, i); - sparSetAdd(cur, ip); + int32_t ip = *(int32_t *)taosArrayGet(t->insts, i); + bool succ = sparSetAdd(cur, ip, NULL); + assert(succ == true); } dfaRun(builder->dfa, cur, next, byte); @@ -126,7 +127,9 @@ bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t * bool isMatch = false; for (int i = 0; i < sparSetLen(set); i++) { - uint32_t ip = sparSetGet(set, i); + int32_t ip; + + if (false == sparSetGet(set, i, &ip)) continue; Inst *inst = taosArrayGet(builder->dfa->insts, ip); if (inst->ty == JUMP || inst->ty == SPLIT) { @@ -186,7 +189,8 @@ void dfaAdd(FstDfa *dfa, FstSparseSet *set, uint32_t ip) { if (sparSetContains(set, ip)) { return; } - sparSetAdd(set, ip); + bool succ = sparSetAdd(set, ip, NULL); + assert(succ == true); Inst *inst = taosArrayGet(dfa->insts, ip); if (inst->ty == MATCH || inst->ty == RANGE) { // do nothing @@ -203,7 +207,8 @@ bool dfaRun(FstDfa *dfa, FstSparseSet *from, FstSparseSet *to, uint8_t byte) { bool isMatch = false; sparSetClear(to); for (int i = 0; i < sparSetLen(from); i++) { - uint32_t ip = sparSetGet(from, i); + int32_t ip; + if (false == sparSetGet(from, i, &ip)) continue; Inst *inst = taosArrayGet(dfa->insts, ip); if (inst->ty == JUMP || inst->ty == SPLIT) { diff --git a/source/libs/index/src/indexFstSparse.c b/source/libs/index/src/indexFstSparse.c index 71d8854dcc..99ed5d6429 100644 --- a/source/libs/index/src/indexFstSparse.c +++ b/source/libs/index/src/indexFstSparse.c @@ -21,8 +21,12 @@ FstSparseSet *sparSetCreate(int32_t sz) { return NULL; } - ss->dense = (uint32_t *)taosMemoryCalloc(sz, sizeof(uint32_t)); - ss->sparse = (uint32_t *)taosMemoryCalloc(sz, sizeof(uint32_t)); + ss->dense = (int32_t *)taosMemoryMalloc(sz * sizeof(int32_t)); + memset(ss->dense, -1, sz * sizeof(int32_t)); + ss->sparse = (int32_t *)taosMemoryMalloc(sz * sizeof(int32_t)); + memset(ss->sparse, -1, sz * sizeof(int32_t)); + ss->cap = sz; + ss->size = 0; return ss; } @@ -38,23 +42,39 @@ uint32_t sparSetLen(FstSparseSet *ss) { // Get occupied size return ss == NULL ? 0 : ss->size; } -uint32_t sparSetAdd(FstSparseSet *ss, uint32_t ip) { +bool sparSetAdd(FstSparseSet *ss, int32_t ip, int32_t *idx) { if (ss == NULL) { - return 0; + return false; + } + if (ip >= ss->cap) { + return false; } uint32_t i = ss->size; ss->dense[i] = ip; ss->sparse[ip] = i; ss->size += 1; - return i; + + if (idx != NULL) *idx = i; + + return true; } -uint32_t sparSetGet(FstSparseSet *ss, uint32_t i) { - // check later - return ss->dense[i]; +bool sparSetGet(FstSparseSet *ss, int32_t idx, int32_t *ip) { + if (idx >= ss->cap || idx >= ss->size) { + return false; + } + int32_t val = ss->dense[idx]; + if (ip != NULL) { + *ip = val; + } + return val == -1 ? false : true; } -bool sparSetContains(FstSparseSet *ss, uint32_t ip) { - uint32_t i = ss->sparse[ip]; - if (i < ss->size && ss->dense[i] == ip) { +bool sparSetContains(FstSparseSet *ss, int32_t ip) { + if (ip >= ss->cap) { + return false; + } + int32_t i = ss->sparse[ip]; + + if (i < ss->cap && i < ss->size && ss->dense[i] == ip) { return true; } else { return false; @@ -64,5 +84,7 @@ void sparSetClear(FstSparseSet *ss) { if (ss == NULL) { return; } + memset(ss->dense, -1, ss->cap * sizeof(int32_t)); + memset(ss->sparse, -1, ss->cap * sizeof(int32_t)); ss->size = 0; } diff --git a/source/libs/index/test/fstUtilUT.cc b/source/libs/index/test/fstUtilUT.cc index 2c29758756..22fe1a9150 100644 --- a/source/libs/index/test/fstUtilUT.cc +++ b/source/libs/index/test/fstUtilUT.cc @@ -51,10 +51,18 @@ class FstSparseSetEnv : public ::testing::Test { }; // test FstDfaBuilder -TEST_F(FstUtilEnv, test1) {} -TEST_F(FstUtilEnv, test2) {} -TEST_F(FstUtilEnv, test3) {} -TEST_F(FstUtilEnv, test4) {} +TEST_F(FstUtilEnv, test1) { + // test +} +TEST_F(FstUtilEnv, test2) { + // test +} +TEST_F(FstUtilEnv, test3) { + // test +} +TEST_F(FstUtilEnv, test4) { + // test +} // test FstRegex @@ -64,7 +72,38 @@ TEST_F(FstRegexEnv, test3) {} TEST_F(FstRegexEnv, test4) {} // test FstSparseSet -TEST_F(FstSparseSetEnv, test1) {} -TEST_F(FstSparseSetEnv, test2) {} -TEST_F(FstSparseSetEnv, test3) {} -TEST_F(FstSparseSetEnv, test4) {} +TEST_F(FstSparseSetEnv, test1) { + for (int8_t i = 0; i < 20; i++) { + int32_t val = -1; + bool succ = sparSetAdd(set, 'a' + i, &val); + } + EXPECT_EQ(sparSetLen(set), 20); + for (int8_t i = 0; i < 20; i++) { + int val = -1; + bool find = sparSetGet(set, i, &val); + EXPECT_EQ(find, true); + EXPECT_EQ(val, i + 'a'); + } + for (int8_t i = 'a'; i < 'a' + 20; i++) { + EXPECT_EQ(sparSetContains(set, i), true); + } + + for (int8_t i = 'A'; i < 20; i++) { + EXPECT_EQ(sparSetContains(set, 'A'), false); + } + + for (int i = 512; i < 1000; i++) { + EXPECT_EQ(sparSetAdd(set, i, NULL), false); + + EXPECT_EQ(sparSetGet(set, i, NULL), false); + EXPECT_EQ(sparSetContains(set, i), false); + } + sparSetClear(set); + + for (int i = 'a'; i < 'a' + 20; i++) { + EXPECT_EQ(sparSetGet(set, i, NULL), false); + } + for (int i = 1000; i < 2000; i++) { + EXPECT_EQ(sparSetGet(set, i, NULL), false); + } +} From 2225411edcd4e578984554fbc6acdd1d602ab62b Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sun, 10 Jul 2022 13:56:31 +0800 Subject: [PATCH 12/38] refactor(stream) --- source/dnode/vnode/src/inc/tq.h | 2 +- source/dnode/vnode/src/tq/tq.c | 15 +++++++++------ source/dnode/vnode/src/tq/tqExec.c | 24 +++++++++++++++++------- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executorMain.c | 2 -- source/libs/executor/src/scanoperator.c | 3 ++- 6 files changed, 30 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index b2ea08d50d..f58bbe0d68 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -129,7 +129,7 @@ typedef struct { static STqMgmt tqMgmt = {0}; // tqRead -int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* offset); +int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* offset); int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum); // tqExec diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0c72d35027..2aef9e7175 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -112,7 +112,8 @@ int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, }; tmsgSendRsp(&resp); - tqDebug("vgId:%d from consumer:%" PRId64 ", (epoch %d) send rsp, res msg type %d, reqOffset:%" PRId64 ", rspOffset:%" PRId64, + tqDebug("vgId:%d from consumer:%" PRId64 ", (epoch %d) send rsp, res msg type %d, reqOffset:%" PRId64 + ", rspOffset:%" PRId64, TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->reqOffset, pRsp->rspOffset); return 0; @@ -179,8 +180,8 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { tDecoderClear(&decoder); if (offset.val.type == TMQ_OFFSET__SNAPSHOT_DATA) { - tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64, offset.subKey, - TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts); + tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64, + offset.subKey, TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts); } else if (offset.val.type == TMQ_OFFSET__LOG) { tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, offset.subKey, TD_VID(pTq->pVnode), offset.val.version); @@ -316,9 +317,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } // 3.query - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN && fetchOffsetNew.type == TMQ_OFFSET__LOG) { + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { fetchOffsetNew.version++; - if (tqScanLog(pTq, &pHandle->execHandle, &dataRsp, &fetchOffsetNew) < 0) { + if (tqScan(pTq, &pHandle->execHandle, &dataRsp, &fetchOffsetNew) < 0) { ASSERT(0); code = -1; goto OVER; @@ -333,7 +334,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { goto OVER; } - if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN && fetchOffsetNew.type == TMQ_OFFSET__LOG) { + if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN) { int64_t fetchVer = fetchOffsetNew.version + 1; SWalCkHead* pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); if (pCkHead == NULL) { @@ -411,6 +412,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } taosMemoryFree(pCkHead); +#if 0 } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) { tqInfo("retrieve using snapshot actual offset: uid %" PRId64 " ts %" PRId64, fetchOffsetNew.uid, fetchOffsetNew.ts); if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, fetchOffsetNew, workerId) < 0) { @@ -421,6 +423,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { code = -1; } +#endif } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_META) { ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 4ecd88b1fd..b2fd5a5ddc 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -59,15 +59,17 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp) { return 0; } -int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* pOffset) { +int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* pOffset) { qTaskInfo_t task = pExec->execCol.task[0]; if (qStreamPrepareScan(task, pOffset) < 0) { + ASSERT(pOffset->type == TMQ_OFFSET__LOG); pRsp->rspOffset = *pOffset; pRsp->rspOffset.version--; return 0; } + int32_t rowCnt = 0; while (1) { SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; @@ -77,11 +79,19 @@ int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOff if (pDataBlock != NULL) { tqAddBlockDataToRsp(pDataBlock, pRsp); - if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader[0]->msgIter.uid; - tqAddTbNameToRsp(pTq, uid, pRsp); - } pRsp->blockNum++; + if (pRsp->withTbName) { + if (pOffset->type == TMQ_OFFSET__LOG) { + int64_t uid = pExec->pExecReader[0]->msgIter.uid; + tqAddTbNameToRsp(pTq, uid, pRsp); + } else { + pRsp->withTbName = 0; + } + } + if (pOffset->type == TMQ_OFFSET__LOG) { + rowCnt += pDataBlock->info.rows; + if (rowCnt >= 4096) break; + } continue; } @@ -94,12 +104,12 @@ int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOff ASSERT(0); } + ASSERT(pRsp->rspOffset.type != 0); + if (pRsp->rspOffset.type == TMQ_OFFSET__LOG) { ASSERT(pRsp->rspOffset.version + 1 >= pRsp->reqOffset.version); } - ASSERT(pRsp->rspOffset.type != 0); - break; } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 82387119b6..21eb7680d5 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -144,6 +144,7 @@ typedef struct { void* metaBlk; // for tmq fetching meta SSDataBlock* pullOverBlk; // for streaming SWalFilterCond cond; + int64_t lastScanUid; } SStreamTaskInfo; typedef struct SExecTaskInfo { diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index ca1d8b2b10..42ad173c74 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -336,8 +336,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts, pTableScanInfo->currentTable, tableSz); - } else { - // switch to log } } else { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index d2f63849d0..57e74415a7 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1273,6 +1273,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { /*pTaskInfo->streamInfo.lastStatus.version = pTaskInfo->streamInfo.prepareStatus.version - 1;*/ /*} else {*/ pTaskInfo->streamInfo.lastStatus = ret.offset; + /*pTaskInfo->streamInfo.lastScanUid */ ASSERT(pTaskInfo->streamInfo.lastStatus.version + 1 >= pTaskInfo->streamInfo.prepareStatus.version); /*}*/ return NULL; @@ -1490,7 +1491,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; } else if (pInfo->blockType == STREAM_INPUT__TABLE_SCAN) { - /*ASSERT(0);*/ + ASSERT(0); // check reader last status // if not match, reset status SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); From 3169b5e6c3acdaef0e358a9763be6afcd3b02559 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sun, 10 Jul 2022 13:58:30 +0800 Subject: [PATCH 13/38] fix: fix hb crash issue --- source/client/src/clientHb.c | 6 ++++-- source/libs/catalog/inc/catalogInt.h | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 96f0eb8e2b..e2d75d39e3 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -266,8 +266,10 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { static int32_t emptyRspNum = 0; char *key = (char *)param; SClientHbBatchRsp pRsp = {0}; - tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp); - + if (TSDB_CODE_SUCCESS == code) { + tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp); + } + int32_t rspNum = taosArrayGetSize(pRsp.rsps); taosThreadMutexLock(&appInfo.mutex); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 9d0e3871cc..7637c66b84 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -166,7 +166,7 @@ typedef struct SCtgDBCache { int8_t deleted; SCtgVgCache vgCache; SHashObj *tbCache; // key:tbname, value:SCtgTbCache - SHashObj *stbCache; // key:suid, value:STableMeta* + SHashObj *stbCache; // key:suid, value:char* } SCtgDBCache; typedef struct SCtgRentSlot { From 77cb2c0b847102839cae022bad96e81aa484e9b5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 10 Jul 2022 13:45:26 +0800 Subject: [PATCH 14/38] feat:write meta from tmq to taosd --- source/dnode/mnode/impl/src/mndStb.c | 66 ++++++++++++++++------------ 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 224c1dbb5d..8ce22c2b2e 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -45,6 +45,7 @@ static int32_t mndProcessTableMetaReq(SRpcMsg *pReq); static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); static int32_t mndProcessTableCfgReq(SRpcMsg *pReq); +static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp); int32_t mndInitStb(SMnode *pMnode) { SSdbTable table = { @@ -854,6 +855,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { SStbObj *pStb = NULL; SDbObj *pDb = NULL; SMCreateStbReq createReq = {0}; + bool isAlter = false; if (tDeserializeSMCreateStbReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -889,6 +891,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { code = 0; goto _OVER; } else if ((tagDelta == 1 || colDelta == 1) && (verDelta == 1)) { + isAlter = true; mInfo("stb:%s, schema version is only increased by 1 number, do alter operation", createReq.name); } else { mError("stb:%s, schema version increase more than 1 number, error is returned", createReq.name); @@ -929,7 +932,12 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { goto _OVER; } - code = mndCreateStb(pMnode, pReq, &createReq, pDb); + if (isAlter) { + bool needRsp = false; + code = mndAlterStbImp(pMnode, pReq, pDb, pStb, needRsp); + } else { + code = mndCreateStb(pMnode, pReq, &createReq, pDb); + } if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: @@ -1495,14 +1503,13 @@ static int32_t mndBuildStbCfg(SMnode *pMnode, const char *dbFName, const char *t return code; } -static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont, - int32_t *pLen) { +static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, SStbObj *pObj, void **pCont, int32_t *pLen) { int32_t ret; SEncoder ec = {0}; uint32_t contLen = 0; SMAlterStbRsp alterRsp = {0}; SName name = {0}; - tNameFromString(&name, pAlter->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + tNameFromString(&name, pObj->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); alterRsp.pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); if (NULL == alterRsp.pMeta) { @@ -1535,10 +1542,36 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, S return 0; } +static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq); + if (pTrans == NULL) goto _OVER; + + mDebug("trans:%d, used to alter stb:%s", pTrans->id, pStb->name); + mndTransSetDbName(pTrans, pDb->name, NULL); + + if (needRsp) { + void *pCont = NULL; + int32_t contLen = 0; + if (mndBuildSMAlterStbRsp(pDb, pStb, &pCont, &contLen) != 0) goto _OVER; + mndTransSetRpcRsp(pTrans, pCont, contLen); + } + + if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + code = 0; + +_OVER: + mndTransDrop(pTrans); + return code; +} + static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { bool needRsp = true; int32_t code = -1; - STrans *pTrans = NULL; SField *pField0 = NULL; SStbObj stbObj = {0}; @@ -1587,30 +1620,9 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p } if (code != 0) goto _OVER; - - code = -1; - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq); - if (pTrans == NULL) goto _OVER; - - mDebug("trans:%d, used to alter stb:%s", pTrans->id, pAlter->name); - mndTransSetDbName(pTrans, pDb->name, NULL); - - if (needRsp) { - void *pCont = NULL; - int32_t contLen = 0; - if (mndBuildSMAlterStbRsp(pDb, pAlter, &stbObj, &pCont, &contLen) != 0) goto _OVER; - mndTransSetRpcRsp(pTrans, pCont, contLen); - } - - if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; - if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; - if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - - code = 0; + code = mndAlterStbImp(pMnode, pReq, pDb, &stbObj, needRsp); _OVER: - mndTransDrop(pTrans); taosMemoryFreeClear(stbObj.pTags); taosMemoryFreeClear(stbObj.pColumns); return code; From bafa54778a7173473c2e784896d10a3b91b0c87d Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sun, 10 Jul 2022 14:10:39 +0800 Subject: [PATCH 15/38] refactor(stream) --- source/dnode/vnode/src/tq/tq.c | 4 +++- source/libs/executor/src/scanoperator.c | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 2aef9e7175..e0afc6c80f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -318,7 +318,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { // 3.query if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - fetchOffsetNew.version++; + if (fetchOffsetNew.type == TMQ_OFFSET__LOG) { + fetchOffsetNew.version++; + } if (tqScan(pTq, &pHandle->execHandle, &dataRsp, &fetchOffsetNew) < 0) { ASSERT(0); code = -1; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 57e74415a7..3532c1d367 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -427,8 +427,11 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { pOperator->cost.totalCost = pTableScanInfo->readRecorder.elapsedTime; // todo refactor - pTableScanInfo->lastStatus.uid = pBlock->info.uid; - pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey; + /*pTableScanInfo->lastStatus.uid = pBlock->info.uid;*/ + /*pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;*/ + pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; + pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.uid; + pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; ASSERT(pBlock->info.uid != 0); return pBlock; From 4d4ec8e7173cf28cc9ea4cecf13047d46c6a4704 Mon Sep 17 00:00:00 2001 From: tomchon Date: Sun, 10 Jul 2022 14:16:33 +0800 Subject: [PATCH 16/38] test:modify testcase of mnode --- tests/system-test/6-cluster/5dnode1mnode.py | 6 +- .../6-cluster/5dnode3mnodeAdd1Ddnoe.py | 229 ++++++++++++++++++ .../5dnode3mnodeRestartDnodeInsertData.py | 223 +++++++++++++++++ .../5dnode3mnodeSep1VnodeStopDnodeCreateDb.py | 61 ++--- ...5dnode3mnodeSep1VnodeStopDnodeCreateStb.py | 5 +- ...dnode3mnodeSep1VnodeStopDnodeInsertData.py | 221 +++++++++++++++++ .../5dnode3mnodeSep1VnodeStopMnodeCreateDb.py | 3 +- ...5dnode3mnodeSep1VnodeStopVnodeCreateStb.py | 9 +- .../6-cluster/clusterCommonCreate.py | 8 +- tests/system-test/fulltest.sh | 5 +- 10 files changed, 725 insertions(+), 45 deletions(-) create mode 100644 tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py create mode 100644 tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py create mode 100644 tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index 5f4ab7357b..4611726c14 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -124,9 +124,9 @@ class TDTestCase: tdSql.query('show databases;') tdSql.checkData(2,5,'no_strict') tdSql.error('alter database db strict 0') - tdSql.execute('alter database db strict 1') - tdSql.query('show databases;') - tdSql.checkData(2,5,'strict') + # tdSql.execute('alter database db strict 1') + # tdSql.query('show databases;') + # tdSql.checkData(2,5,'strict') def getConnection(self, dnode): host = dnode.cfgDict["fqdn"] diff --git a/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py b/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py new file mode 100644 index 0000000000..d3de31eb04 --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py @@ -0,0 +1,229 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +from numpy import row_stack +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def insertData(self,countstart,countstop): + # fisrt add data : db\stable\childtable\general table + + for couti in range(countstart,countstop): + tdLog.debug("drop database if exists db%d" %couti) + tdSql.execute("drop database if exists db%d" %couti) + print("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("use db%d" %couti) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + + def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db0_0', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'stbNumbers': 2, + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 200, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + "rowsPerTbl": 1000, + "batchNum": 5000 + } + hostname = socket.gethostname() + dnodeNumbers=int(dnodeNumbers) + mnodeNums=int(mnodeNums) + vnodeNumbers = int(dnodeNumbers-mnodeNums) + allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"]) + rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"] + rowsall=rowsPerStb*paraDict['stbNumbers'] + dbNumbers = 1 + + tdLog.info("first check dnode and mnode") + tdSql.query("show dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodeNumbers) + clusterComCheck.checkMnodeStatus(1) + + # fisr add three mnodes; + tdLog.info("fisr add three mnodes and check mnode status") + tdSql.execute("create mnode on dnode 2") + clusterComCheck.checkMnodeStatus(2) + tdSql.execute("create mnode on dnode 3") + clusterComCheck.checkMnodeStatus(3) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("show dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodeNumbers) + + # create database and stable + clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']) + tdLog.info("Take turns stopping Mnodes ") + + tdDnodes=cluster.dnodes + # dnode6=cluster.addDnode(6) + # tdDnodes.append(dnode6) + # tdDnodes = ClusterDnodes(tdDnodes) + stopcount =0 + threads=[] + + # create stable:stb_0 + stableName= paraDict['stbName'] + newTdSql=tdCom.newTdSql() + clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']) + #create child table:ctb_0 + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum']) + #insert date + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) + for tr in threads: + tr.start() + dnode6Port=int(6030+5*100) + tdSql.execute("create dnode '%s:%d'"%(hostname,dnode6Port)) + clusterComCheck.checkDnodes(dnodeNumbers) + + while stopcount < restartNumbers: + tdLog.info(" restart loop: %d"%stopcount ) + if stopRole == "mnode": + for i in range(mnodeNums): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + elif stopRole == "vnode": + for i in range(vnodeNumbers): + tdDnodes[i+mnodeNums].stoptaosd() + # sleep(10) + tdDnodes[i+mnodeNums].starttaosd() + # sleep(10) + elif stopRole == "dnode": + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + + # dnodeNumbers don't include database of schema + if clusterComCheck.checkDnodes(dnodeNumbers): + tdLog.info("123") + else: + print("456") + + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + for tr in threads: + tr.join() + + + clusterComCheck.checkDnodes(dnodeNumbers) + clusterComCheck.checkDbRows(dbNumbers) + # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"]) + + tdSql.execute("use %s" %(paraDict["dbName"])) + tdSql.query("show stables") + tdSql.checkRows(paraDict["stbNumbers"]) + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + tdSql.query("select * from %s"%stableName) + tdSql.checkRows(rowsPerStb) + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='dnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py new file mode 100644 index 0000000000..e5946342d2 --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py @@ -0,0 +1,223 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +from numpy import row_stack +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def insertData(self,countstart,countstop): + # fisrt add data : db\stable\childtable\general table + + for couti in range(countstart,countstop): + tdLog.debug("drop database if exists db%d" %couti) + tdSql.execute("drop database if exists db%d" %couti) + print("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("use db%d" %couti) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + + def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db0_0', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'stbNumbers': 2, + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 200, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + "rowsPerTbl": 1000, + "batchNum": 5000 + } + + dnodeNumbers=int(dnodeNumbers) + mnodeNums=int(mnodeNums) + vnodeNumbers = int(dnodeNumbers-mnodeNums) + allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"]) + rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"] + rowsall=rowsPerStb*paraDict['stbNumbers'] + dbNumbers = 1 + + tdLog.info("first check dnode and mnode") + tdSql.query("show dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodeNumbers) + clusterComCheck.checkMnodeStatus(1) + + # fisr add three mnodes; + tdLog.info("fisr add three mnodes and check mnode status") + tdSql.execute("create mnode on dnode 2") + clusterComCheck.checkMnodeStatus(2) + tdSql.execute("create mnode on dnode 3") + clusterComCheck.checkMnodeStatus(3) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("show dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodeNumbers) + + # create database and stable + clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']) + tdLog.info("Take turns stopping Mnodes ") + + tdDnodes=cluster.dnodes + stopcount =0 + threads=[] + + # create stable:stb_0 + stableName= paraDict['stbName'] + newTdSql=tdCom.newTdSql() + clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']) + #create child table:ctb_0 + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum']) + #insert date + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) + for tr in threads: + tr.start() + for tr in threads: + tr.join() + + while stopcount < restartNumbers: + tdLog.info(" restart loop: %d"%stopcount ) + if stopRole == "mnode": + for i in range(mnodeNums): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + elif stopRole == "vnode": + for i in range(vnodeNumbers): + tdDnodes[i+mnodeNums].stoptaosd() + # sleep(10) + tdDnodes[i+mnodeNums].starttaosd() + # sleep(10) + elif stopRole == "dnode": + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + + # dnodeNumbers don't include database of schema + if clusterComCheck.checkDnodes(dnodeNumbers): + tdLog.info("123") + else: + print("456") + + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + + + clusterComCheck.checkDnodes(dnodeNumbers) + clusterComCheck.checkDbRows(dbNumbers) + # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"]) + + tdSql.execute("use %s" %(paraDict["dbName"])) + tdSql.query("show stables") + tdSql.checkRows(paraDict["stbNumbers"]) + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + tdSql.query("select * from %s"%stableName) + tdSql.checkRows(rowsPerStb) + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='dnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py index 96fad487d1..8971a51ef3 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py @@ -143,42 +143,43 @@ class TDTestCase: threads=[] for i in range(restartNumbers): dbNameIndex = '%s%d'%(paraDict["dbName"],i) - threads.append(threading.Thread(target=clusterComCreate.create_databases, args=(tdSql, dbNameIndex,paraDict["dbNumbers"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.create_databases, args=(newTdSql, dbNameIndex,paraDict["dbNumbers"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) for tr in threads: tr.start() tdLog.info("Take turns stopping Mnodes ") - # while stopcount < restartNumbers: - # tdLog.info(" restart loop: %d"%stopcount ) - # if stopRole == "mnode": - # for i in range(mnodeNums): - # tdDnodes[i].stoptaosd() - # # sleep(10) - # tdDnodes[i].starttaosd() - # # sleep(10) - # elif stopRole == "vnode": - # for i in range(vnodeNumbers): - # tdDnodes[i+mnodeNums].stoptaosd() - # # sleep(10) - # tdDnodes[i+mnodeNums].starttaosd() - # # sleep(10) - # elif stopRole == "dnode": - # for i in range(dnodeNumbers): - # tdDnodes[i].stoptaosd() - # # sleep(10) - # tdDnodes[i].starttaosd() - # # sleep(10) + while stopcount < restartNumbers: + tdLog.info(" restart loop: %d"%stopcount ) + if stopRole == "mnode": + for i in range(mnodeNums): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + elif stopRole == "vnode": + for i in range(vnodeNumbers): + tdDnodes[i+mnodeNums].stoptaosd() + # sleep(10) + tdDnodes[i+mnodeNums].starttaosd() + # sleep(10) + elif stopRole == "dnode": + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) - # # dnodeNumbers don't include database of schema - # if clusterComCheck.checkDnodes(dnodeNumbers): - # tdLog.info("check dnodes status is ready") - # else: - # tdLog.info("check dnodes status is not ready") - # self.stopThread(threads) - # tdLog.exit("one or more of dnodes failed to start ") - # # self.check3mnode() - # stopcount+=1 + # dnodeNumbers don't include database of schema + if clusterComCheck.checkDnodes(dnodeNumbers): + tdLog.info("check dnodes status is ready") + else: + tdLog.info("check dnodes status is not ready") + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 for tr in threads: tr.join() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py index 4ab9aa64e1..6db1a9fddd 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py @@ -92,7 +92,7 @@ class TDTestCase: def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): tdLog.printNoPrefix("======== test case 1: ") - paraDict = {'dbName': 'db0_0', + paraDict = {'dbName': 'db', 'dropFlag': 1, 'event': '', 'vgroups': 4, @@ -143,7 +143,8 @@ class TDTestCase: threads=[] for i in range(restartNumbers): stableName= '%s%d'%(paraDict['stbName'],i) - threads.append(threading.Thread(target=clusterComCreate.create_stables, args=(tdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']))) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.create_stables, args=(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']))) for tr in threads: tr.start() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py new file mode 100644 index 0000000000..8f99ef0b5c --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py @@ -0,0 +1,221 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +from numpy import row_stack +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def insertData(self,countstart,countstop): + # fisrt add data : db\stable\childtable\general table + + for couti in range(countstart,countstop): + tdLog.debug("drop database if exists db%d" %couti) + tdSql.execute("drop database if exists db%d" %couti) + print("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("use db%d" %couti) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + + def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db0_0', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'stbNumbers': 2, + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 200, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + "rowsPerTbl": 10000, + "batchNum": 5000 + } + + dnodeNumbers=int(dnodeNumbers) + mnodeNums=int(mnodeNums) + vnodeNumbers = int(dnodeNumbers-mnodeNums) + allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"]) + rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"] + rowsall=rowsPerStb*paraDict['stbNumbers'] + dbNumbers = 1 + + tdLog.info("first check dnode and mnode") + tdSql.query("show dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodeNumbers) + clusterComCheck.checkMnodeStatus(1) + + # fisr add three mnodes; + tdLog.info("fisr add three mnodes and check mnode status") + tdSql.execute("create mnode on dnode 2") + clusterComCheck.checkMnodeStatus(2) + tdSql.execute("create mnode on dnode 3") + clusterComCheck.checkMnodeStatus(3) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("show dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodeNumbers) + + # create database and stable + clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']) + tdLog.info("Take turns stopping Mnodes ") + + tdDnodes=cluster.dnodes + stopcount =0 + threads=[] + + # create stable:stb_0 + stableName= paraDict['stbName'] + newTdSql=tdCom.newTdSql() + clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']) + #create child table:ctb_0 + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum']) + #insert date + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) + for tr in threads: + tr.start() + while stopcount < restartNumbers: + tdLog.info(" restart loop: %d"%stopcount ) + if stopRole == "mnode": + for i in range(mnodeNums): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + elif stopRole == "vnode": + for i in range(vnodeNumbers): + tdDnodes[i+mnodeNums].stoptaosd() + # sleep(10) + tdDnodes[i+mnodeNums].starttaosd() + # sleep(10) + elif stopRole == "dnode": + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + + # dnodeNumbers don't include database of schema + if clusterComCheck.checkDnodes(dnodeNumbers): + tdLog.info("123") + else: + print("456") + + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + + for tr in threads: + tr.join() + clusterComCheck.checkDnodes(dnodeNumbers) + clusterComCheck.checkDbRows(dbNumbers) + # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"]) + + tdSql.execute("use %s" %(paraDict["dbName"])) + tdSql.query("show stables") + tdSql.checkRows(paraDict["stbNumbers"]) + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + tdSql.query("select * from %s"%stableName) + tdSql.checkRows(rowsPerStb) + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='dnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py index 17c344e341..f820d812ec 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py @@ -116,7 +116,8 @@ class TDTestCase: threads=[] for i in range(restartNumbers): dbNameIndex = '%s%d'%(paraDict["dbName"],i) - threads.append(threading.Thread(target=clusterComCreate.create_databases, args=(tdSql, dbNameIndex,paraDict["dbNumbers"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.create_databases, args=(newTdSql, dbNameIndex,paraDict["dbNumbers"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) for tr in threads: tr.start() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py index 2f1c1368d1..128dc10b37 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py @@ -30,8 +30,8 @@ class TDTestCase: self.TDDnodes = None tdSql.init(conn.cursor()) self.host = socket.gethostname() - - + print(tdSql) + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -113,6 +113,7 @@ class TDTestCase: allStbNumbers=(paraDict['stbNumbers']*restartNumbers) dbNumbers = 1 + print(tdSql) tdLog.info("first check dnode and mnode") tdSql.query("show dnodes;") tdSql.checkData(0,1,'%s:6030'%self.host) @@ -141,9 +142,11 @@ class TDTestCase: tdDnodes=cluster.dnodes stopcount =0 threads=[] + for i in range(restartNumbers): stableName= '%s%d'%(paraDict['stbName'],i) - threads.append(threading.Thread(target=clusterComCreate.create_stables, args=(tdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']))) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.create_stables, args=(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']))) for tr in threads: tr.start() diff --git a/tests/system-test/6-cluster/clusterCommonCreate.py b/tests/system-test/6-cluster/clusterCommonCreate.py index 9d3f74a2e3..851fe3b51c 100644 --- a/tests/system-test/6-cluster/clusterCommonCreate.py +++ b/tests/system-test/6-cluster/clusterCommonCreate.py @@ -152,7 +152,7 @@ class ClusterComCreate: if (i % 2 == 0): tagValue = 'shanghai' - sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i,stbName,i+1, tagValue) + sql += " %s_%d using %s tags(%d, '%s')"%(ctbPrefix,i,stbName,i+1, tagValue) if (i > 0) and (i%100 == 0): tsql.execute(sql) sql = pre_create @@ -173,13 +173,13 @@ class ClusterComCreate: startTs = int(round(t * 1000)) #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) for i in range(ctbNum): - sql += " %s%d values "%(stbName,i) + sql += " %s_%d values "%(stbName,i) for j in range(rowsPerTbl): - sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + sql += "(%d, %d, %d, 'mnode_%d') "%(startTs + j, j, j,j) if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): tsql.execute(sql) if j < rowsPerTbl - 1: - sql = "insert into %s%d values " %(stbName,i) + sql = "insert into %s_%d values " %(stbName,i) else: sql = "insert into " #end sql diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 1e305f2518..0aa9e395db 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -122,7 +122,7 @@ python3 ./test.py -f 2-query/irate.py python3 ./test.py -f 2-query/function_null.py python3 ./test.py -f 2-query/queryQnode.py -#python3 ./test.py -f 6-cluster/5dnode1mnode.py +python3 ./test.py -f 6-cluster/5dnode1mnode.py #python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 #python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 #python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 @@ -135,7 +135,8 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 - # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py # python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 # python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 - +python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 5 -M 3 python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py From 0521bd5afe552a93c7fb4a7d2560ab032cc5cc79 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 10 Jul 2022 14:28:46 +0800 Subject: [PATCH 17/38] refactor(sync): add vnodeSnapWriterClose IsApply --- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 8d0ae4785b..d49c307233 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -478,7 +478,7 @@ static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void static int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) { SVnode *pVnode = pFsm->data; - int32_t code = vnodeSnapWriterClose(pWriter, isApply); + int32_t code = vnodeSnapWriterClose(pWriter, !isApply); return code; } From 6a9304e2eb347deb2c5150a1662a5ee72a11666d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 10 Jul 2022 14:41:37 +0800 Subject: [PATCH 18/38] test: comment out unstable case --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 409fb5e930..9ed34daf69 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -120,7 +120,7 @@ python3 ./test.py -f 2-query/irate.py python3 ./test.py -f 2-query/and_or_for_byte.py python3 ./test.py -f 2-query/function_null.py -python3 ./test.py -f 2-query/queryQnode.py +#python3 ./test.py -f 2-query/queryQnode.py #python3 ./test.py -f 6-cluster/5dnode1mnode.py #python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 From 5640036f6690e46b968f3ae1219e889d9010c721 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sun, 10 Jul 2022 14:48:16 +0800 Subject: [PATCH 19/38] refactor(stream) --- source/client/src/tmq.c | 398 ++++++++++++------------ source/dnode/vnode/src/tq/tqExec.c | 6 +- source/libs/executor/inc/executorimpl.h | 2 + source/libs/executor/src/executorMain.c | 2 + source/libs/executor/src/executorimpl.c | 37 ++- 5 files changed, 232 insertions(+), 213 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 6710f73fc6..c9d949300b 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "cJSON.h" #include "clientInt.h" #include "clientLog.h" #include "parser.h" @@ -23,7 +24,6 @@ #include "tqueue.h" #include "tref.h" #include "ttimer.h" -#include "cJSON.h" int32_t tmqAskEp(tmq_t* tmq, bool async); @@ -106,8 +106,8 @@ struct tmq_t { tsem_t rspSem; }; -struct tmq_raw_data{ - void *raw_meta; +struct tmq_raw_data { + void* raw_meta; int32_t raw_meta_len; int16_t raw_meta_type; }; @@ -953,6 +953,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { goto FAIL; } + tscInfo("consumer %ld is setup, consumer group %s", pTmq->consumerId, pTmq->groupId); + return pTmq; FAIL: @@ -1194,10 +1196,10 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { for (int32_t j = 0; j < vgNumCur; j++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); - char buf[50]; - tFormatOffset(buf, 50, &pVgCur->currentOffsetNew); - tscDebug("consumer:%" PRId64 ", epoch %d vgId:%d vgKey is %s, offset is %s", tmq->consumerId, epoch, pVgCur->vgId, - vgKey, buf); + char buf[80]; + tFormatOffset(buf, 80, &pVgCur->currentOffsetNew); + tscDebug("consumer:%" PRId64 ", epoch %d vgId:%d vgKey is %s, offset is %s", tmq->consumerId, epoch, + pVgCur->vgId, vgKey, buf); taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffsetNew, sizeof(STqOffsetVal)); } } @@ -1564,7 +1566,8 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus != TMQ_VG_STATUS__IDLE) { int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1); - tscTrace("consumer:%" PRId64 ", epoch %d skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, vgSkipCnt); + tscTrace("consumer:%" PRId64 ", epoch %d skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, + vgSkipCnt); continue; /*if (vgSkipCnt < 10000) continue;*/ #if 0 @@ -1620,8 +1623,8 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { char offsetFormatBuf[80]; tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffsetNew); - tscDebug("consumer:%" PRId64 ", send poll to %s vgId:%d, epoch %d, req offset:%s, reqId:%" PRIu64, tmq->consumerId, - pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId); + tscDebug("consumer:%" PRId64 ", send poll to %s vgId:%d, epoch %d, req offset:%s, reqId:%" PRIu64, + tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId); /*printf("send vgId:%d %" PRId64 "\n", pVg->vgId, pVg->currentOffset);*/ asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; @@ -1669,7 +1672,8 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { int32_t consumerEpoch = atomic_load_32(&tmq->epoch); if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; - /*printf("vgId:%d offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ + /*printf("vgId:%d offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, + * rspMsg->msg.rspOffset);*/ pVg->currentOffsetNew = pollRspWrapper->dataRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); if (pollRspWrapper->dataRsp.blockNum == 0) { @@ -1691,7 +1695,8 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { int32_t consumerEpoch = atomic_load_32(&tmq->epoch); if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; - /*printf("vgId:%d offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ + /*printf("vgId:%d offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, + * rspMsg->msg.rspOffset);*/ pVg->currentOffsetNew.version = pollRspWrapper->metaRsp.rspOffset; pVg->currentOffsetNew.type = TMQ_OFFSET__LOG; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); @@ -1848,9 +1853,9 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } -tmq_raw_data *tmq_get_raw_meta(TAOS_RES* res) { +tmq_raw_data* tmq_get_raw_meta(TAOS_RES* res) { if (TD_RES_TMQ_META(res)) { - tmq_raw_data *raw = taosMemoryCalloc(1, sizeof(tmq_raw_data)); + tmq_raw_data* raw = taosMemoryCalloc(1, sizeof(tmq_raw_data)); SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; raw->raw_meta = pMetaRspObj->metaRsp.metaRsp; raw->raw_meta_len = pMetaRspObj->metaRsp.metaRspLen; @@ -1860,7 +1865,8 @@ tmq_raw_data *tmq_get_raw_meta(TAOS_RES* res) { return NULL; } -static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t){ +static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, + int8_t t) { char* string = NULL; cJSON* json = cJSON_CreateObject(); if (json == NULL) { @@ -1870,31 +1876,31 @@ static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* sch cJSON_AddItemToObject(json, "type", type); char uid[32] = {0}; - sprintf(uid, "%"PRIi64, id); + sprintf(uid, "%" PRIi64, id); cJSON* id_ = cJSON_CreateString(uid); cJSON_AddItemToObject(json, "id", id_); cJSON* tableName = cJSON_CreateString(name); cJSON_AddItemToObject(json, "tableName", tableName); cJSON* tableType = cJSON_CreateString(t == TSDB_NORMAL_TABLE ? "normal" : "super"); cJSON_AddItemToObject(json, "tableType", tableType); -// cJSON* version = cJSON_CreateNumber(1); -// cJSON_AddItemToObject(json, "version", version); + // cJSON* version = cJSON_CreateNumber(1); + // cJSON_AddItemToObject(json, "version", version); cJSON* columns = cJSON_CreateArray(); - for(int i = 0; i < schemaRow->nCols; i++){ - cJSON* column = cJSON_CreateObject(); - SSchema *s = schemaRow->pSchema + i; - cJSON* cname = cJSON_CreateString(s->name); + for (int i = 0; i < schemaRow->nCols; i++) { + cJSON* column = cJSON_CreateObject(); + SSchema* s = schemaRow->pSchema + i; + cJSON* cname = cJSON_CreateString(s->name); cJSON_AddItemToObject(column, "name", cname); cJSON* ctype = cJSON_CreateNumber(s->type); cJSON_AddItemToObject(column, "type", ctype); - if(s->type == TSDB_DATA_TYPE_BINARY){ + if (s->type == TSDB_DATA_TYPE_BINARY) { int32_t length = s->bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(column, "length", cbytes); - }else if (s->type == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (s->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (s->type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(column, "length", cbytes); } cJSON_AddItemToArray(columns, column); @@ -1902,20 +1908,20 @@ static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* sch cJSON_AddItemToObject(json, "columns", columns); cJSON* tags = cJSON_CreateArray(); - for(int i = 0; schemaTag && i < schemaTag->nCols; i++){ - cJSON* tag = cJSON_CreateObject(); - SSchema *s = schemaTag->pSchema + i; - cJSON* tname = cJSON_CreateString(s->name); + for (int i = 0; schemaTag && i < schemaTag->nCols; i++) { + cJSON* tag = cJSON_CreateObject(); + SSchema* s = schemaTag->pSchema + i; + cJSON* tname = cJSON_CreateString(s->name); cJSON_AddItemToObject(tag, "name", tname); cJSON* ttype = cJSON_CreateNumber(s->type); cJSON_AddItemToObject(tag, "type", ttype); - if(s->type == TSDB_DATA_TYPE_BINARY){ + if (s->type == TSDB_DATA_TYPE_BINARY) { int32_t length = s->bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(tag, "length", cbytes); - }else if (s->type == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (s->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (s->type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(tag, "length", cbytes); } cJSON_AddItemToArray(tags, tag); @@ -1927,13 +1933,13 @@ static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* sch return string; } -static char *processCreateStb(SMqMetaRsp *metaRsp){ +static char* processCreateStb(SMqMetaRsp* metaRsp) { SVCreateStbReq req = {0}; SDecoder coder; - char* string = NULL; + char* string = NULL; // decode and process req - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); @@ -1949,7 +1955,7 @@ _err: return string; } -static char *buildCreateCTableJson(STag* pTag, int64_t sid, char* name, int64_t id){ +static char* buildCreateCTableJson(STag* pTag, int64_t sid, char* name, int64_t id) { char* string = NULL; cJSON* json = cJSON_CreateObject(); if (json == NULL) { @@ -1958,7 +1964,7 @@ static char *buildCreateCTableJson(STag* pTag, int64_t sid, char* name, int64_t cJSON* type = cJSON_CreateString("create"); cJSON_AddItemToObject(json, "type", type); char cid[32] = {0}; - sprintf(cid, "%"PRIi64, id); + sprintf(cid, "%" PRIi64, id); cJSON* cid_ = cJSON_CreateString(cid); cJSON_AddItemToObject(json, "id", cid_); @@ -1968,19 +1974,19 @@ static char *buildCreateCTableJson(STag* pTag, int64_t sid, char* name, int64_t cJSON_AddItemToObject(json, "tableType", tableType); char sid_[32] = {0}; - sprintf(sid_, "%"PRIi64, sid); + sprintf(sid_, "%" PRIi64, sid); cJSON* using = cJSON_CreateString(sid_); cJSON_AddItemToObject(json, "using", using); -// cJSON* version = cJSON_CreateNumber(1); -// cJSON_AddItemToObject(json, "version", version); + // cJSON* version = cJSON_CreateNumber(1); + // cJSON_AddItemToObject(json, "version", version); cJSON* tags = cJSON_CreateArray(); - if (tTagIsJson(pTag)) { // todo + if (tTagIsJson(pTag)) { // todo char* pJson = parseTagDatatoJson(pTag); cJSON* tag = cJSON_CreateObject(); - cJSON* tname = cJSON_CreateString("unknown"); // todo + cJSON* tname = cJSON_CreateString("unknown"); // todo cJSON_AddItemToObject(tag, "name", tname); cJSON* ttype = cJSON_CreateNumber(TSDB_DATA_TYPE_JSON); cJSON_AddItemToObject(tag, "type", ttype); @@ -1999,12 +2005,12 @@ static char *buildCreateCTableJson(STag* pTag, int64_t sid, char* name, int64_t goto end; } - for(int i = 0; i < taosArrayGetSize(pTagVals); i++){ + for (int i = 0; i < taosArrayGetSize(pTagVals); i++) { STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, i); cJSON* tag = cJSON_CreateObject(); -// cJSON* tname = cJSON_CreateNumber(pTagVal->cid); - cJSON* tname = cJSON_CreateString("unkonwn"); // todo + // cJSON* tname = cJSON_CreateNumber(pTagVal->cid); + cJSON* tname = cJSON_CreateString("unkonwn"); // todo cJSON_AddItemToObject(tag, "name", tname); cJSON* ttype = cJSON_CreateNumber(pTagVal->type); cJSON_AddItemToObject(tag, "type", ttype); @@ -2032,13 +2038,13 @@ end: return string; } -static char *processCreateTable(SMqMetaRsp *metaRsp){ +static char* processCreateTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVCreateTbBatchReq req = {0}; - SVCreateTbReq *pCreateReq; - char *string = NULL; + SVCreateTbReq* pCreateReq; + char* string = NULL; // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&decoder, data, len); if (tDecodeSVCreateTbBatchReq(&decoder, &req) < 0) { @@ -2048,27 +2054,29 @@ static char *processCreateTable(SMqMetaRsp *metaRsp){ // loop to create table for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; - if(pCreateReq->type == TSDB_CHILD_TABLE){ - string = buildCreateCTableJson((STag*)pCreateReq->ctb.pTag, pCreateReq->ctb.suid, pCreateReq->name, pCreateReq->uid); - }else if(pCreateReq->type == TSDB_NORMAL_TABLE){ - string = buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE); + if (pCreateReq->type == TSDB_CHILD_TABLE) { + string = + buildCreateCTableJson((STag*)pCreateReq->ctb.pTag, pCreateReq->ctb.suid, pCreateReq->name, pCreateReq->uid); + } else if (pCreateReq->type == TSDB_NORMAL_TABLE) { + string = + buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE); } } tDecoderClear(&decoder); - _exit: +_exit: tDecoderClear(&decoder); return string; } -static char *processAlterTable(SMqMetaRsp *metaRsp){ - SDecoder decoder = {0}; - SVAlterTbReq vAlterTbReq = {0}; - char *string = NULL; +static char* processAlterTable(SMqMetaRsp* metaRsp) { + SDecoder decoder = {0}; + SVAlterTbReq vAlterTbReq = {0}; + char* string = NULL; // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&decoder, data, len); if (tDecodeSVAlterTbReq(&decoder, &vAlterTbReq) < 0) { @@ -2081,8 +2089,8 @@ static char *processAlterTable(SMqMetaRsp *metaRsp){ } cJSON* type = cJSON_CreateString("alter"); cJSON_AddItemToObject(json, "type", type); -// cJSON* uid = cJSON_CreateNumber(id); -// cJSON_AddItemToObject(json, "uid", uid); + // cJSON* uid = cJSON_CreateNumber(id); + // cJSON_AddItemToObject(json, "uid", uid); cJSON* tableName = cJSON_CreateString(vAlterTbReq.tbName); cJSON_AddItemToObject(json, "tableName", tableName); cJSON* tableType = cJSON_CreateString(vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_TAG_VAL ? "child" : "normal"); @@ -2097,43 +2105,43 @@ static char *processAlterTable(SMqMetaRsp *metaRsp){ cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type); cJSON_AddItemToObject(json, "colType", colType); - if(vAlterTbReq.type == TSDB_DATA_TYPE_BINARY){ + if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY) { int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); - }else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); } break; } - case TSDB_ALTER_TABLE_DROP_COLUMN:{ + case TSDB_ALTER_TABLE_DROP_COLUMN: { cJSON* alterType = cJSON_CreateNumber(TSDB_ALTER_TABLE_DROP_COLUMN); cJSON_AddItemToObject(json, "alterType", alterType); cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); cJSON_AddItemToObject(json, "colName", colName); break; } - case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:{ + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: { cJSON* alterType = cJSON_CreateNumber(TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES); cJSON_AddItemToObject(json, "alterType", alterType); cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); cJSON_AddItemToObject(json, "colName", colName); cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type); cJSON_AddItemToObject(json, "colType", colType); - if(vAlterTbReq.type == TSDB_DATA_TYPE_BINARY){ + if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY) { int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); - }else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); } break; } - case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:{ + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { cJSON* alterType = cJSON_CreateNumber(TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME); cJSON_AddItemToObject(json, "alterType", alterType); cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); @@ -2142,12 +2150,12 @@ static char *processAlterTable(SMqMetaRsp *metaRsp){ cJSON_AddItemToObject(json, "colNewName", colNewName); break; } - case TSDB_ALTER_TABLE_UPDATE_TAG_VAL:{ + case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: { cJSON* alterType = cJSON_CreateNumber(TSDB_ALTER_TABLE_UPDATE_TAG_VAL); cJSON_AddItemToObject(json, "alterType", alterType); cJSON* tagName = cJSON_CreateString(vAlterTbReq.tagName); cJSON_AddItemToObject(json, "colName", tagName); - cJSON* colValue = cJSON_CreateString("invalid, todo"); // todo + cJSON* colValue = cJSON_CreateString("invalid, todo"); // todo cJSON_AddItemToObject(json, "colValue", colValue); cJSON* isNull = cJSON_CreateBool(vAlterTbReq.isNull); cJSON_AddItemToObject(json, "colValueNull", isNull); @@ -2158,18 +2166,18 @@ static char *processAlterTable(SMqMetaRsp *metaRsp){ } string = cJSON_PrintUnformatted(json); - _exit: +_exit: tDecoderClear(&decoder); return string; } -static char *processDropSTable(SMqMetaRsp *metaRsp){ - SDecoder decoder = {0}; - SVDropStbReq req = {0}; - char *string = NULL; +static char* processDropSTable(SMqMetaRsp* metaRsp) { + SDecoder decoder = {0}; + SVDropStbReq req = {0}; + char* string = NULL; // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&decoder, data, len); if (tDecodeSVDropStbReq(&decoder, &req) < 0) { @@ -2183,7 +2191,7 @@ static char *processDropSTable(SMqMetaRsp *metaRsp){ cJSON* type = cJSON_CreateString("drop"); cJSON_AddItemToObject(json, "type", type); char uid[32] = {0}; - sprintf(uid, "%"PRIi64, req.suid); + sprintf(uid, "%" PRIi64, req.suid); cJSON* id = cJSON_CreateString(uid); cJSON_AddItemToObject(json, "id", id); cJSON* tableName = cJSON_CreateString(req.name); @@ -2193,18 +2201,18 @@ static char *processDropSTable(SMqMetaRsp *metaRsp){ string = cJSON_PrintUnformatted(json); - _exit: +_exit: tDecoderClear(&decoder); return string; } -static char *processDropTable(SMqMetaRsp *metaRsp){ - SDecoder decoder = {0}; - SVDropTbBatchReq req = {0}; - char *string = NULL; +static char* processDropTable(SMqMetaRsp* metaRsp) { + SDecoder decoder = {0}; + SVDropTbBatchReq req = {0}; + char* string = NULL; // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&decoder, data, len); if (tDecodeSVDropTbBatchReq(&decoder, &req) < 0) { @@ -2217,57 +2225,57 @@ static char *processDropTable(SMqMetaRsp *metaRsp){ } cJSON* type = cJSON_CreateString("drop"); cJSON_AddItemToObject(json, "type", type); -// cJSON* uid = cJSON_CreateNumber(id); -// cJSON_AddItemToObject(json, "uid", uid); -// cJSON* tableType = cJSON_CreateString("normal"); -// cJSON_AddItemToObject(json, "tableType", tableType); + // cJSON* uid = cJSON_CreateNumber(id); + // cJSON_AddItemToObject(json, "uid", uid); + // cJSON* tableType = cJSON_CreateString("normal"); + // cJSON_AddItemToObject(json, "tableType", tableType); cJSON* tableNameList = cJSON_CreateArray(); for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { SVDropTbReq* pDropTbReq = req.pReqs + iReq; - cJSON* tableName = cJSON_CreateString(pDropTbReq->name); // todo + cJSON* tableName = cJSON_CreateString(pDropTbReq->name); // todo cJSON_AddItemToArray(tableNameList, tableName); } cJSON_AddItemToObject(json, "tableNameList", tableNameList); string = cJSON_PrintUnformatted(json); - _exit: +_exit: tDecoderClear(&decoder); return string; } -char *tmq_get_json_meta(TAOS_RES *res){ +char* tmq_get_json_meta(TAOS_RES* res) { if (!TD_RES_TMQ_META(res)) { return NULL; } SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_STB){ + if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_STB) { return processCreateStb(&pMetaRspObj->metaRsp); - }else if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_STB){ + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_STB) { return processCreateStb(&pMetaRspObj->metaRsp); - }else if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_STB){ + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_STB) { return processDropSTable(&pMetaRspObj->metaRsp); - }else if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_TABLE){ + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_TABLE) { return processCreateTable(&pMetaRspObj->metaRsp); - }else if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_TABLE){ + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_TABLE) { return processAlterTable(&pMetaRspObj->metaRsp); - }else if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE){ + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE) { return processDropTable(&pMetaRspObj->metaRsp); } return NULL; } -static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ +static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { SVCreateStbReq req = {0}; SDecoder coder; SMCreateStbReq pReq = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SRequestObj* pRequest = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = NULL; - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); + STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); if (NULL == pTscObj) { code = TSDB_CODE_TSC_DISCONNECTED; goto end; @@ -2278,12 +2286,12 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ goto end; } - if(!pRequest->pDb){ + if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVCreateStbReq(&coder, &req) < 0) { @@ -2292,16 +2300,16 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ } // build create stable pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SField)); - for(int32_t i = 0; i < req.schemaRow.nCols; i++){ + for (int32_t i = 0; i < req.schemaRow.nCols; i++) { SSchema* pSchema = req.schemaRow.pSchema + i; - SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; + SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; strcpy(field.name, pSchema->name); taosArrayPush(pReq.pColumns, &field); } pReq.pTags = taosArrayInit(req.schemaTag.nCols, sizeof(SField)); - for(int32_t i = 0; i < req.schemaTag.nCols; i++){ + for (int32_t i = 0; i < req.schemaTag.nCols; i++) { SSchema* pSchema = req.schemaTag.pSchema + i; - SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; + SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; strcpy(field.name, pSchema->name); taosArrayPush(pReq.pTags, &field); } @@ -2327,7 +2335,7 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ } tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - SQuery pQuery = {0}; + SQuery pQuery = {0}; pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; @@ -2337,21 +2345,21 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ code = pRequest->code; taosMemoryFree(pCmdMsg.pMsg); - end: +end: destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); tDecoderClear(&coder); return code; } -static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ +static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { SVDropStbReq req = {0}; SDecoder coder; SMDropStbReq pReq = {0}; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SRequestObj* pRequest = NULL; - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); + STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); if (NULL == pTscObj) { code = TSDB_CODE_TSC_DISCONNECTED; goto end; @@ -2362,12 +2370,12 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ goto end; } - if(!pRequest->pDb){ + if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVDropStbReq(&coder, &req) < 0) { @@ -2393,7 +2401,7 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ } tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - SQuery pQuery = {0}; + SQuery pQuery = {0}; pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; @@ -2403,7 +2411,7 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ code = pRequest->code; taosMemoryFree(pCmdMsg.pMsg); - end: +end: destroyRequest(pRequest); tDecoderClear(&coder); return code; @@ -2416,18 +2424,18 @@ typedef struct SVgroupCreateTableBatch { } SVgroupCreateTableBatch; static void destroyCreateTbReqBatch(void* data) { - SVgroupCreateTableBatch* pTbBatch = (SVgroupCreateTableBatch*) data; + SVgroupCreateTableBatch* pTbBatch = (SVgroupCreateTableBatch*)data; taosArrayDestroy(pTbBatch->req.pArray); } -static int32_t taosCreateTable(TAOS *taos, void *meta, int32_t metaLen){ - SVCreateTbBatchReq req = {0}; - SDecoder coder = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SRequestObj *pRequest = NULL; - SQuery *pQuery = NULL; - SHashObj *pVgroupHashmap = NULL; - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); +static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { + SVCreateTbBatchReq req = {0}; + SDecoder coder = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = NULL; + SQuery* pQuery = NULL; + SHashObj* pVgroupHashmap = NULL; + STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); if (NULL == pTscObj) { code = TSDB_CODE_TSC_DISCONNECTED; @@ -2439,12 +2447,12 @@ static int32_t taosCreateTable(TAOS *taos, void *meta, int32_t metaLen){ goto end; } - if(!pRequest->pDb){ + if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVCreateTbBatchReq(&coder, &req) < 0) { @@ -2452,8 +2460,8 @@ static int32_t taosCreateTable(TAOS *taos, void *meta, int32_t metaLen){ goto end; } - SVCreateTbReq *pCreateReq = NULL; - SCatalog* pCatalog = NULL; + SVCreateTbReq* pCreateReq = NULL; + SCatalog* pCatalog = NULL; code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { goto end; @@ -2467,15 +2475,15 @@ static int32_t taosCreateTable(TAOS *taos, void *meta, int32_t metaLen){ taosHashSetFreeFp(pVgroupHashmap, destroyCreateTbReqBatch); SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; // loop to create table for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; SVgroupInfo pInfo = {0}; - SName pName; + SName pName; toName(pTscObj->acctId, pRequest->pDb, pCreateReq->name, &pName); code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); if (code != TSDB_CODE_SUCCESS) { @@ -2507,7 +2515,7 @@ static int32_t taosCreateTable(TAOS *taos, void *meta, int32_t metaLen){ pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->msgType = TDMT_VND_CREATE_TABLE; pQuery->stableQuery = false; - pQuery->pRoot = nodesMakeNode(QUERY_NODE_CREATE_TABLE_STMT); + pQuery->pRoot = nodesMakeNode(QUERY_NODE_CREATE_TABLE_STMT); code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray); if (code != TSDB_CODE_SUCCESS) { @@ -2515,10 +2523,10 @@ static int32_t taosCreateTable(TAOS *taos, void *meta, int32_t metaLen){ } launchQueryImpl(pRequest, pQuery, false, NULL); - pQuery = NULL; // no need to free in the end - code = pRequest->code; + pQuery = NULL; // no need to free in the end + code = pRequest->code; - end: +end: taosHashCleanup(pVgroupHashmap); destroyRequest(pRequest); tDecoderClear(&coder); @@ -2537,14 +2545,14 @@ static void destroyDropTbReqBatch(void* data) { taosArrayDestroy(pTbBatch->req.pArray); } -static int32_t taosDropTable(TAOS *taos, void *meta, int32_t metaLen){ - SVDropTbBatchReq req = {0}; - SDecoder coder = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SRequestObj *pRequest = NULL; - SQuery *pQuery = NULL; - SHashObj *pVgroupHashmap = NULL; - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); +static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { + SVDropTbBatchReq req = {0}; + SDecoder coder = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = NULL; + SQuery* pQuery = NULL; + SHashObj* pVgroupHashmap = NULL; + STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); if (NULL == pTscObj) { code = TSDB_CODE_TSC_DISCONNECTED; @@ -2556,12 +2564,12 @@ static int32_t taosDropTable(TAOS *taos, void *meta, int32_t metaLen){ goto end; } - if(!pRequest->pDb){ + if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVDropTbBatchReq(&coder, &req) < 0) { @@ -2569,8 +2577,8 @@ static int32_t taosDropTable(TAOS *taos, void *meta, int32_t metaLen){ goto end; } - SVDropTbReq *pDropReq = NULL; - SCatalog *pCatalog = NULL; + SVDropTbReq* pDropReq = NULL; + SCatalog* pCatalog = NULL; code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { goto end; @@ -2584,15 +2592,15 @@ static int32_t taosDropTable(TAOS *taos, void *meta, int32_t metaLen){ taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch); SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; // loop to create table for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pDropReq = req.pReqs + iReq; SVgroupInfo pInfo = {0}; - SName pName; + SName pName; toName(pTscObj->acctId, pRequest->pDb, pDropReq->name, &pName); code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); if (code != TSDB_CODE_SUCCESS) { @@ -2622,7 +2630,7 @@ static int32_t taosDropTable(TAOS *taos, void *meta, int32_t metaLen){ pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->msgType = TDMT_VND_DROP_TABLE; pQuery->stableQuery = false; - pQuery->pRoot = nodesMakeNode(QUERY_NODE_DROP_TABLE_STMT); + pQuery->pRoot = nodesMakeNode(QUERY_NODE_DROP_TABLE_STMT); code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray); if (code != TSDB_CODE_SUCCESS) { @@ -2630,10 +2638,10 @@ static int32_t taosDropTable(TAOS *taos, void *meta, int32_t metaLen){ } launchQueryImpl(pRequest, pQuery, false, NULL); - pQuery = NULL; // no need to free in the end - code = pRequest->code; + pQuery = NULL; // no need to free in the end + code = pRequest->code; - end: +end: taosHashCleanup(pVgroupHashmap); destroyRequest(pRequest); tDecoderClear(&coder); @@ -2641,15 +2649,15 @@ static int32_t taosDropTable(TAOS *taos, void *meta, int32_t metaLen){ return code; } -static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ - SVAlterTbReq req = {0}; - SDecoder coder = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SRequestObj *pRequest = NULL; - SQuery *pQuery = NULL; - SArray *pArray = NULL; - SVgDataBlocks *pVgData = NULL; - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); +static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { + SVAlterTbReq req = {0}; + SDecoder coder = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = NULL; + SQuery* pQuery = NULL; + SArray* pArray = NULL; + SVgDataBlocks* pVgData = NULL; + STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); if (NULL == pTscObj) { code = TSDB_CODE_TSC_DISCONNECTED; @@ -2661,12 +2669,12 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ goto end; } - if(!pRequest->pDb){ + if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVAlterTbReq(&coder, &req) < 0) { @@ -2675,23 +2683,23 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ } // do not deal TSDB_ALTER_TABLE_UPDATE_OPTIONS - if(req.action == TSDB_ALTER_TABLE_UPDATE_OPTIONS){ + if (req.action == TSDB_ALTER_TABLE_UPDATE_OPTIONS) { goto end; } - SCatalog *pCatalog = NULL; + SCatalog* pCatalog = NULL; code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { goto end; } SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; SVgroupInfo pInfo = {0}; - SName pName = {0}; + SName pName = {0}; toName(pTscObj->acctId, pRequest->pDb, req.tbName, &pName); code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); if (code != TSDB_CODE_SUCCESS) { @@ -2725,7 +2733,7 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->msgType = TDMT_VND_ALTER_TABLE; pQuery->stableQuery = false; - pQuery->pRoot = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + pQuery->pRoot = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); code = rewriteToVnodeModifyOpStmt(pQuery, pArray); if (code != TSDB_CODE_SUCCESS) { @@ -2733,14 +2741,14 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ } launchQueryImpl(pRequest, pQuery, false, NULL); - pQuery = NULL; // no need to free in the end + pQuery = NULL; // no need to free in the end pVgData = NULL; - pArray = NULL; - code = pRequest->code; + pArray = NULL; + code = pRequest->code; end: taosArrayDestroy(pArray); - if(pVgData) taosMemoryFreeClear(pVgData->pData); + if (pVgData) taosMemoryFreeClear(pVgData->pData); taosMemoryFreeClear(pVgData); destroyRequest(pRequest); tDecoderClear(&coder); @@ -2748,22 +2756,22 @@ end: return code; } -int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data *raw_meta){ +int32_t taos_write_raw_meta(TAOS* taos, tmq_raw_data* raw_meta) { if (!taos || !raw_meta) { return TSDB_CODE_INVALID_PARA; } - if(raw_meta->raw_meta_type == TDMT_VND_CREATE_STB) { + if (raw_meta->raw_meta_type == TDMT_VND_CREATE_STB) { return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_ALTER_STB){ + } else if (raw_meta->raw_meta_type == TDMT_VND_ALTER_STB) { return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_DROP_STB){ + } else if (raw_meta->raw_meta_type == TDMT_VND_DROP_STB) { return taosDropStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_CREATE_TABLE){ + } else if (raw_meta->raw_meta_type == TDMT_VND_CREATE_TABLE) { return taosCreateTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_ALTER_TABLE){ + } else if (raw_meta->raw_meta_type == TDMT_VND_ALTER_TABLE) { return taosAlterTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_DROP_TABLE){ + } else if (raw_meta->raw_meta_type == TDMT_VND_DROP_TABLE) { return taosDropTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); } return TSDB_CODE_INVALID_PARA; diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index b2fd5a5ddc..4422c3f180 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -89,8 +89,10 @@ int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffset } } if (pOffset->type == TMQ_OFFSET__LOG) { + continue; + } else { rowCnt += pDataBlock->info.rows; - if (rowCnt >= 4096) break; + if (rowCnt <= 4096) continue; } continue; } @@ -116,6 +118,7 @@ int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffset return 0; } +#if 0 int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset, int32_t workerId) { ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN); qTaskInfo_t task = pExec->execCol.task[workerId]; @@ -163,6 +166,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, S return 0; } +#endif int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataRsp* pRsp, int32_t workerId) { if (pExec->subType == TOPIC_SUB_TYPE__COLUMN) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 21eb7680d5..ae0d0bebbd 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -299,10 +299,12 @@ typedef struct STableScanInfo { uint64_t queryId; // todo remove it uint64_t taskId; // todo remove it +#if 0 struct { uint64_t uid; int64_t ts; } lastStatus; +#endif int8_t scanMode; int8_t noTable; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 42ad173c74..53ebc322a8 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -351,6 +351,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { return 0; } +#if 0 int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; @@ -370,3 +371,4 @@ int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts) { return doGetScanStatus(pTaskInfo->pRoot, uid, ts); } +#endif diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 4b0f29e106..f2f696b21f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2463,7 +2463,7 @@ static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) { blockDataDestroy(pInfo->binfo.pRes); cleanupAggSup(&pInfo->aggSup); - + taosMemoryFreeClear(param); } @@ -2844,7 +2844,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan } } } - +#if 0 int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) { uint8_t type = pOperator->operatorType; @@ -2930,6 +2930,7 @@ int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts) { return TSDB_CODE_SUCCESS; } +#endif // this is a blocking operator static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { @@ -3341,8 +3342,8 @@ static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo* pInfo, SResult SExecTaskInfo* pTaskInfo) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; - int64_t ekey = Q_STATUS_EQUAL(pTaskInfo->status, TASK_COMPLETED) ? pInfo->win.ekey - : pInfo->existNewGroupBlock->info.window.ekey; + int64_t ekey = + Q_STATUS_EQUAL(pTaskInfo->status, TASK_COMPLETED) ? pInfo->win.ekey : pInfo->existNewGroupBlock->info.window.ekey; taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey); @@ -3678,14 +3679,14 @@ void cleanupBasicInfo(SOptrBasicInfo* pInfo) { void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { SOptrBasicInfo* pInfo = (SOptrBasicInfo*)param; cleanupBasicInfo(pInfo); - + taosMemoryFreeClear(param); } void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param; - cleanupBasicInfo(&pInfo->binfo); - + cleanupBasicInfo(&pInfo->binfo); + taosMemoryFreeClear(param); } @@ -3694,7 +3695,7 @@ void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); pInfo->pRes = blockDataDestroy(pInfo->pRes); taosMemoryFreeClear(pInfo->p); - + taosMemoryFreeClear(param); } @@ -3706,7 +3707,7 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); taosArrayDestroy(pInfo->pPseudoColInfo); - + taosMemoryFreeClear(param); } @@ -3724,7 +3725,7 @@ static void destroyIndefinitOperatorInfo(void* param, int32_t numOfOutput) { taosArrayDestroy(pInfo->pPseudoColInfo); cleanupAggSup(&pInfo->aggSup); cleanupExprSupp(&pInfo->scalarSup); - + taosMemoryFreeClear(param); } @@ -3743,7 +3744,7 @@ void doDestroyExchangeOperatorInfo(void* param) { } tsem_destroy(&pExInfo->ready); - + taosMemoryFreeClear(param); } @@ -3972,7 +3973,7 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t pInfo->pFillInfo = taosCreateFillInfo(order, w.skey, 0, capacity, numOfCols, pInterval, fillType, pColInfo, id); pInfo->win = win; - pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES); + pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES); if (pInfo->pFillInfo == NULL || pInfo->p == NULL) { taosMemoryFree(pInfo->pFillInfo); taosMemoryFree(pInfo->p); @@ -4465,7 +4466,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision}; int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; - pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, pPhyNode->pConditions, pTaskInfo); + pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, + pPhyNode->pConditions, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL == type) { SMergeIntervalPhysiNode* pIntervalPhyNode = (SMergeIntervalPhysiNode*)pPhyNode; @@ -4504,8 +4506,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId; - pOptr = - createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, pPhyNode->pConditions, pTaskInfo); + pOptr = createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, + pPhyNode->pConditions, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION == type) { pOptr = createStreamSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION == type) { @@ -4527,7 +4529,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; SColumn col = extractColumnFromColumnNode(pColNode); - pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, &col, pPhyNode->pConditions, pTaskInfo); + pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, &col, pPhyNode->pConditions, + pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE == type) { pOptr = createStreamStateAggOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN == type) { @@ -4798,7 +4801,7 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT return TSDB_CODE_OUT_OF_MEMORY; } pInserterParam->readHandle = readHandle; - + *pParam = pInserterParam; break; } From c25c4964bebe52f60adf7c9c3edbfe3a74f23162 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 10 Jul 2022 15:03:57 +0800 Subject: [PATCH 20/38] fix(query): fix syntax error. --- source/libs/executor/src/executorMain.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index ca1d8b2b10..5c482e89b3 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -327,12 +327,11 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { ASSERT(found); tsdbSetTableId(pTableScanInfo->dataReader, uid); - int64_t oldSkey = pTableScanInfo->cond.twindows[0].skey; - pTableScanInfo->cond.twindows[0].skey = ts + 1; - tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); - pTableScanInfo->cond.twindows[0].skey = oldSkey; + int64_t oldSkey = pTableScanInfo->cond.twindows.skey; + pTableScanInfo->cond.twindows.skey = ts + 1; + tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); + pTableScanInfo->cond.twindows.skey = oldSkey; pTableScanInfo->scanTimes = 0; - pTableScanInfo->curTWinIdx = 0; qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts, pTableScanInfo->currentTable, tableSz); From a64a0320a0d9169b5dd5e3ff9a99df3f906f2595 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 10 Jul 2022 15:21:16 +0800 Subject: [PATCH 21/38] fix: some problem of parser and planner --- include/libs/nodes/querynodes.h | 2 + source/libs/function/src/functionMgt.c | 2 +- source/libs/parser/src/parTranslater.c | 106 +++++++----------- source/libs/parser/test/parSelectTest.cpp | 6 +- source/libs/planner/test/planPartByTest.cpp | 2 + tests/system-test/1-insert/time_range_wise.py | 10 +- 6 files changed, 56 insertions(+), 72 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index b9a0b90a9a..bbff34c66f 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -250,6 +250,7 @@ typedef struct SSelectStmt { SLimitNode* pSlimit; char stmtName[TSDB_TABLE_NAME_LEN]; uint8_t precision; + int32_t selectFuncNum; bool isEmptyResult; bool isTimeLineResult; bool hasAggFuncs; @@ -257,6 +258,7 @@ typedef struct SSelectStmt { bool hasIndefiniteRowsFunc; bool hasSelectFunc; bool hasSelectValFunc; + bool hasOtherVectorFunc; bool hasUniqueFunc; bool hasTailFunc; bool hasInterpFunc; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 262adc5d6f..1da9d4a70f 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -139,7 +139,7 @@ bool fmIsAggFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MG bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); } -bool fmIsVectorFunc(int32_t funcId) { return !fmIsScalarFunc(funcId); } +bool fmIsVectorFunc(int32_t funcId) { return !fmIsScalarFunc(funcId) && !fmIsScanPseudoColumnFunc(funcId); } bool fmIsSelectFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SELECT_FUNC); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6beb06bf45..8e6798dcd2 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1186,6 +1186,12 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId); pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId); pSelect->hasIndefiniteRowsFunc = pSelect->hasIndefiniteRowsFunc ? true : fmIsIndefiniteRowsFunc(pFunc->funcId); + if (fmIsSelectFunc(pFunc->funcId)) { + pSelect->hasSelectFunc = true; + ++(pSelect->selectFuncNum); + } else if (fmIsAggFunc(pFunc->funcId) || fmIsIndefiniteRowsFunc(pFunc->funcId)) { + pSelect->hasOtherVectorFunc = true; + } pSelect->hasUniqueFunc = pSelect->hasUniqueFunc ? true : (FUNCTION_TYPE_UNIQUE == pFunc->funcType); pSelect->hasTailFunc = pSelect->hasTailFunc ? true : (FUNCTION_TYPE_TAIL == pFunc->funcType); pSelect->hasInterpFunc = pSelect->hasInterpFunc ? true : (FUNCTION_TYPE_INTERP == pFunc->funcType); @@ -1395,16 +1401,12 @@ static int32_t getGroupByErrorCode(STranslateContext* pCxt) { if (isDistinctOrderBy(pCxt)) { return TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION; } - return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION; + if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) { + return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION; + } + return TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN; } -typedef struct SCheckExprForGroupByCxt { - STranslateContext* pTranslateCxt; - int32_t selectFuncNum; - bool hasSelectValFunc; - bool hasOtherAggFunc; -} SCheckExprForGroupByCxt; - static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode) { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); if (NULL == pFunc) { @@ -1445,67 +1447,49 @@ static EDealRes rewriteExprToGroupKeyFunc(STranslateContext* pCxt, SNode** pNode } static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { - SCheckExprForGroupByCxt* pCxt = (SCheckExprForGroupByCxt*)pContext; + STranslateContext* pCxt = (STranslateContext*)pContext; + SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; if (!nodesIsExprNode(*pNode) || isAliasColumn(*pNode)) { return DEAL_RES_CONTINUE; } - if (isSelectFunc(*pNode)) { - ++(pCxt->selectFuncNum); - } else if (isAggFunc(*pNode)) { - pCxt->hasOtherAggFunc = true; - } - if ((pCxt->selectFuncNum > 1 && pCxt->hasSelectValFunc) || (pCxt->hasOtherAggFunc && pCxt->hasSelectValFunc)) { - return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); - } - if (isAggFunc(*pNode) && !isDistinctOrderBy(pCxt->pTranslateCxt)) { + if (isVectorFunc(*pNode) && !isDistinctOrderBy(pCxt)) { return DEAL_RES_IGNORE_CHILD; } SNode* pGroupNode = NULL; - FOREACH(pGroupNode, getGroupByList(pCxt->pTranslateCxt)) { + FOREACH(pGroupNode, getGroupByList(pCxt)) { if (nodesEqualNode(getGroupByNode(pGroupNode), *pNode)) { return DEAL_RES_IGNORE_CHILD; } } SNode* pPartKey = NULL; - FOREACH(pPartKey, ((SSelectStmt*)pCxt->pTranslateCxt->pCurrStmt)->pPartitionByList) { + FOREACH(pPartKey, pSelect->pPartitionByList) { if (nodesEqualNode(pPartKey, *pNode)) { - return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode); + return rewriteExprToGroupKeyFunc(pCxt, pNode); } } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { - if (pCxt->selectFuncNum > 1 || pCxt->hasOtherAggFunc) { - return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); + if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc) { + return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt)); } else { - pCxt->hasSelectValFunc = true; - return rewriteColToSelectValFunc(pCxt->pTranslateCxt, pNode); + return rewriteColToSelectValFunc(pCxt, pNode); } } - if (isAggFunc(*pNode) && isDistinctOrderBy(pCxt->pTranslateCxt)) { - return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); + if (isVectorFunc(*pNode) && isDistinctOrderBy(pCxt)) { + return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt)); } return DEAL_RES_CONTINUE; } static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode** pNode) { - SCheckExprForGroupByCxt cxt = { - .pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false, .hasOtherAggFunc = false}; - nodesRewriteExpr(pNode, doCheckExprForGroupBy, &cxt); - if (cxt.selectFuncNum != 1 && cxt.hasSelectValFunc) { - return generateSyntaxErrMsg(&pCxt->msgBuf, getGroupByErrorCode(pCxt)); - } + nodesRewriteExpr(pNode, doCheckExprForGroupBy, pCxt); return pCxt->errCode; } -static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList) { - if (NULL == getGroupByList(pCxt)) { +static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect, SNodeList* pList) { + if (NULL == getGroupByList(pCxt) && NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; } - SCheckExprForGroupByCxt cxt = { - .pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false, .hasOtherAggFunc = false}; - nodesRewriteExprs(pList, doCheckExprForGroupBy, &cxt); - if (cxt.selectFuncNum != 1 && cxt.hasSelectValFunc) { - return generateSyntaxErrMsg(&pCxt->msgBuf, getGroupByErrorCode(pCxt)); - } + nodesRewriteExprs(pList, doCheckExprForGroupBy, pCxt); return pCxt->errCode; } @@ -1529,7 +1513,6 @@ static int32_t rewriteColsToSelectValFunc(STranslateContext* pCxt, SSelectStmt* typedef struct CheckAggColCoexistCxt { STranslateContext* pTranslateCxt; - bool existVectorFunc; bool existCol; int32_t selectFuncNum; bool existOtherVectorFunc; @@ -1537,13 +1520,12 @@ typedef struct CheckAggColCoexistCxt { static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext; - if (isSelectFunc(*pNode)) { - ++(pCxt->selectFuncNum); - } else if (isAggFunc(*pNode)) { - pCxt->existOtherVectorFunc = true; - } if (isVectorFunc(*pNode)) { - pCxt->existVectorFunc = true; + if (isSelectFunc(*pNode)) { + ++(pCxt->selectFuncNum); + } else { + pCxt->existOtherVectorFunc = true; + } return DEAL_RES_IGNORE_CHILD; } SNode* pPartKey = NULL; @@ -1559,14 +1541,12 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { } static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) { - if (NULL != pSelect->pGroupByList) { + if (NULL != pSelect->pGroupByList || NULL != pSelect->pWindow || + (!pSelect->hasAggFuncs && !pSelect->hasIndefiniteRowsFunc)) { return TSDB_CODE_SUCCESS; } - CheckAggColCoexistCxt cxt = {.pTranslateCxt = pCxt, - .existVectorFunc = false, - .existCol = false, - .selectFuncNum = 0, - .existOtherVectorFunc = false}; + CheckAggColCoexistCxt cxt = { + .pTranslateCxt = pCxt, .existCol = false, .selectFuncNum = 0, .existOtherVectorFunc = false}; nodesRewriteExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); if (!pSelect->isDistinct) { nodesRewriteExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); @@ -1574,7 +1554,7 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) if (1 == cxt.selectFuncNum && !cxt.existOtherVectorFunc) { return rewriteColsToSelectValFunc(pCxt, pSelect); } - if ((cxt.selectFuncNum > 1 || cxt.existVectorFunc || NULL != pSelect->pWindow) && cxt.existCol) { + if (cxt.existCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SINGLE_GROUP); } return TSDB_CODE_SUCCESS; @@ -2056,7 +2036,7 @@ static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->currClause = SQL_CLAUSE_ORDER_BY; code = translateExprList(pCxt, pSelect->pOrderByList); if (TSDB_CODE_SUCCESS == code) { - code = checkExprListForGroupBy(pCxt, pSelect->pOrderByList); + code = checkExprListForGroupBy(pCxt, pSelect, pSelect->pOrderByList); } } return code; @@ -2069,7 +2049,7 @@ static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect code = translateStar(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { - code = checkExprListForGroupBy(pCxt, pSelect->pProjectionList); + code = checkExprListForGroupBy(pCxt, pSelect, pSelect->pProjectionList); } return code; } @@ -5264,8 +5244,8 @@ static int32_t serializeVgroupCreateTableBatch(SVgroupCreateTableBatch* pTbBatch } static void destroyCreateTbReqBatch(void* data) { - SVgroupCreateTableBatch* pTbBatch = (SVgroupCreateTableBatch*) data; - size_t size = taosArrayGetSize(pTbBatch->req.pArray); + SVgroupCreateTableBatch* pTbBatch = (SVgroupCreateTableBatch*)data; + size_t size = taosArrayGetSize(pTbBatch->req.pArray); for (int32_t i = 0; i < size; ++i) { SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i); taosMemoryFreeClear(pTableReq->name); @@ -5347,10 +5327,10 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, SCreateSubTableClause* pStmt, const STag* pTag, uint64_t suid, SVgroupInfo* pVgInfo) { -// char dbFName[TSDB_DB_FNAME_LEN] = {0}; -// SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; -// strcpy(name.dbname, pStmt->dbName); -// tNameGetFullDbName(&name, dbFName); + // char dbFName[TSDB_DB_FNAME_LEN] = {0}; + // SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; + // strcpy(name.dbname, pStmt->dbName); + // tNameGetFullDbName(&name, dbFName); struct SVCreateTbReq req = {0}; req.type = TD_CHILD_TABLE; diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 4ca2dec299..c6d74c5305 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -144,9 +144,9 @@ TEST_F(ParserSelectTest, IndefiniteRowsFunc) { TEST_F(ParserSelectTest, IndefiniteRowsFuncSemanticCheck) { useDb("root", "test"); - run("SELECT DIFF(c1), c2 FROM t1", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); + run("SELECT DIFF(c1), c2 FROM t1", TSDB_CODE_PAR_NOT_SINGLE_GROUP); - run("SELECT DIFF(c1), tbname FROM t1", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); + run("SELECT DIFF(c1), tbname FROM t1", TSDB_CODE_PAR_NOT_SINGLE_GROUP); run("SELECT DIFF(c1), count(*) FROM t1", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); @@ -273,7 +273,7 @@ TEST_F(ParserSelectTest, interval) { TEST_F(ParserSelectTest, intervalSemanticCheck) { useDb("root", "test"); - run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NOT_SINGLE_GROUP); + run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN); run("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 3 INTERVAL(1d) FILL(NEXT)", TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE); run("SELECT HISTOGRAM(c1, 'log_bin', '{\"start\": -33,\"factor\": 55,\"count\": 5,\"infinity\": false}', 1) FROM t1 " "WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' INTERVAL(10s) FILL(NULL)", diff --git a/source/libs/planner/test/planPartByTest.cpp b/source/libs/planner/test/planPartByTest.cpp index 2abcb44dfd..3b71ae3b88 100644 --- a/source/libs/planner/test/planPartByTest.cpp +++ b/source/libs/planner/test/planPartByTest.cpp @@ -35,6 +35,8 @@ TEST_F(PlanPartitionByTest, withAggFunc) { run("select count(*) from t1 partition by c1"); + run("select count(*) from st1 partition by c1"); + run("select count(*), c1 from t1 partition by c1"); } diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py index a620a4b51a..2596f82476 100644 --- a/tests/system-test/1-insert/time_range_wise.py +++ b/tests/system-test/1-insert/time_range_wise.py @@ -614,12 +614,12 @@ class TDTestCase: self.__insert_data() self.all_test() - tdLog.printNoPrefix("==========step2:create table in rollup database") - tdSql.execute("create database db3 retentions 1s:4m,2s:8m,3s:12m") - tdSql.execute("use db3") + #tdLog.printNoPrefix("==========step2:create table in rollup database") + #tdSql.execute("create database db3 retentions 1s:4m,2s:8m,3s:12m") + #tdSql.execute("use db3") # self.__create_tb() - tdSql.execute(f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s max_delay 1m sma({INT_COL}) ") - self.all_test() + #tdSql.execute(f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s max_delay 1m sma({INT_COL}) ") + #self.all_test() # self.__insert_data() From ff5f1aa1a5fcb2cad5d5a59bc94d92c809341d08 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sun, 10 Jul 2022 15:23:34 +0800 Subject: [PATCH 22/38] refactor(stream) --- source/dnode/vnode/src/tq/tqExec.c | 1 - source/libs/executor/src/scanoperator.c | 81 ++----------------------- 2 files changed, 6 insertions(+), 76 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 4422c3f180..49cf42b083 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -94,7 +94,6 @@ int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffset rowCnt += pDataBlock->info.rows; if (rowCnt <= 4096) continue; } - continue; } void* meta = qStreamExtractMetaMsg(task); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 3532c1d367..e42510ceba 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1234,9 +1234,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock doFilter(pInfo->pCondition, pInfo->pRes); blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); - if (pBlockInfo->rows > 0) { - return 0; - } return 0; } @@ -1262,7 +1259,9 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { /*pTaskInfo->streamInfo.lastStatus = ret.offset;*/ if (pInfo->pRes->info.rows > 0) { return pInfo->pRes; - /*} else {*/ + } else { + // data is filtered out, do clean + /*tDeleteSSDataBlock(&ret.data);*/ } } else if (ret.fetchType == FETCH_TYPE__META) { @@ -1271,14 +1270,8 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTaskInfo->streamInfo.metaBlk = ret.meta; return NULL; } else if (ret.fetchType == FETCH_TYPE__NONE) { - /*if (ret.offset.version == -1) {*/ - /*pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__LOG;*/ - /*pTaskInfo->streamInfo.lastStatus.version = pTaskInfo->streamInfo.prepareStatus.version - 1;*/ - /*} else {*/ pTaskInfo->streamInfo.lastStatus = ret.offset; - /*pTaskInfo->streamInfo.lastScanUid */ ASSERT(pTaskInfo->streamInfo.lastStatus.version + 1 >= pTaskInfo->streamInfo.prepareStatus.version); - /*}*/ return NULL; } else { ASSERT(0); @@ -1398,72 +1391,8 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { return NULL; } - pInfo->pRes->info.rows = block.info.rows; - pInfo->pRes->info.uid = block.info.uid; - pInfo->pRes->info.type = STREAM_NORMAL; - pInfo->pRes->info.capacity = block.info.rows; + setBlockIntoRes(pInfo, &block); - uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &block.info.uid, sizeof(int64_t)); - if (groupIdPre) { - pInfo->pRes->info.groupId = *groupIdPre; - } else { - pInfo->pRes->info.groupId = 0; - } - - // for generating rollup SMA result, each time is an independent time serie. - // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this - if (pInfo->assignBlockUid) { - pInfo->pRes->info.groupId = block.info.uid; - } - - // todo extract method - for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) { - SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); - if (!pColMatchInfo->output) { - continue; - } - - bool colExists = false; - for (int32_t j = 0; j < blockDataGetNumOfCols(&block); ++j) { - SColumnInfoData* pResCol = bdGetColumnInfoData(&block, j); - if (pResCol->info.colId == pColMatchInfo->colId) { - taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol); - colExists = true; - break; - } - } - - // the required column does not exists in submit block, let's set it to be all null value - if (!colExists) { - SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId); - colDataAppendNNULL(pDst, 0, pBlockInfo->rows); - } - } - - taosArrayDestroy(block.pDataBlock); - - ASSERT(pInfo->pRes->pDataBlock != NULL); -#if 0 - if (pInfo->pRes->pDataBlock == NULL) { - // TODO add log - updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); - pOperator->status = OP_EXEC_DONE; - pTaskInfo->code = terrno; - return NULL; - } -#endif - - // currently only the tbname pseudo column - if (pInfo->numOfPseudoExpr > 0) { - code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, - GET_TASKID(pTaskInfo)); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } - } - - doFilter(pInfo->pCondition, pInfo->pRes); - blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); if (pBlockInfo->rows > 0) { break; } @@ -1493,12 +1422,14 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; +#if 0 } else if (pInfo->blockType == STREAM_INPUT__TABLE_SCAN) { ASSERT(0); // check reader last status // if not match, reset status SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); return pResult && pResult->info.rows > 0 ? pResult : NULL; +#endif } else { ASSERT(0); From ee3ceccb0aabb57f08a566d04d04e3bcb8f1c228 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 10 Jul 2022 16:05:21 +0800 Subject: [PATCH 23/38] fix(sync): vnode snapshot sender --- source/dnode/vnode/src/vnd/vnodeSync.c | 40 +++++++++++++++++++ source/libs/sync/src/syncAppendEntriesReply.c | 8 ++-- .../tsim/sync/vnodesnapshot-restart.sim | 2 +- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index d49c307233..1508b67f5e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -450,42 +450,81 @@ static void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType)); } +#define USE_TSDB_SNAPSHOT + static int32_t vnodeSnapshotStartRead(struct SSyncFSM *pFsm, void *pParam, void **ppReader) { +#ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; SSnapshotParam *pSnapshotParam = pParam; int32_t code = vnodeSnapReaderOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapReader **)ppReader); return code; +#else + *ppReader = taosMemoryMalloc(32); + return 0; +#endif } static int32_t vnodeSnapshotStopRead(struct SSyncFSM *pFsm, void *pReader) { +#ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; int32_t code = vnodeSnapReaderClose(pReader); return code; +#else + taosMemoryFree(pReader); + return 0; +#endif } static int32_t vnodeSnapshotDoRead(struct SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) { +#ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; int32_t code = vnodeSnapRead(pReader, (uint8_t **)ppBuf, len); return code; +#else + static int32_t times = 0; + if (times++ < 5) { + *len = 64; + *ppBuf = taosMemoryMalloc(*len); + snprintf(*ppBuf, *len, "snapshot block %d", times); + } else { + *len = 0; + *ppBuf = NULL; + } + return 0; +#endif } static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void **ppWriter) { +#ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; SSnapshotParam *pSnapshotParam = pParam; int32_t code = vnodeSnapWriterOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapWriter **)ppWriter); return code; +#else + *ppWriter = taosMemoryMalloc(32); + return 0; +#endif } static int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) { +#ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; int32_t code = vnodeSnapWriterClose(pWriter, !isApply); return code; +#else + taosMemoryFree(pWriter); + return 0; +#endif } static int32_t vnodeSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) { +#ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; int32_t code = vnodeSnapWrite(pWriter, pBuf, len); return code; +#else + return 0; +#endif } static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { @@ -509,6 +548,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { SSyncInfo syncInfo = { + //.snapshotStrategy = SYNC_STRATEGY_WAL_FIRST, .snapshotStrategy = SYNC_STRATEGY_NO_SNAPSHOT, .batchSize = 10, .vgId = pVnode->config.vgId, diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 153f18da86..c18c2b4d38 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -235,10 +235,10 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie // do nothing } else { - SSyncRaftEntry* pEntry; - int32_t code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, nextIndex, &pEntry); - ASSERT(code == 0); - syncNodeStartSnapshotOnce(ths, SYNC_INDEX_BEGIN, nextIndex, pEntry->term, pMsg); + SSnapshot oldSnapshot; + ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &oldSnapshot); + SyncTerm newSnapshotTerm = oldSnapshot.lastApplyTerm; + syncNodeStartSnapshotOnce(ths, SYNC_INDEX_BEGIN, nextIndex, newSnapshotTerm, pMsg); // get sender SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId)); diff --git a/tests/script/tsim/sync/vnodesnapshot-restart.sim b/tests/script/tsim/sync/vnodesnapshot-restart.sim index 3ed82fdfe7..b44191071d 100644 --- a/tests/script/tsim/sync/vnodesnapshot-restart.sim +++ b/tests/script/tsim/sync/vnodesnapshot-restart.sim @@ -1,6 +1,6 @@ system sh/stop_dnodes.sh -system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start From 4d99188ee84fe8787bbaedd49d35ad44861a190e Mon Sep 17 00:00:00 2001 From: tomchon Date: Sun, 10 Jul 2022 16:12:09 +0800 Subject: [PATCH 24/38] test:modify testcase of qnode --- tests/pytest/util/cluster.py | 5 +++-- tests/system-test/2-query/queryQnode.py | 29 ++++++++++--------------- tests/system-test/fulltest.sh | 6 ++--- tests/system-test/test.py | 24 +++++++++++++++----- 4 files changed, 36 insertions(+), 28 deletions(-) diff --git a/tests/pytest/util/cluster.py b/tests/pytest/util/cluster.py index efa83323a4..e892e1cc02 100644 --- a/tests/pytest/util/cluster.py +++ b/tests/pytest/util/cluster.py @@ -58,9 +58,10 @@ class ConfigureyCluster: self.dnodes.append(dnode) return self.dnodes - def create_dnode(self,conn): + def create_dnode(self,conn,dnodeNum): tdSql.init(conn.cursor()) - for dnode in self.dnodes[1:]: + dnodeNum=int(dnodeNum) + for dnode in self.dnodes[1:dnodeNum]: # print(dnode.cfgDict) dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] tdSql.execute(" create dnode '%s';"%dnode_id) diff --git a/tests/system-test/2-query/queryQnode.py b/tests/system-test/2-query/queryQnode.py index d9976d8f3e..8b893a93d7 100644 --- a/tests/system-test/2-query/queryQnode.py +++ b/tests/system-test/2-query/queryQnode.py @@ -278,22 +278,7 @@ class TDTestCase: tdSql.checkData(0,0,rowsPerSTable) return - # test case1 base - def test_case1(self): - #stableCount=threadNumbersCtb - parameterDict = {'vgroups': 1, \ - 'threadNumbersCtb': 5, \ - 'threadNumbersIda': 5, \ - 'stableCount': 5, \ - 'tablesPerStb': 50, \ - 'rowsPerTable': 10, \ - 'dbname': 'db', \ - 'stbname': 'stb', \ - 'host': 'localhost', \ - 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 - - tdLog.debug("-----create database and muti-thread create tables test------- ") - + # test case : Switch back and forth among the three queryPolicy（1\2\3） def test_case1(self): self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 1, 2, 1*10) tdSql.execute("use db1;") @@ -407,6 +392,7 @@ class TDTestCase: tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") assert unionallQnode==tdSql.queryResult + # test case : queryPolicy = 2 def test_case2(self): self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 10, 2, 1*10) tdSql.query("show qnodes") @@ -438,8 +424,9 @@ class TDTestCase: tdSql.query("select max(c1) from stb10_0;") tdSql.query("select min(c1) from stb11_0;") - def test_case3(self): + # test case : queryPolicy = 3 + def test_case3(self): tdSql.execute('alter local "queryPolicy" "3"') tdLog.debug("create qnode on dnode 1") tdSql.execute("create qnode on dnode 1") @@ -472,10 +459,16 @@ class TDTestCase: # run case def run(self): # test qnode + tdLog.debug(" test_case1 ............ [start]") self.test_case1() + tdLog.debug(" test_case1 ............ [OK]") + tdLog.debug(" test_case2 ............ [start]") self.test_case2() - + tdLog.debug(" test_case2 ............ [OK]") + tdLog.debug(" test_case3 ............ [start]") self.test_case3() + tdLog.debug(" test_case3 ............ [OK]") + # tdLog.debug(" LIMIT test_case3 ............ [OK]") def stop(self): diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 9f165d6404..f6f902e890 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -120,12 +120,12 @@ python3 ./test.py -f 2-query/irate.py python3 ./test.py -f 2-query/and_or_for_byte.py python3 ./test.py -f 2-query/function_null.py -python3 ./test.py -f 2-query/queryQnode.py +#python3 ./test.py -f 2-query/queryQnode.py python3 ./test.py -f 6-cluster/5dnode1mnode.py -#python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 +#BUG python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 #python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 -#python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 9596efdfc8..0a891759d0 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -64,8 +64,9 @@ if __name__ == "__main__": updateCfgDict = {} execCmd = "" queryPolicy = 1 - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy']) + createDnodeNums = 1 + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -81,9 +82,11 @@ if __name__ == "__main__": tdLog.printNoPrefix('-d update cfg dict, base64 json str') tdLog.printNoPrefix('-k not kill valgrind processer') tdLog.printNoPrefix('-e eval str to run') - tdLog.printNoPrefix('-N create dnodes numbers in clusters') + tdLog.printNoPrefix('-N start dnodes numbers in clusters') tdLog.printNoPrefix('-M create mnode numbers in clusters') tdLog.printNoPrefix('-Q set queryPolicy in one dnode') + tdLog.printNoPrefix('-C create Dnode Numbers in one cluster') + sys.exit(0) @@ -143,6 +146,9 @@ if __name__ == "__main__": if key in ['-Q', '--queryPolicy']: queryPolicy = value + if key in ['-C', '--createDnodeNums']: + createDnodeNums = value + if not execCmd == "": tdDnodes.init(deployPath) print(execCmd) @@ -239,7 +245,11 @@ if __name__ == "__main__": host, config=tdDnodes.getSimCfgPath()) print(tdDnodes.getSimCfgPath(),host) - cluster.create_dnode(conn) + if createDnodeNums == 1: + createDnodeNums=dnodeNums + else: + createDnodeNums=createDnodeNums + cluster.create_dnode(conn,createDnodeNums) try: if cluster.check_dnode(conn) : print("check dnode ready") @@ -314,7 +324,11 @@ if __name__ == "__main__": host, config=tdDnodes.getSimCfgPath()) print(tdDnodes.getSimCfgPath(),host) - cluster.create_dnode(conn) + if createDnodeNums == 1: + createDnodeNums=dnodeNums + else: + createDnodeNums=createDnodeNums + cluster.create_dnode(conn,createDnodeNums) try: if cluster.check_dnode(conn) : print("check dnode ready") From 06fe7f6a5a853c91c55fa6a8a01c6e7d94e6d73d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 10 Jul 2022 16:17:42 +0800 Subject: [PATCH 25/38] update test case --- source/libs/index/inc/indexFstDfa.h | 2 +- source/libs/index/src/indexFstDfa.c | 12 +++++------- source/libs/index/src/indexFstRegex.c | 7 +++++-- source/libs/index/src/indexFstSparse.c | 14 ++++++++++---- source/libs/index/test/fstUtilUT.cc | 5 ++++- 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/source/libs/index/inc/indexFstDfa.h b/source/libs/index/inc/indexFstDfa.h index 9ca10897fd..5a5622e280 100644 --- a/source/libs/index/inc/indexFstDfa.h +++ b/source/libs/index/inc/indexFstDfa.h @@ -51,7 +51,7 @@ FstDfa *dfaBuilderBuild(FstDfaBuilder *builder); bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t bytes, uint32_t *result); -bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result); +bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result); /* * dfa related func diff --git a/source/libs/index/src/indexFstDfa.c b/source/libs/index/src/indexFstDfa.c index 275580ebdc..b8ac3bd3f5 100644 --- a/source/libs/index/src/indexFstDfa.c +++ b/source/libs/index/src/indexFstDfa.c @@ -64,7 +64,7 @@ void dfaBuilderDestroy(FstDfaBuilder *builder) { taosMemoryFree(builder); } -FstDfa *dfaBuilder(FstDfaBuilder *builder) { +FstDfa *dfaBuilderBuild(FstDfaBuilder *builder) { uint32_t sz = taosArrayGetSize(builder->dfa->insts); FstSparseSet *cur = sparSetCreate(sz); FstSparseSet *nxt = sparSetCreate(sz); @@ -73,7 +73,7 @@ FstDfa *dfaBuilder(FstDfaBuilder *builder) { SArray *states = taosArrayInit(0, sizeof(uint32_t)); uint32_t result; - if (dfaBuilderCachedState(builder, cur, &result)) { + if (dfaBuilderCacheState(builder, cur, &result)) { taosArrayPush(states, &result); } SHashObj *seen = taosHashInit(12, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -98,8 +98,6 @@ FstDfa *dfaBuilder(FstDfaBuilder *builder) { return builder->dfa; } -FstDfa *dfaBuilderBuild(FstDfaBuilder *builer) { return NULL; } - bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t byte, uint32_t *result) { sparSetClear(cur); @@ -114,7 +112,7 @@ bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet t = taosArrayGet(builder->dfa->states, state); uint32_t nxtState; - if (dfaBuilderCachedState(builder, next, &nxtState)) { + if (dfaBuilderCacheState(builder, next, &nxtState)) { t->next[byte] = nxtState; *result = nxtState; return true; @@ -122,7 +120,7 @@ bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet return false; } -bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result) { +bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result) { SArray *tinsts = taosArrayInit(4, sizeof(uint32_t)); bool isMatch = false; @@ -190,7 +188,7 @@ void dfaAdd(FstDfa *dfa, FstSparseSet *set, uint32_t ip) { return; } bool succ = sparSetAdd(set, ip, NULL); - assert(succ == true); + // assert(succ == true); Inst *inst = taosArrayGet(dfa->insts, ip); if (inst->ty == MATCH || inst->ty == RANGE) { // do nothing diff --git a/source/libs/index/src/indexFstRegex.c b/source/libs/index/src/indexFstRegex.c index 37cb58996f..0189e9fe6e 100644 --- a/source/libs/index/src/indexFstRegex.c +++ b/source/libs/index/src/indexFstRegex.c @@ -29,8 +29,11 @@ FstRegex *regexCreate(const char *str) { regex->orig = orig; // construct insts based on str - SArray *insts = NULL; - + SArray *insts = taosArrayInit(256, sizeof(uint8_t)); + for (int i = 0; i < strlen(str); i++) { + uint8_t v = str[i]; + taosArrayPush(insts, &v); + } FstDfaBuilder *builder = dfaBuilderCreate(insts); regex->dfa = dfaBuilderBuild(builder); return regex; diff --git a/source/libs/index/src/indexFstSparse.c b/source/libs/index/src/indexFstSparse.c index 99ed5d6429..5fdd797aa8 100644 --- a/source/libs/index/src/indexFstSparse.c +++ b/source/libs/index/src/indexFstSparse.c @@ -15,6 +15,11 @@ #include "indexFstSparse.h" +static void sparSetUtil(int32_t *buf, int32_t cap) { + for (int32_t i = 0; i < cap; i++) { + buf[i] = -1; + } +} FstSparseSet *sparSetCreate(int32_t sz) { FstSparseSet *ss = taosMemoryCalloc(1, sizeof(FstSparseSet)); if (ss == NULL) { @@ -22,9 +27,10 @@ FstSparseSet *sparSetCreate(int32_t sz) { } ss->dense = (int32_t *)taosMemoryMalloc(sz * sizeof(int32_t)); - memset(ss->dense, -1, sz * sizeof(int32_t)); ss->sparse = (int32_t *)taosMemoryMalloc(sz * sizeof(int32_t)); - memset(ss->sparse, -1, sz * sizeof(int32_t)); + sparSetUtil(ss->dense, sz); + sparSetUtil(ss->sparse, sz); + ss->cap = sz; ss->size = 0; @@ -84,7 +90,7 @@ void sparSetClear(FstSparseSet *ss) { if (ss == NULL) { return; } - memset(ss->dense, -1, ss->cap * sizeof(int32_t)); - memset(ss->sparse, -1, ss->cap * sizeof(int32_t)); + sparSetUtil(ss->dense, ss->cap); + sparSetUtil(ss->sparse, ss->cap); ss->size = 0; } diff --git a/source/libs/index/test/fstUtilUT.cc b/source/libs/index/test/fstUtilUT.cc index 22fe1a9150..593a312c9e 100644 --- a/source/libs/index/test/fstUtilUT.cc +++ b/source/libs/index/test/fstUtilUT.cc @@ -66,7 +66,10 @@ TEST_F(FstUtilEnv, test4) { // test FstRegex -TEST_F(FstRegexEnv, test1) {} +TEST_F(FstRegexEnv, test1) { + // + EXPECT_EQ(regex != NULL, true); +} TEST_F(FstRegexEnv, test2) {} TEST_F(FstRegexEnv, test3) {} TEST_F(FstRegexEnv, test4) {} From 5819b616ce73f22a0d7390ca867b4a363943faaa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 10 Jul 2022 16:33:38 +0800 Subject: [PATCH 26/38] fix(query): update the number of query threads. --- source/common/src/tglobal.c | 2 +- source/util/src/tworker.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5b3993dd40..ab5ce3742a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -418,7 +418,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, 0) != 0) return -1; tsNumOfVnodeQueryThreads = tsNumOfCores / 2; - tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 1); + tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 2); if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, 0) != 0) return -1; tsNumOfVnodeFetchThreads = TRANGE(tsNumOfVnodeFetchThreads, 1, 1); diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 68f96c0385..88bd36f0cb 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -39,7 +39,7 @@ int32_t tQWorkerInit(SQWorkerPool *pool) { worker->pool = pool; } - uDebug("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max); + uInfo("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max); return 0; } From 9189f886a9994b13d5ddf0951f003cf2ccee95ee Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sun, 10 Jul 2022 16:34:45 +0800 Subject: [PATCH 27/38] feat(tmq): support consume from tsdb then wal --- include/libs/executor/executor.h | 1 + source/dnode/vnode/src/inc/tq.h | 2 ++ source/dnode/vnode/src/tq/tq.c | 3 +++ source/dnode/vnode/src/tq/tqExec.c | 8 +++++++- source/libs/executor/src/scanoperator.c | 3 +++ 5 files changed, 16 insertions(+), 1 deletion(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index cff4b0234c..a95293a5b1 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -36,6 +36,7 @@ typedef struct SReadHandle { void* vnode; void* mnd; SMsgCb* pMsgCb; + int64_t version; bool initMetaReader; bool initTableReader; bool initTqReader; diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index f58bbe0d68..8abaac6dff 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -89,6 +89,8 @@ typedef struct { STqExecTb execTb; STqExecDb execDb; }; + // TODO remove it + int64_t tsdbEndVer; } STqExecHandle; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e0afc6c80f..ae0f7f56a2 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -483,6 +483,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { /*for (int32_t i = 0; i < 5; i++) {*/ /*pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);*/ /*}*/ + int64_t ver = walGetCommittedVer(pTq->pVnode->pWal); if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { pHandle->execHandle.execCol.qmsg = req.qmsg; req.qmsg = NULL; @@ -493,6 +494,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { .vnode = pTq->pVnode, .initTableReader = true, .initTqReader = true, + .version = ver, }; pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle); ASSERT(pHandle->execHandle.execCol.task[i]); @@ -501,6 +503,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { ASSERT(scanner); pHandle->execHandle.pExecReader[i] = qExtractReaderFromStreamScanner(scanner); ASSERT(pHandle->execHandle.pExecReader[i]); + pHandle->execHandle.tsdbEndVer = ver; } } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { for (int32_t i = 0; i < 5; i++) { diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 49cf42b083..3ee274ced1 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -96,6 +96,12 @@ int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffset } } + if (pRsp->blockNum == 0 && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { + tqOffsetResetToLog(pOffset, pExec->tsdbEndVer + 1); + qStreamPrepareScan(task, pOffset); + continue; + } + void* meta = qStreamExtractMetaMsg(task); if (meta != NULL) { // tq add meta to rsp @@ -107,7 +113,7 @@ int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffset ASSERT(pRsp->rspOffset.type != 0); - if (pRsp->rspOffset.type == TMQ_OFFSET__LOG) { + if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { ASSERT(pRsp->rspOffset.version + 1 >= pRsp->reqOffset.version); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e42510ceba..f7155b2431 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1493,6 +1493,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pHandle) { SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanOp->info; + if (pHandle->version > 0) { + pSTInfo->cond.endVersion = pHandle->version; + } SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, 0); if (pHandle->initTableReader) { From 72738040a9b117475005f4118dd7774e9fda0ae7 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 10 Jul 2022 16:44:50 +0800 Subject: [PATCH 28/38] fix: some problem of parser and planner --- source/libs/nodes/src/nodesCodeFuncs.c | 53 +++++++++++- source/libs/parser/src/parInsert.c | 84 +------------------- source/libs/parser/src/parser.c | 2 + source/libs/planner/src/planOptimizer.c | 10 ++- source/libs/planner/src/planSpliter.c | 1 + source/libs/planner/test/planOrderByTest.cpp | 6 ++ source/libs/planner/test/planPartByTest.cpp | 2 + 7 files changed, 71 insertions(+), 87 deletions(-) diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 5d5e95cd2b..294488a38d 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -543,11 +543,40 @@ static int32_t jsonToLogicPlanNode(const SJson* pJson, void* pObj) { static const char* jkScanLogicPlanScanCols = "ScanCols"; static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols"; -static const char* jkScanLogicPlanTableId = "TableId"; static const char* jkScanLogicPlanTableType = "TableType"; +static const char* jkScanLogicPlanTableId = "TableId"; +static const char* jkScanLogicPlanStableId = "StableId"; +static const char* jkScanLogicPlanScanCount = "ScanCount"; +static const char* jkScanLogicPlanReverseScanCount = "ReverseScanCount"; static const char* jkScanLogicPlanTagCond = "TagCond"; static const char* jkScanLogicPlanGroupTags = "GroupTags"; +// typedef struct SScanLogicNode { +// uint64_t stableId; +// SVgroupsInfo* pVgroupList; +// EScanType scanType; +// uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count +// STimeWindow scanRange; +// SName tableName; +// bool showRewrite; +// double ratio; +// SNodeList* pDynamicScanFuncs; +// int32_t dataRequired; +// int64_t interval; +// int64_t offset; +// int64_t sliding; +// int8_t intervalUnit; +// int8_t slidingUnit; +// SNode* pTagCond; +// SNode* pTagIndexCond; +// int8_t triggerType; +// int64_t watermark; +// int8_t igExpired; +// SArray* pSmaIndexes; +// SNodeList* pGroupTags; +// bool groupSort; +// } SScanLogicNode; + static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; @@ -558,11 +587,20 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkScanLogicPlanScanPseudoCols, pNode->pScanPseudoCols); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableType, pNode->tableType); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableId, pNode->tableId); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableType, pNode->tableType); + code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanStableId, pNode->stableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanScanCount, pNode->scanSeq[0]); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanReverseScanCount, pNode->scanSeq[1]); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkScanLogicPlanTagCond, nodeToJson, pNode->pTagCond); @@ -585,11 +623,20 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkScanLogicPlanScanPseudoCols, &pNode->pScanPseudoCols); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkScanLogicPlanTableType, &pNode->tableType); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetUBigIntValue(pJson, jkScanLogicPlanTableId, &pNode->tableId); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetTinyIntValue(pJson, jkScanLogicPlanTableType, &pNode->tableType); + code = tjsonGetUBigIntValue(pJson, jkScanLogicPlanStableId, &pNode->stableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUTinyIntValue(pJson, jkScanLogicPlanScanCount, &pNode->scanSeq[0]); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUTinyIntValue(pJson, jkScanLogicPlanReverseScanCount, &pNode->scanSeq[1]); } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkScanLogicPlanTagCond, &pNode->pTagCond); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index a5cf755a74..2494c5a8a7 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -120,86 +120,6 @@ static int32_t skipInsertInto(char** pSql, SMsgBuf* pMsg) { return TSDB_CODE_SUCCESS; } -static int32_t parserValidateIdToken(SToken* pToken) { - if (pToken == NULL || pToken->z == NULL || pToken->type != TK_NK_ID) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // it is a token quoted with escape char '`' - if (pToken->z[0] == TS_ESCAPE_CHAR && pToken->z[pToken->n - 1] == TS_ESCAPE_CHAR) { - return TSDB_CODE_SUCCESS; - } - - char* sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); - if (sep == NULL) { // It is a single part token, not a complex type - if (isNumber(pToken)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - strntolower(pToken->z, pToken->z, pToken->n); - } else { // two part - int32_t oldLen = pToken->n; - char* pStr = pToken->z; - - if (pToken->type == TK_NK_SPACE) { - pToken->n = (uint32_t)strtrim(pToken->z); - } - - pToken->n = tGetToken(pToken->z, &pToken->type); - if (pToken->z[pToken->n] != TS_PATH_DELIMITER[0]) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (pToken->type != TK_NK_ID) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - int32_t firstPartLen = pToken->n; - - pToken->z = sep + 1; - pToken->n = (uint32_t)(oldLen - (sep - pStr) - 1); - int32_t len = tGetToken(pToken->z, &pToken->type); - if (len != pToken->n || pToken->type != TK_NK_ID) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // re-build the whole name string - if (pStr[firstPartLen] == TS_PATH_DELIMITER[0]) { - // first part do not have quote do nothing - } else { - pStr[firstPartLen] = TS_PATH_DELIMITER[0]; - memmove(&pStr[firstPartLen + 1], pToken->z, pToken->n); - uint32_t offset = (uint32_t)(pToken->z - (pStr + firstPartLen + 1)); - memset(pToken->z + pToken->n - offset, ' ', offset); - } - - pToken->n += (firstPartLen + sizeof(TS_PATH_DELIMITER[0])); - pToken->z = pStr; - - strntolower(pToken->z, pToken->z, pToken->n); - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullDbName, char* tableName) { - if (parserValidateIdToken(pStname) != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid table name", pStname->z); - } - - char* p = strnchr(pStname->z, TS_PATH_DELIMITER[0], pStname->n, false); - if (NULL != p) { // db.table - int32_t n = sprintf(fullDbName, "%d.", pCxt->pComCxt->acctId); - strncpy(fullDbName + n, pStname->z, p - pStname->z); - strncpy(tableName, p + 1, pStname->n - (p - pStname->z) - 1); - } else { - snprintf(fullDbName, TSDB_DB_FNAME_LEN, "%d.%s", pCxt->pComCxt->acctId, pCxt->pComCxt->db); - strncpy(tableName, pStname->z, pStname->n); - } - - return TSDB_CODE_SUCCESS; -} - static int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf) { const char* msg1 = "name too long"; const char* msg2 = "invalid database name"; @@ -978,11 +898,11 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 case TSDB_DATA_TYPE_NCHAR: { int32_t output = 0; - void* p = taosMemoryCalloc(1, pToken->n * TSDB_NCHAR_SIZE); + void* p = taosMemoryCalloc(1, pSchema->bytes - VARSTR_HEADER_SIZE); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), pToken->n * TSDB_NCHAR_SIZE, &output)) { + if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { if (errno == E2BIG) { taosMemoryFree(p); return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index b36adbe5d4..218b5d9f4b 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -38,6 +38,8 @@ bool qIsInsertValuesSql(const char* pStr, size_t length) { t = tStrGetToken((char*)pStr, &index, false); if (TK_USING == t.type || TK_VALUES == t.type) { return true; + } else if (TK_SELECT == t.type) { + return false; } if (0 == t.type) { break; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index fb3c420a19..d4c470be3c 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1010,8 +1010,13 @@ static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicS SNodeList* pScanNodes) { EOrder order = sortPriKeyOptGetPriKeyOrder(pSort); if (ORDER_DESC == order) { - SNode* pScan = NULL; - FOREACH(pScan, pScanNodes) { TSWAP(((SScanLogicNode*)pScan)->scanSeq[0], ((SScanLogicNode*)pScan)->scanSeq[1]); } + SNode* pScanNode = NULL; + FOREACH(pScanNode, pScanNodes) { + SScanLogicNode* pScan = (SScanLogicNode*)pScanNode; + if (pScan->scanSeq[0] > 0) { + TSWAP(pScan->scanSeq[0], pScan->scanSeq[1]); + } + } } int32_t code = @@ -1020,6 +1025,7 @@ static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicS NODES_CLEAR_LIST(pSort->node.pChildren); nodesDestroyNode((SNode*)pSort); } + pCxt->optimized = true; return code; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 2137108386..d7eccf4b8e 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -945,6 +945,7 @@ static int32_t stbSplSplitPartitionNode(SSplitContext* pCxt, SStableSplitInfo* p code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, (SNode*)splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT)); } + pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; ++(pCxt->groupId); return code; } diff --git a/source/libs/planner/test/planOrderByTest.cpp b/source/libs/planner/test/planOrderByTest.cpp index e542f4772f..13dfbad78c 100644 --- a/source/libs/planner/test/planOrderByTest.cpp +++ b/source/libs/planner/test/planOrderByTest.cpp @@ -53,6 +53,12 @@ TEST_F(PlanOrderByTest, withGroupBy) { run("SELECT SUM(c1) AS a FROM t1 GROUP BY c2 ORDER BY a"); } +TEST_F(PlanOrderByTest, withSubquery) { + useDb("root", "test"); + + run("SELECT ts FROM (SELECT * FROM t1 ORDER BY ts DESC) ORDER BY ts DESC"); +} + TEST_F(PlanOrderByTest, stable) { useDb("root", "test"); diff --git a/source/libs/planner/test/planPartByTest.cpp b/source/libs/planner/test/planPartByTest.cpp index 3b71ae3b88..48a4c12577 100644 --- a/source/libs/planner/test/planPartByTest.cpp +++ b/source/libs/planner/test/planPartByTest.cpp @@ -37,6 +37,8 @@ TEST_F(PlanPartitionByTest, withAggFunc) { run("select count(*) from st1 partition by c1"); + run("select sample(c1, 2) from st1 partition by c1"); + run("select count(*), c1 from t1 partition by c1"); } From 6cd21fe04b555d55665aacb61d683edee18e68cc Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sun, 10 Jul 2022 16:52:11 +0800 Subject: [PATCH 29/38] enh: enhance in operator --- include/libs/scalar/scalar.h | 2 +- source/libs/scalar/inc/sclInt.h | 2 +- source/libs/scalar/inc/sclvector.h | 2 +- source/libs/scalar/src/filter.c | 12 +- source/libs/scalar/src/scalar.c | 14 ++- source/libs/scalar/src/sclvector.c | 117 +++++++++++++++--- tests/script/jenkins/basic.txt | 4 + .../insert/insert_select.sim | 0 tests/script/tsim/scalar/in.sim | 83 +++++++++++++ 9 files changed, 206 insertions(+), 30 deletions(-) rename tests/script/{general => tsim}/insert/insert_select.sim (100%) create mode 100644 tests/script/tsim/scalar/in.sim diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 2a1a75c431..c81c474366 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -42,7 +42,7 @@ int32_t scalarGetOperatorParamNum(EOperatorType type); int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); int32_t vectorGetConvertType(int32_t type1, int32_t type2); -int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut); +int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* overflow); /* Math functions */ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 1794d8f5ed..4422da1b81 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -56,7 +56,7 @@ typedef struct SScalarCtx { #define SCL_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) -int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out); +int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out, int32_t* overflow); int32_t sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows, SScalarParam* pParam); int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index f80ffc70a2..a1bf1ce1ef 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -94,7 +94,7 @@ static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) return p; } -typedef void (*_bufConverteFunc)(char *buf, SScalarParam* pOut, int32_t outType); +typedef void (*_bufConverteFunc)(char *buf, SScalarParam* pOut, int32_t outType, int32_t* overflow); typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *output, int32_t order); _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 57819db0b8..42121e8813 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1042,12 +1042,18 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { SValueNode *valueNode = (SValueNode *)cell->pNode; - if (valueNode->node.resType.type != type) { - code = doConvertDataType(valueNode, &out); + if (valueNode->node.resType.type != type) { + int32_t overflow = 0; + code = doConvertDataType(valueNode, &out, &overflow); if (code) { // fltError("convert from %d to %d failed", in.type, out.type); FLT_ERR_RET(code); } + + if (overflow) { + cell = cell->pNext; + continue; + } len = tDataTypes[type].bytes; @@ -1835,7 +1841,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } // todo refactor the convert - int32_t code = doConvertDataType(var, &out); + int32_t code = doConvertDataType(var, &out, NULL); if (code != TSDB_CODE_SUCCESS) { qError("convert value to type[%d] failed", type); return TSDB_CODE_TSC_INVALID_OPERATION; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 9bd82fb1a4..e99690c556 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -59,7 +59,7 @@ int32_t sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows, SScalarPara return TSDB_CODE_SUCCESS; } -int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) { +int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out, int32_t* overflow) { SScalarParam in = {.numOfRows = 1}; int32_t code = sclCreateColumnInfoData(&pValueNode->node.resType, 1, &in); if (code != TSDB_CODE_SUCCESS) { @@ -69,7 +69,7 @@ int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) { colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); colInfoDataEnsureCapacity(out->columnData, 1); - code = vectorConvertImpl(&in, out); + code = vectorConvertImpl(&in, out, overflow); sclFreeParam(&in); return code; @@ -107,15 +107,21 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { out.columnData->info.bytes = tDataTypes[type].bytes; } - code = doConvertDataType(valueNode, &out); + int32_t overflow = 0; + code = doConvertDataType(valueNode, &out, &overflow); if (code != TSDB_CODE_SUCCESS) { // sclError("convert data from %d to %d failed", in.type, out.type); SCL_ERR_JRET(code); } + if (overflow) { + cell = cell->pNext; + continue; + } + if (IS_VAR_DATA_TYPE(type)) { buf = colDataGetVarData(out.columnData, 0); - len = varDataTLen(data); + len = varDataTLen(buf); } else { len = tDataTypes[type].bytes; buf = out.columnData->pData; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 98cb923b49..bf457d07eb 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -276,7 +276,7 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { return p; } -static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam* pOut, int32_t rowIndex) { +static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam* pOut, int32_t rowIndex, int32_t* overflow) { int64_t value = 0; if (taosParseTime(buf, &value, strlen(buf), pOut->columnData->info.precision, tsDaylight) != TSDB_CODE_SUCCESS) { value = 0; @@ -285,10 +285,26 @@ static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam* pOut, int32_t r colDataAppendInt64(pOut->columnData, rowIndex, &value); } -static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { +static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowIndex, int32_t* overflow) { + if (overflow) { + int64_t minValue = tDataTypes[pOut->columnData->info.type].minValue; + int64_t maxValue = tDataTypes[pOut->columnData->info.type].maxValue; + int64_t value = (int64_t)taosStr2Int64(buf, NULL, 10); + if (value > maxValue) { + *overflow = 1; + return; + } else if (value < minValue) { + *overflow = -1; + return; + } else { + *overflow = 0; + } + } + switch (pOut->columnData->info.type) { case TSDB_DATA_TYPE_TINYINT: { int8_t value = (int8_t)taosStr2Int8(buf, NULL, 10); + colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value); break; } @@ -310,7 +326,22 @@ static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowI } } -static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { +static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t rowIndex, int32_t* overflow) { + if (overflow) { + uint64_t minValue = (uint64_t)tDataTypes[pOut->columnData->info.type].minValue; + uint64_t maxValue = (uint64_t)tDataTypes[pOut->columnData->info.type].maxValue; + uint64_t value = (uint64_t)taosStr2UInt64(buf, NULL, 10); + if (value > maxValue) { + *overflow = 1; + return; + } else if (value < minValue) { + *overflow = -1; + return; + } else { + *overflow = 0; + } + } + switch (pOut->columnData->info.type) { case TSDB_DATA_TYPE_UTINYINT: { uint8_t value = (uint8_t)taosStr2UInt8(buf, NULL, 10); @@ -335,18 +366,24 @@ static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t ro } } -static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t rowIndex) { +static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t rowIndex, int32_t* overflow) { + if (TSDB_DATA_TYPE_FLOAT == pOut->columnData->info.type) { + float value = taosStr2Float(buf, NULL); + colDataAppendFloat(pOut->columnData, rowIndex, &value); + return; + } + double value = taosStr2Double(buf, NULL); colDataAppendDouble(pOut->columnData, rowIndex, &value); } -static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowIndex) { +static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowIndex, int32_t* overflow) { int64_t value = taosStr2Int64(buf, NULL, 10); bool v = (value != 0)? true:false; colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*) &v); } -static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex) { +static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex, int32_t* overflow) { int32_t len = 0; int32_t inputLen = varDataLen(buf); int32_t outputMaxLen = (inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; @@ -359,7 +396,7 @@ static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIn taosMemoryFree(t); } -static FORCE_INLINE void ncharToVar(char* buf, SScalarParam* pOut, int32_t rowIndex) { +static FORCE_INLINE void ncharToVar(char* buf, SScalarParam* pOut, int32_t rowIndex, int32_t* overflow) { int32_t inputLen = varDataLen(buf); char* t = taosMemoryCalloc(1, inputLen + VARSTR_HEADER_SIZE); @@ -376,7 +413,7 @@ static FORCE_INLINE void ncharToVar(char* buf, SScalarParam* pOut, int32_t rowIn //TODO opt performance, tmp is not needed. -int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { +int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType, int32_t* overflow) { bool vton = false; _bufConverteFunc func = NULL; @@ -415,8 +452,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in if(inType == TSDB_DATA_TYPE_JSON){ if(*data == TSDB_DATA_TYPE_NULL) { ASSERT(0); - } - else if(*data == TSDB_DATA_TYPE_NCHAR) { + } else if(*data == TSDB_DATA_TYPE_NCHAR) { data += CHAR_BYTES; convertType = TSDB_DATA_TYPE_NCHAR; } else if(tTagIsJson(data)){ @@ -453,7 +489,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in } } - (*func)(tmp, pOut, i); + (*func)(tmp, pOut, i, overflow); taosMemoryFreeClear(tmp); } @@ -606,7 +642,7 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1 int32_t len = sprintf(varDataVal(tmp), "%" PRId64, value); varDataLen(tmp) = len; if (outType == TSDB_DATA_TYPE_NCHAR) { - varToNchar(tmp, pOut, i); + varToNchar(tmp, pOut, i, NULL); } else { colDataAppend(pOutputCol, i, (char *)tmp, false); } @@ -623,7 +659,7 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1 int32_t len = sprintf(varDataVal(tmp), "%" PRIu64, value); varDataLen(tmp) = len; if (outType == TSDB_DATA_TYPE_NCHAR) { - varToNchar(tmp, pOut, i); + varToNchar(tmp, pOut, i, NULL); } else { colDataAppend(pOutputCol, i, (char *)tmp, false); } @@ -640,7 +676,7 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1 int32_t len = sprintf(varDataVal(tmp), "%lf", value); varDataLen(tmp) = len; if (outType == TSDB_DATA_TYPE_NCHAR) { - varToNchar(tmp, pOut, i); + varToNchar(tmp, pOut, i, NULL); } else { colDataAppend(pOutputCol, i, (char *)tmp, false); } @@ -653,9 +689,8 @@ int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int1 return TSDB_CODE_SUCCESS; } - // TODO opt performance -int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { +int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut, int32_t* overflow) { SColumnInfoData* pInputCol = pIn->columnData; SColumnInfoData* pOutputCol = pOut->columnData; @@ -668,7 +703,47 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { int16_t outType = pOutputCol->info.type; if (IS_VAR_DATA_TYPE(inType)) { - return vectorConvertFromVarData(pIn, pOut, inType, outType); + return vectorConvertFromVarData(pIn, pOut, inType, outType, overflow); + } + + if (overflow) { + ASSERT(1 == pIn->numOfRows); + + pOut->numOfRows = 0; + + if (IS_SIGNED_NUMERIC_TYPE(outType)) { + int64_t minValue = tDataTypes[outType].minValue; + int64_t maxValue = tDataTypes[outType].maxValue; + + double value = 0; + GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, 0)); + + if (value > maxValue) { + *overflow = 1; + return TSDB_CODE_SUCCESS; + } else if (value < minValue) { + *overflow = -1; + return TSDB_CODE_SUCCESS; + } else { + *overflow = 0; + } + } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) { + uint64_t minValue = (uint64_t)tDataTypes[outType].minValue; + uint64_t maxValue = (uint64_t)tDataTypes[outType].maxValue; + + double value = 0; + GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, 0)); + + if (value > maxValue) { + *overflow = 1; + return TSDB_CODE_SUCCESS; + } else if (value < minValue) { + *overflow = -1; + return TSDB_CODE_SUCCESS; + } else { + *overflow = 0; + } + } } pOut->numOfRows = pIn->numOfRows; @@ -829,6 +904,8 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { return TSDB_CODE_SUCCESS; } + + int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = { /* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB */ /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -873,7 +950,7 @@ int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int3 return TSDB_CODE_OUT_OF_MEMORY; } - code = vectorConvertImpl(input, output); + code = vectorConvertImpl(input, output, NULL); if (code) { // taosMemoryFreeClear(paramOut1->data); return code; @@ -944,7 +1021,7 @@ static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SSca return code; } - code = vectorConvertImpl(pParam, pDest); + code = vectorConvertImpl(pParam, pDest, NULL); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1714,7 +1791,7 @@ void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut } void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { - vectorConvertImpl(pLeft, pOut); + vectorConvertImpl(pLeft, pOut, NULL); for(int32_t i = 0; i < pOut->numOfRows; ++i) { if(colDataIsNull_s(pOut->columnData, i)) { int8_t v = 0; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 79f0034052..906c2b8792 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -52,6 +52,7 @@ ./test.sh -f tsim/insert/null.sim ./test.sh -f tsim/insert/update0.sim ./test.sh -f tsim/insert/commit-merge0.sim +./test.sh -f tsim/insert/insert_select.sim # ---- parser ./test.sh -f tsim/parser/groupby-basic.sim @@ -193,4 +194,7 @@ # --- catalog ./test.sh -f tsim/catalog/alterInCurrent.sim +# --- scalar +./test.sh -f tsim/scalar/in.sim + #======================b1-end=============== diff --git a/tests/script/general/insert/insert_select.sim b/tests/script/tsim/insert/insert_select.sim similarity index 100% rename from tests/script/general/insert/insert_select.sim rename to tests/script/tsim/insert/insert_select.sim diff --git a/tests/script/tsim/scalar/in.sim b/tests/script/tsim/scalar/in.sim new file mode 100644 index 0000000000..60c12a00c2 --- /dev/null +++ b/tests/script/tsim/scalar/in.sim @@ -0,0 +1,83 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== step1 +sql drop database if exists db1; +sql create database db1 vgroups 3; +sql use db1; +sql create stable st1 (fts timestamp, fbool bool, ftiny tinyint, fsmall smallint, fint int, fbig bigint, futiny tinyint unsigned, fusmall smallint unsigned, fuint int unsigned, fubig bigint unsigned, ffloat float, fdouble double, fbin binary(10), fnchar nchar(10)) tags(tts timestamp, tbool bool, ttiny tinyint, tsmall smallint, tint int, tbig bigint, tutiny tinyint unsigned, tusmall smallint unsigned, tuint int unsigned, tubig bigint unsigned, tfloat float, tdouble double, tbin binary(10), tnchar nchar(10)); +sql create table tb1 using st1 tags('2022-07-10 16:31:00', true, 1, 1, 1, 1, 1, 1, 1, 1, 1.0, 1.0, 'a', 'a'); +sql create table tb2 using st1 tags('2022-07-10 16:32:00', false, 2, 2, 2, 2, 2, 2, 2, 2, 2.0, 2.0, 'b', 'b'); +sql create table tb3 using st1 tags('2022-07-10 16:33:00', true, 3, 3, 3, 3, 3, 3, 3, 3, 3.0, 3.0, 'c', 'c'); + +sql insert into tb1 values ('2022-07-10 16:31:01', false, 1, 1, 1, 1, 1, 1, 1, 1, 1.0, 1.0, 'a', 'a'); +sql insert into tb1 values ('2022-07-10 16:31:02', true, 2, 2, 2, 2, 2, 2, 2, 2, 2.0, 2.0, 'b', 'b'); +sql insert into tb1 values ('2022-07-10 16:31:03', false, 3, 3, 3, 3, 3, 3, 3, 3, 3.0, 3.0, 'c', 'c'); +sql insert into tb1 values ('2022-07-10 16:31:04', true, 4, 4, 4, 4, 4, 4, 4, 4, 4.0, 4.0, 'd', 'd'); +sql insert into tb1 values ('2022-07-10 16:31:05', false, 5, 5, 5, 5, 5, 5, 5, 5, 5.0, 5.0, 'e', 'e'); + +sql insert into tb2 values ('2022-07-10 16:32:01', false, 1, 1, 1, 1, 1, 1, 1, 1, 1.0, 1.0, 'a', 'a'); +sql insert into tb2 values ('2022-07-10 16:32:02', true, 2, 2, 2, 2, 2, 2, 2, 2, 2.0, 2.0, 'b', 'b'); +sql insert into tb2 values ('2022-07-10 16:32:03', false, 3, 3, 3, 3, 3, 3, 3, 3, 3.0, 3.0, 'c', 'c'); +sql insert into tb2 values ('2022-07-10 16:32:04', true, 4, 4, 4, 4, 4, 4, 4, 4, 4.0, 4.0, 'd', 'd'); +sql insert into tb2 values ('2022-07-10 16:32:05', false, 5, 5, 5, 5, 5, 5, 5, 5, 5.0, 5.0, 'e', 'e'); + +sql insert into tb3 values ('2022-07-10 16:33:01', false, 1, 1, 1, 1, 1, 1, 1, 1, 1.0, 1.0, 'a', 'a'); +sql insert into tb3 values ('2022-07-10 16:33:02', true, 2, 2, 2, 2, 2, 2, 2, 2, 2.0, 2.0, 'b', 'b'); +sql insert into tb3 values ('2022-07-10 16:33:03', false, 3, 3, 3, 3, 3, 3, 3, 3, 3.0, 3.0, 'c', 'c'); +sql insert into tb3 values ('2022-07-10 16:33:04', true, 4, 4, 4, 4, 4, 4, 4, 4, 4.0, 4.0, 'd', 'd'); +sql insert into tb3 values ('2022-07-10 16:33:05', false, 5, 5, 5, 5, 5, 5, 5, 5, 5.0, 5.0, 'e', 'e'); + +sql select * from tb1 where fts in ('2022-07-10 16:31:01', '2022-07-10 16:31:03', 1657441865000); +if $rows != 3 then + return -1 +endi + +sql select * from tb1 where fbool in (0, 3); +if $rows != 5 then + return -1 +endi + +sql select * from tb1 where ftiny in (257); +if $rows != 0 then + return -1 +endi + +sql select * from tb1 where ftiny in (2, 257); +if $rows != 1 then + return -1 +endi + +sql select * from tb1 where futiny in (0, 257); +if $rows != 0 then + return -1 +endi + +sql select * from st1 where tts in ('2022-07-10 16:31:00', '2022-07-10 16:33:00', 1657441865000); +if $rows != 10 then + return -1 +endi + +sql select * from st1 where tbool in (0, 3); +if $rows != 15 then + return -1 +endi + +sql select * from st1 where ttiny in (257); +if $rows != 0 then + return -1 +endi + +sql select * from st1 where ttiny in (2, 257); +if $rows != 5 then + return -1 +endi + +sql select * from st1 where tutiny in (0, 257); +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From bb760de1cfedd40089cdb5df7b3e9784c3d96788 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 10 Jul 2022 17:08:33 +0800 Subject: [PATCH 30/38] add test case --- source/libs/index/src/indexFstDfa.c | 1 - source/libs/index/src/indexFstRegex.c | 5 +---- source/libs/index/src/indexFstSparse.c | 8 ++++---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/source/libs/index/src/indexFstDfa.c b/source/libs/index/src/indexFstDfa.c index b8ac3bd3f5..046ed0f4f4 100644 --- a/source/libs/index/src/indexFstDfa.c +++ b/source/libs/index/src/indexFstDfa.c @@ -126,7 +126,6 @@ bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *r for (int i = 0; i < sparSetLen(set); i++) { int32_t ip; - if (false == sparSetGet(set, i, &ip)) continue; Inst *inst = taosArrayGet(builder->dfa->insts, ip); diff --git a/source/libs/index/src/indexFstRegex.c b/source/libs/index/src/indexFstRegex.c index 0189e9fe6e..e148f211f2 100644 --- a/source/libs/index/src/indexFstRegex.c +++ b/source/libs/index/src/indexFstRegex.c @@ -22,11 +22,8 @@ FstRegex *regexCreate(const char *str) { if (regex == NULL) { return NULL; } - int32_t sz = (int32_t)strlen(str); - char *orig = taosMemoryCalloc(1, sz); - memcpy(orig, str, sz); - regex->orig = orig; + regex->orig = tstrdup(str); // construct insts based on str SArray *insts = taosArrayInit(256, sizeof(uint8_t)); diff --git a/source/libs/index/src/indexFstSparse.c b/source/libs/index/src/indexFstSparse.c index 5fdd797aa8..60eb7afd90 100644 --- a/source/libs/index/src/indexFstSparse.c +++ b/source/libs/index/src/indexFstSparse.c @@ -52,7 +52,7 @@ bool sparSetAdd(FstSparseSet *ss, int32_t ip, int32_t *idx) { if (ss == NULL) { return false; } - if (ip >= ss->cap) { + if (ip >= ss->cap || ip < 0) { return false; } uint32_t i = ss->size; @@ -65,7 +65,7 @@ bool sparSetAdd(FstSparseSet *ss, int32_t ip, int32_t *idx) { return true; } bool sparSetGet(FstSparseSet *ss, int32_t idx, int32_t *ip) { - if (idx >= ss->cap || idx >= ss->size) { + if (idx >= ss->cap || idx >= ss->size || idx < 0) { return false; } int32_t val = ss->dense[idx]; @@ -75,12 +75,12 @@ bool sparSetGet(FstSparseSet *ss, int32_t idx, int32_t *ip) { return val == -1 ? false : true; } bool sparSetContains(FstSparseSet *ss, int32_t ip) { - if (ip >= ss->cap) { + if (ip >= ss->cap || ip < 0) { return false; } int32_t i = ss->sparse[ip]; - if (i < ss->cap && i < ss->size && ss->dense[i] == ip) { + if (i >= 0 && i < ss->cap && i < ss->size && ss->dense[i] == ip) { return true; } else { return false; From b542b8f49a1f00357cd9d47965a98770c21c786e Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 10 Jul 2022 17:31:15 +0800 Subject: [PATCH 31/38] fix: some problem of parser and planner --- source/libs/executor/inc/executorimpl.h | 11 ++ source/libs/executor/src/executorimpl.c | 7 +- source/libs/executor/src/timewindowoperator.c | 177 ++++++++---------- source/libs/planner/src/planPhysiCreater.c | 5 +- 4 files changed, 100 insertions(+), 100 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index ae0d0bebbd..41c244f3de 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -473,6 +473,17 @@ typedef struct SIntervalAggOperatorInfo { SNode *pCondition; } SIntervalAggOperatorInfo; +typedef struct SMergeAlignedIntervalAggOperatorInfo { + SIntervalAggOperatorInfo *intervalAggOperatorInfo; + + bool hasGroupId; + uint64_t groupId; + SSDataBlock* prefetchedBlock; + bool inputBlocksFinished; + + SNode* pCondition; +} SMergeAlignedIntervalAggOperatorInfo; + typedef struct SStreamFinalIntervalOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode SOptrBasicInfo binfo; // basic info diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index f2f696b21f..58c7caaf8e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3994,8 +3994,11 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* int32_t num = 0; SSDataBlock* pResBlock = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc); SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pTargets, NULL, &num); - SInterval* pInterval = &((SIntervalAggOperatorInfo*)downstream->info)->interval; - int32_t type = convertFillType(pPhyFillNode->mode); + SInterval* pInterval = + QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType + ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval + : &((SIntervalAggOperatorInfo*)downstream->info)->interval; + int32_t type = convertFillType(pPhyFillNode->mode); SResultInfo* pResultInfo = &pOperator->resultInfo; initResultSizeInfo(pOperator, 4096); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index f9810913e3..d060a7a888 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -80,11 +80,11 @@ static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, T } static STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order) { - int32_t factor = (order == TSDB_ORDER_ASC)? -1:1; + int32_t factor = (order == TSDB_ORDER_ASC) ? -1 : 1; STimeWindow win = *pWindow; STimeWindow save = win; - while(win.skey <= ts && win.ekey >= ts) { + while (win.skey <= ts && win.ekey >= ts) { save = win; win.skey = taosTimeAdd(win.skey, factor * pInterval->sliding, pInterval->slidingUnit, pInterval->precision); win.ekey = taosTimeAdd(win.ekey, factor * pInterval->sliding, pInterval->slidingUnit, pInterval->precision); @@ -133,7 +133,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI if (order == TSDB_ORDER_ASC) { if (w.skey <= ts && w.ekey >= ts) { // ts is resident in current time window, but we need to find the first - //qualified time window that cover this timestamp + // qualified time window that cover this timestamp w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order); } else { // todo refactor: @@ -889,7 +889,7 @@ static void removeResults(SArray* pWins, SArray* pUpdated) { } int64_t getWinReskey(void* data, int32_t index) { - SArray* res = (SArray*)data; + SArray* res = (SArray*)data; SWinRes* pos = taosArrayGet(res, index); return pos->ts; } @@ -899,15 +899,14 @@ static void removeDeleteResults(SArray* pUpdated, SArray* pDelWins) { int32_t delSize = taosArrayGetSize(pDelWins); for (int32_t i = 0; i < upSize; i++) { SResKeyPos* pResKey = taosArrayGetP(pUpdated, i); - int64_t key = *(int64_t*)pResKey->key; - int32_t index = binarySearch(pDelWins, delSize, key, TSDB_ORDER_DESC, getWinReskey); + int64_t key = *(int64_t*)pResKey->key; + int32_t index = binarySearch(pDelWins, delSize, key, TSDB_ORDER_DESC, getWinReskey); if (index >= 0 && key == getWinReskey(pDelWins, index)) { taosArrayRemove(pDelWins, index); } } } - bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) { ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0); return pSup->maxTs != INT64_MIN && ts < pSup->maxTs - pSup->waterMark; @@ -1206,7 +1205,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - while(1) { + while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doFilter(pInfo->pCondition, pBInfo->pRes); @@ -1221,7 +1220,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { } } pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows; - return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes; + return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes; } int32_t order = TSDB_ORDER_ASC; @@ -1247,7 +1246,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - while(1) { + while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doFilter(pInfo->pCondition, pBInfo->pRes); @@ -1262,7 +1261,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { } } pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows; - return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes; + return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes; } static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { @@ -1384,13 +1383,14 @@ bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* pUpWins, SInterval* pInterval) { SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - TSKEY* tsStarts = (TSKEY*)pStartCol->pData; + TSKEY* tsStarts = (TSKEY*)pStartCol->pData; SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - uint64_t* groupIds = (uint64_t*)pGroupCol->pData; + uint64_t* groupIds = (uint64_t*)pGroupCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, pInterval->precision, TSDB_ORDER_ASC); + STimeWindow win = + getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, pInterval->precision, TSDB_ORDER_ASC); doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]); if (pUpWins) { SWinRes winRes = {.ts = win.skey, .groupId = groupIds[i]}; @@ -1399,8 +1399,8 @@ void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, } } -static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, - int32_t numOfOutput, SSDataBlock* pBlock, SArray* pUpWins) { +static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t numOfOutput, + SSDataBlock* pBlock, SArray* pUpWins) { SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* tsCols = (TSKEY*)pTsCol->pData; uint64_t* pGpDatas = NULL; @@ -1415,7 +1415,7 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, TSDB_ORDER_ASC); step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); uint64_t winGpId = pGpDatas ? pGpDatas[i] : pBlock->info.groupId; - bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), winGpId, numOfOutput); + bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), winGpId, numOfOutput); if (pUpWins && res) { SWinRes winRes = {.ts = win.skey, .groupId = winGpId}; taosArrayPush(pUpWins, &winRes); @@ -1440,9 +1440,9 @@ static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) { return TSDB_CODE_SUCCESS; } -static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, - SInterval* pInterval, SHashObj* pPullDataMap, SArray* closeWins, - SArray* pRecyPages, SDiskbasedBuf* pDiscBuf) { +static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, + SHashObj* pPullDataMap, SArray* closeWins, SArray* pRecyPages, + SDiskbasedBuf* pDiscBuf) { void* pIte = NULL; size_t keyLen = 0; while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { @@ -1497,8 +1497,8 @@ static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) { SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs); - closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, - &pChInfo->interval, NULL, NULL, NULL, pChInfo->aggSup.pResultBuf); + closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL, NULL, + pChInfo->aggSup.pResultBuf); } } @@ -1544,7 +1544,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } if (pOperator->status == OP_RES_TO_RETURN) { - doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex ,pInfo->pDelRes); + doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); if (pInfo->pDelRes->info.rows > 0) { return pInfo->pDelRes; } @@ -1559,7 +1559,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); // SResKeyPos + SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); // SResKeyPos while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -1568,11 +1568,12 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { printDataBlock(pBlock, "single interval recv"); if (pBlock->info.type == STREAM_CLEAR) { - doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, - pOperator->exprSupp.numOfExprs, pBlock, NULL); + doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, + NULL); qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); continue; - } if (pBlock->info.type == STREAM_DELETE_DATA) { + } + if (pBlock->info.type == STREAM_DELETE_DATA) { doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { @@ -1597,8 +1598,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated); } pOperator->status = OP_RES_TO_RETURN; - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, - &pInfo->interval, NULL, pUpdated, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdated, + pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); @@ -1617,7 +1618,7 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); taosMemoryFreeClear(pInfo->stateKey.pData); - + taosMemoryFreeClear(param); } @@ -1626,7 +1627,7 @@ void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); taosArrayDestroy(pInfo->pRecycledPages); - + taosMemoryFreeClear(param); } @@ -1650,7 +1651,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { } } nodesDestroyNode((SNode*)pInfo->pPhyNode); - + taosMemoryFreeClear(param); } @@ -1793,8 +1794,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pInfo->pDelWins = taosArrayInit(4, sizeof(SWinRes)); pInfo->delIndex = 0; // pInfo->pDelRes = createPullDataBlock(); todo(liuyao) for delete - pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete - pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete + pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false); // todo(liuyao) for delete + pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; // todo(liuyao) for delete initResultRowInfo(&pInfo->binfo.resultRowInfo); @@ -1962,7 +1963,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; if (pOperator->status == OP_RES_TO_RETURN) { - while(1) { + while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doFilter(pInfo->pCondition, pBInfo->pRes); @@ -1977,7 +1978,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { } } pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows; - return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes; + return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes; } int64_t st = taosGetTimestampUs(); @@ -2006,7 +2007,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - while(1) { + while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doFilter(pInfo->pCondition, pBInfo->pRes); @@ -2021,7 +2022,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { } } pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows; - return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes; + return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes; } static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) { @@ -2385,13 +2386,14 @@ _error: void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); - + taosMemoryFreeClear(param); } SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, - STimeWindowAggSupp* pTwAggSupp, SNode* pCondition, SExecTaskInfo* pTaskInfo) { + STimeWindowAggSupp* pTwAggSupp, SNode* pCondition, + SExecTaskInfo* pTaskInfo) { SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -2470,20 +2472,20 @@ bool hasIntervalWindow(SAggSupporter* pSup, TSKEY ts, uint64_t groupId) { return p1 != NULL; } -static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, - SArray* pWinArray, int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SArray* pUpdated) { +static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray, + int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SArray* pUpdated) { int32_t size = taosArrayGetSize(pWinArray); if (!pInfo->pChildren) { return; } for (int32_t i = 0; i < size; i++) { - SWinRes* pWinRes = taosArrayGet(pWinArray, i); - SResultRow* pCurResult = NULL; - STimeWindow ParentWin = {.skey = pWinRes->ts, .ekey = pWinRes->ts+1}; - setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &ParentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx, numOfOutput, - pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + SWinRes* pWinRes = taosArrayGet(pWinArray, i); + SResultRow* pCurResult = NULL; + STimeWindow ParentWin = {.skey = pWinRes->ts, .ekey = pWinRes->ts + 1}; + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &ParentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx, + numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); - bool find = true; + bool find = true; for (int32_t j = 0; j < numOfChildren; j++) { SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, j); SIntervalAggOperatorInfo* pChInfo = pChildOp->info; @@ -2493,8 +2495,9 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr } find = true; SResultRow* pChResult = NULL; - setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &ParentWin, true, &pChResult, pWinRes->groupId, pChildSup->pCtx, - pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup, pTaskInfo); + setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &ParentWin, true, &pChResult, pWinRes->groupId, + pChildSup->pCtx, pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup, + pTaskInfo); compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo); } if (find && pUpdated) { @@ -2783,18 +2786,16 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->binfo.pRes->info.type = pBlock->info.type; } else if (pBlock->info.type == STREAM_CLEAR) { SArray* pUpWins = taosArrayInit(8, sizeof(SWinRes)); - doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, - pBlock, pUpWins); + doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info; SExprSupp* pChildSup = &pChildOp->exprSupp; - doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, - pChildSup->numOfExprs, pBlock, NULL); - rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, - pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, NULL); + doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildSup->numOfExprs, pBlock, NULL); + rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs, + pOperator->pTaskInfo, NULL); taosArrayDestroy(pUpWins); continue; } @@ -2812,7 +2813,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SExprSupp* pChildSup = &pChildOp->exprSupp; doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, pBlock, NULL, &pChildInfo->interval); rebuildIntervalWindow(pInfo, pSup, pInfo->pDelWins, pInfo->binfo.pRes->info.groupId, - pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, pUpdated); + pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, pUpdated); continue; } removeResults(pInfo->pDelWins, pUpdated); @@ -2862,8 +2863,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); if (IS_FINAL_OP(pInfo)) { - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, - &pInfo->interval, pInfo->pPullDataMap, pUpdated, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap, + pUpdated, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); } @@ -3005,8 +3006,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pPullDataRes = createPullDataBlock(); pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; // pInfo->pDelRes = createPullDataBlock(); // todo(liuyao) for delete - pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete - pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete + pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false); // todo(liuyao) for delete + pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; // todo(liuyao) for delete pInfo->delIndex = 0; pInfo->pDelWins = taosArrayInit(4, sizeof(SWinRes)); @@ -3063,7 +3064,7 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFreeClear(pChInfo); } } - + taosMemoryFreeClear(param); } @@ -3152,8 +3153,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); pInfo->pDelIterator = NULL; // pInfo->pDelRes = createPullDataBlock(); - pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete - pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete + pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false); // todo(liuyao) for delete + pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; // todo(liuyao) for delete pInfo->pChildren = NULL; pInfo->isFinal = false; pInfo->pPhyNode = pPhyNode; @@ -3206,9 +3207,7 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) { return false; } -bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { - return isInTimeWindow(&pWinInfo->win, ts, gap); -} +bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { return isInTimeWindow(&pWinInfo->win, ts, gap); } static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, int32_t index) { SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; @@ -3234,7 +3233,7 @@ SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) { // don't add new window SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, - int64_t gap, int32_t* pIndex) { + int64_t gap, int32_t* pIndex) { SArray* pWinInfos = getWinInfos(pAggSup, groupId); pAggSup->pCurWins = pWinInfos; @@ -3243,7 +3242,7 @@ SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY start return NULL; } // find the first position which is smaller than the key - int32_t index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey); + int32_t index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey); SResultWindowInfo* pWin = NULL; if (index >= 0) { pWin = taosArrayGet(pWinInfos, index); @@ -3514,16 +3513,15 @@ void deleteWindow(SArray* pWinInfos, int32_t index) { static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result) { SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; + TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); - TSKEY* endDatas = (TSKEY*)pEndTsCol->pData; + TSKEY* endDatas = (TSKEY*)pEndTsCol->pData; SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); - uint64_t* gpDatas = (uint64_t*)pGroupCol->pData; + uint64_t* gpDatas = (uint64_t*)pGroupCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { int32_t winIndex = 0; - while(1) { - SResultWindowInfo* pCurWin = - getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex); + while (1) { + SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex); if (!pCurWin) { break; } @@ -3755,8 +3753,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); - doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs, - 0, pWins); + doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs, 0, + pWins); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); @@ -3910,7 +3908,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { break; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { // gap must be 0 - doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, NULL); + doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, NULL); copyDataBlock(pInfo->pDelRes, pBlock); pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; break; @@ -4398,8 +4396,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); pInfo->pDelIterator = NULL; // pInfo->pDelRes = createPullDataBlock(); // todo(liuyao) for delete - pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);// todo(liuyao) for delete - pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;// todo(liuyao) for delete + pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false); // todo(liuyao) for delete + pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; // todo(liuyao) for delete pInfo->pChildren = NULL; pInfo->ignoreExpiredData = pStateNode->window.igExpired; @@ -4428,17 +4426,6 @@ _error: return NULL; } -typedef struct SMergeAlignedIntervalAggOperatorInfo { - SIntervalAggOperatorInfo *intervalAggOperatorInfo; - - bool hasGroupId; - uint64_t groupId; - SSDataBlock* prefetchedBlock; - bool inputBlocksFinished; - - SNode* pCondition; -} SMergeAlignedIntervalAggOperatorInfo; - void destroyMergeAlignedIntervalOperatorInfo(void* param, int32_t numOfOutput) { SMergeAlignedIntervalAggOperatorInfo* miaInfo = (SMergeAlignedIntervalAggOperatorInfo*)param; destroyIntervalOperatorInfo(miaInfo->intervalAggOperatorInfo, numOfOutput); @@ -4514,7 +4501,8 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR currTs = tsCols[currPos]; currWin.skey = currTs; currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, - iaInfo->interval.precision) - 1; + iaInfo->interval.precision) - + 1; startPos = currPos; ret = setTimeWindowOutputBuf(pResultRowInfo, &currWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); @@ -4593,7 +4581,8 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, - int32_t primaryTsSlotId, SNode* pCondition, SExecTaskInfo* pTaskInfo) { + int32_t primaryTsSlotId, SNode* pCondition, + SExecTaskInfo* pTaskInfo) { SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (miaInfo == NULL || pOperator == NULL) { @@ -4683,7 +4672,7 @@ void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) { SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param; tdListFree(miaInfo->groupIntervals); destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput); - + taosMemoryFreeClear(param); } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 18d69d21d8..d10fe1ce0c 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1317,10 +1317,7 @@ static int32_t createFillPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren } if (TSDB_CODE_SUCCESS == code) { - pFill->pWStartTs = nodesCloneNode(pFillNode->pWStartTs); - if (NULL == pFill->pWStartTs) { - code = TSDB_CODE_OUT_OF_MEMORY; - } + code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pFillNode->pWStartTs, &pFill->pWStartTs); } if (TSDB_CODE_SUCCESS == code && NULL != pFillNode->pValues) { From 7dc7f478de6eb8c18dab10f9a9c2dedd7484e003 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 10 Jul 2022 17:34:21 +0800 Subject: [PATCH 32/38] refactor: do some internal refactor and fix some memory leak. --- include/libs/executor/executor.h | 2 +- source/client/inc/clientInt.h | 14 ++-- source/client/src/clientEnv.c | 29 ++++--- source/client/src/clientImpl.c | 101 ++++++++++-------------- source/client/src/clientMain.c | 36 +++------ source/client/src/clientSml.c | 16 +--- source/client/src/clientStmt.c | 37 ++++----- source/client/src/tmq.c | 53 ++++--------- source/dnode/vnode/src/tsdb/tsdbRead.c | 63 +++++++++++++-- source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/executorimpl.c | 26 +++--- source/libs/qworker/src/qwUtil.c | 2 +- 12 files changed, 186 insertions(+), 195 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index cff4b0234c..0aa48ef2aa 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -109,7 +109,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, * @param tversion * @return */ -int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion, +int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion, int32_t* tversion); /** diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 9380b73d2d..91f21f6e6a 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -222,8 +222,8 @@ typedef struct SRequestObj { SArray* tableList; SQueryExecMetric metric; SRequestSendRecvBody body; - bool stableQuery; - bool validateOnly; + bool stableQuery; // todo refactor + bool validateOnly; // todo refactor bool killed; uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog @@ -247,9 +247,9 @@ void doFreeReqResultInfo(SReqResultInfo* pResInfo); int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq); void syncCatalogFn(SMetaData* pResult, void* param, int32_t code); -SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly); +SRequestObj* execQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly); TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly); -void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly); +void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly); static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; @@ -297,7 +297,7 @@ int32_t releaseTscObj(int64_t rid); uint64_t generateRequestId(); -void* createRequest(STscObj* pObj, int32_t type); +void* createRequest(uint64_t connId, int32_t type); void destroyRequest(SRequestObj* pRequest); SRequestObj* acquireRequest(int64_t rid); int32_t releaseRequest(int64_t rid); @@ -318,13 +318,13 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, uint16_t port, int connType); -SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly); +SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly); int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb); int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList); -int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); +int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, bool validateSql, SRequestObj** pRequest); void taos_close_internal(void* taos); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 89ecf16b40..ba92ed238b 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -49,13 +49,8 @@ void cleanupTscQhandle() { // destroy handle taosCleanUpScheduler(tscQhandle); } -static int32_t registerRequest(SRequestObj *pRequest) { - STscObj *pTscObj = acquireTscObj(pRequest->pTscObj->id); - if (NULL == pTscObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return terrno; - } +static int32_t registerRequest(SRequestObj *pRequest, STscObj* pTscObj) { // connection has been released already, abort creating request. pRequest->self = taosAddRef(clientReqRefPool, pRequest); @@ -246,29 +241,34 @@ STscObj *acquireTscObj(int64_t rid) { return (STscObj *)taosAcquireRef(clientCon int32_t releaseTscObj(int64_t rid) { return taosReleaseRef(clientConnRefPool, rid); } -void *createRequest(STscObj *pObj, int32_t type) { - assert(pObj != NULL); - +void *createRequest(uint64_t connId, int32_t type) { SRequestObj *pRequest = (SRequestObj *)taosMemoryCalloc(1, sizeof(SRequestObj)); if (NULL == pRequest) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } + STscObj* pTscObj = acquireTscObj(connId); + if (pTscObj == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; + } + pRequest->resType = RES_TYPE__QUERY; - pRequest->pDb = getDbOfConnection(pObj); pRequest->requestId = generateRequestId(); pRequest->metric.start = taosGetTimestampUs(); pRequest->body.resInfo.convertUcs4 = true; // convert ucs4 by default - pRequest->type = type; - pRequest->pTscObj = pObj; + + pRequest->pDb = getDbOfConnection(pTscObj); + pRequest->pTscObj = pTscObj; + pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); pRequest->msgBufLen = ERROR_MSG_BUF_DEFAULT_SIZE; tsem_init(&pRequest->body.rspSem, 0, 0); - if (registerRequest(pRequest)) { + if (registerRequest(pRequest, pTscObj)) { doDestroyRequest(pRequest); return NULL; } @@ -327,8 +327,8 @@ void doDestroyRequest(void *p) { if (pRequest->self) { deregisterRequest(pRequest); } - taosMemoryFree(pRequest); + taosMemoryFree(pRequest); tscTrace("end to destroy request %" PRIx64 " p:%p", reqId, pRequest); } @@ -338,7 +338,6 @@ void destroyRequest(SRequestObj *pRequest) { } taos_stop_query(pRequest); - removeRequest(pRequest->self); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6fb3d98027..a4a5ec7499 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -148,29 +148,49 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType); } -int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest) { - *pRequest = createRequest(pTscObj, TSDB_SQL_SELECT); +int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, bool validateSql, SRequestObj** pRequest) { + *pRequest = createRequest(connId, TSDB_SQL_SELECT); if (*pRequest == NULL) { - tscError("failed to malloc sqlObj"); + tscError("failed to malloc sqlObj, %s", sql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } (*pRequest)->sqlstr = taosMemoryMalloc(sqlLen + 1); if ((*pRequest)->sqlstr == NULL) { - tscError("0x%" PRIx64 " failed to prepare sql string buffer", (*pRequest)->self); - (*pRequest)->msgBuf = strdup("failed to prepare sql string buffer"); + tscError("0x%" PRIx64 " failed to prepare sql string buffer, %s", (*pRequest)->self, sql); + destroyRequest(*pRequest); + *pRequest = NULL; return TSDB_CODE_TSC_OUT_OF_MEMORY; } strntolower((*pRequest)->sqlstr, sql, (int32_t)sqlLen); (*pRequest)->sqlstr[sqlLen] = 0; - (*pRequest)->sqlLen = sqlLen; + (*pRequest)->sqlLen = sqlLen; + (*pRequest)->validateOnly = validateSql; + if (param == NULL) { + SSyncQueryParam* pParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); + if (pParam == NULL) { + destroyRequest(*pRequest); + *pRequest = NULL; + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + tsem_init(&pParam->sem, 0, 0); + pParam->pRequest = (*pRequest); + param = pParam; + } + + (*pRequest)->body.param = param; + + STscObj* pTscObj = (*pRequest)->pTscObj; if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, sizeof((*pRequest)->self))) { + tscError("%d failed to add to request container, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self, + (*pRequest)->requestId, pTscObj->id, sql); + destroyRequest(*pRequest); *pRequest = NULL; - tscError("put request to request hash failed"); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -882,18 +902,16 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue return pRequest; } -SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly) { +SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly) { SRequestObj* pRequest = NULL; SQuery* pQuery = NULL; - int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); + int32_t code = buildRequest(connId, sql, sqlLen, NULL, validateOnly, &pRequest); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; } - pRequest->validateOnly = validateOnly; - code = parseSql(pRequest, false, &pQuery, NULL); if (code != TSDB_CODE_SUCCESS) { pRequest->code = code; @@ -1044,19 +1062,20 @@ int32_t removeMeta(STscObj* pTscObj, SArray* tbList) { return TSDB_CODE_SUCCESS; } -SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly) { +// todo remove it soon +SRequestObj* execQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly) { SRequestObj* pRequest = NULL; int32_t retryNum = 0; int32_t code = 0; do { destroyRequest(pRequest); - pRequest = launchQuery(pTscObj, sql, sqlLen, validateOnly); + pRequest = launchQuery(connId, sql, sqlLen, validateOnly); if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) { break; } - code = refreshMeta(pTscObj, pRequest); + code = refreshMeta(pRequest->pTscObj, pRequest); if (code) { pRequest->code = code; break; @@ -1064,7 +1083,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool valid } while (retryNum++ < REQUEST_TOTAL_EXEC_TIMES); if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) { - removeMeta(pTscObj, pRequest->tableList); + removeMeta(pRequest->pTscObj, pRequest->tableList); } return pRequest; @@ -1119,7 +1138,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t return pTscObj; } - SRequestObj* pRequest = createRequest(pTscObj, TDMT_MND_CONNECT); + SRequestObj* pRequest = createRequest(pTscObj->id, TDMT_MND_CONNECT); if (pRequest == NULL) { destroyTscObj(pTscObj); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1446,15 +1465,10 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU return NULL; } - SSyncQueryParam* pParam = pRequest->body.param; - if (NULL == pParam) { - pParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); - tsem_init(&pParam->sem, 0, 0); - } - // convert ucs4 to native multi-bytes string pResultInfo->convertUcs4 = convertUcs4; + SSyncQueryParam* pParam = pRequest->body.param; taos_fetch_rows_a(pRequest, syncFetchFn, pParam); tsem_wait(&pParam->sem); } @@ -2026,22 +2040,9 @@ void syncQueryFn(void* param, void* res, int32_t code) { tsem_post(&pParam->sem); } -void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - fp(param, NULL, terrno); - return; - } - - int64_t rid = *(int64_t*)taos; - STscObj* pTscObj = acquireTscObj(rid); - if (pTscObj == NULL || sql == NULL || NULL == fp) { +void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) { + if (sql == NULL || NULL == fp) { terrno = TSDB_CODE_INVALID_PARA; - if (pTscObj) { - releaseTscObj(rid); - } else { - terrno = TSDB_CODE_TSC_DISCONNECTED; - } fp(param, NULL, terrno); return; } @@ -2050,26 +2051,20 @@ void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - releaseTscObj(rid); - fp(param, NULL, terrno); return; } SRequestObj* pRequest = NULL; - int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); + int32_t code = buildRequest(connId, sql, sqlLen, param, validateOnly, &pRequest); if (code != TSDB_CODE_SUCCESS) { terrno = code; - releaseTscObj(rid); fp(param, NULL, terrno); return; } - pRequest->validateOnly = validateOnly; pRequest->body.queryFp = fp; - pRequest->body.param = param; doAsyncQuery(pRequest, false); - releaseTscObj(rid); } TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { @@ -2078,36 +2073,22 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { return NULL; } - int64_t rid = *(int64_t*)taos; - STscObj* pTscObj = acquireTscObj(rid); - if (pTscObj == NULL || sql == NULL) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - #if SYNC_ON_TOP_OF_ASYNC SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); tsem_init(¶m->sem, 0, 0); - taosAsyncQueryImpl((TAOS*)&rid, sql, syncQueryFn, param, validateOnly); + taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly); tsem_wait(¶m->sem); - - releaseTscObj(rid); - return param->pRequest; #else size_t sqlLen = strlen(sql); if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { - releaseTscObj(rid); tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; return NULL; } - TAOS_RES* pRes = execQuery(pTscObj, sql, sqlLen, validateOnly); - - releaseTscObj(rid); - + TAOS_RES* pRes = execQuery(connId, sql, sqlLen, validateOnly); return pRes; #endif } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index e908046b1e..12de522cbc 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -479,7 +479,6 @@ void taos_stop_query(TAOS_RES *res) { } schedulerFreeJob(&pRequest->body.queryJob, TSDB_CODE_TSC_QUERY_KILLED); - tscDebug("request %" PRIx64 " killed", pRequest->requestId); } @@ -706,7 +705,8 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { } void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { - taosAsyncQueryImpl(taos, sql, fp, param, false); + int64_t connId = *(int64_t*)taos; + taosAsyncQueryImpl(connId, sql, fp, param, false); } int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { @@ -915,7 +915,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { return terrno; } - int64_t rid = *(int64_t *)taos; + int64_t connId = *(int64_t *)taos; const int32_t MAX_TABLE_NAME_LENGTH = 12 * 1024 * 1024; // 12MB list int32_t code = 0; SRequestObj * pRequest = NULL; @@ -933,12 +933,14 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { return TSDB_CODE_TSC_INVALID_OPERATION; } - STscObj *pTscObj = acquireTscObj(rid); - if (pTscObj == NULL) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return terrno; + char *sql = "taos_load_table_info"; + code = buildRequest(connId, sql, strlen(sql), NULL, false, &pRequest); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + goto _return; } + STscObj* pTscObj = pRequest->pTscObj; code = transferTableNameList(tableNameList, pTscObj->acctId, pTscObj->db, &catalogReq.pTableMeta); if (code) { goto _return; @@ -950,36 +952,22 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { goto _return; } - char *sql = "taos_load_table_info"; - code = buildRequest(pTscObj, sql, strlen(sql), &pRequest); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - goto _return; - } - - SSyncQueryParam param = {0}; - tsem_init(¶m.sem, 0, 0); - param.pRequest = pRequest; - SRequestConnInfo conn = { .pTrans = pTscObj->pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, ¶m, NULL); + code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, NULL, NULL); if (code) { goto _return; } - tsem_wait(¶m.sem); + SSyncQueryParam* pParam = pRequest->body.param; + tsem_wait(&pParam->sem); _return: - taosArrayDestroy(catalogReq.pTableMeta); destroyRequest(pRequest); - - releaseTscObj(rid); - return code; } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index b6972e4670..6e689adf95 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2442,22 +2442,15 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr return NULL; } - int64_t rid = *(int64_t*)taos; - STscObj* pTscObj = acquireTscObj(rid); - if (NULL == pTscObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - uError("SML:taos_schemaless_insert invalid taos"); - return NULL; - } - - SRequestObj* request = (SRequestObj*)createRequest(pTscObj, TSDB_SQL_INSERT); + SRequestObj* request = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); if(!request){ - releaseTscObj(rid); uError("SML:taos_schemaless_insert error request is null"); return NULL; } int batchs = 0; + STscObj* pTscObj = request->pTscObj; + pTscObj->schemalessType = 1; SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; @@ -2507,7 +2500,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr batchs = ceil(((double)numLines) / LINE_BATCH); for (int i = 0; i < batchs; ++i) { - SRequestObj* req = (SRequestObj*)createRequest(pTscObj, TSDB_SQL_INSERT); + SRequestObj* req = (SRequestObj*)createRequest(pTscObj->id, TSDB_SQL_INSERT); if(!req){ request->code = TSDB_CODE_OUT_OF_MEMORY; uError("SML:taos_schemaless_insert error request is null"); @@ -2549,6 +2542,5 @@ end: // ((STscObj *)taos)->schemalessType = 0; pTscObj->schemalessType = 1; uDebug("resultend:%s", request->msgBuf); - releaseTscObj(rid); return (TAOS_RES*)request; } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 1e0f30695d..d653b8720f 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -5,6 +5,14 @@ #include "clientStmt.h" +static int32_t stmtCreateRequest(STscStmt* pStmt) { + if (pStmt->exec.pRequest == NULL) { + return buildRequest(pStmt->taos->id, pStmt->sql.sqlStr, pStmt->sql.sqlLen, NULL, false, &pStmt->exec.pRequest); + } else { + return TSDB_CODE_SUCCESS; + } +} + int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { int32_t code = 0; @@ -217,9 +225,7 @@ int32_t stmtParseSql(STscStmt* pStmt) { .getExecInfoFn = stmtGetExecInfo, }; - if (NULL == pStmt->exec.pRequest) { - STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); - } + STMT_ERR_RET(stmtCreateRequest(pStmt)); STMT_ERR_RET(parseSql(pStmt->exec.pRequest, false, &pStmt->sql.pQuery, &stmtCb)); @@ -532,9 +538,7 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) { STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - if (NULL == pStmt->exec.pRequest) { - STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); - } + STMT_ERR_RET(stmtCreateRequest(pStmt)); STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); @@ -625,9 +629,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { pStmt->exec.pRequest = NULL; } - if (NULL == pStmt->exec.pRequest) { - STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); - } + STMT_ERR_RET(stmtCreateRequest(pStmt)); if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); @@ -873,9 +875,7 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { pStmt->exec.pRequest = NULL; } - if (NULL == pStmt->exec.pRequest) { - STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); - } + STMT_ERR_RET(stmtCreateRequest(pStmt)); if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); @@ -905,9 +905,7 @@ int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { pStmt->exec.pRequest = NULL; } - if (NULL == pStmt->exec.pRequest) { - STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); - } + STMT_ERR_RET(stmtCreateRequest(pStmt)); if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); @@ -933,10 +931,7 @@ int stmtGetParamNum(TAOS_STMT* stmt, int* nums) { pStmt->exec.pRequest = NULL; } - if (NULL == pStmt->exec.pRequest) { - STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); - } - + STMT_ERR_RET(stmtCreateRequest(pStmt)); if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); } @@ -969,9 +964,7 @@ int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) { pStmt->exec.pRequest = NULL; } - if (NULL == pStmt->exec.pRequest) { - STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); - } + STMT_ERR_RET(stmtCreateRequest(pStmt)); if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index cea6905c69..614174d97a 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -2267,13 +2267,7 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ int32_t code = TSDB_CODE_SUCCESS; SRequestObj* pRequest = NULL; - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); - if (NULL == pTscObj) { - code = TSDB_CODE_TSC_DISCONNECTED; - goto end; - } - - code = buildRequest(pTscObj, "", 0, &pRequest); + code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -2313,6 +2307,8 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ pReq.suid = req.suid; pReq.source = 1; + STscObj* pTscObj = pRequest->pTscObj; + SName tableName; tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); @@ -2351,13 +2347,7 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ int32_t code = TSDB_CODE_SUCCESS; SRequestObj* pRequest = NULL; - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); - if (NULL == pTscObj) { - code = TSDB_CODE_TSC_DISCONNECTED; - goto end; - } - - code = buildRequest(pTscObj, "", 0, &pRequest); + code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -2379,6 +2369,9 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ pReq.igNotExists = true; pReq.source = 1; pReq.suid = req.suid; + + STscObj* pTscObj = pRequest->pTscObj; + SName tableName; tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); @@ -2427,14 +2420,8 @@ static int32_t taosCreateTable(TAOS *taos, void *meta, int32_t metaLen){ SRequestObj *pRequest = NULL; SQuery *pQuery = NULL; SHashObj *pVgroupHashmap = NULL; - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); - if (NULL == pTscObj) { - code = TSDB_CODE_TSC_DISCONNECTED; - goto end; - } - - code = buildRequest(pTscObj, "", 0, &pRequest); + code = buildRequest(*(int64_t*) taos, "", 0, NULL, false, &pRequest); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -2452,7 +2439,9 @@ static int32_t taosCreateTable(TAOS *taos, void *meta, int32_t metaLen){ goto end; } - SVCreateTbReq *pCreateReq = NULL; + STscObj* pTscObj = pRequest->pTscObj; + + SVCreateTbReq *pCreateReq = NULL; SCatalog* pCatalog = NULL; code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { @@ -2544,14 +2533,8 @@ static int32_t taosDropTable(TAOS *taos, void *meta, int32_t metaLen){ SRequestObj *pRequest = NULL; SQuery *pQuery = NULL; SHashObj *pVgroupHashmap = NULL; - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); - if (NULL == pTscObj) { - code = TSDB_CODE_TSC_DISCONNECTED; - goto end; - } - - code = buildRequest(pTscObj, "", 0, &pRequest); + code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -2569,6 +2552,8 @@ static int32_t taosDropTable(TAOS *taos, void *meta, int32_t metaLen){ goto end; } + STscObj* pTscObj = pRequest->pTscObj; + SVDropTbReq *pDropReq = NULL; SCatalog *pCatalog = NULL; code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); @@ -2649,14 +2634,9 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ SQuery *pQuery = NULL; SArray *pArray = NULL; SVgDataBlocks *pVgData = NULL; - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); - if (NULL == pTscObj) { - code = TSDB_CODE_TSC_DISCONNECTED; - goto end; - } - code = buildRequest(pTscObj, "", 0, &pRequest); + code = buildRequest(*(int64_t*) taos, "", 0, NULL, false, &pRequest); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -2679,7 +2659,8 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ goto end; } - SCatalog *pCatalog = NULL; + STscObj* pTscObj = pRequest->pTscObj; + SCatalog* pCatalog = NULL; code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { goto end; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f650480ff2..51f7f23776 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -59,7 +59,6 @@ typedef struct SBlockLoadSuppInfo { SColumnDataAgg tsColAgg; SColumnDataAgg** plist; int16_t* colIds; // column ids for loading file block data - int32_t* slotIds; // colId to slotId char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. } SBlockLoadSuppInfo; @@ -183,7 +182,6 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK return NULL; } - // todo apply the lastkey of table check to avoid to load header file for (int32_t j = 0; j < numOfTables; ++j) { STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid}; if (ASCENDING_TRAVERSE(pTsdbReader->order)) { @@ -218,6 +216,30 @@ static void resetDataBlockScanInfo(SHashObj* pTableMap) { } } +static void destroyBlockScanInfo(SHashObj* pTableMap) { + STableBlockScanInfo* p = NULL; + + while ((p = taosHashIterate(pTableMap, p)) != NULL) { + p->iterInit = false; + p->iiter.hasVal = false; + + if (p->iter.iter != NULL) { + tsdbTbDataIterDestroy(p->iter.iter); + p->iter.iter = NULL; + } + + if (p->iiter.iter != NULL) { + tsdbTbDataIterDestroy(p->iiter.iter); + p->iiter.iter = NULL; + } + + taosArrayDestroy(p->delSkyline); + p->delSkyline = NULL; + } + + taosHashCleanup(pTableMap); +} + static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) { ASSERT(pWindow != NULL); return pWindow->skey > pWindow->ekey; @@ -265,6 +287,10 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, const STsdbFSState* pFSt return TSDB_CODE_SUCCESS; } +static void cleanupFilesetIterator(SFilesetIter* pIter) { + taosArrayDestroy(pIter->pFileList); +} + static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) { bool asc = ASCENDING_TRAVERSE(pIter->order); int32_t step = asc ? 1 : -1; @@ -313,7 +339,15 @@ static void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order) { pIter->order = order; pIter->index = -1; pIter->numOfBlocks = -1; - pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); + if (pIter->blockList == NULL) { + pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); + } else { + taosArrayClear(pIter->blockList); + } +} + +static void cleanupDataBlockIterator(SDataBlockIter* pIter) { + taosArrayDestroy(pIter->blockList); } static void initReaderStatus(SReaderStatus* pStatus) { @@ -2515,6 +2549,7 @@ void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDe tRowMergerInit(&merge, pRow, pReader->pSchema); doMergeRowsInBuf(pIter, k.ts, pDelList, &merge, pReader); tRowMergerGetRow(&merge, pTSRow); + tRowMergerClear(&merge); } void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, @@ -2651,6 +2686,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e } doAppendOneRow(pBlock, pReader, pTSRow); + taosMemoryFree(pTSRow); // no data in buffer, return immediately if (!(pBlockScanInfo->iter.hasVal || pBlockScanInfo->iiter.hasVal)) { @@ -2778,11 +2814,24 @@ void tsdbReaderClose(STsdbReader* pReader) { return; } - blockDataDestroy(pReader->pResBlock); - taosMemoryFreeClear(pReader->suppInfo.plist); + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; + + taosMemoryFreeClear(pSupInfo->plist); + taosMemoryFree(pSupInfo->colIds); + + taosArrayDestroy(pSupInfo->pColAgg); + for(int32_t i = 0; i < blockDataGetNumOfCols(pReader->pResBlock); ++i) { + if (pSupInfo->buildBuf[i] != NULL) { + taosMemoryFreeClear(pSupInfo->buildBuf[i]); + } + } + taosMemoryFree(pSupInfo->buildBuf); + + cleanupFilesetIterator(&pReader->status.fileIter); + cleanupDataBlockIterator(&pReader->status.blockIter); + destroyBlockScanInfo(pReader->status.pTableMap); + blockDataDestroy(pReader->pResBlock); - taosArrayDestroy(pReader->suppInfo.pColAgg); - taosMemoryFree(pReader->suppInfo.slotIds); #if 0 // if (pReader->status.pTableScanInfo != NULL) { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index fc35fba935..5a1a46a9b8 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -184,7 +184,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo return code; } -int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion, +int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion, int32_t* tversion) { ASSERT(tinfo != NULL && dbName != NULL && tableName != NULL); SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b209684388..8171c7e42e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include "filter.h" #include "function.h" #include "functionMgt.h" @@ -4052,7 +4053,7 @@ static STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SRea static SArray* extractColumnInfo(SNodeList* pNodeList); -int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskInfo* pTaskInfo) { +int32_t extractTableSchemaInfo(SReadHandle* pHandle, uint64_t uid, SExecTaskInfo* pTaskInfo) { SMetaReader mr = {0}; metaReaderInit(&mr, pHandle->meta, 0); int32_t code = metaGetTableEntryByUid(&mr, uid); @@ -4076,10 +4077,20 @@ int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskI } metaReaderClear(&mr); - return TSDB_CODE_SUCCESS; } +static void cleanupTableSchemaInfo(SExecTaskInfo* pTaskInfo) { + taosMemoryFreeClear(pTaskInfo->schemaVer.dbname); + if (pTaskInfo->schemaVer.sw == NULL) { + return; + } + + taosMemoryFree(pTaskInfo->schemaVer.sw->pSchema); + taosMemoryFree(pTaskInfo->schemaVer.sw); + taosMemoryFree(pTaskInfo->schemaVer.tablename); +} + static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum) { taosArrayClear(pTableListInfo->pGroupList); SArray* sortSupport = taosArrayInit(groupNum, sizeof(uint64_t)); @@ -4253,7 +4264,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); + code = extractTableSchemaInfo(pHandle, pTableScanNode->scan.uid, pTaskInfo); if (code) { pTaskInfo->code = terrno; return NULL; @@ -4271,7 +4282,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pTaskInfo->code = code; return NULL; } - code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); + code = extractTableSchemaInfo(pHandle, pTableScanNode->scan.uid, pTaskInfo); if (code) { pTaskInfo->code = terrno; return NULL; @@ -4368,7 +4379,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo // return NULL; // } - int32_t code = extractTableSchemaVersion(pHandle, pScanNode->uid, pTaskInfo); + int32_t code = extractTableSchemaInfo(pHandle, pScanNode->uid, pTaskInfo); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; return NULL; @@ -4867,11 +4878,8 @@ void doDestroyTask(SExecTaskInfo* pTaskInfo) { doDestroyTableList(&pTaskInfo->tableqinfoList); destroyOperatorInfo(pTaskInfo->pRoot); - // taosArrayDestroy(pTaskInfo->summary.queryProfEvents); - // taosHashCleanup(pTaskInfo->summary.operatorProfResults); + cleanupTableSchemaInfo(pTaskInfo); - taosMemoryFree(pTaskInfo->schemaVer.dbname); - taosMemoryFree(pTaskInfo->schemaVer.tablename); taosMemoryFreeClear(pTaskInfo->sql); taosMemoryFreeClear(pTaskInfo->id.str); taosMemoryFreeClear(pTaskInfo); diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 5aaf2b8038..9c49cbcb1f 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -436,7 +436,7 @@ void qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; char tbName[TSDB_TABLE_NAME_LEN] = {0}; - qGetQueriedTableSchemaVersion(pTaskInfo, dbFName, tbName, &ctx->tbInfo.sversion, &ctx->tbInfo.tversion); + qGetQueryTableSchemaVersion(pTaskInfo, dbFName, tbName, &ctx->tbInfo.sversion, &ctx->tbInfo.tversion); if (dbFName[0] && tbName[0]) { sprintf(ctx->tbInfo.tbFName, "%s.%s", dbFName, tbName); From 75ed474f0968b90d0ca2d8178f822b7b970cad03 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 8 Jul 2022 19:55:06 +0800 Subject: [PATCH 33/38] ci(stream): add session test --- source/libs/executor/inc/executil.h | 2 ++ source/libs/executor/src/executil.c | 2 +- source/libs/executor/src/timewindowoperator.c | 1 + tests/script/jenkins/basic.txt | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 2c8fbe9206..f9aba30a46 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -126,4 +126,6 @@ void cleanupQueryTableDataCond(SQueryTableDataCond* pCond); int32_t convertFillType(int32_t mode); +int32_t resultrowComparAsc(const void* p1, const void* p2); + #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index e3579f9dcb..adbfa2c4ba 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -77,7 +77,7 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { pGroupResInfo->index = 0; } -static int32_t resultrowComparAsc(const void* p1, const void* p2) { +int32_t resultrowComparAsc(const void* p1, const void* p2) { SResKeyPos* pp1 = *(SResKeyPos**)p1; SResKeyPos* pp2 = *(SResKeyPos**)p2; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 64c27fdb28..ed8ea56329 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3441,6 +3441,7 @@ static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated) { *(int64_t*)pos->key = ((SWinRes*)pData)->ts; taosArrayPush(pUpdated, &pos); } + taosArraySort(pUpdated, resultrowComparAsc); return TSDB_CODE_SUCCESS; } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 906c2b8792..94414edbf2 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -98,7 +98,7 @@ ./test.sh -f tsim/stream/distributeInterval0.sim # ./test.sh -f tsim/stream/distributeIntervalRetrive0.sim # ./test.sh -f tsim/stream/distributesession0.sim -# ./test.sh -f tsim/stream/session0.sim +./test.sh -f tsim/stream/session0.sim ./test.sh -f tsim/stream/session1.sim # ./test.sh -f tsim/stream/state0.sim ./test.sh -f tsim/stream/triggerInterval0.sim From 246562024590f03e0dbaaa22cb5c34647a5cc31e Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Sun, 10 Jul 2022 16:09:54 +0800 Subject: [PATCH 34/38] feat(stream): support expressions --- source/common/src/tdatablock.c | 2 +- source/libs/executor/inc/executorimpl.h | 6 +- source/libs/executor/src/timewindowoperator.c | 64 +++++++++++++++++-- 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4e17a9aea0..3bb829f77a 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1746,7 +1746,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) for (int32_t k = 0; k < colNum; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - if (colDataIsNull(pColInfoData, rows, j, NULL)) { + if (colDataIsNull(pColInfoData, rows, j, NULL) || !var) { len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL"); if (len >= size -1) return dumpBuf; continue; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 5f13c83eda..3c674e8d10 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -481,7 +481,7 @@ typedef struct SStreamFinalIntervalOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode SOptrBasicInfo binfo; // basic info SAggSupporter aggSup; // aggregate supporter - + SExprSupp scalarSupp; // supporter for perform scalar function SGroupResInfo groupResInfo; // multiple results build supporter SInterval interval; // interval info int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. @@ -629,6 +629,7 @@ typedef struct SStateWindowInfo { typedef struct SStreamSessionAggOperatorInfo { SOptrBasicInfo binfo; SStreamAggSupporter streamAggSup; + SExprSupp scalarSupp; // supporter for perform scalar function SGroupResInfo groupResInfo; int64_t gap; // session window gap int32_t primaryTsIndex; // primary timestamp slot id @@ -679,11 +680,12 @@ typedef struct SStateWindowOperatorInfo { typedef struct SStreamStateAggOperatorInfo { SOptrBasicInfo binfo; SStreamAggSupporter streamAggSup; + SExprSupp scalarSupp; // supporter for perform scalar function SGroupResInfo groupResInfo; int32_t primaryTsIndex; // primary timestamp slot id int32_t order; // current SSDataBlock scan order STimeWindowAggSupp twAggSup; - SColumn stateCol; // start row index + SColumn stateCol; SqlFunctionCtx* pDummyCtx; // for combine SSDataBlock* pDelRes; SHashObj* pSeDeleted; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index ed8ea56329..78775073a4 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2401,6 +2401,12 @@ void addPullWindow(SHashObj* pMap, SWinRes* pWinRes, int32_t size) { static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } +STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) { + STimeWindow w = {.skey = ts, .ekey = INT64_MAX}; + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + return w; +} + static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, SArray* pUpdated) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; @@ -2420,8 +2426,12 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); - STimeWindow nextWin = - getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->order); + STimeWindow nextWin = {0}; + if (IS_FINAL_OP(pInfo)) { + nextWin = getFinalTimeWindow(ts, &pInfo->interval); + } else { + nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->order); + } while (1) { bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); if (pInfo->ignoreExpiredData && isClosed) { @@ -2478,8 +2488,12 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, + if (IS_FINAL_OP(pInfo)) { + forwardRows = 1; + } else { + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) { saveResultRow(pResult, tableGroupId, pUpdated); } @@ -2645,7 +2659,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { clearSpecialDataBlock(pInfo->pUpdateRes); removeDeleteResults(pUpdated, pInfo->pDelWins); pOperator->status = OP_RES_TO_RETURN; - qInfo("Stream Final Interval return data"); + qInfo("%s return data", IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); break; } printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval Semi recv"); @@ -2705,6 +2719,10 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { continue; } + if (pInfo->scalarSupp.pExprInfo != NULL) { + SExprSupp* pExprSup = &pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + } setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true); doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdated); if (IS_FINAL_OP(pInfo)) { @@ -2822,6 +2840,15 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, 4096); + if (pIntervalPhyNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); + int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } + int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); @@ -2988,6 +3015,14 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh } initResultSizeInfo(pOperator, 4096); + if (pSessionNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar); + int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } SExprSupp* pSup = &pOperator->exprSupp; code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); @@ -3656,6 +3691,10 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { continue; } + if (pInfo->scalarSupp.pExprInfo != NULL) { + SExprSupp* pExprSup = &pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo)); @@ -3788,6 +3827,10 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { continue; } + if (pInfo->scalarSupp.pExprInfo != NULL) { + SExprSupp* pExprSup = &pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, false); @@ -4184,6 +4227,10 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { continue; } + if (pInfo->scalarSupp.pExprInfo != NULL) { + SExprSupp* pExprSup = &pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted); @@ -4237,6 +4284,15 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->stateCol = extractColumnFromColumnNode(pColNode); initResultSizeInfo(pOperator, 4096); + if (pStateNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar); + int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } + initResultRowInfo(&pInfo->binfo.resultRowInfo); pInfo->twAggSup = (STimeWindowAggSupp){ .waterMark = pStateNode->window.watermark, From 9fe73acc86ecab132c322d29cd889a05a0d6024d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 10 Jul 2022 18:53:52 +0800 Subject: [PATCH 35/38] refactor: adjust logs --- source/common/src/tglobal.c | 22 +++++++++------------ source/libs/qworker/src/qworker.c | 2 +- source/util/src/tlog.c | 11 +++++------ tests/script/sh/deploy.sh | 33 ++++++++++++++++--------------- 4 files changed, 32 insertions(+), 36 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index ab5ce3742a..98cd6e861e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -292,15 +292,14 @@ int32_t taosAddClientLogCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, 1) != 0) return -1; if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, 1) != 0) return -1; if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, 1) != 0) return -1; if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, 1) != 0) return -1; if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, 1) != 0) return -1; if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, 1) != 0) return -1; return 0; } @@ -308,7 +307,6 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0) != 0) return -1; @@ -485,20 +483,18 @@ static void taosSetClientLogCfg(SConfig *pCfg) { tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; - cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; - uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; - qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; - rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; + uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; - idxDebugFlag = cfgGetItem(pCfg, "idxDebugFlag")->i32; + rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; + qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; + cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; } static void taosSetServerLogCfg(SConfig *pCfg) { dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32; vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32; mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32; - qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32; sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32; tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 757279269a..3e8ced318c 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1041,7 +1041,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW *qWorkerMgmt = mgmt; - qDebug("qworker initialized for node, type:%d, id:%d, handle:%p", mgmt->nodeType, mgmt->nodeId, mgmt); + qDebug("qworker initialized, type:%d, id:%d, handle:%p", mgmt->nodeType, mgmt->nodeId, mgmt); return TSDB_CODE_SUCCESS; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 9d7656de35..490c6f29bf 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -745,14 +745,14 @@ cmp_end: void taosSetAllDebugFlag(int32_t flag) { if (flag <= 0) return; + uDebugFlag = flag; + rpcDebugFlag = flag; + jniDebugFlag = flag; + qDebugFlag = flag; + cDebugFlag = flag; dDebugFlag = flag; vDebugFlag = flag; mDebugFlag = flag; - cDebugFlag = flag; - jniDebugFlag = flag; - uDebugFlag = flag; - rpcDebugFlag = flag; - qDebugFlag = flag; wDebugFlag = flag; sDebugFlag = flag; tsdbDebugFlag = flag; @@ -761,6 +761,5 @@ void taosSetAllDebugFlag(int32_t flag) { udfDebugFlag = flag; smaDebugFlag = flag; idxDebugFlag = flag; - uInfo("all debug flag are set to %d", flag); } diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 4d93878a98..de7b8ecfbf 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -122,28 +122,29 @@ echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG echo "fqdn ${HOSTNAME}" >> $TAOS_CFG echo "serverPort ${NODE}" >> $TAOS_CFG echo "supportVnodes 1024" >> $TAOS_CFG +echo "statusInterval 1" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG echo "debugFlag 0" >> $TAOS_CFG -echo "mDebugFlag 143" >> $TAOS_CFG -echo "dDebugFlag 143" >> $TAOS_CFG -echo "vDebugFlag 143" >> $TAOS_CFG -echo "tqDebugFlag 143" >> $TAOS_CFG -echo "tsdbDebugFlag 143" >> $TAOS_CFG -echo "cDebugFlag 143" >> $TAOS_CFG -echo "jniDebugFlag 143" >> $TAOS_CFG -echo "qDebugFlag 143" >> $TAOS_CFG -echo "rpcDebugFlag 143" >> $TAOS_CFG -echo "sDebugFlag 143" >> $TAOS_CFG -echo "wDebugFlag 143" >> $TAOS_CFG -echo "idxDebugFlag 143" >> $TAOS_CFG -echo "fsDebugFlag 143" >> $TAOS_CFG -echo "udfDebugFlag 143" >> $TAOS_CFG -echo "smaDebugFlag 143" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "uDebugFlag 131" >> $TAOS_CFG +echo "rpcDebugFlag 131" >> $TAOS_CFG +echo "jniDebugFlag 143" >> $TAOS_CFG +echo "qDebugFlag 143" >> $TAOS_CFG +echo "cDebugFlag 143" >> $TAOS_CFG +echo "dDebugFlag 143" >> $TAOS_CFG +echo "vDebugFlag 143" >> $TAOS_CFG +echo "mDebugFlag 143" >> $TAOS_CFG +echo "wDebugFlag 143" >> $TAOS_CFG +echo "sDebugFlag 143" >> $TAOS_CFG +echo "tsdbDebugFlag 143" >> $TAOS_CFG +echo "tqDebugFlag 143" >> $TAOS_CFG +echo "fsDebugFlag 143" >> $TAOS_CFG +echo "idxDebugFlag 143" >> $TAOS_CFG +echo "udfDebugFlag 143" >> $TAOS_CFG +echo "smaDebugFlag 143" >> $TAOS_CFG +echo "idxDebugFlag 143" >> $TAOS_CFG echo "numOfLogLines 20000000" >> $TAOS_CFG -echo "statusInterval 1" >> $TAOS_CFG echo "asyncLog 0" >> $TAOS_CFG echo "locale en_US.UTF-8" >> $TAOS_CFG echo "telemetryReporting 0" >> $TAOS_CFG From 9e4c4c50a0212fdb2275b6f1b5de0109a19df855 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 10 Jul 2022 19:10:06 +0800 Subject: [PATCH 36/38] refactor(sync): update to SYNC_STRATEGY_WAL_FIRST --- source/dnode/vnode/src/vnd/vnodeSync.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 1508b67f5e..97ce8eaab7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -548,8 +548,8 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { SSyncInfo syncInfo = { - //.snapshotStrategy = SYNC_STRATEGY_WAL_FIRST, - .snapshotStrategy = SYNC_STRATEGY_NO_SNAPSHOT, + .snapshotStrategy = SYNC_STRATEGY_WAL_FIRST, + //.snapshotStrategy = SYNC_STRATEGY_NO_SNAPSHOT, .batchSize = 10, .vgId = pVnode->config.vgId, .isStandBy = pVnode->config.standby, From e13dd96782c6a4504add74e13600aabf4083b669 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 10 Jul 2022 19:02:53 +0800 Subject: [PATCH 37/38] test: valgrind case --- tests/script/tsim/valgrind/basic2.sim | 2 +- tests/script/tsim/valgrind/checkError2.sim | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/tsim/valgrind/basic2.sim b/tests/script/tsim/valgrind/basic2.sim index eb61f85444..154617bc18 100644 --- a/tests/script/tsim/valgrind/basic2.sim +++ b/tests/script/tsim/valgrind/basic2.sim @@ -48,7 +48,7 @@ sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s print =============== step6: select data sql select * from ct1 -#sql select * from stb +sql select * from stb _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/checkError2.sim b/tests/script/tsim/valgrind/checkError2.sim index f98cd0df1d..3a2819776d 100644 --- a/tests/script/tsim/valgrind/checkError2.sim +++ b/tests/script/tsim/valgrind/checkError2.sim @@ -47,7 +47,7 @@ sql insert into ct1 values(now+0s, 10, 2.0, 3.0) sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) print =============== step6: select data -#sql select * from ct1 +sql select * from ct1 #sql select * from stb _OVER: @@ -58,7 +58,7 @@ print ----> start to check if there are ERRORS in vagrind log file for each dnod system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content <= 0 then +if $system_content <= 2 then return 0 endi From bcacc1d6b53ff97e4cdda6e613747fca9f28324f Mon Sep 17 00:00:00 2001 From: tomchon Date: Sun, 10 Jul 2022 19:15:36 +0800 Subject: [PATCH 38/38] test:comment mnode case --- tests/system-test/fulltest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index f6f902e890..3b0dd76a30 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -135,8 +135,8 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 - # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py # python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 # python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 -python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 5 -M 3 +# BUG Redict python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 +# python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 5 -M 3 python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py