From f670b6cd36d5b18f1bfee096daab2fdd6390fe53 Mon Sep 17 00:00:00 2001 From: danielclow <106956386+danielclow@users.noreply.github.com> Date: Fri, 26 Jan 2024 18:30:43 +0800 Subject: [PATCH 01/17] docs: update taosdata links to tdengine for 3.0 branch --- docs/en/08-client-libraries/50-odbc.mdx | 2 +- docs/en/10-deployment/02-docker.md | 6 +++--- docs/en/10-deployment/03-k8s.md | 2 +- docs/en/14-reference/06-taosdump.md | 2 +- docs/en/20-third-party/75-powerbi.md | 4 ++-- docs/en/25-application/_03-immigrate.md | 2 -- docs/en/28-releases/02-tools.md | 2 +- 7 files changed, 9 insertions(+), 11 deletions(-) diff --git a/docs/en/08-client-libraries/50-odbc.mdx b/docs/en/08-client-libraries/50-odbc.mdx index 08b2c031c6..4d9161473a 100644 --- a/docs/en/08-client-libraries/50-odbc.mdx +++ b/docs/en/08-client-libraries/50-odbc.mdx @@ -49,7 +49,7 @@ TDengine ODBC driver supports two kinds of connections to TDengine cluster, nati 4.2 [Connection Type]: required field, we choose "WebSocket" - 4.3 [URL]: required field, the URL for the ODBC data source, for example, `http://localhost:6041` is the URL for a local TDengine cluster, `https://gw.cloud.taosdata.com?token=your_token` is the URL for a TDengine cloud service. + 4.3 [URL]: required field, the URL for the ODBC data source, for example, `http://localhost:6041` is the URL for a local TDengine cluster. 4.4 [Database]: optional field, the default database to access diff --git a/docs/en/10-deployment/02-docker.md b/docs/en/10-deployment/02-docker.md index 2a4511c7b1..2c281ec408 100644 --- a/docs/en/10-deployment/02-docker.md +++ b/docs/en/10-deployment/02-docker.md @@ -128,7 +128,7 @@ If you want to start your application in a container, you need to add the corres FROM ubuntu:20.04 RUN apt-get update && apt-get install -y wget ENV TDENGINE_VERSION=3.0.0.0 -RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ +RUN wget -c https://tdengine.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && cd TDengine-client-${TDENGINE_VERSION} \ && ./install_client.sh \ @@ -229,7 +229,7 @@ Here is the full Dockerfile: ```docker FROM golang:1.17.6-buster as builder ENV TDENGINE_VERSION=3.0.0.0 -RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ +RUN wget -c https://tdengine.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && cd TDengine-client-${TDENGINE_VERSION} \ && ./install_client.sh \ @@ -245,7 +245,7 @@ RUN go build FROM ubuntu:20.04 RUN apt-get update && apt-get install -y wget ENV TDENGINE_VERSION=3.0.0.0 -RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ +RUN wget -c https://tdengine.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && cd TDengine-client-${TDENGINE_VERSION} \ && ./install_client.sh \ diff --git a/docs/en/10-deployment/03-k8s.md b/docs/en/10-deployment/03-k8s.md index 10c0341598..affb492169 100644 --- a/docs/en/10-deployment/03-k8s.md +++ b/docs/en/10-deployment/03-k8s.md @@ -8,7 +8,7 @@ description: This document describes how to deploy TDengine on Kubernetes. As a time series database for Cloud Native architecture design, TDengine supports Kubernetes deployment. Firstly we introduce how to use YAML files to create a highly available TDengine cluster from scratch step by step for production usage, and highlight the common operations of TDengine in Kubernetes environment. -To meet [high availability ](https://docs.taosdata.com/tdinternal/high-availability/)requirements, clusters need to meet the following requirements: +To meet [high availability](../../tdinternal/high-availability/) requirements, clusters need to meet the following requirements: - 3 or more dnodes: multiple vnodes in the same vgroup of TDengine are not allowed to be distributed in one dnode at the same time, so if you create a database with 3 replicas, the number of dnodes is greater than or equal to 3 - 3 mnodes: mnode is responsible for the management of the entire TDengine cluster. The default number of mnode in TDengine cluster is only one. If the dnode where the mnode located is dropped, the entire cluster is unavailable. diff --git a/docs/en/14-reference/06-taosdump.md b/docs/en/14-reference/06-taosdump.md index 5ec26df1fa..263834dba1 100644 --- a/docs/en/14-reference/06-taosdump.md +++ b/docs/en/14-reference/06-taosdump.md @@ -19,7 +19,7 @@ Users should not use taosdump to back up raw data, environment settings, hardwar There are two ways to install taosdump: -- Install the taosTools official installer. Please find taosTools from [Release History](https://docs.taosdata.com/releases/tools/) page and download and install it. +- Install the taosTools official installer. Please find taosTools from [Release History](../../releases/tools/) page and download and install it. - Compile taos-tools separately and install it. Please refer to the [taos-tools](https://github.com/taosdata/taos-tools) repository for details. diff --git a/docs/en/20-third-party/75-powerbi.md b/docs/en/20-third-party/75-powerbi.md index 4744467f30..03268a5610 100644 --- a/docs/en/20-third-party/75-powerbi.md +++ b/docs/en/20-third-party/75-powerbi.md @@ -19,7 +19,7 @@ With TDengine ODBC driver, PowerBI can access time series data stored in TDengin ## Install Driver -Depending on your TDengine server version, download appropriate version of TDengine client package from TDengine website [Download Link](https://docs.taosdata.com/get-started/package/), or TDengine explorer if you are using a local TDengine cluster. Install the TDengine client package on same Windows machine where PowerBI is running. +Depending on your TDengine server version, download appropriate version of TDengine client package from TDengine website [Download Link](../../get-started/package/), or TDengine explorer if you are using a local TDengine cluster. Install the TDengine client package on same Windows machine where PowerBI is running. ### Configure Data Source @@ -38,7 +38,7 @@ To better use Power BI to analyze the data stored in TDengine, you need to under 1. Dimention: it's normally category (text) data to describe such information as device, collection point, model. In the supertable template of TDengine, we use tag columns to store the dimention information. You can use SQL like `select distinct tbname, tag1, tag2 from supertable` to get dimentions. -2. Metric: quantitive (numeric) fileds that can be calculated, like SUM, AVERAGE, MINIMUM. If the collecting frequency is 1 second, then there are 31,536,000 records in one year, it will be too low efficient to import so big data into Power BI. In TDengine, you can use data partition query, window partition query, in combination with pseudo columns related to window, to import downsampled data into Power BI. For more details, please refer to [TDengine Specialized Queries](https://docs.taosdata.com/taos-sql/distinguished/)。 +2. Metric: quantitive (numeric) fileds that can be calculated, like SUM, AVERAGE, MINIMUM. If the collecting frequency is 1 second, then there are 31,536,000 records in one year, it will be too low efficient to import so big data into Power BI. In TDengine, you can use data partition query, window partition query, in combination with pseudo columns related to window, to import downsampled data into Power BI. For more details, please refer to [TDengine Specialized Queries](../../taos-sql/distinguished/)。 - Window partition query: for example, thermal meters collect one data per second, but you need to query the average temperature every 10 minutes, you can use window subclause to get the downsampling data you need. The corresponding SQL is like `select tbname, _wstart date，avg(temperature) temp from table interval(10m)`, in which _wstart is a pseudo column indicting the start time of a widow, 10m is the duration of the window, `avg(temperature)` indicates the aggregate value inside a window. diff --git a/docs/en/25-application/_03-immigrate.md b/docs/en/25-application/_03-immigrate.md index 4dc7ee711c..af31b31e3c 100644 --- a/docs/en/25-application/_03-immigrate.md +++ b/docs/en/25-application/_03-immigrate.md @@ -184,8 +184,6 @@ TDengine supports the standard JDBC 3.0 interface for manipulating databases, bu To facilitate historical data migration, we provide a plug-in for the data synchronization tool DataX, which can automatically write data into TDengine.The automatic data migration of DataX can only support the data migration process of a single value model. -For the specific usage of DataX and how to use DataX to write data to TDengine, please refer to [DataX-based TDengine Data Migration Tool](https://www.taosdata.com/engineering/16401.html). - After migrating via DataX, we found that we can significantly improve the efficiency of migrating historical data by starting multiple processes and migrating numerous metrics simultaneously. The following are some records of the migration process. We provide these as a reference for application migration. | Number of datax instances (number of concurrent processes) | Migration record speed (pieces/second) | diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index f3099b13b4..4def3df638 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -6,7 +6,7 @@ description: This document provides download links for all released versions of taosTools installation packages can be downloaded at the following links: -For other historical version installers, please visit [here](https://www.taosdata.com/all-downloads). +For other historical version installers, please visit [here](https://tdengine.com/downloads/historical/). import Release from "/components/ReleaseV3"; From 843ecffc6628e6e16bc55f9eca1408849633957d Mon Sep 17 00:00:00 2001 From: Chris Zhai Date: Tue, 19 Mar 2024 19:19:25 +0800 Subject: [PATCH 02/17] add test cases for ts-4211 - v1 --- tests/pytest/util/sql.py | 3 + .../1-insert/insert_column_value.py | 155 +++++++++++++----- 2 files changed, 118 insertions(+), 40 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 92074161b6..7ccb290d62 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -85,6 +85,7 @@ class TDSql: i=1 while i <= queryTimes: try: + tdLog.info(sql) self.affectedRows = self.cursor.execute(sql) return self.affectedRows except Exception as e: @@ -120,8 +121,10 @@ class TDSql: expectErrNotOccured = True try: + tdLog.info("sql:%s" % (sql)) self.cursor.execute(sql) except BaseException as e: + tdLog.info("err:%s" % (e)) expectErrNotOccured = False self.errno = e.errno error_info = repr(e) diff --git a/tests/system-test/1-insert/insert_column_value.py b/tests/system-test/1-insert/insert_column_value.py index 49b77f4199..73d9aaeeda 100644 --- a/tests/system-test/1-insert/insert_column_value.py +++ b/tests/system-test/1-insert/insert_column_value.py @@ -1,4 +1,4 @@ -import datetime +import datetime, time from enum import Enum from util.log import * from util.sql import * @@ -55,14 +55,30 @@ class TDTestCase: f"create table {dbname}.stb_in (ts timestamp, c0 int, c1 smallint) tags(t0 bigint, t1 tinyint);", f"create table {dbname}.stb_ui (ts timestamp, c0 int unsigned, c1 smallint unsigned) tags(t0 bigint unsigned, t1 tinyint unsigned);", f"create table {dbname}.stb_fl (ts timestamp, c0 float, c1 float) tags(t0 float, t1 float);", - f"create table {dbname}.stb_db (ts timestamp, c0 float, c1 float) tags(t0 float, t1 float);", + f"create table {dbname}.stb_db (ts timestamp, c0 double, c1 double) tags(t0 double, t1 double);", f"create table {dbname}.stb_ge (ts timestamp, c0 geometry(512), c1 geometry(512)) tags(t0 geometry(512), t1 geometry(512));", - f"create table {dbname}.stb_js (ts timestamp, c0 int) tags(t0 json);" ] + f"create table {dbname}.stb_js (ts timestamp, c0 int) tags(t0 json);"] + + CREATE_NTB_LIST = [ f"create table {dbname}.ntb_vc (ts timestamp, c0 binary(50), c1 varchar(50));", + f"create table {dbname}.ntb_nc (ts timestamp, c0 nchar(50), c1 nchar(50));", + f"create table {dbname}.ntb_ts (ts timestamp, c0 timestamp, c1 timestamp);", + f"create table {dbname}.ntb_bo (ts timestamp, c0 bool, c1 bool);", + f"create table {dbname}.ntb_vb (ts timestamp, c0 varbinary(50), c1 varbinary(50));", + f"create table {dbname}.ntb_in (ts timestamp, c0 int, c1 smallint);", + f"create table {dbname}.ntb_ui (ts timestamp, c0 int unsigned, c1 smallint unsigned);", + f"create table {dbname}.ntb_fl (ts timestamp, c0 float, c1 float);", + f"create table {dbname}.ntb_db (ts timestamp, c0 double, c1 double);", + f"create table {dbname}.ntb_ge (ts timestamp, c0 geometry(512), c1 geometry(512));"] for _stb in CREATE_STB_LIST: tdSql.execute(_stb) tdSql.query(f'show {dbname}.stables') tdSql.checkRows(len(CREATE_STB_LIST)) + for _stb in CREATE_NTB_LIST: + tdSql.execute(_stb) + tdSql.query(f'show {dbname}.tables') + tdSql.checkRows(len(CREATE_NTB_LIST)) + def _query_check_varchar(self, result, okv, nv, row = 0, col = 0): for i in range(row): for j in range(1, col): @@ -147,12 +163,18 @@ class TDTestCase: tdSql.checkEqual(check_result, True) - def _query_check(self, dbname="db", stbname="", ctbname="", nRows = 0, okv = None, nv = None, dtype = TDDataType.NULL): + def _query_check(self, dbname="db", stbname="", ctbname="", ntbname="",nRows = 0, okv = None, nv = None, dtype = TDDataType.NULL): result = None - if dtype != TDDataType.GEOMETRY: # geometry query by py connector need to be supported - tdSql.query(f'select * from {dbname}.{stbname}') + if stbname: + if dtype != TDDataType.GEOMETRY: # geometry query by py connector need to be supported + tdSql.query(f'select * from {dbname}.{stbname}') + tdSql.checkRows(nRows) + result = tdSql.queryResult + else: + tdSql.query(f'select * from {dbname}.{ntbname}') tdSql.checkRows(nRows) result = tdSql.queryResult + if dtype == TDDataType.VARCHAR or dtype == TDDataType.NCHAR: self._query_check_varchar(result, okv, nv, nRows, 4) @@ -176,48 +198,101 @@ class TDTestCase: if ctbname != "": tdSql.execute(f'drop table {dbname}.{ctbname}') - def __insert_query_common(self, dbname="db", stbname="", ctbname="", oklist=[], kolist=[], okv=None, dtype = TDDataType.NULL): + def __insert_query_common(self, dbname="db", stbname="", ctbname="", ntbname="", oklist=[], kolist=[], okv=None, dtype = TDDataType.NULL): tdLog.info(f'{dbname}.{stbname} {ctbname}, oklist:%d, kolist:%d'%(len(oklist), len(kolist))) tdSql.checkEqual(34, len(oklist) + len(kolist)) + # while True: + # tdSql.execute(f'insert into db.ntb_ge values(now, "POINT (1.000000 1.000000)", "POINT (1.000000 1.000000)")') + # # tdSql.execute(f'insert into db.ntb_ge values(now + 1s, "POINT (1.000000 1.000000)", "POINT (1.000000 1.000000)")') + # # tdSql.execute(f'insert into db.ntb_ge values(now + 2s, "POINT (1.000000 1.000000)", "POINT (1.000000 1.000000)")') + + # # self._query_check(dbname, None, None, ntbname, 3, okv, _e, dtype) + # # time.sleep(1) + # # print('sleep 1s') + # tdSql.query(f'select * from db.ntb_ge', queryTimes=1) + # tdSql.checkRows(3) + # tdSql.query(f'delete from db.ntb_ge') + for _l in kolist: for _e in _l: + # create sub-table manually, check tag tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, {okv})' %(_e)) tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, %s)' %(_e)) tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, %s)' %(_e, _e)) + + # create sub-table automatically, check tag tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, %s) values(now, {okv}, {okv})' %(_e)) tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, {okv}) values(now, {okv}, {okv})' %(_e)) + + # create sub-table automatically, check value tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, %s, {okv})' %(_e)) tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, {okv}, %s)' %(_e)) tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, %s, %s)' %(_e, _e)) + + # check alter table tag tdSql.execute(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, {okv}, {okv})') - self._query_check(dbname,stbname, "", 1, okv, _e, dtype) + self._query_check(dbname,stbname, "", None, 1, okv, _e, dtype) tdSql.execute(f'alter table {dbname}.{ctbname} set tag t0 = {okv}') tdSql.error(f'alter table {dbname}.{ctbname} set tag t0 = %s' %(_e)) tdSql.error(f'alter table {dbname}.{ctbname} set tag t1 = %s' %(_e)) tdSql.execute(f'drop table {dbname}.{ctbname}') + + # insert into normal table, check value + tdSql.error(f'insert into {dbname}.{ntbname} values(now, %s, {okv})' %(_e)) + tdSql.error(f'insert into {dbname}.{ntbname} values(now, {okv}, %s)' %(_e)) + tdSql.error(f'insert into {dbname}.{ntbname} values(now, %s, %s)' %(_e, _e)) + for _l in oklist: for _e in _l: - tdLog.info(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, {okv})' %(_e)) - tdSql.execute(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, {okv})' %(_e)) - tdSql.execute(f'insert into {dbname}.{ctbname} values(now + 0s, %s, {okv})' %(_e)) - tdSql.execute(f'insert into {dbname}.{ctbname} values(now + 1s, {okv}, %s)' %(_e)) - tdSql.execute(f'insert into {dbname}.{ctbname} values(now + 2s, %s, %s)' %(_e, _e)) - tdLog.info(f'insert into {dbname}.{ctbname} values(now + 0s, %s, {okv})' %(_e)) - tdLog.info(f'insert into {dbname}.{ctbname} values(now + 1s, {okv}, %s)' %(_e)) - tdLog.info(f'insert into {dbname}.{ctbname} values(now + 2s, %s, %s)' %(_e, _e)) - tdSql.execute(f'alter table {dbname}.{ctbname} set tag t0 = %s' %(_e)) - tdSql.execute(f'alter table {dbname}.{ctbname} set tag t1 = %s' %(_e)) - self._query_check(dbname,stbname, ctbname, 3, okv, _e, dtype) - tdSql.execute(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, %s)' %(_e, _e)) - tdSql.execute(f'insert into {dbname}.{ctbname} values(now, %s, %s)' %(_e, _e)) - self._query_check(dbname,stbname, ctbname, 1, okv, _e, dtype) - tdSql.execute(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, {okv}) values(now, %s, {okv})' %(_e, _e)) - self._query_check(dbname,stbname, ctbname, 1, okv, _e, dtype) - tdSql.execute(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, %s) values(now, {okv}, %s)' %(_e, _e)) - self._query_check(dbname,stbname, ctbname, 1, okv, _e, dtype) - tdSql.execute(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, %s) values(now, %s, %s)' %(_e, _e, _e, _e)) - self._query_check(dbname,stbname, ctbname, 1, okv, _e, dtype) + # 1. create sub-table manually, check tag + tdSql.execute(f'create table {dbname}.{ctbname}_1 using {dbname}.{stbname} tags(%s, {okv})' %(_e)) + tdSql.execute(f'create table {dbname}.{ctbname}_2 using {dbname}.{stbname} tags({okv}, %s)' %(_e)) + tdSql.execute(f'create table {dbname}.{ctbname}_3 using {dbname}.{stbname} tags(%s, %s)' %(_e, _e)) + + + # 1.1 insert into sub-table, check value + tdSql.execute(f'insert into {dbname}.{ctbname}_1 values(now + 0s, %s, {okv})' %(_e)) + tdSql.execute(f'insert into {dbname}.{ctbname}_2 values(now + 1s, {okv}, %s)' %(_e)) + tdSql.execute(f'insert into {dbname}.{ctbname}_3 values(now + 2s, %s, %s)' %(_e, _e)) + + # 1.2 check alter table tag + tdSql.execute(f'alter table {dbname}.{ctbname}_1 set tag t0 = %s' %(_e)) + tdSql.execute(f'alter table {dbname}.{ctbname}_2 set tag t1 = %s' %(_e)) + + # 1.3 check table data + self._query_check(dbname, stbname, f'{ctbname}_1', None, 3, okv, _e, dtype) + self._query_check(dbname, stbname, f'{ctbname}_2', None, 2, okv, _e, dtype) + self._query_check(dbname, stbname, f'{ctbname}_3', None, 1, okv, _e, dtype) + + # 2. insert into value by creating sub-table automatically, check tag & value + tdSql.execute(f'insert into {dbname}.{ctbname}_1 using {dbname}.{stbname} tags(%s, {okv}) values(now, %s, {okv})' %(_e, _e)) + tdSql.execute(f'insert into {dbname}.{ctbname}_2 using {dbname}.{stbname} tags({okv}, %s) values(now + 1s, {okv}, %s)' %(_e, _e)) + tdSql.execute(f'insert into {dbname}.{ctbname}_3 using {dbname}.{stbname} tags(%s, %s) values(now + 2s, %s, %s)' %(_e, _e, _e, _e)) + + self._query_check(dbname, stbname, f'{ctbname}_1', None, 3, okv, _e, dtype) + self._query_check(dbname, stbname, f'{ctbname}_2', None, 2, okv, _e, dtype) + self._query_check(dbname, stbname, f'{ctbname}_3', None, 1, okv, _e, dtype) + + # 3. insert into value by supper-table, check tag & value + tdSql.execute(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_1", %s, {okv}, now, %s, {okv})' %(_e, _e)) + tdSql.execute(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_2", {okv}, %s, now + 1s, {okv}, %s)' %(_e, _e)) + tdSql.execute(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_3", %s, %s, now + 2s, %s, %s)' %(_e, _e, _e, _e)) + + self._query_check(dbname, stbname, f'{ctbname}_1', None, 3, okv, _e, dtype) + self._query_check(dbname, stbname, f'{ctbname}_2', None, 2, okv, _e, dtype) + self._query_check(dbname, stbname, f'{ctbname}_3', None, 1, okv, _e, dtype) + + # 4. insert value into normal table + tdSql.execute(f'insert into {dbname}.{ntbname} values(now, %s, {okv})' %(_e)) + tdSql.execute(f'insert into {dbname}.{ntbname} values(now + 1s, {okv}, %s)' %(_e)) + tdSql.execute(f'insert into {dbname}.{ntbname} values(now + 2s, %s, %s)' %(_e, _e)) + + if dtype != TDDataType.GEOMETRY: + # self._query_check(dbname, None, None, ntbname, 3, okv, _e, dtype) + tdSql.query(f'select * from {dbname}.{ntbname}') + tdSql.checkRows(3) + tdSql.query(f'delete from {dbname}.{ntbname}') def __insert_query_json(self, dbname="db", stbname="", ctbname="", oklist=[], kolist=[], okv=None): tdLog.info(f'{dbname}.{stbname} {ctbname}, oklist:%d, kolist:%d'%(len(oklist), len(kolist))) @@ -244,7 +319,7 @@ class TDTestCase: def __insert_query_exec(self): STR_EMPTY = ['\'\'', "\"\"", '\' \'', "\" \""] - STR_INTEGER_P = ["\"42\"", '\'+42\'', '\'+0\'', '\'-0\'', '\'0x2A\'', '\'-0X0\'', '\'+0x0\'', '\'0B00101010\'', '\'-0b00\''] + STR_INTEGER_P = ["\"42\"", '\'+42\'', '\'+0\'', '\'1\'','\'-0\'', '\'0x2A\'', '\'-0X0\'', '\'+0x0\'', '\'0B00101010\'', '\'-0b00\''] STR_INTEGER_M = ['\'-128\'', '\'-0X1\'', '\"-0x34\"', '\'-0b01\'', '\'-0B00101010\''] STR_FLOAT_P = ['\'42.1\'', "\"+0.003\"", "\'-0.0\'"] STR_FLOAT_M = ["\"-32.001\""] @@ -320,21 +395,21 @@ class TDTestCase: RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M] PARAM_LIST = [ - ["db", "stb_vc", "ctb_vc", OK_VC, KO_VC, "\'vc\'", TDDataType.VARCHAR], - ["db", "stb_nc", "ctb_nc", OK_NC, KO_NC, "\'nc\'", TDDataType.NCHAR], - ["db", "stb_ts", "ctb_ts", OK_TS, KO_TS, "now", TDDataType.TIMESTAMP], - ["db", "stb_bo", "ctb_bo", OK_BO, KO_BO, "true", TDDataType.BOOL], - ["db", "stb_vb", "ctb_vb", OK_VB, KO_VB, "\'\\x12\'", TDDataType.VARBINARY], - ["db", "stb_in", "ctb_in", OK_IN, KO_IN, "-1", TDDataType.UINT], - ["db", "stb_ui", "ctb_ui", OK_UI, KO_UI, "1", TDDataType.UINT], - ["db", "stb_fl", "ctb_fl", OK_FL, KO_FL, "1.0", TDDataType.FLOAT], - ["db", "stb_db", "ctb_db", OK_DB, KO_DB, "1.0", TDDataType.DOUBLE], - ["db", "stb_ge", "ctb_ge", OK_GE, KO_GE, "\'POINT(1.0 1.0)\'", TDDataType.GEOMETRY] + ["db", "stb_vc", "ctb_vc", "ntb_vc", OK_VC, KO_VC, "\'vc\'", TDDataType.VARCHAR], + ["db", "stb_nc", "ctb_nc", "ntb_nc", OK_NC, KO_NC, "\'nc\'", TDDataType.NCHAR], + ["db", "stb_ts", "ctb_ts", "ntb_ts", OK_TS, KO_TS, "now", TDDataType.TIMESTAMP], + ["db", "stb_bo", "ctb_bo", "ntb_bo", OK_BO, KO_BO, "true", TDDataType.BOOL], + ["db", "stb_vb", "ctb_vb", "ntb_vb", OK_VB, KO_VB, "\'\\x12\'", TDDataType.VARBINARY], + ["db", "stb_in", "ctb_in", "ntb_in", OK_IN, KO_IN, "-1", TDDataType.UINT], + ["db", "stb_ui", "ctb_ui", "ntb_ui", OK_UI, KO_UI, "1", TDDataType.UINT], + ["db", "stb_fl", "ctb_fl", "ntb_fl", OK_FL, KO_FL, "1.0", TDDataType.FLOAT], + ["db", "stb_db", "ctb_db", "ntb_db", OK_DB, KO_DB, "1.0", TDDataType.DOUBLE], + ["db", "stb_ge", "ctb_ge", "ntb_ge", OK_GE, KO_GE, "\'POINT(1.0 1.0)\'", TDDataType.GEOMETRY] ] # check with common function for _pl in PARAM_LIST: - self.__insert_query_common(_pl[0], _pl[1], _pl[2], _pl[3], _pl[4], _pl[5], _pl[6]) + self.__insert_query_common(_pl[0], _pl[1], _pl[2], _pl[3], _pl[4], _pl[5], _pl[6], _pl[7]) # check json self.__insert_query_json("db", "stb_js", "ctb_js", OK_JS, KO_JS, "\'{\"k1\":\"v1\",\"k2\":\"v2\"}\'") From 701ac8b8657784a7271c823e45ba90ff1b4b0b86 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 19 Mar 2024 19:17:30 +0800 Subject: [PATCH 03/17] fix: arb alter database check --- source/dnode/mnode/impl/src/mndDb.c | 21 +++++++++++++-------- source/libs/parser/src/parAstCreater.c | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 93f9f0eab8..a1f3a24661 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -397,11 +397,11 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1; if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1; if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1; - #ifdef TD_ENTERPRISE +#ifdef TD_ENTERPRISE if ((pCfg->replications == 2) ^ (pCfg->withArbitrator == TSDB_MAX_DB_WITH_ARBITRATOR)) return -1; - #else +#else if (pCfg->replications != 1 && pCfg->replications != 3) return -1; - #endif +#endif if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; if (pCfg->schemaless < TSDB_DB_SCHEMALESS_OFF || pCfg->schemaless > TSDB_DB_SCHEMALESS_ON) return -1; if (pCfg->cacheLast < TSDB_CACHE_MODEL_NONE || pCfg->cacheLast > TSDB_CACHE_MODEL_BOTH) return -1; @@ -438,13 +438,14 @@ static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pOldCfg, SDbCfg *pN if (pNewCfg->daysToKeep0 < pNewCfg->daysPerFile) return -1; if (pNewCfg->daysToKeep0 > pNewCfg->daysToKeep1) return -1; if (pNewCfg->daysToKeep1 > pNewCfg->daysToKeep2) return -1; - if (pNewCfg->keepTimeOffset < TSDB_MIN_KEEP_TIME_OFFSET || pNewCfg->keepTimeOffset > TSDB_MAX_KEEP_TIME_OFFSET) return -1; + if (pNewCfg->keepTimeOffset < TSDB_MIN_KEEP_TIME_OFFSET || pNewCfg->keepTimeOffset > TSDB_MAX_KEEP_TIME_OFFSET) + return -1; if (pNewCfg->walFsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pNewCfg->walFsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1; if (pNewCfg->walLevel < TSDB_MIN_WAL_LEVEL || pNewCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1; if (pNewCfg->cacheLast < TSDB_CACHE_MODEL_NONE || pNewCfg->cacheLast > TSDB_CACHE_MODEL_BOTH) return -1; if (pNewCfg->cacheLastSize < TSDB_MIN_DB_CACHE_SIZE || pNewCfg->cacheLastSize > TSDB_MAX_DB_CACHE_SIZE) return -1; if (pNewCfg->replications < TSDB_MIN_DB_REPLICA || pNewCfg->replications > TSDB_MAX_DB_REPLICA) return -1; - #ifdef TD_ENTERPRISE +#ifdef TD_ENTERPRISE if ((pNewCfg->replications == 2) ^ (pNewCfg->withArbitrator == TSDB_MAX_DB_WITH_ARBITRATOR)) return -1; if (pNewCfg->replications == 2 && pNewCfg->withArbitrator == TSDB_MAX_DB_WITH_ARBITRATOR) { if (pOldCfg->replications != 1 && pOldCfg->replications != 2) { @@ -452,9 +453,13 @@ static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pOldCfg, SDbCfg *pN return -1; } } - #else + if (pNewCfg->replications != 2 && pOldCfg->replications == 2) { + terrno = TSDB_CODE_OPS_NOT_SUPPORT; + return -1; + } +#else if (pNewCfg->replications != 1 && pNewCfg->replications != 3) return -1; - #endif +#endif if (pNewCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pNewCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1; if (pNewCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pNewCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1; if (pNewCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pNewCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1; @@ -940,7 +945,7 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = 0; } - if (pAlter->withArbitrator != pDb->cfg.withArbitrator) { + if (pAlter->withArbitrator >= TSDB_MIN_DB_WITH_ARBITRATOR && pAlter->withArbitrator != pDb->cfg.withArbitrator) { pDb->cfg.withArbitrator = pAlter->withArbitrator; pDb->vgVersion++; terrno = 0; diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 4ef14ad93d..4c2921d5c5 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1299,7 +1299,7 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { pOptions->sstTrigger = -1; pOptions->tablePrefix = -1; pOptions->tableSuffix = -1; - pOptions->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; + pOptions->withArbitrator = -1; return (SNode*)pOptions; } From 9bf540a1b2e9d45d79e4c0759e294b824bfc04e5 Mon Sep 17 00:00:00 2001 From: Chris Zhai Date: Wed, 20 Mar 2024 17:35:21 +0800 Subject: [PATCH 04/17] update test case for ts-4421 - v2 --- tests/pytest/util/sql.py | 1 + .../1-insert/insert_column_value.py | 82 ++++++++++++------- 2 files changed, 53 insertions(+), 30 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 7ccb290d62..783009b3fb 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -169,6 +169,7 @@ class TDSql: i=1 while i <= queryTimes: try: + tdLog.info(sql) self.cursor.execute(sql) self.queryResult = self.cursor.fetchall() self.queryRows = len(self.queryResult) diff --git a/tests/system-test/1-insert/insert_column_value.py b/tests/system-test/1-insert/insert_column_value.py index 73d9aaeeda..6726a33ea4 100644 --- a/tests/system-test/1-insert/insert_column_value.py +++ b/tests/system-test/1-insert/insert_column_value.py @@ -47,8 +47,9 @@ class TDTestCase: tdSql.execute(f'create database if not exists db vgroups 1') def __create_tb(self, dbname="db"): - CREATE_STB_LIST = [ f"create table {dbname}.stb_vc (ts timestamp, c0 binary(50), c1 varchar(50)) tags(t0 varchar(50), t1 binary(50));", + CREATE_STB_LIST = [ f"create table {dbname}.stb_vc (ts timestamp, c0 varchar(50), c1 varchar(50)) tags(t0 varchar(50), t1 varchar(50));", f"create table {dbname}.stb_nc (ts timestamp, c0 nchar(50), c1 nchar(50)) tags(t0 nchar(50), t1 nchar(50));", + f"create table {dbname}.stb_bi (ts timestamp, c0 binary(50), c1 binary(50)) tags(t0 binary(50), t1 binary(50));", f"create table {dbname}.stb_ts (ts timestamp, c0 timestamp, c1 timestamp) tags(t0 timestamp, t1 timestamp);", f"create table {dbname}.stb_bo (ts timestamp, c0 bool, c1 bool) tags(t0 bool, t1 bool);", f"create table {dbname}.stb_vb (ts timestamp, c0 varbinary(50), c1 varbinary(50)) tags(t0 varbinary(50), t1 varbinary(50));", @@ -61,6 +62,7 @@ class TDTestCase: CREATE_NTB_LIST = [ f"create table {dbname}.ntb_vc (ts timestamp, c0 binary(50), c1 varchar(50));", f"create table {dbname}.ntb_nc (ts timestamp, c0 nchar(50), c1 nchar(50));", + f"create table {dbname}.ntb_bi (ts timestamp, c0 binary(50), c1 binary(50));", f"create table {dbname}.ntb_ts (ts timestamp, c0 timestamp, c1 timestamp);", f"create table {dbname}.ntb_bo (ts timestamp, c0 bool, c1 bool);", f"create table {dbname}.ntb_vb (ts timestamp, c0 varbinary(50), c1 varbinary(50));", @@ -200,37 +202,29 @@ class TDTestCase: def __insert_query_common(self, dbname="db", stbname="", ctbname="", ntbname="", oklist=[], kolist=[], okv=None, dtype = TDDataType.NULL): tdLog.info(f'{dbname}.{stbname} {ctbname}, oklist:%d, kolist:%d'%(len(oklist), len(kolist))) - tdSql.checkEqual(34, len(oklist) + len(kolist)) - - # while True: - # tdSql.execute(f'insert into db.ntb_ge values(now, "POINT (1.000000 1.000000)", "POINT (1.000000 1.000000)")') - # # tdSql.execute(f'insert into db.ntb_ge values(now + 1s, "POINT (1.000000 1.000000)", "POINT (1.000000 1.000000)")') - # # tdSql.execute(f'insert into db.ntb_ge values(now + 2s, "POINT (1.000000 1.000000)", "POINT (1.000000 1.000000)")') - - # # self._query_check(dbname, None, None, ntbname, 3, okv, _e, dtype) - # # time.sleep(1) - # # print('sleep 1s') - # tdSql.query(f'select * from db.ntb_ge', queryTimes=1) - # tdSql.checkRows(3) - # tdSql.query(f'delete from db.ntb_ge') + # tdSql.checkEqual(34, len(oklist) + len(kolist)) for _l in kolist: for _e in _l: # create sub-table manually, check tag + tdLog.info('ko:create sub-table manually, check tag') tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, {okv})' %(_e)) tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, %s)' %(_e)) tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, %s)' %(_e, _e)) # create sub-table automatically, check tag + tdLog.info('ko:create sub-table automatically, check tag') tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, %s) values(now, {okv}, {okv})' %(_e)) tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, {okv}) values(now, {okv}, {okv})' %(_e)) # create sub-table automatically, check value + tdLog.info('ko:create sub-table automatically, check value') tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, %s, {okv})' %(_e)) tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, {okv}, %s)' %(_e)) tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, %s, %s)' %(_e, _e)) # check alter table tag + tdLog.info('ko:check alter table tag') tdSql.execute(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, {okv}, {okv})') self._query_check(dbname,stbname, "", None, 1, okv, _e, dtype) tdSql.execute(f'alter table {dbname}.{ctbname} set tag t0 = {okv}') @@ -239,6 +233,7 @@ class TDTestCase: tdSql.execute(f'drop table {dbname}.{ctbname}') # insert into normal table, check value + tdLog.info('ko:insert into normal table, check value') tdSql.error(f'insert into {dbname}.{ntbname} values(now, %s, {okv})' %(_e)) tdSql.error(f'insert into {dbname}.{ntbname} values(now, {okv}, %s)' %(_e)) tdSql.error(f'insert into {dbname}.{ntbname} values(now, %s, %s)' %(_e, _e)) @@ -246,26 +241,31 @@ class TDTestCase: for _l in oklist: for _e in _l: # 1. create sub-table manually, check tag + tdLog.info('ok:create sub-table manually, check tag') tdSql.execute(f'create table {dbname}.{ctbname}_1 using {dbname}.{stbname} tags(%s, {okv})' %(_e)) tdSql.execute(f'create table {dbname}.{ctbname}_2 using {dbname}.{stbname} tags({okv}, %s)' %(_e)) tdSql.execute(f'create table {dbname}.{ctbname}_3 using {dbname}.{stbname} tags(%s, %s)' %(_e, _e)) # 1.1 insert into sub-table, check value + tdLog.info('ok:insert into sub-table, check value') tdSql.execute(f'insert into {dbname}.{ctbname}_1 values(now + 0s, %s, {okv})' %(_e)) tdSql.execute(f'insert into {dbname}.{ctbname}_2 values(now + 1s, {okv}, %s)' %(_e)) tdSql.execute(f'insert into {dbname}.{ctbname}_3 values(now + 2s, %s, %s)' %(_e, _e)) # 1.2 check alter table tag - tdSql.execute(f'alter table {dbname}.{ctbname}_1 set tag t0 = %s' %(_e)) - tdSql.execute(f'alter table {dbname}.{ctbname}_2 set tag t1 = %s' %(_e)) + tdLog.info('ok:check alter table tag') + tdSql.execute(f'alter table {dbname}.{ctbname}_1 set tag t1 = %s' %(_e)) + tdSql.execute(f'alter table {dbname}.{ctbname}_2 set tag t0 = %s' %(_e)) # 1.3 check table data + tdLog.info('ok:check table data') self._query_check(dbname, stbname, f'{ctbname}_1', None, 3, okv, _e, dtype) self._query_check(dbname, stbname, f'{ctbname}_2', None, 2, okv, _e, dtype) self._query_check(dbname, stbname, f'{ctbname}_3', None, 1, okv, _e, dtype) # 2. insert into value by creating sub-table automatically, check tag & value + tdLog.info('ok:insert into value by creating sub-table automatically, check tag & value') tdSql.execute(f'insert into {dbname}.{ctbname}_1 using {dbname}.{stbname} tags(%s, {okv}) values(now, %s, {okv})' %(_e, _e)) tdSql.execute(f'insert into {dbname}.{ctbname}_2 using {dbname}.{stbname} tags({okv}, %s) values(now + 1s, {okv}, %s)' %(_e, _e)) tdSql.execute(f'insert into {dbname}.{ctbname}_3 using {dbname}.{stbname} tags(%s, %s) values(now + 2s, %s, %s)' %(_e, _e, _e, _e)) @@ -275,6 +275,7 @@ class TDTestCase: self._query_check(dbname, stbname, f'{ctbname}_3', None, 1, okv, _e, dtype) # 3. insert into value by supper-table, check tag & value + tdLog.info('ok:insert into value by supper-table, check tag & value') tdSql.execute(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_1", %s, {okv}, now, %s, {okv})' %(_e, _e)) tdSql.execute(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_2", {okv}, %s, now + 1s, {okv}, %s)' %(_e, _e)) tdSql.execute(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_3", %s, %s, now + 2s, %s, %s)' %(_e, _e, _e, _e)) @@ -284,6 +285,7 @@ class TDTestCase: self._query_check(dbname, stbname, f'{ctbname}_3', None, 1, okv, _e, dtype) # 4. insert value into normal table + tdLog.info('ok:insert value into normal table') tdSql.execute(f'insert into {dbname}.{ntbname} values(now, %s, {okv})' %(_e)) tdSql.execute(f'insert into {dbname}.{ntbname} values(now + 1s, {okv}, %s)' %(_e)) tdSql.execute(f'insert into {dbname}.{ntbname} values(now + 2s, %s, %s)' %(_e, _e)) @@ -296,7 +298,7 @@ class TDTestCase: def __insert_query_json(self, dbname="db", stbname="", ctbname="", oklist=[], kolist=[], okv=None): tdLog.info(f'{dbname}.{stbname} {ctbname}, oklist:%d, kolist:%d'%(len(oklist), len(kolist))) - tdSql.checkEqual(34, len(oklist) + len(kolist)) + # tdSql.checkEqual(34, len(oklist) + len(kolist)) for _l in kolist: for _e in _l: @@ -319,6 +321,8 @@ class TDTestCase: def __insert_query_exec(self): STR_EMPTY = ['\'\'', "\"\"", '\' \'', "\" \""] + STR_CHINESE = ['\'年年岁岁，花相似\''] + STR_CHINESE_ILLEGAL = ['\'洛阳城东桃李花，飞来飞去落谁家。幽闺女儿爱颜色，坐见落花长叹息。今岁花开君不待。明年花开复谁在。故人不共洛阳东，今来空对落花风。年年岁岁花相似，岁岁年年人不同\''] STR_INTEGER_P = ["\"42\"", '\'+42\'', '\'+0\'', '\'1\'','\'-0\'', '\'0x2A\'', '\'-0X0\'', '\'+0x0\'', '\'0B00101010\'', '\'-0b00\''] STR_INTEGER_M = ['\'-128\'', '\'-0X1\'', '\"-0x34\"', '\'-0b01\'', '\'-0B00101010\''] STR_FLOAT_P = ['\'42.1\'', "\"+0.003\"", "\'-0.0\'"] @@ -334,11 +338,16 @@ class TDTestCase: STR_TSKP_MISC = ['\'now()+1s\'', '\' now()\'', '\'now( )\'', '\'today() \'', '\'today())\'', '\'today()+1m\'', '\'today()-1w\''] STR_BOOL = ['\'true\'', '\'false\'', '\'TRUE\'', '\'FALSE\'', '\'tRuE\'', '\'falsE\''] STR_TS = ["\"2024-02-01 00:00:01.001-08:00\"", "\'2024-02-01T00:00:01.001+09:00\'", "\"2024-02-01\"", "\'2024-02-02 00:00:01\'", "\'2024-02-02 00:00:01.009\'"] + STR_TS_ILLEGAL = ["\"2023-2024-02-01 00:00:01.001-08:00\"", "\'2024-02-01T99:00:01.001+09:00\'", "\"2024-02-31\"", "\'2024-02-02 00:88:01\'", "\'2024-02-02 00:00:77.009\'"] STR_VARBIN = ['\'\\x12\'', '\'\\x13\'', '\' \\x14 \'', '\'\\x12ab\''] STR_JSON_O = ['\'{\"k1\":\"v1\"}\'', '\' {} \''] - STR_JSON_A = ['\'[]\''] + STR_JSON_A = ['\'[]\'', '\"{\'k1\': \'v1\',\'k2\'}\"', '\"{\'k1\': \'v1\'}}\"'] STR_GEO = ['\' POINT(1.0 1.0)\'', '\'LINESTRING(1.00 +2.0, 2.1 -3.2, 5.00 5.01) \'', '\'POLYGON((1.0 1.0, -2.0 +2.0, 1.0 1.0))\'' ] + STR_GEO_ILLEGAL = ['\' POINT(1.0)\'', '\'LINESTRING(1.00 +2.0, -3.2, 5.00 5.01) \'', '\'POLYGON((-2.0 +2.0, 1.0 1.0))\'' ] STR_NULL = ['\'NuLl\'', '\'null\'', '\'NULL\''] + STR_NONE = ['\'NoNe\'', '\'none\'', '\'NONE\''] + STR_BINARY = ['\'\\x7f8290\'', '\'\\X7f8290\'', '\'x7f8290\'', '\'\\x\''] # bug TD-29193 + STR_BINARY_ILLEGAL = ['\'\\x7f829\''] RAW_INTEGER_P = [' 42 ', '+042 ', ' +0', '0 ', '-0', '0', ' 0X2A', ' -0x0 ', '+0x0 ', ' 0B00101010', ' -0b00'] RAW_INTEGER_M = [' -42 ', ' -0128',' -0x1', ' -0X2A', '-0b01 ', ' -0B00101010 '] @@ -346,6 +355,7 @@ class TDTestCase: RAW_FLOAT_M = ['-128.001 '] RAW_FLOAT_E_P = [' 1e-100', ' +0.1E+2', ' -0.1E-10'] RAW_FLOAT_E_M = [" -1E2 "] + RAW_FLOAT_E_M_SPE = ["-1e-100"] RAW_MISC = ['123abc', "123c", '-123d', '+', '-', ' *', ' /', '% ', '&', "|", "^", "&&", "||", "!", " =", ' None ', 'NONE', 'now+1 s', 'now-1','now-1y','now+2 d', 'today+1 s', 'today-1','today-1y','today+2 d', 'now()+1 s', 'now()-1','now()-1y','now()+2 d', 'today()+1 s', 'today()-1','today()-1y','today()+2 d'] RAW_OPTR = ['1*10', '1+2', '-2-0','1%2', '2/0', '1&31'] @@ -355,48 +365,60 @@ class TDTestCase: RAW_TSKP_OPTR = [' noW ( ) + 1s', 'nOw( ) + 2D', 'NOW () + 000s', ' today()+1M', 'today( ) - 1w ', 'TodaY ( ) - 1U '] RAW_BOOL = ['true', 'false', ' TRUE ', 'FALSE ', ' tRuE', ' falsE '] RAW_NULL = ['NuLl', 'null ', ' NULL', ' NULL '] + RAW_NONE = ['None', 'none ', ' NoNe', ' NONE '] + RAW_BINARY_ILLEGAL = ['\\x7f8290', '\\X7f8290', 'x7f8290', '\\x', '\\x7f829'] OK_VC = [STR_EMPTY, STR_INTEGER_P, STR_INTEGER_M, STR_FLOAT_P, STR_FLOAT_M, STR_FLOAT_E_P, STR_FLOAT_E_M, STR_MISC, STR_OPTR, STR_TSK, STR_TSK_MISC, STR_TSKP, STR_TSKP_MISC, STR_BOOL, STR_TS, STR_VARBIN, STR_JSON_O, STR_JSON_A, STR_GEO, STR_NULL, RAW_INTEGER_P, RAW_INTEGER_M, RAW_FLOAT_P, RAW_FLOAT_M, - RAW_FLOAT_E_P, RAW_FLOAT_E_M, RAW_TSK, RAW_BOOL, RAW_NULL] - KO_VC = [RAW_MISC, RAW_OPTR, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR] + RAW_FLOAT_E_P, RAW_FLOAT_E_M, RAW_TSK, RAW_BOOL, RAW_NULL, STR_CHINESE, STR_NONE] + KO_VC = [RAW_MISC, RAW_OPTR, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, STR_CHINESE_ILLEGAL, RAW_NONE] OK_NC = OK_VC KO_NC = KO_VC + OK_BI = OK_VC + KO_BI = KO_VC OK_TS = [STR_TSK, STR_INTEGER_P, STR_INTEGER_M, STR_TSKP, STR_TS, STR_NULL, RAW_INTEGER_P, RAW_INTEGER_M, RAW_TSK, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, RAW_NULL] KO_TS = [STR_EMPTY, STR_FLOAT_P, STR_FLOAT_M, STR_FLOAT_E_P, STR_FLOAT_E_M, STR_MISC, STR_OPTR, STR_TSK_MISC, STR_TSKP_MISC, STR_BOOL, STR_VARBIN, - STR_JSON_O, STR_JSON_A, STR_GEO, RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M, RAW_MISC, RAW_OPTR, RAW_BOOL] + STR_JSON_O, STR_JSON_A, STR_GEO, RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M, RAW_MISC, RAW_OPTR, RAW_BOOL, STR_CHINESE, STR_NONE, STR_TS_ILLEGAL, + STR_BINARY, RAW_NONE] OK_BO = [STR_BOOL, STR_INTEGER_P, STR_INTEGER_M, STR_FLOAT_P, STR_FLOAT_M, STR_FLOAT_E_P, STR_FLOAT_E_M,RAW_BOOL, STR_NULL, RAW_INTEGER_P, RAW_INTEGER_M, RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M, RAW_NULL] KO_BO = [STR_EMPTY, STR_TSK, STR_TSKP, STR_TS, STR_MISC, STR_OPTR, STR_TSK_MISC, STR_TSKP_MISC, STR_VARBIN, STR_JSON_O, STR_JSON_A, STR_GEO, RAW_TSK, - RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, RAW_MISC, RAW_OPTR] + RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, RAW_MISC, RAW_OPTR, STR_CHINESE, STR_NONE, STR_BINARY, RAW_NONE] OK_VB = [STR_EMPTY, STR_INTEGER_P, STR_INTEGER_M, STR_FLOAT_P, STR_FLOAT_M, STR_FLOAT_E_P, STR_FLOAT_E_M, STR_MISC, STR_OPTR, STR_TSK, STR_TSK_MISC, STR_TSKP, - STR_TSKP_MISC, STR_BOOL, STR_TS, STR_VARBIN, STR_JSON_O, STR_JSON_A, STR_GEO, STR_NULL, RAW_NULL] - KO_VB = [RAW_INTEGER_P, RAW_INTEGER_M, RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M, RAW_TSK, RAW_BOOL, RAW_MISC, RAW_OPTR, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR] + STR_TSKP_MISC, STR_BOOL, STR_TS, STR_VARBIN, STR_JSON_O, STR_JSON_A, STR_GEO, STR_NULL, RAW_NULL, STR_CHINESE, STR_NONE, STR_BINARY] + KO_VB = [RAW_INTEGER_P, RAW_INTEGER_M, RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M, RAW_TSK, RAW_BOOL, RAW_MISC, RAW_OPTR, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, + STR_BINARY_ILLEGAL, STR_BINARY_ILLEGAL, RAW_BINARY_ILLEGAL, RAW_NONE] OK_IN = [STR_INTEGER_P, STR_INTEGER_M, STR_FLOAT_P, STR_FLOAT_M, STR_FLOAT_E_P, STR_FLOAT_E_M, STR_NULL, RAW_INTEGER_P, RAW_INTEGER_M, RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M, RAW_NULL] KO_IN = [STR_EMPTY, STR_MISC, STR_OPTR, STR_TSK, STR_TSK_MISC, STR_TSKP, STR_TSKP_MISC, STR_BOOL, STR_TS, STR_VARBIN, STR_JSON_O, STR_JSON_A, STR_GEO, RAW_TSK, - RAW_BOOL, RAW_MISC, RAW_OPTR, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR] - OK_UI = [STR_INTEGER_P, STR_FLOAT_P, STR_FLOAT_E_P, STR_NULL, RAW_INTEGER_P, RAW_FLOAT_P, RAW_FLOAT_E_P, RAW_NULL] + RAW_BOOL, RAW_MISC, RAW_OPTR, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, STR_CHINESE, STR_NONE, STR_BINARY, RAW_NONE] + OK_UI = [STR_INTEGER_P, STR_FLOAT_P, STR_FLOAT_E_P, STR_NULL, RAW_INTEGER_P, RAW_FLOAT_P, RAW_FLOAT_E_P, RAW_NULL, RAW_FLOAT_E_M_SPE] KO_UI = [STR_EMPTY, STR_MISC, STR_INTEGER_M, STR_FLOAT_M, STR_FLOAT_E_M, STR_OPTR, STR_TSK, STR_TSK_MISC, STR_TSKP, STR_TSKP_MISC, STR_BOOL, STR_TS, STR_VARBIN, - STR_JSON_O, STR_JSON_A, STR_GEO, RAW_TSK, RAW_BOOL, RAW_INTEGER_M, RAW_FLOAT_M, RAW_FLOAT_E_M, RAW_MISC, RAW_OPTR, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR] + STR_JSON_O, STR_JSON_A, STR_GEO, RAW_TSK, RAW_BOOL, RAW_INTEGER_M, RAW_FLOAT_M, RAW_FLOAT_E_M, RAW_MISC, RAW_OPTR, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, + STR_CHINESE, STR_NONE, RAW_NONE] OK_FL = [RAW_INTEGER_P, STR_INTEGER_P, STR_INTEGER_M, STR_FLOAT_P, STR_FLOAT_M, STR_FLOAT_E_P, STR_FLOAT_E_M, STR_NULL, RAW_INTEGER_M, RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M, RAW_NULL] KO_FL = [STR_EMPTY, STR_MISC, STR_OPTR, STR_TSK, STR_TSK_MISC, STR_TSKP, STR_TSKP_MISC, STR_BOOL, STR_TS, STR_VARBIN, STR_JSON_O, STR_JSON_A, STR_GEO, RAW_TSK, - RAW_BOOL, RAW_MISC, RAW_OPTR, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR] + RAW_BOOL, RAW_MISC, RAW_OPTR, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, STR_CHINESE, STR_NONE, STR_BINARY, RAW_NONE] OK_DB = OK_FL KO_DB = KO_FL OK_GE = [STR_GEO, STR_NULL, RAW_NULL] KO_GE = [STR_EMPTY, STR_MISC, STR_OPTR, STR_TSK, STR_TSK_MISC, STR_TSKP, STR_TSKP_MISC, STR_BOOL, STR_TS, STR_JSON_O, STR_JSON_A, STR_VARBIN, RAW_TSK, RAW_BOOL, RAW_MISC, RAW_OPTR, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, STR_INTEGER_P, STR_INTEGER_M, STR_FLOAT_P, STR_FLOAT_M, STR_FLOAT_E_P, STR_FLOAT_E_M, RAW_INTEGER_P, RAW_INTEGER_M, - RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M] + RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M, STR_CHINESE, STR_NONE, STR_BINARY, STR_GEO_ILLEGAL, RAW_NONE] OK_JS = [STR_EMPTY, STR_JSON_O, STR_NULL, RAW_NULL] KO_JS = [STR_JSON_A, STR_MISC, STR_OPTR, STR_TSK, STR_TSK_MISC, STR_TSKP, STR_TSKP_MISC, STR_BOOL, STR_TS, STR_GEO, STR_VARBIN, RAW_TSK, RAW_BOOL, RAW_MISC, RAW_OPTR, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, STR_INTEGER_P, STR_INTEGER_M, STR_FLOAT_P, STR_FLOAT_M, STR_FLOAT_E_P, STR_FLOAT_E_M, RAW_INTEGER_P, RAW_INTEGER_M, - RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M] + RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M, STR_CHINESE, STR_NONE, STR_BINARY, RAW_NONE] + + # PARAM_LIST = [ + # ["db", "stb_vc", "ctb_vc", "ntb_vc", OK_VC, KO_VC, "\'vc\'", TDDataType.VARCHAR] + # ] PARAM_LIST = [ ["db", "stb_vc", "ctb_vc", "ntb_vc", OK_VC, KO_VC, "\'vc\'", TDDataType.VARCHAR], ["db", "stb_nc", "ctb_nc", "ntb_nc", OK_NC, KO_NC, "\'nc\'", TDDataType.NCHAR], + ["db", "stb_bi", "ctb_bi", "ntb_bi", OK_BI, KO_BI, "\'bi\'", TDDataType.BINARY], ["db", "stb_ts", "ctb_ts", "ntb_ts", OK_TS, KO_TS, "now", TDDataType.TIMESTAMP], ["db", "stb_bo", "ctb_bo", "ntb_bo", OK_BO, KO_BO, "true", TDDataType.BOOL], ["db", "stb_vb", "ctb_vb", "ntb_vb", OK_VB, KO_VB, "\'\\x12\'", TDDataType.VARBINARY], From e18f056c9f5b0a4639a2f2d6c16a9f52706a1280 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 21 Mar 2024 10:52:00 +0800 Subject: [PATCH 05/17] enh: meta pause/resume for sys table scan --- source/libs/executor/src/sysscanoperator.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 5735ee298b..390cde96f0 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -522,6 +522,8 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); + } else { + pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0); } if (pInfo->pSchema == NULL) { @@ -599,6 +601,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { pInfo->restore = true; if (pInfo->pRes->info.rows > 0) { + pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); break; } } else { @@ -607,6 +610,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { } if (numOfRows > 0) { + pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); numOfRows = 0; } @@ -702,6 +706,8 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); + } else { + pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0); } bool blockFull = false; @@ -741,6 +747,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { numOfRows = 0; if (pInfo->pRes->info.rows > 0) { + pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); break; } @@ -749,6 +756,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } if (numOfRows > 0) { + pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); numOfRows = 0; } From facca231f869f9d850405f55c9ee25c162ea6d40 Mon Sep 17 00:00:00 2001 From: Chris Zhai Date: Thu, 21 Mar 2024 14:39:22 +0800 Subject: [PATCH 06/17] update test cases for ts-4421 - v3 --- .../1-insert/insert_column_value.py | 103 +++++++++++++----- 1 file changed, 74 insertions(+), 29 deletions(-) diff --git a/tests/system-test/1-insert/insert_column_value.py b/tests/system-test/1-insert/insert_column_value.py index 6726a33ea4..696401fd6a 100644 --- a/tests/system-test/1-insert/insert_column_value.py +++ b/tests/system-test/1-insert/insert_column_value.py @@ -45,6 +45,8 @@ class TDTestCase: tdSql.init(conn.cursor()) tdSql.execute(f'drop database if exists db') tdSql.execute(f'create database if not exists db vgroups 1') + tdLog.printNoPrefix("create table") + self.__create_tb() def __create_tb(self, dbname="db"): CREATE_STB_LIST = [ f"create table {dbname}.stb_vc (ts timestamp, c0 varchar(50), c1 varchar(50)) tags(t0 varchar(50), t1 varchar(50));", @@ -53,8 +55,14 @@ class TDTestCase: f"create table {dbname}.stb_ts (ts timestamp, c0 timestamp, c1 timestamp) tags(t0 timestamp, t1 timestamp);", f"create table {dbname}.stb_bo (ts timestamp, c0 bool, c1 bool) tags(t0 bool, t1 bool);", f"create table {dbname}.stb_vb (ts timestamp, c0 varbinary(50), c1 varbinary(50)) tags(t0 varbinary(50), t1 varbinary(50));", - f"create table {dbname}.stb_in (ts timestamp, c0 int, c1 smallint) tags(t0 bigint, t1 tinyint);", - f"create table {dbname}.stb_ui (ts timestamp, c0 int unsigned, c1 smallint unsigned) tags(t0 bigint unsigned, t1 tinyint unsigned);", + f"create table {dbname}.stb_in (ts timestamp, c0 int, c1 int) tags(t0 int, t1 int);", + f"create table {dbname}.stb_ui (ts timestamp, c0 int unsigned, c1 int unsigned) tags(t0 int unsigned, t1 int unsigned);", + f"create table {dbname}.stb_bin (ts timestamp, c0 bigint, c1 bigint) tags(t0 bigint, t1 bigint);", + f"create table {dbname}.stb_bui (ts timestamp, c0 bigint unsigned, c1 bigint unsigned) tags(t0 bigint unsigned, t1 bigint unsigned);", + f"create table {dbname}.stb_sin (ts timestamp, c0 smallint, c1 smallint) tags(t0 smallint, t1 smallint);", + f"create table {dbname}.stb_sui (ts timestamp, c0 smallint unsigned, c1 smallint unsigned) tags(t0 smallint unsigned, t1 smallint unsigned);", + f"create table {dbname}.stb_tin (ts timestamp, c0 tinyint, c1 tinyint) tags(t0 tinyint, t1 tinyint);", + f"create table {dbname}.stb_tui (ts timestamp, c0 tinyint unsigned, c1 tinyint unsigned) tags(t0 tinyint unsigned, t1 tinyint unsigned);", f"create table {dbname}.stb_fl (ts timestamp, c0 float, c1 float) tags(t0 float, t1 float);", f"create table {dbname}.stb_db (ts timestamp, c0 double, c1 double) tags(t0 double, t1 double);", f"create table {dbname}.stb_ge (ts timestamp, c0 geometry(512), c1 geometry(512)) tags(t0 geometry(512), t1 geometry(512));", @@ -66,8 +74,14 @@ class TDTestCase: f"create table {dbname}.ntb_ts (ts timestamp, c0 timestamp, c1 timestamp);", f"create table {dbname}.ntb_bo (ts timestamp, c0 bool, c1 bool);", f"create table {dbname}.ntb_vb (ts timestamp, c0 varbinary(50), c1 varbinary(50));", - f"create table {dbname}.ntb_in (ts timestamp, c0 int, c1 smallint);", - f"create table {dbname}.ntb_ui (ts timestamp, c0 int unsigned, c1 smallint unsigned);", + f"create table {dbname}.ntb_in (ts timestamp, c0 int, c1 int);", + f"create table {dbname}.ntb_ui (ts timestamp, c0 int unsigned, c1 int unsigned);", + f"create table {dbname}.ntb_bin (ts timestamp, c0 bigint, c1 bigint);", + f"create table {dbname}.ntb_bui (ts timestamp, c0 bigint unsigned, c1 bigint unsigned);", + f"create table {dbname}.ntb_sin (ts timestamp, c0 smallint, c1 smallint);", + f"create table {dbname}.ntb_sui (ts timestamp, c0 smallint unsigned, c1 smallint unsigned);", + f"create table {dbname}.ntb_tin (ts timestamp, c0 tinyint, c1 tinyint);", + f"create table {dbname}.ntb_tui (ts timestamp, c0 tinyint unsigned, c1 tinyint unsigned);", f"create table {dbname}.ntb_fl (ts timestamp, c0 float, c1 float);", f"create table {dbname}.ntb_db (ts timestamp, c0 double, c1 double);", f"create table {dbname}.ntb_ge (ts timestamp, c0 geometry(512), c1 geometry(512));"] @@ -201,30 +215,30 @@ class TDTestCase: tdSql.execute(f'drop table {dbname}.{ctbname}') def __insert_query_common(self, dbname="db", stbname="", ctbname="", ntbname="", oklist=[], kolist=[], okv=None, dtype = TDDataType.NULL): - tdLog.info(f'{dbname}.{stbname} {ctbname}, oklist:%d, kolist:%d'%(len(oklist), len(kolist))) + tdLog.info(f'[Begin]{dbname}.{stbname} {ctbname}, oklist:%d, kolist:%d, TDDataType:%s'%(len(oklist), len(kolist), dtype)) # tdSql.checkEqual(34, len(oklist) + len(kolist)) for _l in kolist: for _e in _l: # create sub-table manually, check tag - tdLog.info('ko:create sub-table manually, check tag') + tdLog.info('[ko:create sub-table manually, check tag]') tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, {okv})' %(_e)) tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, %s)' %(_e)) - tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, %s)' %(_e, _e)) + # tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, %s)' %(_e, _e)) # create sub-table automatically, check tag - tdLog.info('ko:create sub-table automatically, check tag') + tdLog.info('[ko:create sub-table automatically, check tag]') tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, %s) values(now, {okv}, {okv})' %(_e)) tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, {okv}) values(now, {okv}, {okv})' %(_e)) # create sub-table automatically, check value - tdLog.info('ko:create sub-table automatically, check value') + tdLog.info('[ko:create sub-table automatically, check value]') tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, %s, {okv})' %(_e)) tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, {okv}, %s)' %(_e)) - tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, %s, %s)' %(_e, _e)) + # tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, %s, %s)' %(_e, _e)) # check alter table tag - tdLog.info('ko:check alter table tag') + tdLog.info('[ko:check alter table tag]') tdSql.execute(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, {okv}, {okv})') self._query_check(dbname,stbname, "", None, 1, okv, _e, dtype) tdSql.execute(f'alter table {dbname}.{ctbname} set tag t0 = {okv}') @@ -232,40 +246,46 @@ class TDTestCase: tdSql.error(f'alter table {dbname}.{ctbname} set tag t1 = %s' %(_e)) tdSql.execute(f'drop table {dbname}.{ctbname}') + # insert into value by supper-table, check tag & value + tdLog.info('[ko:insert into value by supper-table, check tag & value]') + tdSql.error(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_1", %s, {okv}, now, {okv}, {okv})' %(_e)) + tdSql.error(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_2", {okv},{okv}, now + 1s, {okv}, %s)' %(_e)) + # tdSql.error(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_3", %s, %s, now + 2s, %s, %s)' %(_e, _e, _e, _e)) + # insert into normal table, check value - tdLog.info('ko:insert into normal table, check value') + tdLog.info('[ko:insert into normal table, check value]') tdSql.error(f'insert into {dbname}.{ntbname} values(now, %s, {okv})' %(_e)) tdSql.error(f'insert into {dbname}.{ntbname} values(now, {okv}, %s)' %(_e)) - tdSql.error(f'insert into {dbname}.{ntbname} values(now, %s, %s)' %(_e, _e)) + # tdSql.error(f'insert into {dbname}.{ntbname} values(now, %s, %s)' %(_e, _e)) for _l in oklist: for _e in _l: # 1. create sub-table manually, check tag - tdLog.info('ok:create sub-table manually, check tag') + tdLog.info('[ok:create sub-table manually, check tag]') tdSql.execute(f'create table {dbname}.{ctbname}_1 using {dbname}.{stbname} tags(%s, {okv})' %(_e)) tdSql.execute(f'create table {dbname}.{ctbname}_2 using {dbname}.{stbname} tags({okv}, %s)' %(_e)) tdSql.execute(f'create table {dbname}.{ctbname}_3 using {dbname}.{stbname} tags(%s, %s)' %(_e, _e)) # 1.1 insert into sub-table, check value - tdLog.info('ok:insert into sub-table, check value') + tdLog.info('[ok:insert into sub-table, check value]') tdSql.execute(f'insert into {dbname}.{ctbname}_1 values(now + 0s, %s, {okv})' %(_e)) tdSql.execute(f'insert into {dbname}.{ctbname}_2 values(now + 1s, {okv}, %s)' %(_e)) tdSql.execute(f'insert into {dbname}.{ctbname}_3 values(now + 2s, %s, %s)' %(_e, _e)) # 1.2 check alter table tag - tdLog.info('ok:check alter table tag') + tdLog.info('[ok:check alter table tag]') tdSql.execute(f'alter table {dbname}.{ctbname}_1 set tag t1 = %s' %(_e)) tdSql.execute(f'alter table {dbname}.{ctbname}_2 set tag t0 = %s' %(_e)) # 1.3 check table data - tdLog.info('ok:check table data') + tdLog.info('[ok:check table data]') self._query_check(dbname, stbname, f'{ctbname}_1', None, 3, okv, _e, dtype) self._query_check(dbname, stbname, f'{ctbname}_2', None, 2, okv, _e, dtype) self._query_check(dbname, stbname, f'{ctbname}_3', None, 1, okv, _e, dtype) # 2. insert into value by creating sub-table automatically, check tag & value - tdLog.info('ok:insert into value by creating sub-table automatically, check tag & value') + tdLog.info('[ok:insert into value by creating sub-table automatically, check tag & value]') tdSql.execute(f'insert into {dbname}.{ctbname}_1 using {dbname}.{stbname} tags(%s, {okv}) values(now, %s, {okv})' %(_e, _e)) tdSql.execute(f'insert into {dbname}.{ctbname}_2 using {dbname}.{stbname} tags({okv}, %s) values(now + 1s, {okv}, %s)' %(_e, _e)) tdSql.execute(f'insert into {dbname}.{ctbname}_3 using {dbname}.{stbname} tags(%s, %s) values(now + 2s, %s, %s)' %(_e, _e, _e, _e)) @@ -275,7 +295,7 @@ class TDTestCase: self._query_check(dbname, stbname, f'{ctbname}_3', None, 1, okv, _e, dtype) # 3. insert into value by supper-table, check tag & value - tdLog.info('ok:insert into value by supper-table, check tag & value') + tdLog.info('[ok:insert into value by supper-table, check tag & value]') tdSql.execute(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_1", %s, {okv}, now, %s, {okv})' %(_e, _e)) tdSql.execute(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_2", {okv}, %s, now + 1s, {okv}, %s)' %(_e, _e)) tdSql.execute(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_3", %s, %s, now + 2s, %s, %s)' %(_e, _e, _e, _e)) @@ -285,7 +305,7 @@ class TDTestCase: self._query_check(dbname, stbname, f'{ctbname}_3', None, 1, okv, _e, dtype) # 4. insert value into normal table - tdLog.info('ok:insert value into normal table') + tdLog.info('[ok:insert value into normal table]') tdSql.execute(f'insert into {dbname}.{ntbname} values(now, %s, {okv})' %(_e)) tdSql.execute(f'insert into {dbname}.{ntbname} values(now + 1s, {okv}, %s)' %(_e)) tdSql.execute(f'insert into {dbname}.{ntbname} values(now + 2s, %s, %s)' %(_e, _e)) @@ -318,7 +338,7 @@ class TDTestCase: tdSql.query(f'select * from {dbname}.{stbname}') tdSql.checkRows(1) tdSql.execute(f'drop table {dbname}.{ctbname}') - + def __insert_query_exec(self): STR_EMPTY = ['\'\'', "\"\"", '\' \'', "\" \""] STR_CHINESE = ['\'年年岁岁，花相似\''] @@ -351,6 +371,14 @@ class TDTestCase: RAW_INTEGER_P = [' 42 ', '+042 ', ' +0', '0 ', '-0', '0', ' 0X2A', ' -0x0 ', '+0x0 ', ' 0B00101010', ' -0b00'] RAW_INTEGER_M = [' -42 ', ' -0128',' -0x1', ' -0X2A', '-0b01 ', ' -0B00101010 '] + RAW_INTEGER_INT_BOUNDARY_ILLEGAL =['2147483648', '-2147483649'] + RAW_INTEGER_UINT_BOUNDARY_ILLEGAL =['4294967296', '-1'] + RAW_INTEGER_BINT_BOUNDARY_ILLEGAL =['9223372036854775808', '-9223372036854775809'] + RAW_INTEGER_UBINT_BOUNDARY_ILLEGAL =['18446744073709551616', '-1'] + RAW_INTEGER_SINT_BOUNDARY_ILLEGAL =['32768', '-32769'] + RAW_INTEGER_USINT_BOUNDARY_ILLEGAL =['65536', '-1'] + RAW_INTEGER_TINT_BOUNDARY_ILLEGAL =['128', '-129'] + RAW_INTEGER_UTINT_BOUNDARY_ILLEGAL =['256', '-1'] RAW_FLOAT_P = [' 123.012', ' 0.0', ' +0.0', ' -0.0 '] RAW_FLOAT_M = ['-128.001 '] RAW_FLOAT_E_P = [' 1e-100', ' +0.1E+2', ' -0.1E-10'] @@ -390,12 +418,26 @@ class TDTestCase: STR_BINARY_ILLEGAL, STR_BINARY_ILLEGAL, RAW_BINARY_ILLEGAL, RAW_NONE] OK_IN = [STR_INTEGER_P, STR_INTEGER_M, STR_FLOAT_P, STR_FLOAT_M, STR_FLOAT_E_P, STR_FLOAT_E_M, STR_NULL, RAW_INTEGER_P, RAW_INTEGER_M, RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M, RAW_NULL] - KO_IN = [STR_EMPTY, STR_MISC, STR_OPTR, STR_TSK, STR_TSK_MISC, STR_TSKP, STR_TSKP_MISC, STR_BOOL, STR_TS, STR_VARBIN, STR_JSON_O, STR_JSON_A, STR_GEO, RAW_TSK, + BASE_KO_IN = [STR_EMPTY, STR_MISC, STR_OPTR, STR_TSK, STR_TSK_MISC, STR_TSKP, STR_TSKP_MISC, STR_BOOL, STR_TS, STR_VARBIN, STR_JSON_O, STR_JSON_A, STR_GEO, RAW_TSK, RAW_BOOL, RAW_MISC, RAW_OPTR, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, STR_CHINESE, STR_NONE, STR_BINARY, RAW_NONE] + KO_IN = BASE_KO_IN + [RAW_INTEGER_INT_BOUNDARY_ILLEGAL] + OK_BIN = OK_IN + KO_BIN = BASE_KO_IN + [RAW_INTEGER_BINT_BOUNDARY_ILLEGAL] + OK_SIN = OK_IN + KO_SIN = BASE_KO_IN + [RAW_INTEGER_SINT_BOUNDARY_ILLEGAL] + OK_TIN = OK_IN + KO_TIN = BASE_KO_IN + [RAW_INTEGER_TINT_BOUNDARY_ILLEGAL] OK_UI = [STR_INTEGER_P, STR_FLOAT_P, STR_FLOAT_E_P, STR_NULL, RAW_INTEGER_P, RAW_FLOAT_P, RAW_FLOAT_E_P, RAW_NULL, RAW_FLOAT_E_M_SPE] - KO_UI = [STR_EMPTY, STR_MISC, STR_INTEGER_M, STR_FLOAT_M, STR_FLOAT_E_M, STR_OPTR, STR_TSK, STR_TSK_MISC, STR_TSKP, STR_TSKP_MISC, STR_BOOL, STR_TS, STR_VARBIN, + BASE_KO_UI = [STR_EMPTY, STR_MISC, STR_INTEGER_M, STR_FLOAT_M, STR_FLOAT_E_M, STR_OPTR, STR_TSK, STR_TSK_MISC, STR_TSKP, STR_TSKP_MISC, STR_BOOL, STR_TS, STR_VARBIN, STR_JSON_O, STR_JSON_A, STR_GEO, RAW_TSK, RAW_BOOL, RAW_INTEGER_M, RAW_FLOAT_M, RAW_FLOAT_E_M, RAW_MISC, RAW_OPTR, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, STR_CHINESE, STR_NONE, RAW_NONE] + KO_UI = BASE_KO_UI + [RAW_INTEGER_UINT_BOUNDARY_ILLEGAL] + OK_UBINT = OK_UI + KO_UBINT = BASE_KO_UI + [RAW_INTEGER_UBINT_BOUNDARY_ILLEGAL] + OK_USINT = OK_UI + KO_USINT = BASE_KO_UI + [RAW_INTEGER_USINT_BOUNDARY_ILLEGAL] + OK_UTINT = OK_UI + KO_UTINT = BASE_KO_UI + [RAW_INTEGER_UTINT_BOUNDARY_ILLEGAL] OK_FL = [RAW_INTEGER_P, STR_INTEGER_P, STR_INTEGER_M, STR_FLOAT_P, STR_FLOAT_M, STR_FLOAT_E_P, STR_FLOAT_E_M, STR_NULL, RAW_INTEGER_M, RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M, RAW_NULL] KO_FL = [STR_EMPTY, STR_MISC, STR_OPTR, STR_TSK, STR_TSK_MISC, STR_TSKP, STR_TSKP_MISC, STR_BOOL, STR_TS, STR_VARBIN, STR_JSON_O, STR_JSON_A, STR_GEO, RAW_TSK, @@ -411,9 +453,6 @@ class TDTestCase: RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, STR_INTEGER_P, STR_INTEGER_M, STR_FLOAT_P, STR_FLOAT_M, STR_FLOAT_E_P, STR_FLOAT_E_M, RAW_INTEGER_P, RAW_INTEGER_M, RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M, STR_CHINESE, STR_NONE, STR_BINARY, RAW_NONE] - # PARAM_LIST = [ - # ["db", "stb_vc", "ctb_vc", "ntb_vc", OK_VC, KO_VC, "\'vc\'", TDDataType.VARCHAR] - # ] PARAM_LIST = [ ["db", "stb_vc", "ctb_vc", "ntb_vc", OK_VC, KO_VC, "\'vc\'", TDDataType.VARCHAR], @@ -422,8 +461,16 @@ class TDTestCase: ["db", "stb_ts", "ctb_ts", "ntb_ts", OK_TS, KO_TS, "now", TDDataType.TIMESTAMP], ["db", "stb_bo", "ctb_bo", "ntb_bo", OK_BO, KO_BO, "true", TDDataType.BOOL], ["db", "stb_vb", "ctb_vb", "ntb_vb", OK_VB, KO_VB, "\'\\x12\'", TDDataType.VARBINARY], - ["db", "stb_in", "ctb_in", "ntb_in", OK_IN, KO_IN, "-1", TDDataType.UINT], + + ["db", "stb_in", "ctb_in", "ntb_in", OK_IN, KO_IN, "-1", TDDataType.INT], ["db", "stb_ui", "ctb_ui", "ntb_ui", OK_UI, KO_UI, "1", TDDataType.UINT], + ["db", "stb_bin", "ctb_bin", "ntb_bin", OK_BIN, KO_BIN, "-1", TDDataType.BIGINT], + ["db", "stb_bui", "ctb_bui", "ntb_bui", OK_UBINT, KO_UBINT, "1", TDDataType.UBIGINT], + ["db", "stb_sin", "ctb_sin", "ntb_sin", OK_SIN, KO_SIN, "-1", TDDataType.SMALLINT], + ["db", "stb_sui", "ctb_sui", "ntb_sui", OK_USINT, KO_USINT, "1", TDDataType.USMALLINT], + ["db", "stb_tin", "ctb_tin", "ntb_tin", OK_TIN, KO_TIN, "-1", TDDataType.TINYINT], + ["db", "stb_tui", "ctb_tui", "ntb_tui", OK_UTINT, KO_UTINT, "1", TDDataType.UTINYINT], + ["db", "stb_fl", "ctb_fl", "ntb_fl", OK_FL, KO_FL, "1.0", TDDataType.FLOAT], ["db", "stb_db", "ctb_db", "ntb_db", OK_DB, KO_DB, "1.0", TDDataType.DOUBLE], ["db", "stb_ge", "ctb_ge", "ntb_ge", OK_GE, KO_GE, "\'POINT(1.0 1.0)\'", TDDataType.GEOMETRY] @@ -437,8 +484,6 @@ class TDTestCase: def run(self): - tdLog.printNoPrefix("==========step1:create table") - self.__create_tb() self.__insert_query_exec() From 6b2821cd288b6f611517ecd20f9c62b529bbf0eb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 21 Mar 2024 15:45:28 +0800 Subject: [PATCH 07/17] fix(query): not lock when reading meta, and do some internal refactor. --- source/libs/executor/src/sysscanoperator.c | 183 +++++++++++---------- 1 file changed, 100 insertions(+), 83 deletions(-) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 390cde96f0..5e8c98c5c4 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -433,93 +433,109 @@ static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) { return false; } -static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pTaskInfo->storageAPI; - +static SSDataBlock* doOptimizeTableNameFilter(SOperatorInfo* pOperator, SSDataBlock* dataBlock, char* dbname) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SSysTableScanInfo* pInfo = pOperator->info; - if (pOperator->status == OP_EXEC_DONE) { + int32_t numOfRows = 0; + + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tableName, pInfo->req.filterTb); + + SMetaReader smrTable = {0}; + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); + int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrTable, pInfo->req.filterTb); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly + pAPI->metaReaderFn.clearReader(&smrTable); + pInfo->loadInfo.totalRows = 0; return NULL; } - blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; + if (smrTable.me.type == TSDB_SUPER_TABLE) { + pAPI->metaReaderFn.clearReader(&smrTable); + pInfo->loadInfo.totalRows = 0; + return NULL; + } - SSDataBlock* dataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_COLS); - blockDataEnsureCapacity(dataBlock, pOperator->resultInfo.capacity); + if (smrTable.me.type == TSDB_CHILD_TABLE) { + int64_t suid = smrTable.me.ctbEntry.suid; + pAPI->metaReaderFn.clearReader(&smrTable); + pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); + code = pAPI->metaReaderFn.getTableEntryByUid(&smrTable, suid); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly + pAPI->metaReaderFn.clearReader(&smrTable); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + } + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + SSchemaWrapper* schemaRow = NULL; + if (smrTable.me.type == TSDB_SUPER_TABLE) { + schemaRow = &smrTable.me.stbEntry.schemaRow; + STR_TO_VARSTR(typeName, "CHILD_TABLE"); + } else if (smrTable.me.type == TSDB_NORMAL_TABLE) { + schemaRow = &smrTable.me.ntbEntry.schemaRow; + STR_TO_VARSTR(typeName, "NORMAL_TABLE"); + } + + sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); + pAPI->metaReaderFn.clearReader(&smrTable); + + if (numOfRows > 0) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + } + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + setOperatorCompleted(pOperator); + + qDebug("get cols success, total rows:%" PRIu64 ", current:%d %s", pInfo->loadInfo.totalRows, pInfo->pRes->info.rows, + GET_TASKID(pTaskInfo)); + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +int32_t doExtractDbName(char* dbname, SSysTableScanInfo* pInfo, SStorageAPI* pAPI) { + SName sn = {0}; const char* db = NULL; int32_t vgId = 0; pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); tNameGetDbName(&sn, varDataVal(dbname)); varDataSetLen(dbname, strlen(varDataVal(dbname))); - // optimize when sql like where table_name='tablename' and xxx. - if (pInfo->req.filterTb[0]) { - char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(tableName, pInfo->req.filterTb); + return TSDB_CODE_SUCCESS; +} - SMetaReader smrTable = {0}; - pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); - int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrTable, pInfo->req.filterTb); - if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly - pAPI->metaReaderFn.clearReader(&smrTable); - blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows = 0; - return NULL; - } +static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSysTableScanInfo* pInfo = pOperator->info; + int32_t numOfRows = 0; + int32_t ret = 0; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + SSDataBlock* pDataBlock = NULL; - if (smrTable.me.type == TSDB_SUPER_TABLE) { - pAPI->metaReaderFn.clearReader(&smrTable); - blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows = 0; - return NULL; - } - - if (smrTable.me.type == TSDB_CHILD_TABLE) { - int64_t suid = smrTable.me.ctbEntry.suid; - pAPI->metaReaderFn.clearReader(&smrTable); - pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); - code = pAPI->metaReaderFn.getTableEntryByUid(&smrTable, suid); - if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly - pAPI->metaReaderFn.clearReader(&smrTable); - blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows = 0; - return NULL; - } - } - - char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - SSchemaWrapper* schemaRow = NULL; - if (smrTable.me.type == TSDB_SUPER_TABLE) { - schemaRow = &smrTable.me.stbEntry.schemaRow; - STR_TO_VARSTR(typeName, "CHILD_TABLE"); - } else if (smrTable.me.type == TSDB_NORMAL_TABLE) { - schemaRow = &smrTable.me.ntbEntry.schemaRow; - STR_TO_VARSTR(typeName, "NORMAL_TABLE"); - } - - sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); - pAPI->metaReaderFn.clearReader(&smrTable); - - if (numOfRows > 0) { - relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); - numOfRows = 0; - } - blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - setOperatorCompleted(pOperator); - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + blockDataCleanup(pInfo->pRes); + + pDataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_COLS); + blockDataEnsureCapacity(pDataBlock, pOperator->resultInfo.capacity); + doExtractDbName(dbname, pInfo, pAPI); + + // optimize when sql like where table_name='tablename' and xxx. + if (pInfo->req.filterTb[0]) { + SSDataBlock* p = doOptimizeTableNameFilter(pOperator, pDataBlock, dbname); + blockDataDestroy(pDataBlock); + return p; } - int32_t ret = 0; if (pInfo->pCur == NULL) { pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); } else { @@ -534,7 +550,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { if (!pInfo->pCur || !pInfo->pSchema) { terrno = TSDB_CODE_OUT_OF_MEMORY; qError("sysTableScanUserCols failed since %s", terrstr(terrno)); - blockDataDestroy(dataBlock); + blockDataDestroy(pDataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } @@ -553,7 +569,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { SSchemaWrapper* schemaRow = NULL; if (pInfo->pCur->mr.me.type == TSDB_SUPER_TABLE) { - qDebug("sysTableScanUserCols cursor get super table"); + qDebug("sysTableScanUserCols cursor get super table, %s", GET_TASKID(pTaskInfo)); void* schema = taosHashGet(pInfo->pSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t)); if (schema == NULL) { SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&pInfo->pCur->mr.me.stbEntry.schemaRow); @@ -561,7 +577,8 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { } continue; } else if (pInfo->pCur->mr.me.type == TSDB_CHILD_TABLE) { - qDebug("sysTableScanUserCols cursor get child table"); + qDebug("sysTableScanUserCols cursor get child table, %s", GET_TASKID(pTaskInfo)); + STR_TO_VARSTR(typeName, "CHILD_TABLE"); STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); int64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; @@ -570,13 +587,14 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { schemaRow = *(SSchemaWrapper**)schema; } else { SMetaReader smrSuperTable = {0}; - pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn); int code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly - qError("sysTableScanUserCols get meta by suid:%" PRId64 " error, code:%d", suid, code); + qError("sysTableScanUserCols get meta by suid:%" PRId64 " error, code:%d, %s", suid, code, GET_TASKID(pTaskInfo)); + pAPI->metaReaderFn.clearReader(&smrSuperTable); - blockDataDestroy(dataBlock); + blockDataDestroy(pDataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } @@ -586,17 +604,17 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { pAPI->metaReaderFn.clearReader(&smrSuperTable); } } else if (pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE) { - qDebug("sysTableScanUserCols cursor get normal table"); + qDebug("sysTableScanUserCols cursor get normal table, %s", GET_TASKID(pTaskInfo)); schemaRow = &pInfo->pCur->mr.me.ntbEntry.schemaRow; STR_TO_VARSTR(typeName, "NORMAL_TABLE"); STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); } else { - qDebug("sysTableScanUserCols cursor get invalid table"); + qDebug("sysTableScanUserCols cursor get invalid table, %s", GET_TASKID(pTaskInfo)); continue; } if ((numOfRows + schemaRow->nCols) > pOperator->resultInfo.capacity) { - relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, pDataBlock, pOperator->exprSupp.pFilterInfo); numOfRows = 0; pInfo->restore = true; @@ -605,17 +623,17 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { break; } } else { - sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); + sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, pDataBlock, tableName, schemaRow, typeName); } } if (numOfRows > 0) { pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); - relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, pDataBlock, pOperator->exprSupp.pFilterInfo); numOfRows = 0; } - blockDataDestroy(dataBlock); + blockDataDestroy(pDataBlock); if (ret != 0) { pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; @@ -623,8 +641,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { } pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - qDebug("sysTableScanUserCols get cols success, rows:%" PRIu64, pInfo->loadInfo.totalRows); - + qDebug("get cols success, rows:%" PRIu64 " %s", pInfo->loadInfo.totalRows, GET_TASKID(pTaskInfo)); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } From f58902180a78087a63395a53c1574de3622836d7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 21 Mar 2024 15:51:39 +0800 Subject: [PATCH 08/17] fix(query): fix syntax error. --- source/libs/executor/src/sysscanoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 5e8c98c5c4..76103a8fb7 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -492,8 +492,8 @@ static SSDataBlock* doOptimizeTableNameFilter(SOperatorInfo* pOperator, SSDataBl pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; setOperatorCompleted(pOperator); - qDebug("get cols success, total rows:%" PRIu64 ", current:%d %s", pInfo->loadInfo.totalRows, pInfo->pRes->info.rows, - GET_TASKID(pTaskInfo)); + qDebug("get cols success, total rows:%" PRIu64 ", current:%" PRId64 " %s", pInfo->loadInfo.totalRows, + pInfo->pRes->info.rows, GET_TASKID(pTaskInfo)); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } From 4180eea09145938e37c3a6c91e48cd8ed4f11c0f Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 21 Mar 2024 10:04:39 +0800 Subject: [PATCH 09/17] enh: arb add more log --- source/libs/sync/src/syncAppendEntriesReply.c | 1 + source/libs/sync/src/syncMain.c | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 16739647b5..ede4dc07e1 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -91,6 +91,7 @@ int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { taosThreadMutexLock(&ths->arbTokenMutex); syncUtilGenerateArbToken(ths->myNodeInfo.nodeId, ths->vgId, ths->arbToken); + sInfo("vgId:%d, assigned leader to leader, arbToken:%s", ths->vgId, ths->arbToken); taosThreadMutexUnlock(&ths->arbTokenMutex); } } else { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 2a9b05e354..db3aec5f2a 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -304,6 +304,10 @@ int32_t syncBecomeAssignedLeader(SSyncNode* ths, SRpcMsg* pRpcMsg) { syncNodeBecomeAssignedLeader(ths); } errcode = TSDB_CODE_SUCCESS; + } else { + sInfo("vgId:%d, skip to set assigned leader, token mismatch, local:%s, msg:%s", ths->vgId, ths->arbToken, + req.memberToken); + goto _OVER; } SVArbSetAssignedLeaderRsp rsp = {0}; @@ -1056,6 +1060,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion) { pSyncNode->arbTerm = -1; taosThreadMutexInit(&pSyncNode->arbTokenMutex, NULL); syncUtilGenerateArbToken(pSyncNode->myNodeInfo.nodeId, pSyncInfo->vgId, pSyncNode->arbToken); + sInfo("vgId:%d, arb token:%s", pSyncNode->vgId, pSyncNode->arbToken); // init peersNum, peers, peersId pSyncNode->peersNum = pSyncNode->raftCfg.cfg.totalReplicaNum - 1; From acf42a3d7a89dfb295218235609b74cd6d97f4da Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 21 Mar 2024 15:48:10 +0800 Subject: [PATCH 10/17] fix: arb append noop after become assigned --- source/libs/sync/src/syncMain.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index db3aec5f2a..da18bbdea2 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -302,6 +302,10 @@ int32_t syncBecomeAssignedLeader(SSyncNode* ths, SRpcMsg* pRpcMsg) { goto _OVER; } syncNodeBecomeAssignedLeader(ths); + + if (syncNodeAppendNoop(ths) < 0) { + sError("vgId:%d, assigned leader failed to append noop entry since %s", ths->vgId, terrstr()); + } } errcode = TSDB_CODE_SUCCESS; } else { From be547678979394ff68bc342df77b6f876b97f42e Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 21 Mar 2024 17:08:58 +0800 Subject: [PATCH 11/17] fix: arb show null in show arbgroups --- source/dnode/mnode/impl/src/mndArbGroup.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c index 7193058e80..80664b39e6 100644 --- a/source/dnode/mnode/impl/src/mndArbGroup.c +++ b/source/dnode/mnode/impl/src/mndArbGroup.c @@ -1055,13 +1055,21 @@ static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&pGroup->isSync, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pGroup->assignedLeader.dnodeId, false); + if (pGroup->assignedLeader.dnodeId != 0) { + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)&pGroup->assignedLeader.dnodeId, false); - char token[TSDB_ARB_TOKEN_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(token, pGroup->assignedLeader.token, TSDB_ARB_TOKEN_SIZE + VARSTR_HEADER_SIZE); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)token, false); + char token[TSDB_ARB_TOKEN_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(token, pGroup->assignedLeader.token, TSDB_ARB_TOKEN_SIZE + VARSTR_HEADER_SIZE); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)token, false); + } else { + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetNULL(pColInfo, numOfRows); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetNULL(pColInfo, numOfRows); + } taosThreadMutexUnlock(&pGroup->mutex); From e9b115d9bc629e1c018ef6a32073961befadac51 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 21 Mar 2024 18:57:35 +0800 Subject: [PATCH 12/17] fix(test): update test cases. --- tests/pytest/util/dnodes.py | 1 - tests/system-test/0-others/information_schema.py | 2 ++ tests/system-test/2-query/qnodeCluster.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 789e5866ee..67c3d37960 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -130,7 +130,6 @@ class TDDnode: "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", - "DebugFlag": "131", "mDebugFlag": "143", "dDebugFlag": "143", "vDebugFlag": "143", diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index a7995762d8..21ad221ac5 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -19,6 +19,8 @@ from util.common import * from util.sqlset import * class TDTestCase: + updatecfgDict = {'qDebugFlag':135 , 'mDebugFlag':135} + def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) diff --git a/tests/system-test/2-query/qnodeCluster.py b/tests/system-test/2-query/qnodeCluster.py index 851b5bb1d1..fbddcd299c 100644 --- a/tests/system-test/2-query/qnodeCluster.py +++ b/tests/system-test/2-query/qnodeCluster.py @@ -20,9 +20,9 @@ import threading class TDTestCase: clientCfgDict = {'queryproxy': '1','debugFlag': 135} - clientCfgDict["debugFlag"] = 131 + clientCfgDict["debugFlag"] = 135 updatecfgDict = {'clientCfg': {}} - updatecfgDict = {'debugFlag': 131} + updatecfgDict = {'debugFlag': 135} updatecfgDict = {'keepColumnName': 1} updatecfgDict["clientCfg"] = clientCfgDict From fb2667d92d0b107a81802ab045b302583424fabf Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 22 Mar 2024 10:33:46 +0800 Subject: [PATCH 13/17] fix: sys table scan and meta query --- include/libs/executor/storageapi.h | 12 ++++++------ source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/meta/metaQuery.c | 10 ++++++---- source/libs/executor/src/sysscanoperator.c | 16 ++++------------ 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 04bc839d51..b47a162a1a 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -238,12 +238,12 @@ typedef struct SStoreSnapshotFn { } SStoreSnapshotFn; typedef struct SStoreMeta { - SMTbCursor* (*openTableMetaCursor)(void* pVnode); // metaOpenTbCursor - void (*closeTableMetaCursor)(SMTbCursor* pTbCur); // metaCloseTbCursor - void (*pauseTableMetaCursor)(SMTbCursor* pTbCur); // metaPauseTbCursor - void (*resumeTableMetaCursor)(SMTbCursor* pTbCur, int8_t first); // metaResumeTbCursor - int32_t (*cursorNext)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorNext - int32_t (*cursorPrev)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorPrev + SMTbCursor* (*openTableMetaCursor)(void* pVnode); // metaOpenTbCursor + void (*closeTableMetaCursor)(SMTbCursor* pTbCur); // metaCloseTbCursor + void (*pauseTableMetaCursor)(SMTbCursor* pTbCur); // metaPauseTbCursor + void (*resumeTableMetaCursor)(SMTbCursor* pTbCur, int8_t first, int8_t move); // metaResumeTbCursor + int32_t (*cursorNext)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorNext + int32_t (*cursorPrev)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorPrev int32_t (*getTableTags)(void* pVnode, uint64_t suid, SArray* uidList); int32_t (*getTableTagsByUid)(void* pVnode, int64_t suid, SArray* uidList); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9fdb4993bd..d564c5a36e 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -136,7 +136,7 @@ typedef SVCreateTSmaReq SSmaCfg; SMTbCursor* metaOpenTbCursor(void* pVnode); void metaCloseTbCursor(SMTbCursor* pTbCur); void metaPauseTbCursor(SMTbCursor* pTbCur); -void metaResumeTbCursor(SMTbCursor* pTbCur, int8_t first); +void metaResumeTbCursor(SMTbCursor* pTbCur, int8_t first, int8_t move); int32_t metaTbCursorNext(SMTbCursor* pTbCur, ETableType jumpTableType); int32_t metaTbCursorPrev(SMTbCursor* pTbCur, ETableType jumpTableType); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index cc7ae03483..c52e42d945 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -248,7 +248,7 @@ SMTbCursor *metaOpenTbCursor(void *pVnode) { // tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); pTbCur->pMeta = pVnodeObj->pMeta; pTbCur->paused = 1; - metaResumeTbCursor(pTbCur, 1); + metaResumeTbCursor(pTbCur, 1, 0); return pTbCur; } @@ -273,7 +273,7 @@ void metaPauseTbCursor(SMTbCursor *pTbCur) { pTbCur->paused = 1; } } -void metaResumeTbCursor(SMTbCursor *pTbCur, int8_t first) { +void metaResumeTbCursor(SMTbCursor *pTbCur, int8_t first, int8_t move) { if (pTbCur->paused) { metaReaderDoInit(&pTbCur->mr, pTbCur->pMeta, META_READER_LOCK); @@ -282,9 +282,11 @@ void metaResumeTbCursor(SMTbCursor *pTbCur, int8_t first) { if (first) { tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); } else { - int c = 0; + int c = 1; tdbTbcMoveTo(pTbCur->pDbc, pTbCur->pKey, pTbCur->kLen, &c); - if (c < 0) { + if (c == 0) { + if (move) tdbTbcMoveToNext(pTbCur->pDbc); + } else if (c < 0) { tdbTbcMoveToPrev(pTbCur->pDbc); } else { tdbTbcMoveToNext(pTbCur->pDbc); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 76103a8fb7..589c23ed3a 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -539,7 +539,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { if (pInfo->pCur == NULL) { pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); } else { - pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0); + pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0, 0); } if (pInfo->pSchema == NULL) { @@ -555,14 +555,8 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { return NULL; } - int32_t restore = pInfo->restore; - pInfo->restore = false; - - while (restore || ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) { - if (restore) { - restore = false; - } + while (((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) { char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -616,7 +610,6 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { if ((numOfRows + schemaRow->nCols) > pOperator->resultInfo.capacity) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, pDataBlock, pOperator->exprSupp.pFilterInfo); numOfRows = 0; - pInfo->restore = true; if (pInfo->pRes->info.rows > 0) { pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); @@ -724,7 +717,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { if (pInfo->pCur == NULL) { pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); } else { - pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0); + pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0, 0); } bool blockFull = false; @@ -750,7 +743,6 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } if ((smrSuperTable.me.stbEntry.schemaTag.nCols + numOfRows) > pOperator->resultInfo.capacity) { - pAPI->metaFn.cursorPrev(pInfo->pCur, TSDB_TABLE_MAX); blockFull = true; } else { sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, @@ -1354,7 +1346,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { firstMetaCursor = 1; } if (!firstMetaCursor) { - pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0); + pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0, 1); } blockDataCleanup(pInfo->pRes); From 2821e706a572de563bebd790eb5399de71424997 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Fri, 22 Mar 2024 12:59:07 +0800 Subject: [PATCH 14/17] fix: arb send set assigned req --- source/dnode/mnode/impl/src/mndArbGroup.c | 44 ++++++++++++----------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c index 80664b39e6..e056e698f3 100644 --- a/source/dnode/mnode/impl/src/mndArbGroup.c +++ b/source/dnode/mnode/impl/src/mndArbGroup.c @@ -537,15 +537,25 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) { bool member0IsTimeout = mndCheckArbMemberHbTimeout(&arbGroupDup, 0, nowMs); bool member1IsTimeout = mndCheckArbMemberHbTimeout(&arbGroupDup, 1, nowMs); - int32_t currentAssignedDnodeId = arbGroupDup.assignedLeader.dnodeId; + SArbAssignedLeader* pAssignedLeader = &arbGroupDup.assignedLeader; + int32_t currentAssignedDnodeId = pAssignedLeader->dnodeId; - // 1. both of the two members are timeout => skip + // 1. has assigned && is sync => send req + if (currentAssignedDnodeId != 0 && arbGroupDup.isSync == true) { + (void)mndSendArbSetAssignedLeaderReq(pMnode, currentAssignedDnodeId, vgId, arbToken, term, + pAssignedLeader->token); + mInfo("vgId:%d, arb send set assigned leader to dnodeId:%d", vgId, currentAssignedDnodeId); + sdbRelease(pSdb, pArbGroup); + continue; + } + + // 2. both of the two members are timeout => skip if (member0IsTimeout && member1IsTimeout) { sdbRelease(pSdb, pArbGroup); continue; } - // 2. no member is timeout => check sync + // 3. no member is timeout => check sync if (member0IsTimeout == false && member1IsTimeout == false) { // no assigned leader and not sync if (currentAssignedDnodeId == 0 && !arbGroupDup.isSync) { @@ -556,7 +566,7 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) { continue; } - // 3. one of the members is timeout => set assigned leader + // 4. one of the members is timeout => set assigned leader int32_t candidateIndex = member0IsTimeout ? 1 : 0; SArbGroupMember *pMember = &arbGroupDup.members[candidateIndex]; @@ -581,26 +591,18 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) { continue; } - // no assigned leader => write to sdb - if (currentAssignedDnodeId == 0) { - SArbGroup newGroup = {0}; - mndArbGroupDupObj(&arbGroupDup, &newGroup); - mndArbGroupSetAssignedLeader(&newGroup, candidateIndex); - if (mndPullupArbUpdateGroup(pMnode, &newGroup) != 0) { - mError("vgId:%d, arb failed to pullup set assigned leader to dnodeId:%d, since %s", vgId, pMember->info.dnodeId, - terrstr()); - sdbRelease(pSdb, pArbGroup); - return -1; - } - - mInfo("vgId:%d, arb pull up set assigned leader to dnodeId:%d", vgId, pMember->info.dnodeId); + // is sync && no assigned leader => write to sdb + SArbGroup newGroup = {0}; + mndArbGroupDupObj(&arbGroupDup, &newGroup); + mndArbGroupSetAssignedLeader(&newGroup, candidateIndex); + if (mndPullupArbUpdateGroup(pMnode, &newGroup) != 0) { + mError("vgId:%d, arb failed to pullup set assigned leader to dnodeId:%d, since %s", vgId, pMember->info.dnodeId, + terrstr()); sdbRelease(pSdb, pArbGroup); - continue; + return -1; } - // isSync == true && dnodeId match => send request to dnode - (void)mndSendArbSetAssignedLeaderReq(pMnode, pMember->info.dnodeId, vgId, arbToken, term, pMember->state.token); - mInfo("vgId:%d, arb send set assigned leader to dnodeId:%d", vgId, pMember->info.dnodeId); + mInfo("vgId:%d, arb pull up set assigned leader to dnodeId:%d", vgId, pMember->info.dnodeId); sdbRelease(pSdb, pArbGroup); } From cba77f9f728c272cd7080c2aceb43b7ebf643a3c Mon Sep 17 00:00:00 2001 From: Chris Zhai Date: Fri, 22 Mar 2024 14:31:15 +0800 Subject: [PATCH 15/17] update test cases for ts-4221- v5 --- tests/pytest/util/sql.py | 13 +- .../1-insert/insert_column_value.py | 154 +++++++++++------- 2 files changed, 102 insertions(+), 65 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 783009b3fb..d1ffaaca92 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -85,7 +85,6 @@ class TDSql: i=1 while i <= queryTimes: try: - tdLog.info(sql) self.affectedRows = self.cursor.execute(sql) return self.affectedRows except Exception as e: @@ -116,12 +115,14 @@ class TDSql: else: tdLog.info("sql:%s, check passed, no ErrInfo occurred" % (sql)) - def error(self, sql, expectedErrno = None, expectErrInfo = None, fullMatched = True): + def error(self, sql, expectedErrno = None, expectErrInfo = None, fullMatched = True, show = False): caller = inspect.getframeinfo(inspect.stack()[1][0]) expectErrNotOccured = True - try: + if show: tdLog.info("sql:%s" % (sql)) + + try: self.cursor.execute(sql) except BaseException as e: tdLog.info("err:%s" % (e)) @@ -164,12 +165,14 @@ class TDSql: return self.error_info - def query(self, sql, row_tag=None, queryTimes=10, count_expected_res=None): + def query(self, sql, row_tag=None, queryTimes=10, count_expected_res=None, show = False): + if show: + tdLog.info("sql:%s" % (sql)) + self.sql = sql i=1 while i <= queryTimes: try: - tdLog.info(sql) self.cursor.execute(sql) self.queryResult = self.cursor.fetchall() self.queryRows = len(self.queryResult) diff --git a/tests/system-test/1-insert/insert_column_value.py b/tests/system-test/1-insert/insert_column_value.py index 696401fd6a..fcb83e2f97 100644 --- a/tests/system-test/1-insert/insert_column_value.py +++ b/tests/system-test/1-insert/insert_column_value.py @@ -1,5 +1,6 @@ import datetime, time from enum import Enum +import binascii from util.log import * from util.sql import * from util.cases import * @@ -178,6 +179,37 @@ class TDTestCase: check_result = True tdSql.checkEqual(check_result, True) + def _query_check_varbinary(self, result, okv, nv, row = 0, col = 0): + tdLog.info(f'[okv={okv}, nv={nv}') + for i in range(row): + for j in range(1, col): + check_result = False + if result[i][j]: + check_item = result[i][j].decode('utf-8') + else: + check_item = '' + # new_nv = None + + if nv[0:1] == '\'' or nv[0:1] == '\"': + nv = nv[1:-1] + if okv[0:1] == '\'' or okv[0:1] == '\"': + okv = okv[1:-1] + + # if nv[0:2] == '\\x' or nv[0:2] == '\\X': + # nv = nv[1:] + # elif nv == '' or nv =="": + # new_nv = nv + # elif nv.isspace(): + # new_nv = nv + # else: + # hex_text = binascii.hexlify(nv.encode()) + # new_nv = '/x' + hex_text.decode().upper() + # tdLog.info(f"okv={okv}, nv={nv}, check_item={check_item}") + if check_item == None: + check_item = 'null' + if check_item in okv or check_item in nv: + check_result = True + tdSql.checkEqual(check_result, True) def _query_check(self, dbname="db", stbname="", ctbname="", ntbname="",nRows = 0, okv = None, nv = None, dtype = TDDataType.NULL): result = None @@ -192,17 +224,17 @@ class TDTestCase: result = tdSql.queryResult - if dtype == TDDataType.VARCHAR or dtype == TDDataType.NCHAR: + if dtype == TDDataType.VARCHAR or dtype == TDDataType.NCHAR or dtype == TDDataType.BINARY: self._query_check_varchar(result, okv, nv, nRows, 4) elif dtype == TDDataType.TIMESTAMP: self._query_check_timestamp(result, okv, nv, nRows, 4) elif dtype == TDDataType.BOOL: self._query_check_bool(result, okv, nv, nRows, 4) elif dtype == TDDataType.VARBINARY: - pass - elif dtype == TDDataType.INT: + self._query_check_varbinary(result, okv, nv, nRows, 4) + elif dtype == TDDataType.INT or dtype == TDDataType.BIGINT or dtype == TDDataType.SMALLINT or dtype == TDDataType.TINYINT: self._query_check_int(result, okv, nv, nRows, 4) - elif dtype == TDDataType.UINT: + elif dtype == TDDataType.UINT or dtype == TDDataType.UBIGINT or dtype == TDDataType.USMALLINT or dtype == TDDataType.UTINYINT: self._query_check_int(result, okv, nv, nRows, 4) elif dtype == TDDataType.FLOAT or dtype == TDDataType.DOUBLE: self._query_check_int(result, okv, nv, nRows, 4) @@ -220,63 +252,65 @@ class TDTestCase: for _l in kolist: for _e in _l: + # tdLog.info(f'[ko:verify value "{_e}"]') # create sub-table manually, check tag tdLog.info('[ko:create sub-table manually, check tag]') - tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, {okv})' %(_e)) - tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, %s)' %(_e)) - # tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, %s)' %(_e, _e)) + tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, {okv})' %(_e), show=True) + tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, %s)' %(_e), show=True) + tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, %s)' %(_e, _e), show=True) # create sub-table automatically, check tag tdLog.info('[ko:create sub-table automatically, check tag]') - tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, %s) values(now, {okv}, {okv})' %(_e)) - tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, {okv}) values(now, {okv}, {okv})' %(_e)) + tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, %s) values(now, {okv}, {okv})' %(_e), show=True) + tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags(%s, {okv}) values(now, {okv}, {okv})' %(_e), show=True) # create sub-table automatically, check value tdLog.info('[ko:create sub-table automatically, check value]') - tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, %s, {okv})' %(_e)) - tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, {okv}, %s)' %(_e)) - # tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, %s, %s)' %(_e, _e)) + tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, %s, {okv})' %(_e), show=True) + tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, {okv}, %s)' %(_e), show=True) + tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, %s, %s)' %(_e, _e), show=True) # check alter table tag tdLog.info('[ko:check alter table tag]') - tdSql.execute(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, {okv}, {okv})') - self._query_check(dbname,stbname, "", None, 1, okv, _e, dtype) - tdSql.execute(f'alter table {dbname}.{ctbname} set tag t0 = {okv}') - tdSql.error(f'alter table {dbname}.{ctbname} set tag t0 = %s' %(_e)) - tdSql.error(f'alter table {dbname}.{ctbname} set tag t1 = %s' %(_e)) + tdSql.execute(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}, {okv}) values(now, {okv}, {okv})', show=True) + # self._query_check(dbname,stbname, "", None, 1, okv, _e, dtype) + # tdSql.execute(f'alter table {dbname}.{ctbname} set tag t0 = {okv}', show=True) + tdSql.error(f'alter table {dbname}.{ctbname} set tag t0 = %s' %(_e), show=True) + tdSql.error(f'alter table {dbname}.{ctbname} set tag t1 = %s' %(_e), show=True) tdSql.execute(f'drop table {dbname}.{ctbname}') # insert into value by supper-table, check tag & value tdLog.info('[ko:insert into value by supper-table, check tag & value]') - tdSql.error(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_1", %s, {okv}, now, {okv}, {okv})' %(_e)) - tdSql.error(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_2", {okv},{okv}, now + 1s, {okv}, %s)' %(_e)) - # tdSql.error(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_3", %s, %s, now + 2s, %s, %s)' %(_e, _e, _e, _e)) + tdSql.error(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_1", %s, {okv}, now, {okv}, {okv})' %(_e), show=True) + tdSql.error(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_2", {okv},{okv}, now + 1s, {okv}, %s)' %(_e), show=True) + tdSql.error(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_3", %s, %s, now + 2s, %s, %s)' %(_e, _e, _e, _e), show=True) # insert into normal table, check value tdLog.info('[ko:insert into normal table, check value]') - tdSql.error(f'insert into {dbname}.{ntbname} values(now, %s, {okv})' %(_e)) - tdSql.error(f'insert into {dbname}.{ntbname} values(now, {okv}, %s)' %(_e)) - # tdSql.error(f'insert into {dbname}.{ntbname} values(now, %s, %s)' %(_e, _e)) + tdSql.error(f'insert into {dbname}.{ntbname} values(now, %s, {okv})' %(_e), show=True) + tdSql.error(f'insert into {dbname}.{ntbname} values(now, {okv}, %s)' %(_e), show=True) + tdSql.error(f'insert into {dbname}.{ntbname} values(now, %s, %s)' %(_e, _e), show=True) for _l in oklist: for _e in _l: + tdLog.info(f'[ok:verify value "{_e}"]') # 1. create sub-table manually, check tag tdLog.info('[ok:create sub-table manually, check tag]') - tdSql.execute(f'create table {dbname}.{ctbname}_1 using {dbname}.{stbname} tags(%s, {okv})' %(_e)) - tdSql.execute(f'create table {dbname}.{ctbname}_2 using {dbname}.{stbname} tags({okv}, %s)' %(_e)) - tdSql.execute(f'create table {dbname}.{ctbname}_3 using {dbname}.{stbname} tags(%s, %s)' %(_e, _e)) + tdSql.execute(f'create table {dbname}.{ctbname}_1 using {dbname}.{stbname} tags({_e}, {okv})', show=True) + tdSql.execute(f'create table {dbname}.{ctbname}_2 using {dbname}.{stbname} tags({okv}, %s)' %(_e), show=True) + tdSql.execute(f'create table {dbname}.{ctbname}_3 using {dbname}.{stbname} tags(%s, %s)' %(_e, _e), show=True) # 1.1 insert into sub-table, check value tdLog.info('[ok:insert into sub-table, check value]') - tdSql.execute(f'insert into {dbname}.{ctbname}_1 values(now + 0s, %s, {okv})' %(_e)) - tdSql.execute(f'insert into {dbname}.{ctbname}_2 values(now + 1s, {okv}, %s)' %(_e)) - tdSql.execute(f'insert into {dbname}.{ctbname}_3 values(now + 2s, %s, %s)' %(_e, _e)) + tdSql.execute(f'insert into {dbname}.{ctbname}_1 values(now + 0s, %s, {okv})' %(_e), show=True) + tdSql.execute(f'insert into {dbname}.{ctbname}_2 values(now + 1s, {okv}, %s)' %(_e), show=True) + tdSql.execute(f'insert into {dbname}.{ctbname}_3 values(now + 2s, %s, %s)' %(_e, _e), show=True) # 1.2 check alter table tag tdLog.info('[ok:check alter table tag]') - tdSql.execute(f'alter table {dbname}.{ctbname}_1 set tag t1 = %s' %(_e)) - tdSql.execute(f'alter table {dbname}.{ctbname}_2 set tag t0 = %s' %(_e)) + tdSql.execute(f'alter table {dbname}.{ctbname}_1 set tag t1 = %s' %(_e), show=True) + tdSql.execute(f'alter table {dbname}.{ctbname}_2 set tag t0 = %s' %(_e), show=True) # 1.3 check table data tdLog.info('[ok:check table data]') @@ -286,9 +320,9 @@ class TDTestCase: # 2. insert into value by creating sub-table automatically, check tag & value tdLog.info('[ok:insert into value by creating sub-table automatically, check tag & value]') - tdSql.execute(f'insert into {dbname}.{ctbname}_1 using {dbname}.{stbname} tags(%s, {okv}) values(now, %s, {okv})' %(_e, _e)) - tdSql.execute(f'insert into {dbname}.{ctbname}_2 using {dbname}.{stbname} tags({okv}, %s) values(now + 1s, {okv}, %s)' %(_e, _e)) - tdSql.execute(f'insert into {dbname}.{ctbname}_3 using {dbname}.{stbname} tags(%s, %s) values(now + 2s, %s, %s)' %(_e, _e, _e, _e)) + tdSql.execute(f'insert into {dbname}.{ctbname}_1 using {dbname}.{stbname} tags(%s, {okv}) values(now, %s, {okv})' %(_e, _e), show=True) + tdSql.execute(f'insert into {dbname}.{ctbname}_2 using {dbname}.{stbname} tags({okv}, %s) values(now + 1s, {okv}, %s)' %(_e, _e), show=True) + tdSql.execute(f'insert into {dbname}.{ctbname}_3 using {dbname}.{stbname} tags(%s, %s) values(now + 2s, %s, %s)' %(_e, _e, _e, _e), show=True) self._query_check(dbname, stbname, f'{ctbname}_1', None, 3, okv, _e, dtype) self._query_check(dbname, stbname, f'{ctbname}_2', None, 2, okv, _e, dtype) @@ -296,9 +330,9 @@ class TDTestCase: # 3. insert into value by supper-table, check tag & value tdLog.info('[ok:insert into value by supper-table, check tag & value]') - tdSql.execute(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_1", %s, {okv}, now, %s, {okv})' %(_e, _e)) - tdSql.execute(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_2", {okv}, %s, now + 1s, {okv}, %s)' %(_e, _e)) - tdSql.execute(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_3", %s, %s, now + 2s, %s, %s)' %(_e, _e, _e, _e)) + tdSql.execute(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_1", %s, {okv}, now, %s, {okv})' %(_e, _e), show=True) + tdSql.execute(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_2", {okv}, %s, now + 1s, {okv}, %s)' %(_e, _e), show=True) + tdSql.execute(f'insert into {dbname}.{stbname}(tbname, t0, t1, ts, c0, c1) values("{ctbname}_3", %s, %s, now + 2s, %s, %s)' %(_e, _e, _e, _e), show=True) self._query_check(dbname, stbname, f'{ctbname}_1', None, 3, okv, _e, dtype) self._query_check(dbname, stbname, f'{ctbname}_2', None, 2, okv, _e, dtype) @@ -306,9 +340,9 @@ class TDTestCase: # 4. insert value into normal table tdLog.info('[ok:insert value into normal table]') - tdSql.execute(f'insert into {dbname}.{ntbname} values(now, %s, {okv})' %(_e)) - tdSql.execute(f'insert into {dbname}.{ntbname} values(now + 1s, {okv}, %s)' %(_e)) - tdSql.execute(f'insert into {dbname}.{ntbname} values(now + 2s, %s, %s)' %(_e, _e)) + tdSql.execute(f'insert into {dbname}.{ntbname} values(now, %s, {okv})' %(_e), show=True) + tdSql.execute(f'insert into {dbname}.{ntbname} values(now + 1s, {okv}, %s)' %(_e), show=True) + tdSql.execute(f'insert into {dbname}.{ntbname} values(now + 2s, %s, %s)' %(_e, _e), show=True) if dtype != TDDataType.GEOMETRY: # self._query_check(dbname, None, None, ntbname, 3, okv, _e, dtype) @@ -322,22 +356,22 @@ class TDTestCase: for _l in kolist: for _e in _l: - tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s)' %(_e)) - tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags(%s) values(now, 1)' %(_e)) - tdSql.execute(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}) values(now, 1)') - tdSql.query(f'select * from {dbname}.{stbname}') + tdSql.error(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s)' %(_e), show=True) + tdSql.error(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags(%s) values(now, 1)' %(_e), show=True) + tdSql.execute(f'insert into {dbname}.{ctbname} using {dbname}.{stbname} tags({okv}) values(now, 1)', show=True) + tdSql.query(f'select * from {dbname}.{stbname}', show=True) tdSql.checkRows(1) - tdSql.execute(f'alter table {dbname}.{ctbname} set tag t0 = {okv}') - tdSql.error(f'alter table {dbname}.{ctbname} set tag t0 = %s' %(_e)) - tdSql.execute(f'drop table {dbname}.{ctbname}') + tdSql.execute(f'alter table {dbname}.{ctbname} set tag t0 = {okv}', show=True) + tdSql.error(f'alter table {dbname}.{ctbname} set tag t0 = %s' %(_e), show=True) + tdSql.execute(f'drop table {dbname}.{ctbname}', show=True) for _l in oklist: for _e in _l: - tdSql.execute(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s)' %(_e)) - tdSql.execute(f'insert into {dbname}.{ctbname} values(now, 1)') - tdSql.execute(f'alter table {dbname}.{ctbname} set tag t0 = %s' %(_e)) - tdSql.query(f'select * from {dbname}.{stbname}') + tdSql.execute(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(%s)' %(_e), show=True) + tdSql.execute(f'insert into {dbname}.{ctbname} values(now, 1)', show=True) + tdSql.execute(f'alter table {dbname}.{ctbname} set tag t0 = %s' %(_e), show=True) + tdSql.query(f'select * from {dbname}.{stbname}', show=True) tdSql.checkRows(1) - tdSql.execute(f'drop table {dbname}.{ctbname}') + tdSql.execute(f'drop table {dbname}.{ctbname}', show=True) def __insert_query_exec(self): STR_EMPTY = ['\'\'', "\"\"", '\' \'', "\" \""] @@ -359,15 +393,15 @@ class TDTestCase: STR_BOOL = ['\'true\'', '\'false\'', '\'TRUE\'', '\'FALSE\'', '\'tRuE\'', '\'falsE\''] STR_TS = ["\"2024-02-01 00:00:01.001-08:00\"", "\'2024-02-01T00:00:01.001+09:00\'", "\"2024-02-01\"", "\'2024-02-02 00:00:01\'", "\'2024-02-02 00:00:01.009\'"] STR_TS_ILLEGAL = ["\"2023-2024-02-01 00:00:01.001-08:00\"", "\'2024-02-01T99:00:01.001+09:00\'", "\"2024-02-31\"", "\'2024-02-02 00:88:01\'", "\'2024-02-02 00:00:77.009\'"] - STR_VARBIN = ['\'\\x12\'', '\'\\x13\'', '\' \\x14 \'', '\'\\x12ab\''] + STR_VARBIN = ["\'\x12\'", "\'\x13\'", "\' \x14 \'", "\'\x12ab\'"] STR_JSON_O = ['\'{\"k1\":\"v1\"}\'', '\' {} \''] STR_JSON_A = ['\'[]\'', '\"{\'k1\': \'v1\',\'k2\'}\"', '\"{\'k1\': \'v1\'}}\"'] - STR_GEO = ['\' POINT(1.0 1.0)\'', '\'LINESTRING(1.00 +2.0, 2.1 -3.2, 5.00 5.01) \'', '\'POLYGON((1.0 1.0, -2.0 +2.0, 1.0 1.0))\'' ] + STR_GEO = ["\' POINT(1.0 1.0)\'", "\'LINESTRING(1.00 +2.0, 2.1 -3.2, 5.00 5.01) \'", "\'POLYGON((1.0 1.0, -2.0 +2.0, 1.0 1.0))\'" ] STR_GEO_ILLEGAL = ['\' POINT(1.0)\'', '\'LINESTRING(1.00 +2.0, -3.2, 5.00 5.01) \'', '\'POLYGON((-2.0 +2.0, 1.0 1.0))\'' ] STR_NULL = ['\'NuLl\'', '\'null\'', '\'NULL\''] STR_NONE = ['\'NoNe\'', '\'none\'', '\'NONE\''] - STR_BINARY = ['\'\\x7f8290\'', '\'\\X7f8290\'', '\'x7f8290\'', '\'\\x\''] # bug TD-29193 - STR_BINARY_ILLEGAL = ['\'\\x7f829\''] + STR_BINARY = ["\'\x7f8290\'", "\'\X7f8290\'", "\'x7f8290\'", "\'\\x\'"] # bug TD-29193 + STR_BINARY_ILLEGAL = ["\'\\x7f829\'"] RAW_INTEGER_P = [' 42 ', '+042 ', ' +0', '0 ', '-0', '0', ' 0X2A', ' -0x0 ', '+0x0 ', ' 0B00101010', ' -0b00'] RAW_INTEGER_M = [' -42 ', ' -0128',' -0x1', ' -0X2A', '-0b01 ', ' -0B00101010 '] @@ -413,7 +447,7 @@ class TDTestCase: KO_BO = [STR_EMPTY, STR_TSK, STR_TSKP, STR_TS, STR_MISC, STR_OPTR, STR_TSK_MISC, STR_TSKP_MISC, STR_VARBIN, STR_JSON_O, STR_JSON_A, STR_GEO, RAW_TSK, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, RAW_MISC, RAW_OPTR, STR_CHINESE, STR_NONE, STR_BINARY, RAW_NONE] OK_VB = [STR_EMPTY, STR_INTEGER_P, STR_INTEGER_M, STR_FLOAT_P, STR_FLOAT_M, STR_FLOAT_E_P, STR_FLOAT_E_M, STR_MISC, STR_OPTR, STR_TSK, STR_TSK_MISC, STR_TSKP, - STR_TSKP_MISC, STR_BOOL, STR_TS, STR_VARBIN, STR_JSON_O, STR_JSON_A, STR_GEO, STR_NULL, RAW_NULL, STR_CHINESE, STR_NONE, STR_BINARY] + STR_TSKP_MISC, STR_BOOL, STR_TS, STR_VARBIN, STR_JSON_O, STR_JSON_A, STR_GEO, STR_NULL, RAW_NULL, STR_CHINESE, STR_NONE, STR_BINARY] KO_VB = [RAW_INTEGER_P, RAW_INTEGER_M, RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M, RAW_TSK, RAW_BOOL, RAW_MISC, RAW_OPTR, RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, STR_BINARY_ILLEGAL, STR_BINARY_ILLEGAL, RAW_BINARY_ILLEGAL, RAW_NONE] OK_IN = [STR_INTEGER_P, STR_INTEGER_M, STR_FLOAT_P, STR_FLOAT_M, STR_FLOAT_E_P, STR_FLOAT_E_M, STR_NULL, RAW_INTEGER_P, RAW_INTEGER_M, RAW_FLOAT_P, RAW_FLOAT_M, @@ -453,14 +487,14 @@ class TDTestCase: RAW_TSK_OPTR, RAW_TSKP, RAW_TSKP_OPTR, STR_INTEGER_P, STR_INTEGER_M, STR_FLOAT_P, STR_FLOAT_M, STR_FLOAT_E_P, STR_FLOAT_E_M, RAW_INTEGER_P, RAW_INTEGER_M, RAW_FLOAT_P, RAW_FLOAT_M, RAW_FLOAT_E_P, RAW_FLOAT_E_M, STR_CHINESE, STR_NONE, STR_BINARY, RAW_NONE] - + TEST = [["' POINT(1.0 1.0)'"]] PARAM_LIST = [ ["db", "stb_vc", "ctb_vc", "ntb_vc", OK_VC, KO_VC, "\'vc\'", TDDataType.VARCHAR], ["db", "stb_nc", "ctb_nc", "ntb_nc", OK_NC, KO_NC, "\'nc\'", TDDataType.NCHAR], ["db", "stb_bi", "ctb_bi", "ntb_bi", OK_BI, KO_BI, "\'bi\'", TDDataType.BINARY], ["db", "stb_ts", "ctb_ts", "ntb_ts", OK_TS, KO_TS, "now", TDDataType.TIMESTAMP], ["db", "stb_bo", "ctb_bo", "ntb_bo", OK_BO, KO_BO, "true", TDDataType.BOOL], - ["db", "stb_vb", "ctb_vb", "ntb_vb", OK_VB, KO_VB, "\'\\x12\'", TDDataType.VARBINARY], + ["db", "stb_vb", "ctb_vb", "ntb_vb", OK_VB, KO_VB, "'hello'", TDDataType.VARBINARY], ["db", "stb_in", "ctb_in", "ntb_in", OK_IN, KO_IN, "-1", TDDataType.INT], ["db", "stb_ui", "ctb_ui", "ntb_ui", OK_UI, KO_UI, "1", TDDataType.UINT], @@ -473,7 +507,7 @@ class TDTestCase: ["db", "stb_fl", "ctb_fl", "ntb_fl", OK_FL, KO_FL, "1.0", TDDataType.FLOAT], ["db", "stb_db", "ctb_db", "ntb_db", OK_DB, KO_DB, "1.0", TDDataType.DOUBLE], - ["db", "stb_ge", "ctb_ge", "ntb_ge", OK_GE, KO_GE, "\'POINT(1.0 1.0)\'", TDDataType.GEOMETRY] + ["db", "stb_ge", "ctb_ge", "ntb_ge", TEST, KO_GE, "\'POINT(100.0 100.0)\'", TDDataType.GEOMETRY] ] # check with common function From de8670e79af53c19170e420e6780e457045c5a4c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Mar 2024 15:17:20 +0800 Subject: [PATCH 16/17] test: restore debugflag --- tests/pytest/util/dnodes.py | 1 + tests/system-test/2-query/qnodeCluster.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 67c3d37960..789e5866ee 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -130,6 +130,7 @@ class TDDnode: "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", + "DebugFlag": "131", "mDebugFlag": "143", "dDebugFlag": "143", "vDebugFlag": "143", diff --git a/tests/system-test/2-query/qnodeCluster.py b/tests/system-test/2-query/qnodeCluster.py index fbddcd299c..851b5bb1d1 100644 --- a/tests/system-test/2-query/qnodeCluster.py +++ b/tests/system-test/2-query/qnodeCluster.py @@ -20,9 +20,9 @@ import threading class TDTestCase: clientCfgDict = {'queryproxy': '1','debugFlag': 135} - clientCfgDict["debugFlag"] = 135 + clientCfgDict["debugFlag"] = 131 updatecfgDict = {'clientCfg': {}} - updatecfgDict = {'debugFlag': 135} + updatecfgDict = {'debugFlag': 131} updatecfgDict = {'keepColumnName': 1} updatecfgDict["clientCfg"] = clientCfgDict From ea424e822ece6f9aa17ed9e074a1dfc91c594adf Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 25 Mar 2024 11:00:33 +0800 Subject: [PATCH 17/17] fix(tsdb/util): fix missing column value with row iter --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 44621bf4e6..076db8b3ca 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -642,8 +642,8 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { return &pIter->cv; } - if (pIter->iColData < pIter->pRow->pBlockData->nColData) { - tColDataGetValue(&pIter->pRow->pBlockData->aColData[pIter->iColData], pIter->pRow->iRow, &pIter->cv); + if (pIter->iColData <= pIter->pRow->pBlockData->nColData) { + tColDataGetValue(&pIter->pRow->pBlockData->aColData[pIter->iColData - 1], pIter->pRow->iRow, &pIter->cv); ++pIter->iColData; return &pIter->cv; } else {