From 8f5df0b6a3aba15831f8372051315f4005c582a1 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 7 Dec 2023 01:51:23 +0800 Subject: [PATCH] opt double -> int --- source/common/src/tvariant.c | 68 +++++++++++++----- tests/system-test/1-insert/insert_perf.py | 86 +++++++++++++++-------- 2 files changed, 108 insertions(+), 46 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 384df73342..5c7567777f 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -95,19 +95,6 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v return TSDB_CODE_SUCCESS; } -int32_t removeSpace(const char **pp, int32_t n) { - // rm blank space from both head and tail, keep at least one char - const char *z = *pp; - while (n > 1 && *z == ' ') { - z++; - n--; - } - while (n > 1 && z[n-1] == ' ') { - n--; - } - *pp = z; - return n; -} int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { if (n == 0) { @@ -184,6 +171,36 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; + } else if (errno == 0 && *endPtr == '.') { + // pure decimal part + const char *s = endPtr + 1; + const char *end = z + n; + bool pure = true; + while (s < end) { + if (*s < '0' || *s > '9') { + pure = false; + break; + } + s++; + } + if (pure) { + if (endPtr+1 < end && endPtr[1] > '4') { + const char *p = z; + while (*p == ' ') { + p++; + } + if (*p == '-') { + if ( *value > INT64_MIN) { + (*value)--; + } + } else { + if ( *value < INT64_MAX) { + (*value)++; + } + } + } + return TSDB_CODE_SUCCESS; + } } // 2. parse as other @@ -218,9 +235,9 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { } errno = 0; - const char *p = z; char *endPtr = NULL; bool parsed = false; + const char *p = z; while (*p == ' ') { p++; } @@ -264,14 +281,31 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { // 1. parse as integer *value = taosStr2UInt64(p, &endPtr, 10); + if (*p == '-' && *value) { + return TSDB_CODE_FAILED; + } if (endPtr - z == n) { - if (*p == '-' && *value) { - return TSDB_CODE_FAILED; - } if (errno == ERANGE || errno == EINVAL) { return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; + } else if (errno == 0 && *endPtr == '.') { + const char *s = endPtr + 1; + const char *end = z + n; + bool pure = true; + while (s < end) { + if (*s < '0' || *s > '9') { + pure = false; + break; + } + s++; + } + if (pure) { + if (endPtr + 1 < end && endPtr[1] > '4' && *value < UINT64_MAX) { + (*value)++; + } + return TSDB_CODE_SUCCESS; + } } // 2. parse as other diff --git a/tests/system-test/1-insert/insert_perf.py b/tests/system-test/1-insert/insert_perf.py index bf6fb6799d..d96a031458 100644 --- a/tests/system-test/1-insert/insert_perf.py +++ b/tests/system-test/1-insert/insert_perf.py @@ -21,48 +21,70 @@ class TDTestCase: self.testcasefilename = os.path.split(__file__)[-1] self.file1 = f"{self.testcasePath}/int.csv" self.file2 = f"{self.testcasePath}/double.csv" + self.file3 = f"{self.testcasePath}/d+.csv" + self.file4 = f"{self.testcasePath}/uint.csv" self.ts = 1700638570000 # 2023-11-22T07:36:10.000Z self.database = "db1" self.tb1 = "t1" self.tb2 = "t2" - self.stable1 = "st1" - self.stable2 = "st2" - self.tag1 = f'using {self.stable1}(groupId) tags(1)' - self.tag2 = f'using {self.stable2}(groupId) tags(2)' + self.tb3 = "t3" self.once = 1000 tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) def prepare_db(self): - #tdSql.execute(f"drop database if exists {self.database}") - tdSql.execute(f"create database if not exists {self.database}") + tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database {self.database}") tdSql.execute(f"use {self.database}") - tdSql.execute(f"create stable if not exists {self.stable1} (ts timestamp, q_bigint bigint , i1 int, i2 int, i3 int, i4 int, i5 int, i6 int, i7 int, i8 int, i9 int) tags (location binary(64), groupId int);") - tdSql.execute(f"create stable if not exists {self.stable2} (ts timestamp, q_double double, f1 double, f2 double, f3 double, f4 double, f5 double, f6 double, f7 double, f8 double, f9 double) tags (location binary(64), groupId int);") - tdSql.execute(f"drop table if exists {self.tb1};") - tdSql.execute(f"drop table if exists {self.tb2};") - tdSql.execute(f"create table {self.tb1} {self.tag1};") - tdSql.execute(f"create table {self.tb2} {self.tag2};") - - def make_csv(self, filepath, once, isint): + tdSql.execute(f"create table {self.tb1} (ts timestamp, i0 bigint , i1 bigint, i2 bigint, i3 bigint, i4 bigint, i5 bigint, i6 bigint, i7 bigint, i8 bigint, i9 bigint)") + tdSql.execute(f"create table {self.tb2} (ts timestamp, f0 double, f1 double, f2 double, f3 double, f4 double, f5 double, f6 double, f7 double, f8 double, f9 double)") + tdSql.execute(f"create table {self.tb3} (ts timestamp, i0 int unsigned , i1 int unsigned, i2 int unsigned, i3 int unsigned, i4 int unsigned, i5 int unsigned, i6 int unsigned, i7 int unsigned, i8 int unsigned, i9 int unsigned)") + + def make_csv(self, once, intype): + filepath = self.file1 + if intype == 2: + filepath = self.file2 + elif intype == 3: + filepath = self.file3 + elif intype == 4: + filepath = self.file4 + f = open(filepath, 'w') with f: writer = csv.writer(f) rows = [] for i in range(once): r = [] - if isint: + if intype == 1: for k in range(10): r.append(random.randint(-2147483648, 2147483647)) + elif intype == 2: + for k in range(10): + r.append(random.randint(-2147483648, 2147483646) + random.random()) + elif intype == 3: + for k in range(10): + r.append(random.randint(0, 4294967294) + random.random()) else: for k in range(10): - r.append(random.randint(-2147483648, 2147483647) + random.random()) + r.append(random.randint(0, 4294967295)) rows.append(r) writer.writerows(rows) f.close() print(f"{filepath} ready!") - def test_insert(self, tbname, qtime, startts, filepath, isint): + def test_insert(self, tbname, qtime, startts, intype, outtype): + filepath = self.file1 + dinfo = "int" + if intype == 2: + filepath = self.file2 + dinfo = "double" + elif intype == 3: + filepath = self.file3 + dinfo = "+double" + elif intype == 4: + filepath = self.file4 + dinfo = "uint" + f = open(filepath, 'r') rows = [] with f: @@ -89,25 +111,31 @@ class TDTestCase: sum += t2 - t1 sum = sum - tbtype = "10 double col/per row" - if isint: - tbtype = "10 int col/per row" - print(f" insert {self.once} * {qtime} rows: {sum} s, {tbtype}") + tbinfo = "10 bigint col/per row" + if outtype == 2: + tbinfo = "10 double col/per row" + elif outtype == 3: + tbinfo = "10 uint col/per row" + print(f" insert {self.once} * {qtime} rows: {sum} s, {dinfo} -> {tbinfo}") # tdSql.query(f"select count(*) from {self.database}.{tbname};") # tdSql.checkData(0, 0, once*qtime) def run(self): tdSql.prepare(replica = self.replicaVar) - # self.make_csv(self.file1, self.once, True) - # self.make_csv(self.file2, self.once, False) + # self.make_csv(self.once, 1) + # self.make_csv(self.once, 2) + # self.make_csv(self.once, 3) + # self.make_csv(self.once, 4) + self.prepare_db() - self.test_insert(self.tb1, 10000, self.ts, self.file1, True) - self.test_insert(self.tb2, 10000, self.ts, self.file2, False) - self.test_insert(self.tb2, 10000, self.ts, self.file1, False) - - self.test_insert(self.tb1, 10000, self.ts, self.file2, True) - + self.test_insert(self.tb1, 1000, self.ts-10000000, 1, 1) + self.test_insert(self.tb2, 1000, self.ts-10000000, 2, 2) + self.test_insert(self.tb3, 1000, self.ts-10000000, 4, 3) + self.test_insert(self.tb2, 1000, self.ts, 1, 2) + + self.test_insert(self.tb1, 1000, self.ts, 2, 1) + self.test_insert(self.tb3, 1000, self.ts, 3, 3) def stop(self): tdSql.close()