diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index d78c76a543..384df73342 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -81,24 +81,12 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v } if (!parsed) { - bool parse_int = true; - for (int32_t i = 0; i < n; i++) { - if (z[i] < '0' || z[i] > '9') { - parse_int = false; - break; - } - } - if (parse_int) { - // parsing as decimal - *value = taosStr2UInt64(z, &endPtr, 10); - } else { - // parsing as double - double val = taosStr2Double(z, &endPtr); - if (val > UINT64_MAX) { - return TSDB_CODE_FAILED; - } - *value = round(val); + // parsing as double + double val = taosStr2Double(z, &endPtr); + if (val > UINT64_MAX) { + return TSDB_CODE_FAILED; } + *value = round(val); } if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { @@ -145,6 +133,11 @@ int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { } int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + errno = 0; char *endPtr = NULL; bool parsed = false; @@ -179,13 +172,21 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { return TSDB_CODE_SUCCESS; } - // parse as string - if (n == 0) { - *value = 0; + // rm tail space + while (n > 1 && z[n-1] == ' ') { + n--; + } + + // 1. try to parse as integer + *value = taosStr2Int64(z, &endPtr, 10); + if (endPtr - z == n) { + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } return TSDB_CODE_SUCCESS; } - n = removeSpace(&z, n); + // 2. parse as other bool is_neg = false; uint64_t uv = 0; int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv); @@ -211,36 +212,69 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { } int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { - - switch (type) - { - case TK_NK_INTEGER: { - return toUInteger(z, n, 10, value); - } - case TK_NK_FLOAT: { - errno = 0; - char *endPtr = NULL; - double val = round(taosStr2Double(z, &endPtr)); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - return TSDB_CODE_FAILED; - } - if (!IS_VALID_UINT64(val)) { - return TSDB_CODE_FAILED; - } - *value = val; - return TSDB_CODE_SUCCESS; - } - default: - break; - } - - // parse as string if (n == 0) { *value = 0; return TSDB_CODE_SUCCESS; } - n = removeSpace(&z, n); + errno = 0; + const char *p = z; + char *endPtr = NULL; + bool parsed = false; + while (*p == ' ') { + p++; + } + switch (type) { + case TK_NK_INTEGER: { + *value = taosStr2UInt64(p, &endPtr, 10); + if (*p == '-' && *value) { + return TSDB_CODE_FAILED; + } + parsed = true; + } break; + case TK_NK_FLOAT: { + double val = round(taosStr2Double(p, &endPtr)); + if (!IS_VALID_UINT64(val)) { + return TSDB_CODE_FAILED; + } + *value = val; + parsed = true; + } break; + default: + break; + } + + if (parsed) { + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } + while (n > 1 && z[n-1] == ' ') { + n--; + } + if (endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + + // rm tail space + while (n > 1 && z[n-1] == ' ') { + n--; + } + + // 1. parse as integer + *value = taosStr2UInt64(p, &endPtr, 10); + if (endPtr - z == n) { + if (*p == '-' && *value) { + return TSDB_CODE_FAILED; + } + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + + // 2. parse as other bool is_neg = false; int32_t code = parseSignAndUInteger(z, n, &is_neg, value); if (is_neg) { diff --git a/tests/system-test/1-insert/insert_perf.py b/tests/system-test/1-insert/insert_perf.py new file mode 100644 index 0000000000..bf6fb6799d --- /dev/null +++ b/tests/system-test/1-insert/insert_perf.py @@ -0,0 +1,118 @@ +import taos +import sys +import random +import time +import csv + +from datetime import datetime + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + + self.testcasePath = os.path.split(__file__)[0] + self.testcasefilename = os.path.split(__file__)[-1] + self.file1 = f"{self.testcasePath}/int.csv" + self.file2 = f"{self.testcasePath}/double.csv" + self.ts = 1700638570000 # 2023-11-22T07:36:10.000Z + self.database = "db1" + self.tb1 = "t1" + self.tb2 = "t2" + self.stable1 = "st1" + self.stable2 = "st2" + self.tag1 = f'using {self.stable1}(groupId) tags(1)' + self.tag2 = f'using {self.stable2}(groupId) tags(2)' + self.once = 1000 + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepare_db(self): + #tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database if not exists {self.database}") + tdSql.execute(f"use {self.database}") + tdSql.execute(f"create stable if not exists {self.stable1} (ts timestamp, q_bigint bigint , i1 int, i2 int, i3 int, i4 int, i5 int, i6 int, i7 int, i8 int, i9 int) tags (location binary(64), groupId int);") + tdSql.execute(f"create stable if not exists {self.stable2} (ts timestamp, q_double double, f1 double, f2 double, f3 double, f4 double, f5 double, f6 double, f7 double, f8 double, f9 double) tags (location binary(64), groupId int);") + tdSql.execute(f"drop table if exists {self.tb1};") + tdSql.execute(f"drop table if exists {self.tb2};") + tdSql.execute(f"create table {self.tb1} {self.tag1};") + tdSql.execute(f"create table {self.tb2} {self.tag2};") + + def make_csv(self, filepath, once, isint): + f = open(filepath, 'w') + with f: + writer = csv.writer(f) + rows = [] + for i in range(once): + r = [] + if isint: + for k in range(10): + r.append(random.randint(-2147483648, 2147483647)) + else: + for k in range(10): + r.append(random.randint(-2147483648, 2147483647) + random.random()) + rows.append(r) + writer.writerows(rows) + f.close() + print(f"{filepath} ready!") + + def test_insert(self, tbname, qtime, startts, filepath, isint): + f = open(filepath, 'r') + rows = [] + with f: + reader = csv.reader(f, delimiter=',', quotechar='|') + for row in reader: + rows.append(row) + f.close() + self.once = len(rows) + + sum = 0 + for j in range(qtime): + offset = j * self.once + ts = startts + offset + sql = f"insert into {self.database}.{tbname} values" + for i in range(self.once): + r = rows[i] + sql +=f"({ts + i},'{r[0]}','{r[1]}','{r[2]}','{r[3]}','{r[4]}','{r[5]}','{r[6]}','{r[7]}','{r[8]}','{r[9]}')" + + t1 = time.time() + tdSql.execute(f"{sql};", 1) + t2 = time.time() + #print(f"{t2} insert test {j}.") + #print(sql) + sum += t2 - t1 + + sum = sum + tbtype = "10 double col/per row" + if isint: + tbtype = "10 int col/per row" + print(f" insert {self.once} * {qtime} rows: {sum} s, {tbtype}") + + # tdSql.query(f"select count(*) from {self.database}.{tbname};") + # tdSql.checkData(0, 0, once*qtime) + + def run(self): + tdSql.prepare(replica = self.replicaVar) + # self.make_csv(self.file1, self.once, True) + # self.make_csv(self.file2, self.once, False) + self.prepare_db() + self.test_insert(self.tb1, 10000, self.ts, self.file1, True) + self.test_insert(self.tb2, 10000, self.ts, self.file2, False) + self.test_insert(self.tb2, 10000, self.ts, self.file1, False) + + self.test_insert(self.tb1, 10000, self.ts, self.file2, True) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase())