From d2b2dee256812085a464ba3d170779f106e46efd Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 1 Dec 2023 23:44:05 +0800 Subject: [PATCH] opt & add test case --- include/common/tvariant.h | 5 +- source/common/src/tvariant.c | 185 +++++++++++------- source/common/test/commonTests.cpp | 203 +++++++++++++------- source/libs/parser/src/parInsertSql.c | 60 +++--- source/libs/parser/src/parTokenizer.c | 7 +- tests/parallel_test/cases.task | 1 + tests/system-test/1-insert/insert_double.py | 133 +++++++++++++ 7 files changed, 422 insertions(+), 172 deletions(-) create mode 100644 tests/system-test/1-insert/insert_double.py diff --git a/include/common/tvariant.h b/include/common/tvariant.h index 61cf277bb7..66b7302f4e 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -37,8 +37,9 @@ typedef struct SVariant { }; } SVariant; -int32_t toIntegerEx(const char *z, int32_t n, int64_t *value); -int32_t toUIntegerEx(const char *z, int32_t n, uint64_t *value); +int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value); +int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value); +int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double *value); int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value); int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value); diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index adb0964fe6..609bcf1382 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -45,21 +45,6 @@ int32_t parseBinaryUInteger(const char *z, int32_t n, uint64_t *value) { return TSDB_CODE_SUCCESS; } -int32_t parseDecimalUInteger(const char *z, int32_t n, uint64_t *value) { - errno = 0; - char *endPtr = NULL; - while (*z == '0' && n > 1) { - z++; - n--; - } - *value = taosStr2UInt64(z, &endPtr, 10); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - return TSDB_CODE_FAILED; - } - - return TSDB_CODE_SUCCESS; -} - int32_t parseHexUInteger(const char *z, int32_t n, uint64_t *value) { errno = 0; char *endPtr = NULL; @@ -71,10 +56,6 @@ int32_t parseHexUInteger(const char *z, int32_t n, uint64_t *value) { } int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *value) { - *is_neg = false; - if (n < 1) { - return TSDB_CODE_FAILED; - } // parse sign bool has_sign = false; @@ -83,18 +64,18 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v has_sign = true; } else if (z[0] == '+') { has_sign = true; - } else if (z[0] < '0' || z[0] > '9') { + } else if (z[0] != '.' && (z[0] < '0' || z[0] > '9')) { return TSDB_CODE_FAILED; } if (has_sign) { - z++; - n--; - if (n < 1) { + if (n < 2) { return TSDB_CODE_FAILED; } + z++; + n--; } - if (n > 2 && z[0] == '0') { + if (z[0] == '0' && n > 2) { if (z[1] == 'b' || z[1] == 'B') { // paring as binary return parseBinaryUInteger(z, n, value); @@ -106,89 +87,157 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v } } - //rm flag u-unsigned, l-long, f-float(if not in hex str) - char last = tolower(z[n-1]); - if (last == 'u' || last == 'l' || last == 'f') { - n--; - if (n < 1) { - return TSDB_CODE_FAILED; - } - } - - // parsing as decimal - bool parse_int = true; - for (int32_t i = 0; i < n; i++) { - if (z[i] < '0' || z[i] > '9') { - parse_int = false; - break; - } - } - if (parse_int) { - return parseDecimalUInteger(z, n, value); - } - // parsing as double errno = 0; char *endPtr = NULL; double val = taosStr2Double(z, &endPtr); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n || !IS_VALID_UINT64(val)) { + if (errno == ERANGE || errno == EINVAL || endPtr - z != n || val > UINT64_MAX) { return TSDB_CODE_FAILED; } - *value = val; + *value = round(val); return TSDB_CODE_SUCCESS; } int32_t removeSpace(const char **pp, int32_t n) { - // rm blank space from both head and tail + // rm blank space from both head and tail, keep at least one char const char *z = *pp; - while (*z == ' ' && n > 0) { + while (n > 1 && *z == ' ') { z++; n--; } - if (n > 0) { - for (int32_t i = n - 1; i > 0; i--) { - if (z[i] == ' ') { - n--; - } else { - break; - } - } + while (n > 1 && z[n-1] == ' ') { + n--; } *pp = z; return n; } -int32_t toIntegerEx(const char *z, int32_t n, int64_t *value) { - n = removeSpace(&z, n); - if (n < 1) { // fail: all char is space +int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { + if (type != TK_NK_FLOAT) { + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + // rm tail space + while (n > 1 && z[n-1] == ' ') { + n--; + } + } + + errno = 0; + char* endPtr = NULL; + *value = taosStr2Double(z, &endPtr); + + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { return TSDB_CODE_FAILED; } + return TSDB_CODE_SUCCESS; +} + +int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { + errno = 0; + char *endPtr = NULL; + switch (type) + { + case TK_NK_INTEGER: { + *value = taosStr2Int64(z, &endPtr, 10); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + case TK_NK_HEX: { + *value = taosStr2Int64(z, &endPtr, 16); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + case TK_NK_BIN: { + *value = taosStr2Int64(z, &endPtr, 2); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + default: + break; + } + + // parse string + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + n = removeSpace(&z, n); bool is_neg = false; uint64_t uv = 0; int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv); if (code == TSDB_CODE_SUCCESS) { // truncate into int64 - if (uv > INT64_MAX) { - *value = is_neg ? INT64_MIN : INT64_MAX; - return TSDB_CODE_FAILED; + if (is_neg) { + if (uv > 1ull + INT64_MAX) { + *value = INT64_MIN; + return TSDB_CODE_FAILED; + } else { + *value = -uv; + } } else { - *value = is_neg ? -uv : uv; + if (uv > INT64_MAX) { + *value = INT64_MAX; + return TSDB_CODE_FAILED; + } + *value = uv; } } return code; } -int32_t toUIntegerEx(const char *z, int32_t n, uint64_t *value) { - n = removeSpace(&z, n); - if (n < 1) { // fail: all char is space - return TSDB_CODE_FAILED; +int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { + errno = 0; + char *endPtr = NULL; + switch (type) + { + case TK_NK_INTEGER: { + *value = taosStr2UInt64(z, &endPtr, 10); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + case TK_NK_HEX: { + *value = taosStr2UInt64(z, &endPtr, 16); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + case TK_NK_BIN: { + *value = taosStr2UInt64(z, &endPtr, 2); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + default: + break; } + // parse string + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + n = removeSpace(&z, n); + bool is_neg = false; int32_t code = parseSignAndUInteger(z, n, &is_neg, value); if (is_neg) { + if (TSDB_CODE_SUCCESS == code && 0 == *value) { + return TSDB_CODE_SUCCESS; + } return TSDB_CODE_FAILED; } return code; diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 035bca0e15..291941d9b2 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -14,6 +14,7 @@ #include "tdef.h" #include "tvariant.h" #include "ttime.h" +#include "ttokendef.h" namespace { // @@ -25,125 +26,176 @@ int main(int argc, char** argv) { } +TEST(testCase, toUIntegerEx_test) { + uint64_t val = 0; + + char* s = "123"; + int32_t ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 123); + + s = "1000u"; + ret = toUIntegerEx(s, strlen(s), 0, &val); + ASSERT_EQ(ret, -1); + + s = "0x1f"; + ret = toUIntegerEx(s, strlen(s), TK_NK_HEX, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 31); + + s = "0b110"; + ret = toUIntegerEx(s, strlen(s), 0, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 6); + + s = "2567.4787"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 2567); + + s = "1.869895343e4"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 18699); + + s = "-1"; + ret = toUIntegerEx(s, strlen(s),TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); + + s = "-0b10010"; + ret = toUIntegerEx(s, strlen(s), 0, &val); + ASSERT_EQ(ret, -1); + + s = "-0x40"; + ret = toUIntegerEx(s, strlen(s), TK_NK_HEX, &val); + ASSERT_EQ(ret, -1); + + s = "-80.9999"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, -1); + + s = "-5.2343544534e10"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, -1); + + // INT64_MAX + s = "9223372036854775807"; + ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 9223372036854775807); + + // UINT64_MAX + s = "18446744073709551615"; + ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 18446744073709551615u); + + // out of range + s = "18446744073709551616"; + ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); + + s = "5.23e25"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, -1); +} + TEST(testCase, toIntegerEx_test) { int64_t val = 0; char* s = "123"; - int32_t ret = toIntegerEx(s, strlen(s), &val); + int32_t ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 123); - s = "9223372036854775807"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 9223372036854775807); - - s = "9323372036854775807"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, -1); - - s = "-9323372036854775807"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, -1); - s = "-1"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -1); - s = "-9223372036854775807"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, -9223372036854775807); - s = "1000u"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 1000); - - s = "1000l"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 1000); - - s = "1000.0f"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 1000); + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); s = "0x1f"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_HEX, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 31); s = "-0x40"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_HEX, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -64); s = "0b110"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), 0, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 6); s = "-0b10010"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), 0, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -18); + s = "-80.9999"; + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -81); + + s = "2567.8787"; + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 2568); + s = "-5.2343544534e10"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -52343544534); s = "1.869895343e4"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 18698); + ASSERT_EQ(val, 18699); + + // INT64_MAX + s = "9223372036854775807"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 9223372036854775807LL); + + s = "-9223372036854775808"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -9223372036854775808); + + // out of range + s = "9323372036854775807"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); + + s = "-9323372036854775807"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); // UINT64_MAX s = "18446744073709551615"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, -1); - - s = "18446744073709551616"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); ASSERT_EQ(ret, -1); } TEST(testCase, toInteger_test) { - char* s = "123"; - uint32_t type = 0; - int64_t val = 0; + char* s = "123"; int32_t ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 123); - s = "9223372036854775807"; - ret = toInteger(s, strlen(s), 10, &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 9223372036854775807); - - s = "9323372036854775807"; // out of range, > int64_max - ret = toInteger(s, strlen(s), 10, &val); - ASSERT_EQ(ret, -1); - - s = "-9323372036854775807"; - ret = toInteger(s, strlen(s), 10, &val); - ASSERT_EQ(ret, -1); - s = "-1"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -1); - s = "-9223372036854775807"; - ret = toInteger(s, strlen(s), 10, &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, -9223372036854775807); - s = "1000u"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, -1); @@ -163,13 +215,22 @@ TEST(testCase, toInteger_test) { ASSERT_EQ(ret, 0); ASSERT_EQ(val, 72); - // 18446744073709551615 UINT64_MAX - s = "18446744073709551615"; + s = "9223372036854775807"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 18446744073709551615u); + ASSERT_EQ(val, 9223372036854775807); - s = "18446744073709551616"; + s = "-9223372036854775808"; + ret = toInteger(s, strlen(s), 10, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -9223372036854775808); + + // out of range + s = "9323372036854775807"; + ret = toInteger(s, strlen(s), 10, &val); + ASSERT_EQ(ret, -1); + + s = "-9323372036854775807"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, -1); } @@ -225,7 +286,7 @@ TEST(testCase, Datablock_test) { printf("binary column length:%d\n", *(int32_t*)p1->pData); - ASSERT_EQ(blockDataGetNumOfCols(b), 3); + ASSERT_EQ(blockDataGetNumOfCols(b), 2); ASSERT_EQ(blockDataGetNumOfRows(b), 40); char* pData = colDataGetData(p1, 3); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index c993efb9ae..dd62d05440 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -433,7 +433,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_TINYINT: { - code = toIntegerEx(pToken->z, pToken->n, &iv); + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(iv)) { @@ -445,7 +445,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UTINYINT: { - code = toUIntegerEx(pToken->z, pToken->n, &uv); + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); } else if (uv > UINT8_MAX) { @@ -456,7 +456,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_SMALLINT: { - code = toIntegerEx(pToken->z, pToken->n, &iv); + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(iv)) { @@ -467,7 +467,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_USMALLINT: { - code = toUIntegerEx(pToken->z, pToken->n, &uv); + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); } else if (uv > UINT16_MAX) { @@ -478,7 +478,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_INT: { - code = toIntegerEx(pToken->z, pToken->n, &iv); + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z); } else if (!IS_VALID_INT(iv)) { @@ -489,7 +489,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UINT: { - code = toUIntegerEx(pToken->z, pToken->n, &uv); + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); } else if (uv > UINT32_MAX) { @@ -500,7 +500,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_BIGINT: { - code = toIntegerEx(pToken->z, pToken->n, &iv); + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); } @@ -509,7 +509,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UBIGINT: { - code = toUIntegerEx(pToken->z, pToken->n, &uv); + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); } @@ -519,11 +519,11 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, case TSDB_DATA_TYPE_FLOAT: { double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || - isnan(dv)) { + if (dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } *(float*)(&val->i64) = dv; @@ -532,8 +532,9 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, case TSDB_DATA_TYPE_DOUBLE: { double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); + code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { + return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); @@ -1359,7 +1360,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_TINYINT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(pVal->value.val)) { @@ -1368,7 +1369,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_UTINYINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); } else if (pVal->value.val > UINT8_MAX) { @@ -1377,7 +1378,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_SMALLINT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(pVal->value.val)) { @@ -1386,7 +1387,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_USMALLINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); } else if (pVal->value.val > UINT16_MAX) { @@ -1395,7 +1396,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_INT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); } else if (!IS_VALID_INT(pVal->value.val)) { @@ -1404,7 +1405,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_UINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); } else if (pVal->value.val > UINT32_MAX) { @@ -1413,27 +1414,26 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_BIGINT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); } break; } case TSDB_DATA_TYPE_UBIGINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); } break; } case TSDB_DATA_TYPE_FLOAT: { - char* endptr = NULL; double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + int32_t code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || - isnan(dv)) { + if (dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } float f = dv; @@ -1441,12 +1441,12 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_DOUBLE: { - char* endptr = NULL; double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); + int32_t code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { + if (isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); } pVal->value.val = *(int64_t*)&dv; diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 5193bdc47f..5988c68e17 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -609,6 +609,11 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) { break; } + // support float with no decimal part after the decimal point + if (z[i] == '.' && seg == 1) { + *tokenId = TK_NK_FLOAT; + i++; + } if ((z[i] == 'e' || z[i] == 'E') && (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) { i += 2; @@ -751,7 +756,7 @@ SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr, bool* pIgnoreC // support parse the -/+number format if ((isPrevOptr) && (t0.type == TK_NK_MINUS || t0.type == TK_NK_PLUS)) { len = tGetToken(&str[*i + t0.n], &type); - if (type == TK_NK_INTEGER || type == TK_NK_FLOAT) { + if (type == TK_NK_INTEGER || type == TK_NK_FLOAT || type == TK_NK_BIN || type == TK_NK_HEX) { t0.type = type; t0.n += len; } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 981a2411eb..15464eb1b3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -249,6 +249,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_double.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py new file mode 100644 index 0000000000..ce482287fb --- /dev/null +++ b/tests/system-test/1-insert/insert_double.py @@ -0,0 +1,133 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * +import random + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + self.database = "db1" + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepare_db(self): + tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database {self.database}") + tdSql.execute(f"use {self.database}") + + def test_value(self, table_name, dtype, bits): + tdSql.execute(f"drop table if exists {table_name}") + tdSql.execute(f"create table {table_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned)") + + tdSql.execute(f"insert into {table_name} values(now, -16, +6)") + tdSql.execute(f"insert into {table_name} values(now, 80.99 , +0042 )") + tdSql.execute(f"insert into {table_name} values(now, -0042 , +80.99 )") + tdSql.execute(f"insert into {table_name} values(now, 52.34354, 18.6)") + tdSql.execute(f"insert into {table_name} values(now, -12., +3.)") + tdSql.execute(f"insert into {table_name} values(now, -.12, +.3)") + tdSql.execute(f"insert into {table_name} values(now, -2.3e1, +2.324e2)") + tdSql.execute(f"insert into {table_name} values(now, -2e1, +2e2)") + tdSql.execute(f"insert into {table_name} values(now, -2.e1, +2.e2)") + tdSql.execute(f"insert into {table_name} values(now, -0x40, +0b10000)") + tdSql.execute(f"insert into {table_name} values(now, -0b10000, +0x40)") + + # str support + tdSql.execute(f"insert into {table_name} values(now, '-16', '+6')") + tdSql.execute(f"insert into {table_name} values(now, ' -80.99 ', ' +0042 ')") + tdSql.execute(f"insert into {table_name} values(now, ' -0042 ', ' +80.99 ')") + tdSql.execute(f"insert into {table_name} values(now, '52.34354', '18.6')") + tdSql.execute(f"insert into {table_name} values(now, '-12.', '+5.')") + tdSql.execute(f"insert into {table_name} values(now, '-.12', '+.5')") + tdSql.execute(f"insert into {table_name} values(now, '-2.e1', '+2.e2')") + tdSql.execute(f"insert into {table_name} values(now, '-2e1', '+2e2')") + tdSql.execute(f"insert into {table_name} values(now, '-2.3e1', '+2.324e2')") + tdSql.execute(f"insert into {table_name} values(now, '-0x40', '+0b10010')") + tdSql.execute(f"insert into {table_name} values(now, '-0b10010', '+0x40')") + + tdSql.query(f"select * from {table_name}") + tdSql.checkRows(22) + + baseval = 2**(bits/2) + negval = -baseval + 1.645 + posval = baseval + 4.323 + bigval = 2**(bits-1) + max_i = bigval - 1 + min_i = -bigval + max_u = 2*bigval - 1 + min_u = 0 + print("val:", baseval, negval, posval, max_i) + + tdSql.execute(f"insert into {table_name} values(now, {negval}, {posval})") + tdSql.execute(f"insert into {table_name} values(now, -{baseval}, {baseval})") + tdSql.execute(f"insert into {table_name} values(now, {max_i}, {max_u})") + tdSql.execute(f"insert into {table_name} values(now, {min_i}, {min_u})") + + tdSql.query(f"select * from {table_name}") + tdSql.checkRows(26) + + # fail + tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})", "error") + tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})", "error") + tdSql.error(f"insert into {table_name} values(now, {max_i+1}, -1)", "error") + + + def test_tags(self, stable_name, dtype, bits): + tdSql.execute(f"create stable {stable_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned) tags(id {dtype})") + + baseval = 2**(bits/2) + negval = -baseval + 1.645 + posval = baseval + 4.323 + bigval = 2**(bits-1) + max_i = bigval - 1 + min_i = -bigval + max_u = 2*bigval - 1 + min_u = 0 + + tdSql.execute(f"insert into {stable_name}_1 using {stable_name} tags('{negval}') values(now, {negval}, {posval})") + tdSql.execute(f"insert into {stable_name}_2 using {stable_name} tags({posval}) values(now, -{baseval} , {baseval})") + tdSql.execute(f"insert into {stable_name}_3 using {stable_name} tags('0x40') values(now, {max_i}, {max_u})") + tdSql.execute(f"insert into {stable_name}_4 using {stable_name} tags(0b10000) values(now, {min_i}, {min_u})") + + tdSql.execute(f"insert into {stable_name}_5 using {stable_name} tags({max_i}) values(now, '{negval}', '{posval}')") + tdSql.execute(f"insert into {stable_name}_6 using {stable_name} tags('{min_i}') values(now, '-{baseval}' , '{baseval}')") + tdSql.execute(f"insert into {stable_name}_7 using {stable_name} tags(-0x40) values(now, '{max_i}', '{max_u}')") + tdSql.execute(f"insert into {stable_name}_8 using {stable_name} tags('-0b10000') values(now, '{min_i}', '{min_u}')") + + tdSql.execute(f"insert into {stable_name}_9 using {stable_name} tags(12.) values(now, {negval}, {posval})") + tdSql.execute(f"insert into {stable_name}_10 using {stable_name} tags('-8.3') values(now, -{baseval} , {baseval})") + tdSql.execute(f"insert into {stable_name}_11 using {stable_name} tags(2.e1) values(now, {max_i}, {max_u})") + tdSql.execute(f"insert into {stable_name}_12 using {stable_name} tags('-2.3e1') values(now, {min_i}, {min_u})") + + tdSql.query(f"select * from {stable_name}") + tdSql.checkRows(12) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare(replica = self.replicaVar) + self.prepare_db() + + self.test_value("t1", "bigint", 64) + self.test_value("t2", "int", 32) + self.test_value("t3", "smallint", 16) + self.test_value("t4", "tinyint", 8) + tdLog.printNoPrefix("==========end case1 run ...............") + + self.test_tags("t_big", "bigint", 64) + self.test_tags("t_int", "int", 32) + self.test_tags("t_small", "smallint", 16) + self.test_tags("t_tiny", "tinyint", 8) + tdLog.printNoPrefix("==========end case2 run ...............") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase())