From 7e8f3579de3d3e37c9bf9173227bf74bd74f63c4 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 30 Nov 2023 21:06:06 +0800 Subject: [PATCH 01/94] accept float data as int data input --- include/common/ttypes.h | 2 + include/common/tvariant.h | 3 + source/common/src/tvariant.c | 190 ++++++++++++++++++++++++-- source/common/test/commonTests.cpp | 94 ++++++++++++- source/libs/parser/src/parInsertSql.c | 50 ++++--- 5 files changed, 309 insertions(+), 30 deletions(-) diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 279799b172..741e3663db 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -275,9 +275,11 @@ typedef struct { #define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) #define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX) #define IS_VALID_INT(_t) ((_t) >= INT32_MIN && (_t) <= INT32_MAX) +#define IS_VALID_INT64(_t) ((_t) >= INT64_MIN && (_t) <= INT64_MAX) #define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) <= UINT8_MAX) #define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX) #define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) <= UINT32_MAX) +#define IS_VALID_UINT64(_t) ((_t) >= 0 && (_t) <= UINT64_MAX) #define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) #define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) diff --git a/include/common/tvariant.h b/include/common/tvariant.h index 130945cce5..61cf277bb7 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -37,6 +37,9 @@ typedef struct SVariant { }; } SVariant; +int32_t toIntegerEx(const char *z, int32_t n, int64_t *value); +int32_t toUIntegerEx(const char *z, int32_t n, uint64_t *value); + int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value); int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value); diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 5f2796260c..adb0964fe6 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -19,6 +19,181 @@ #include "ttokendef.h" #include "tvariant.h" +int32_t parseBinaryUInteger(const char *z, int32_t n, uint64_t *value) { + // skip head 0b + const char *p = z + 2; + int32_t l = n - 2; + + while (*p == '0' && l > 0) { + p++; + l--; + } + if (l > 64) { // too big + return TSDB_CODE_FAILED; + } + + uint64_t val = 0; + for (int32_t i = 0; i < l; i++) { + val = val << 1; + if (p[i] == '1') { + val |= 1; + } else if (p[i] != '0') { // abnormal char + return TSDB_CODE_FAILED; + } + } + *value = val; + return TSDB_CODE_SUCCESS; +} + +int32_t parseDecimalUInteger(const char *z, int32_t n, uint64_t *value) { + errno = 0; + char *endPtr = NULL; + while (*z == '0' && n > 1) { + z++; + n--; + } + *value = taosStr2UInt64(z, &endPtr, 10); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t parseHexUInteger(const char *z, int32_t n, uint64_t *value) { + errno = 0; + char *endPtr = NULL; + *value = taosStr2UInt64(z, &endPtr, 16); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *value) { + *is_neg = false; + if (n < 1) { + return TSDB_CODE_FAILED; + } + + // parse sign + bool has_sign = false; + if (z[0] == '-') { + *is_neg = true; + has_sign = true; + } else if (z[0] == '+') { + has_sign = true; + } else if (z[0] < '0' || z[0] > '9') { + return TSDB_CODE_FAILED; + } + if (has_sign) { + z++; + n--; + if (n < 1) { + return TSDB_CODE_FAILED; + } + } + + if (n > 2 && z[0] == '0') { + if (z[1] == 'b' || z[1] == 'B') { + // paring as binary + return parseBinaryUInteger(z, n, value); + } + + if (z[1] == 'x' || z[1] == 'X') { + // parsing as hex + return parseHexUInteger(z, n, value); + } + } + + //rm flag u-unsigned, l-long, f-float(if not in hex str) + char last = tolower(z[n-1]); + if (last == 'u' || last == 'l' || last == 'f') { + n--; + if (n < 1) { + return TSDB_CODE_FAILED; + } + } + + // parsing as decimal + bool parse_int = true; + for (int32_t i = 0; i < n; i++) { + if (z[i] < '0' || z[i] > '9') { + parse_int = false; + break; + } + } + if (parse_int) { + return parseDecimalUInteger(z, n, value); + } + + // parsing as double + errno = 0; + char *endPtr = NULL; + double val = taosStr2Double(z, &endPtr); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n || !IS_VALID_UINT64(val)) { + return TSDB_CODE_FAILED; + } + *value = val; + return TSDB_CODE_SUCCESS; +} + +int32_t removeSpace(const char **pp, int32_t n) { + // rm blank space from both head and tail + const char *z = *pp; + while (*z == ' ' && n > 0) { + z++; + n--; + } + if (n > 0) { + for (int32_t i = n - 1; i > 0; i--) { + if (z[i] == ' ') { + n--; + } else { + break; + } + } + } + *pp = z; + return n; +} + +int32_t toIntegerEx(const char *z, int32_t n, int64_t *value) { + n = removeSpace(&z, n); + if (n < 1) { // fail: all char is space + return TSDB_CODE_FAILED; + } + + bool is_neg = false; + uint64_t uv = 0; + int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv); + if (code == TSDB_CODE_SUCCESS) { + // truncate into int64 + if (uv > INT64_MAX) { + *value = is_neg ? INT64_MIN : INT64_MAX; + return TSDB_CODE_FAILED; + } else { + *value = is_neg ? -uv : uv; + } + } + + return code; +} + +int32_t toUIntegerEx(const char *z, int32_t n, uint64_t *value) { + n = removeSpace(&z, n); + if (n < 1) { // fail: all char is space + return TSDB_CODE_FAILED; + } + + bool is_neg = false; + int32_t code = parseSignAndUInteger(z, n, &is_neg, value); + if (is_neg) { + return TSDB_CODE_FAILED; + } + return code; +} + int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) { errno = 0; char *endPtr = NULL; @@ -26,10 +201,10 @@ int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) { *value = taosStr2Int64(z, &endPtr, base); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { errno = 0; - return -1; + return TSDB_CODE_FAILED; } - return 0; + return TSDB_CODE_SUCCESS; } int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) { @@ -39,16 +214,15 @@ int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) { const char *p = z; while (*p == ' ') p++; if (*p == '-') { - return -1; + return TSDB_CODE_FAILED; } *value = taosStr2UInt64(z, &endPtr, base); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { errno = 0; - return -1; + return TSDB_CODE_FAILED; } - - return 0; + return TSDB_CODE_SUCCESS; } /** @@ -147,14 +321,13 @@ void taosVariantDestroy(SVariant *pVar) { taosMemoryFreeClear(pVar->pz); pVar->nLen = 0; } - } void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { if (pSrc == NULL || pDst == NULL) return; pDst->nType = pSrc->nType; - if (pSrc->nType == TSDB_DATA_TYPE_BINARY ||pSrc->nType == TSDB_DATA_TYPE_VARBINARY || + if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_VARBINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR || pSrc->nType == TSDB_DATA_TYPE_JSON || pSrc->nType == TSDB_DATA_TYPE_GEOMETRY) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; @@ -172,7 +345,6 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) { pDst->i = pSrc->i; } - } int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) { diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 107276d7f9..035bca0e15 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -24,6 +24,93 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } + +TEST(testCase, toIntegerEx_test) { + int64_t val = 0; + + char* s = "123"; + int32_t ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 123); + + s = "9223372036854775807"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 9223372036854775807); + + s = "9323372036854775807"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, -1); + + s = "-9323372036854775807"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, -1); + + s = "-1"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -1); + + s = "-9223372036854775807"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -9223372036854775807); + + s = "1000u"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 1000); + + s = "1000l"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 1000); + + s = "1000.0f"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 1000); + + s = "0x1f"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 31); + + s = "-0x40"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -64); + + s = "0b110"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 6); + + s = "-0b10010"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -18); + + s = "-5.2343544534e10"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -52343544534); + + s = "1.869895343e4"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 18698); + + // UINT64_MAX + s = "18446744073709551615"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, -1); + + s = "18446744073709551616"; + ret = toIntegerEx(s, strlen(s), &val); + ASSERT_EQ(ret, -1); +} + TEST(testCase, toInteger_test) { char* s = "123"; uint32_t type = 0; @@ -39,10 +126,9 @@ TEST(testCase, toInteger_test) { ASSERT_EQ(ret, 0); ASSERT_EQ(val, 9223372036854775807); - s = "9323372036854775807"; + s = "9323372036854775807"; // out of range, > int64_max ret = toInteger(s, strlen(s), 10, &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 9323372036854775807u); + ASSERT_EQ(ret, -1); s = "-9323372036854775807"; ret = toInteger(s, strlen(s), 10, &val); @@ -139,7 +225,7 @@ TEST(testCase, Datablock_test) { printf("binary column length:%d\n", *(int32_t*)p1->pData); - ASSERT_EQ(blockDataGetNumOfCols(b), 2); + ASSERT_EQ(blockDataGetNumOfCols(b), 3); ASSERT_EQ(blockDataGetNumOfRows(b), 40); char* pData = colDataGetData(p1, 3); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 31b016458a..c993efb9ae 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -433,7 +433,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_TINYINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + code = toIntegerEx(pToken->z, pToken->n, &iv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z); @@ -444,7 +445,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UTINYINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + code = toUIntegerEx(pToken->z, pToken->n, &uv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); } else if (uv > UINT8_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); @@ -454,7 +456,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_SMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + code = toIntegerEx(pToken->z, pToken->n, &iv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z); @@ -464,7 +467,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_USMALLINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + code = toUIntegerEx(pToken->z, pToken->n, &uv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); } else if (uv > UINT16_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); @@ -474,7 +478,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_INT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + code = toIntegerEx(pToken->z, pToken->n, &iv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z); } else if (!IS_VALID_INT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z); @@ -484,7 +489,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + code = toUIntegerEx(pToken->z, pToken->n, &uv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); } else if (uv > UINT32_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); @@ -494,7 +500,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_BIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + code = toIntegerEx(pToken->z, pToken->n, &iv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); } val->i64 = iv; @@ -502,7 +509,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UBIGINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + code = toUIntegerEx(pToken->z, pToken->n, &uv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); } *(uint64_t*)(&val->i64) = uv; @@ -1351,7 +1359,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_TINYINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "tinyint data overflow", pToken->z); @@ -1359,7 +1368,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_UTINYINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); } else if (pVal->value.val > UINT8_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); @@ -1367,7 +1377,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_SMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); @@ -1375,7 +1386,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_USMALLINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); } else if (pVal->value.val > UINT16_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); @@ -1383,7 +1395,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_INT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); } else if (!IS_VALID_INT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); @@ -1391,7 +1404,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_UINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); } else if (pVal->value.val > UINT32_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); @@ -1399,14 +1413,16 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_BIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { + int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); } break; } case TSDB_DATA_TYPE_UBIGINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); + int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + if (TSDB_CODE_SUCCESS != code) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); } break; } From d368956e3fc438c809338294b899f930351aae95 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 24 Nov 2023 16:24:12 +0800 Subject: [PATCH 02/94] fix timezone range --- source/common/src/ttime.c | 6 +++++- source/os/src/osFile.c | 18 +++++++++--------- source/os/test/osTests.cpp | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 227de7f5fc..cbe700fa2d 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -170,10 +170,14 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { hour = strnatoi(&str[i], len); i += len + 1; } else { - hour = strnatoi(&str[i], 2); + hour = strnatoi(&str[i], strlen(&str[i])); i += 2; } + if (hour > 12 || hour < 0) { + return -1; + } + // return error if there're illegal charaters after min(2 Digits) char* minStr = &str[i]; if (minStr[1] != '\0' && minStr[2] != '\0') { diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 15aca85fc2..39de117339 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -348,15 +348,15 @@ HANDLE taosOpenFileNotStream(const char *path, int32_t tdFileOptions) { if (h != INVALID_HANDLE_VALUE && (tdFileOptions & TD_FILE_APPEND) && (tdFileOptions & TD_FILE_WRITE)) { SetFilePointer(h, 0, NULL, FILE_END); } - if (h == INVALID_HANDLE_VALUE) { - DWORD dwError = GetLastError(); - LPVOID lpMsgBuf; - FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError, - 0, - (LPTSTR)&lpMsgBuf, 0, NULL); - printf("CreateFile failed with error %d: %s", dwError, (char*)lpMsgBuf); - LocalFree(lpMsgBuf); - } + // if (h == INVALID_HANDLE_VALUE) { + // DWORD dwError = GetLastError(); + // LPVOID lpMsgBuf; + // FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError, + // 0, + // (LPTSTR)&lpMsgBuf, 0, NULL); + // printf("CreateFile failed with error %d: %s", dwError, (char*)lpMsgBuf); + // LocalFree(lpMsgBuf); + // } return h; } diff --git a/source/os/test/osTests.cpp b/source/os/test/osTests.cpp index e2185aeac2..9ab58a180a 100644 --- a/source/os/test/osTests.cpp +++ b/source/os/test/osTests.cpp @@ -314,6 +314,6 @@ TEST(osTest, osFilePerformance) { printf("Test OpenForRead & Close file %d times, cost: %" PRId64 "us\n", TESTTIMES, OpenForReadCloseFileCost); } -#endif OSFILE_PERFORMANCE_TEST +#endif // OSFILE_PERFORMANCE_TEST #pragma GCC diagnostic pop From 8180aaeb6739520d80fb852e2373ae2b6761b2fe Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 24 Nov 2023 17:56:50 +0800 Subject: [PATCH 03/94] fix time format --- source/common/src/ttime.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index cbe700fa2d..cdc380dee0 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -170,7 +170,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { hour = strnatoi(&str[i], len); i += len + 1; } else { - hour = strnatoi(&str[i], strlen(&str[i])); + hour = strnatoi(&str[i], 2); i += 2; } @@ -185,7 +185,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { } int64_t minute = strnatoi(&str[i], 2); - if (minute > 59) { + if (minute > 59 || (hour == 12 && minute > 0)) { return -1; } From fc905406d52d79266f27fbf0a9d0e96f6c260994 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 14:08:24 +0800 Subject: [PATCH 04/94] chore: validate tag val --- source/common/src/tdataformat.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index f97f9c0c11..5eebf05596 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1247,6 +1247,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { if (isJson) { n += tPutCStr(p ? p + n : p, pTagVal->pKey); } else { + ASSERTS(pTagVal->cid > 0, "Invalid tag cid %" PRIi16, pTagVal->cid); n += tPutI16v(p ? p + n : p, pTagVal->cid); } From 7d8864ec89ff4f18ff31aba677935500f8300349 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 14:22:49 +0800 Subject: [PATCH 05/94] chore: validate tag val --- source/common/src/tdataformat.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 5eebf05596..fe5374d5da 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1247,8 +1247,8 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { if (isJson) { n += tPutCStr(p ? p + n : p, pTagVal->pKey); } else { - ASSERTS(pTagVal->cid > 0, "Invalid tag cid %" PRIi16, pTagVal->cid); n += tPutI16v(p ? p + n : p, pTagVal->cid); + ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); } // type @@ -1273,6 +1273,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { n += tGetCStr(p + n, &pTagVal->pKey); } else { n += tGetI16v(p + n, &pTagVal->cid); + ASSERTS(pTagVal->cid < 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); } // type From 4cc22a1b90b8bc2860d8694361f05c7444d76189 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 14:26:34 +0800 Subject: [PATCH 06/94] chore: validate tag val --- source/common/src/tdataformat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index fe5374d5da..cf0dd9ecdb 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1273,7 +1273,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { n += tGetCStr(p + n, &pTagVal->pKey); } else { n += tGetI16v(p + n, &pTagVal->cid); - ASSERTS(pTagVal->cid < 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); + ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); } // type From 210a304cc123df9ce01266f9871f3028fba078f0 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 4 Dec 2023 14:41:28 +0800 Subject: [PATCH 07/94] Update tdataformat.c --- source/common/src/tdataformat.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index cf0dd9ecdb..02dfbfebfe 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1273,7 +1273,6 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { n += tGetCStr(p + n, &pTagVal->pKey); } else { n += tGetI16v(p + n, &pTagVal->cid); - ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); } // type From c83b7b80de4559e41d930371ffbf5832d62e310d Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 4 Dec 2023 14:46:28 +0800 Subject: [PATCH 08/94] test case --- tests/system-test/2-query/timezone.py | 51 ++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/timezone.py b/tests/system-test/2-query/timezone.py index 316e776cb1..970ee0ff3b 100644 --- a/tests/system-test/2-query/timezone.py +++ b/tests/system-test/2-query/timezone.py @@ -112,10 +112,59 @@ class TDTestCase: self.data_check(timezone,self.stbname,'stable') for i in range(self.tbnum): self.data_check(timezone,f'{self.stbname}_{i}','child_table') + tdSql.execute(f'drop database {self.dbname}') + + def timezone_format_test(self): + tdSql.execute(f'create database {self.dbname}') + tdSql.execute(self.setsql.set_create_stable_sql(f'{self.dbname}.stb', {'ts':'timestamp','id':'int'}, {'status':'int'})) + + tdSql.execute(f"insert into {self.dbname}.d0 using {self.dbname}.stb tags (1) values ('2021-07-01 00:00:00.000',0);") + tdSql.query(f"select ts from {self.dbname}.d0;") + tdSql.checkData(0, 0, "2021-07-01 00:00:00.000") + + tdSql.execute(f"insert into {self.dbname}.d1 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+07:50',1)") + tdSql.query(f"select ts from {self.dbname}.d1") + tdSql.checkData(0, 0, "2021-07-01 00:10:00.000") + + tdSql.execute(f"insert into {self.dbname}.d2 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+12:00',1)") + tdSql.query(f"select ts from {self.dbname}.d2") + tdSql.checkData(0, 0, "2021-06-30 20:00:00.000") + + tdSql.execute(f"insert into {self.dbname}.d3 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-00:10',1)") + tdSql.query(f"select ts from {self.dbname}.d3") + tdSql.checkData(0, 0, "2021-07-01 08:10:00.000") + + tdSql.execute(f"insert into {self.dbname}.d4 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-12:00',1)") + tdSql.query(f"select ts from {self.dbname}.d4") + tdSql.checkData(0, 0, "2021-07-01 20:00:00.000") + + tdSql.execute(f"insert into {self.dbname}.d5 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-1200',1)") + tdSql.query(f"select ts from {self.dbname}.d5") + tdSql.checkData(0, 0, "2021-07-01 20:00:00.000") + + tdSql.execute(f"insert into {self.dbname}.d6 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-115',1)") + tdSql.query(f"select ts from {self.dbname}.d6") + tdSql.checkData(0, 0, "2021-07-01 19:05:00.000") + + tdSql.execute(f"insert into {self.dbname}.d7 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-1105',1)") + tdSql.query(f"select ts from {self.dbname}.d7") + tdSql.checkData(0, 0, "2021-07-01 19:05:00.000") + + tdSql.error(f"insert into {self.dbname}.d21 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+12:10',1)") + tdSql.error(f"insert into {self.dbname}.d22 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-24:10',1)") + tdSql.error(f"insert into {self.dbname}.d23 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000+12:10',1)") + tdSql.error(f"insert into {self.dbname}.d24 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-24:10',1)") + tdSql.error(f"insert into {self.dbname}.d24 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-24100',1)") + tdSql.error(f"insert into {self.dbname}.d24 using {self.dbname}.stb tags (1) values ('2021-07-01T00:00:00.000-1210',1)") + + tdSql.execute(f'drop database {self.dbname}') + + def run(self): # sourcery skip: extract-duplicate-method timezone = self.get_system_timezone() self.timezone_check_ntb(timezone) - self.timezone_check_stb(timezone) + #self.timezone_check_stb(timezone) + self.timezone_format_test() def stop(self): tdSql.close() From 02ff87b3a3eae2c15c19b51ad338aaad4af2cf7b Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Mon, 4 Dec 2023 19:18:20 +0800 Subject: [PATCH 09/94] fix: revert test case --- tests/system-test/2-query/timezone.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/timezone.py b/tests/system-test/2-query/timezone.py index 970ee0ff3b..33bcb16595 100644 --- a/tests/system-test/2-query/timezone.py +++ b/tests/system-test/2-query/timezone.py @@ -163,7 +163,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method timezone = self.get_system_timezone() self.timezone_check_ntb(timezone) - #self.timezone_check_stb(timezone) + self.timezone_check_stb(timezone) self.timezone_format_test() def stop(self): From 08be3d790cc906727f2c4a3bfa250b37664d4664 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 20:08:46 +0800 Subject: [PATCH 10/94] fix: ignore NULL tag --- source/libs/parser/src/parInsertSql.c | 7 +++++-- tests/script/tsim/parser/columnValue_bigint.sim | 2 ++ tests/script/tsim/parser/columnValue_bool.sim | 2 ++ tests/script/tsim/parser/columnValue_float.sim | 2 ++ tests/script/tsim/parser/columnValue_int.sim | 14 ++++++++++++++ tests/script/tsim/parser/columnValue_smallint.sim | 2 ++ tests/script/tsim/parser/columnValue_tinyint.sim | 2 ++ tests/script/tsim/parser/null_char.sim | 2 ++ 8 files changed, 31 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 9347f94e8d..a3b6f01e3a 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -700,7 +700,8 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SSchema* pTagSchema, SToken* pToken, SArray* pTagName, SArray* pTagVals, STag** pTag) { - if (!isNullValue(pTagSchema->type, pToken)) { + bool isNull = isNullValue(pTagSchema->type, pToken); + if (!isNull) { taosArrayPush(pTagName, pTagSchema->name); } @@ -709,13 +710,15 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z); } - if (isNullValue(pTagSchema->type, pToken)) { + if (isNull) { return tTagNew(pTagVals, 1, true, pTag); } else { return parseJsontoTagData(pToken->z, pTagVals, pTag, &pCxt->msg); } } + if (isNull) return 0; + STagVal val = {0}; int32_t code = parseTagToken(ppSql, pToken, pTagSchema, pStmt->pTableMeta->tableInfo.precision, &val, &pCxt->msg); diff --git a/tests/script/tsim/parser/columnValue_bigint.sim b/tests/script/tsim/parser/columnValue_bigint.sim index 0a024029a5..056855eea2 100644 --- a/tests/script/tsim/parser/columnValue_bigint.sim +++ b/tests/script/tsim/parser/columnValue_bigint.sim @@ -17,6 +17,7 @@ sql create table mt_bigint (ts timestamp, c bigint) tags (tagname bigint) ## case 00: static create table for test tag values sql create table st_bigint_0 using mt_bigint tags (NULL) +sql show create table st_bigint_0 sql show tags from st_bigint_0 if $data05 != NULL then return -1 @@ -173,6 +174,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_bigint_16 using mt_bigint tags (NULL) values (now, NULL) +sql show create table st_bigint_16 sql show tags from st_bigint_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_bool.sim b/tests/script/tsim/parser/columnValue_bool.sim index ad4232d884..2553e6805a 100644 --- a/tests/script/tsim/parser/columnValue_bool.sim +++ b/tests/script/tsim/parser/columnValue_bool.sim @@ -17,6 +17,7 @@ sql create table mt_bool (ts timestamp, c bool) tags (tagname bool) ## case 00: static create table for test tag values sql create table st_bool_0 using mt_bool tags (NULL) +sql show create table st_bool_0 sql show tags from st_bool_0 if $data05 != NULL then print ==1== expect: NULL, actually: $data05 @@ -291,6 +292,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_bool_16 using mt_bool tags (NULL) values (now, NULL) +sql show create table st_bool_16 sql show tags from st_bool_16 if $data05 != NULL then print ==33== expect: NULL, actually: $data00 diff --git a/tests/script/tsim/parser/columnValue_float.sim b/tests/script/tsim/parser/columnValue_float.sim index b2db7dff2b..4dcda33224 100644 --- a/tests/script/tsim/parser/columnValue_float.sim +++ b/tests/script/tsim/parser/columnValue_float.sim @@ -17,6 +17,7 @@ sql create table mt_float (ts timestamp, c float) tags (tagname float) ## case 00: static create table for test tag values sql create table st_float_0 using mt_float tags (NULL) +sql show create table st_float_0 sql show tags from st_float_0 if $data05 != NULL then return -1 @@ -299,6 +300,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_float_16 using mt_float tags (NULL) values (now, NULL) +sql show create table st_float_16 sql show tags from st_float_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_int.sim b/tests/script/tsim/parser/columnValue_int.sim index 4a3b8ebd0b..9f0fb6e294 100644 --- a/tests/script/tsim/parser/columnValue_int.sim +++ b/tests/script/tsim/parser/columnValue_int.sim @@ -17,6 +17,7 @@ sql create table mt_int (ts timestamp, c int) tags (tagname int) ## case 00: static create table for test tag values sql create table st_int_0 using mt_int tags (NULL) +sql show create table st_int_0 sql show tags from st_int_0 if $data05 != NULL then return -1 @@ -27,6 +28,18 @@ if $data05 != NULL then return -1 endi +sql insert into mt_int(tbname,tagname,ts,c) values("st_int_00",NULL,now,1) values("st_int_01", NULL, now,2) +sql show create table st_int_00 +sql show create table st_int_01 +sql show tags from st_int_00 +if $data05 != NULL then + return -1 +endi +sql show tags from st_int_01 +if $data05 != NULL then + return -1 +endi + sql_error create table st_int_2 using mt_int tags ('NULL') sql_error create table st_int_3 using mt_int tags ('NULL') sql_error create table st_int_4 using mt_int tags ("NULL") @@ -172,6 +185,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_int_16 using mt_int tags (NULL) values (now, NULL) +sql show create table st_int_16 sql show tags from st_int_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_smallint.sim b/tests/script/tsim/parser/columnValue_smallint.sim index eb364f3630..f9be6ebd52 100644 --- a/tests/script/tsim/parser/columnValue_smallint.sim +++ b/tests/script/tsim/parser/columnValue_smallint.sim @@ -20,6 +20,7 @@ sql create table mt_smallint (ts timestamp, c smallint) tags (tagname smallint) ## case 00: static create table for test tag values sql create table st_smallint_0 using mt_smallint tags (NULL) +sql show create table st_smallint_0 sql show tags from st_smallint_0 if $data05 != NULL then return -1 @@ -175,6 +176,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_smallint_16 using mt_smallint tags (NULL) values (now, NULL) +sql show create table st_smallint_16 sql show tags from st_smallint_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/columnValue_tinyint.sim b/tests/script/tsim/parser/columnValue_tinyint.sim index d7938aa739..7d0f10a30d 100644 --- a/tests/script/tsim/parser/columnValue_tinyint.sim +++ b/tests/script/tsim/parser/columnValue_tinyint.sim @@ -17,6 +17,7 @@ sql create table mt_tinyint (ts timestamp, c tinyint) tags (tagname tinyint) ## case 00: static create table for test tag values sql create table st_tinyint_0 using mt_tinyint tags (NULL) +sql show create table st_tinyint_0 sql show tags from st_tinyint_0 if $data05 != NULL then print expect NULL, actually: $data05 @@ -173,6 +174,7 @@ endi ## case 02: dynamic create table for test tag values sql insert into st_tinyint_16 using mt_tinyint tags (NULL) values (now, NULL) +sql show create table st_tinyint_16 sql show tags from st_tinyint_16 if $data05 != NULL then return -1 diff --git a/tests/script/tsim/parser/null_char.sim b/tests/script/tsim/parser/null_char.sim index 098f4061c1..9d476b5227 100644 --- a/tests/script/tsim/parser/null_char.sim +++ b/tests/script/tsim/parser/null_char.sim @@ -13,6 +13,7 @@ sql use $db #### case 0: field NULL, or 'NULL' sql create table mt1 (ts timestamp, col1 int, col2 bigint, col3 float, col4 double, col5 binary(8), col6 bool, col7 smallint, col8 tinyint, col9 nchar(8)) tags (tag1 binary(8), tag2 nchar(8), tag3 int, tag4 bigint, tag5 bool, tag6 float) sql create table st1 using mt1 tags (NULL, 'NULL', 100, 1000, 'false', 9.123) +sql show create table st1 sql insert into st1 values ('2019-01-01 09:00:00.000', 123, -123, 3.0, 4.0, 'binary', true, 1000, 121, 'nchar') sql insert into st1 values ('2019-01-01 09:00:01.000', '456', '456', '3.33', '4.444', 'binary', 'true', '1001', '122', 'nchar') sql insert into st1 values ('2019-01-01 09:00:02.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) @@ -117,6 +118,7 @@ if $rows != 0 then endi sql create table st3 using mt2 tags (NULL, 'ABC', 103, 'FALSE') +sql show create table st3 sql insert into st3 (ts, col1) values(now, 1) sql select tag1, tag2, tag3, tag5 from st3 if $rows != 1 then From f3a075ee9e5b897c949fba69e3c2d2f46ff65264 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 21:00:13 +0800 Subject: [PATCH 11/94] test: check null json tag --- tests/system-test/2-query/json_tag.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index ebd580efd4..8db92f38a2 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -394,9 +394,12 @@ class TDTestCase: tdSql.execute(f"create table if not exists {dbname}.jsons3(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") tdSql.execute(f"insert into {dbname}.jsons3_1 using {dbname}.jsons3 tags('{{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}}') values(1591060618000, 3, false, 'json3', '你是3')") tdSql.execute(f"insert into {dbname}.jsons3_2 using {dbname}.jsons3 tags('{{\"tag1\":5,\"tag2\":\"beijing\"}}') values (1591060638000, 2, true, 'json3', 'sss')") + tdSql.execute(f"insert into {dbname}.jsons3_3 using {dbname}.jsons3 tags(NULL) values (1591060638000, 2, true, 'json3', 'sss')") tdSql.query(f"select 'sss',33,a.jtag->'tag3' from {dbname}.jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") tdSql.checkData(0, 0, "sss") tdSql.checkData(0, 2, "true") + tdSql.query(f"show create table jsons3_3") + tdSql.checkNotEqual(tdSql.queryResult[0][1].find("TAGS (null)"), 0) res = tdSql.getColNameList(f"select 'sss',33,a.jtag->'tag3' from {dbname}.jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") cname_list = [] From 646d807a28acd2549c0b6fd731d05e3be7f30e93 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 4 Dec 2023 22:43:04 +0800 Subject: [PATCH 12/94] test: check null tag when insert by stb --- tests/script/tsim/insert/insert_stb.sim | 23 ++++++++++++++++++++ tests/script/tsim/parser/columnValue_int.sim | 12 ---------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/script/tsim/insert/insert_stb.sim b/tests/script/tsim/insert/insert_stb.sim index 4c0797e2a7..14ff2261c2 100644 --- a/tests/script/tsim/insert/insert_stb.sim +++ b/tests/script/tsim/insert/insert_stb.sim @@ -66,6 +66,29 @@ if $rows != 1 then return -1 endi +sql insert into st(tbname, ts, f, t) values('ct2',now,20,NULL)('ct3',now,30,NULL) +sql insert into st(tbname, t, ts, f) values('ct4',NULL, now,20)('ct5',NULL, now,30) +sql show create table ct2 +sql show create table ct3 +sql show create table ct4 +sql show create table ct5 +sql show tags from ct2 +if $data05 != NULL then + return -1 +endi +sql show tags from ct3 +if $data05 != NULL then + return -1 +endi +sql show tags from ct4 +if $data05 != NULL then + return -1 +endi +sql show tags from ct5 +if $data05 != NULL then + return -1 +endi + sql_error insert into d2.st values(now, 1, 1) sql_error insert into d2.st(ts, f) values(now, 1); sql_error insert into d2.st(ts, f, tbname) values(now, 1); diff --git a/tests/script/tsim/parser/columnValue_int.sim b/tests/script/tsim/parser/columnValue_int.sim index 9f0fb6e294..e68ae6f13f 100644 --- a/tests/script/tsim/parser/columnValue_int.sim +++ b/tests/script/tsim/parser/columnValue_int.sim @@ -28,18 +28,6 @@ if $data05 != NULL then return -1 endi -sql insert into mt_int(tbname,tagname,ts,c) values("st_int_00",NULL,now,1) values("st_int_01", NULL, now,2) -sql show create table st_int_00 -sql show create table st_int_01 -sql show tags from st_int_00 -if $data05 != NULL then - return -1 -endi -sql show tags from st_int_01 -if $data05 != NULL then - return -1 -endi - sql_error create table st_int_2 using mt_int tags ('NULL') sql_error create table st_int_3 using mt_int tags ('NULL') sql_error create table st_int_4 using mt_int tags ("NULL") From cb8dfae0cccdabad378c29fad9460f90d05097a2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 5 Dec 2023 10:47:19 +0800 Subject: [PATCH 13/94] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 131 ++++++++++++++++++------ 1 file changed, 99 insertions(+), 32 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index bb80478d73..0753c2454d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -47,6 +47,8 @@ static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScan STsdbReader* pReader, SRow** pTSRow); static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader); +static int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pBlockScanInfo, + STsdbReader* pReader); static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost); static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr, @@ -1741,6 +1743,7 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SRowMerger* pMerger = &pReader->status.merger; + int32_t code = TSDB_CODE_SUCCESS; // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized if (pMerger->pArray == NULL) { @@ -1751,45 +1754,68 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* } } - if (hasDataInFileBlock(pBlockData, pDumpInfo)) { - // no last block available, only data block exists - if (!hasDataInSttBlock(pSttBlockReader)) { - return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); - } - - // row in last file block + bool dataInDataFile = hasDataInFileBlock(pBlockData, pDumpInfo); + bool dataInSttFile = hasDataInSttBlock(pSttBlockReader); + if (dataInDataFile && (!dataInSttFile)) { + // no stt file block available, only data block exists + return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); + } else if ((!dataInDataFile) && dataInSttFile) { + // no data ile block exists + return mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader); + } else { + // row in both stt file blocks and data file blocks TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); int64_t tsLast = getCurrentKeyInSttBlock(pSttBlockReader); if (ASCENDING_TRAVERSE(pReader->info.order)) { - if (key < tsLast) { + if (key < tsLast) { // asc return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key > tsLast) { - return doMergeFileBlockAndLastBlock(pSttBlockReader, pReader, pBlockScanInfo, NULL, false); + return mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader); } - } else { + } else { // desc if (key > tsLast) { return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key < tsLast) { - return doMergeFileBlockAndLastBlock(pSttBlockReader, pReader, pBlockScanInfo, NULL, false); + return mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader); } } + // the following for key == tsLast + // ASC: file block ------> stt block + // DESC: stt block ------> file block SRow* pTSRow = NULL; - int32_t code = tsdbRowMergerAdd(pMerger, &fRow, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; + if (ASCENDING_TRAVERSE(pReader->info.order)) { + code = tsdbRowMergerAdd(pMerger, &fRow, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + + TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); + code = tsdbRowMergerAdd(pMerger, pRow1, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); + } else { + TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); + code = tsdbRowMergerAdd(pMerger, pRow1, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); + + code = tsdbRowMergerAdd(pMerger, &fRow, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); - - TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); - code = tsdbRowMergerAdd(pMerger, pRow1, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); - code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1800,9 +1826,6 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* taosMemoryFree(pTSRow); tsdbRowMergerClear(pMerger); return code; - - } else { // only last block exists - return doMergeFileBlockAndLastBlock(pSttBlockReader, pReader, pBlockScanInfo, NULL, false); } } @@ -1889,8 +1912,8 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } } - // ASC: file block -----> last block -----> imem -----> mem - // DESC: mem -----> imem -----> last block -----> file block + // ASC: file block -----> stt block -----> imem -----> mem + // DESC: mem -----> imem -----> stt block -----> file block if (ASCENDING_TRAVERSE(pReader->info.order)) { if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); @@ -2200,6 +2223,50 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc } } +int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { + bool copied = false; + SRow* pTSRow = NULL; + int64_t tsLastBlock = getCurrentKeyInSttBlock(pSttBlockReader); + + SRowMerger* pMerger = &pReader->status.merger; + TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree); + TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; + + tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " %s", pRow->pBlockData, pRow->iRow, pSttBlockReader->uid, + fRow.pBlockData->aTSKEY[fRow.iRow], pReader->idStr); + + int32_t code = tryCopyDistinctRowFromSttBlock(&fRow, pSttBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); + if (code) { + return code; + } + + if (copied) { + pBlockScanInfo->lastProcKey = tsLastBlock; + return TSDB_CODE_SUCCESS; + } else { + code = tsdbRowMergerAdd(pMerger, &fRow, NULL); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); + tsdbRowMergerAdd(pMerger, pRow1, NULL); + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, + pReader->idStr); + + code = tsdbRowMergerGetRow(pMerger, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); + + taosMemoryFree(pTSRow); + tsdbRowMergerClear(pMerger); + return code; + } +} + static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SSttBlockReader* pSttBlockReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -2221,17 +2288,17 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pSttBlockReader); } - // imem + file + last block + // imem + file + stt block if (pBlockScanInfo->iiter.hasVal) { return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pSttBlockReader); } - // mem + file + last block + // mem + file + stt block if (pBlockScanInfo->iter.hasVal) { return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pSttBlockReader); } - // files data blocks + last block + // files data blocks + stt block return mergeFileBlockAndSttBlock(pReader, pSttBlockReader, key, pBlockScanInfo, pBlockData); } From dd53d570219ade0329a8c6760dc1ffe3f7827c66 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 5 Dec 2023 10:51:19 +0800 Subject: [PATCH 14/94] fix: strncpy replace with memcpy on 3.0 --- source/client/src/clientEnv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 212c42125e..85624c31c5 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -732,7 +732,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { return -1; } newstr[0] = '"'; - strncpy(newstr+1, str, len); + memcpy(newstr+1, str, len); newstr[len + 1] = '"'; newstr[len + 2] = '\0'; str = newstr; From 07301130c5e19676f0b620d0153d392edbdea6f2 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Tue, 5 Dec 2023 11:18:07 +0800 Subject: [PATCH 15/94] enh: event_window + super table need partition by tbname --- source/libs/parser/src/parTranslater.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5c30384a6b..1ef74349b0 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7388,6 +7388,11 @@ static int32_t subtableExprHasColumnOrPseudoColumn(SNode* pNode) { static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + int8_t tableType = ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType; + if (TSDB_SUPER_TABLE == tableType && !hasPartitionByTbname(((SSelectStmt*)pStmt->pQuery)->pPartitionByList)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Stream query on super tables must patitioned by table name"); + } if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || !isTimeLineQuery(pStmt->pQuery) || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList || crossTableWithUdaf(pSelect) || hasJsonTypeProjection(pSelect)) { From e0a14bd359f359c00a8420ad2c422f2e51fa542d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 5 Dec 2023 14:05:16 +0800 Subject: [PATCH 16/94] fix(tsdb/cache): remove debug log --- source/dnode/vnode/src/tsdb/tsdbCache.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 5076599753..4b26697464 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1797,8 +1797,8 @@ static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsRea } if (record.version <= pReader->info.verRange.maxVer) { - tsdbError("tomb xx load/cache: vgId:%d fid:%d commit %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records", - TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid); + /*tsdbError("tomb xx load/cache: vgId:%d fid:%d record %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records", + TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid);*/ SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey}; taosArrayPush(pInfo->pTombData, &delData); From 2019e95049ceb18c41b8ca3b9aec48e6b4e3cad6 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 5 Dec 2023 14:08:17 +0800 Subject: [PATCH 17/94] fix: stream for event window --- source/libs/parser/src/parTranslater.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1ef74349b0..ac92c9a427 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7388,8 +7388,10 @@ static int32_t subtableExprHasColumnOrPseudoColumn(SNode* pNode) { static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; - int8_t tableType = ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType; - if (TSDB_SUPER_TABLE == tableType && !hasPartitionByTbname(((SSelectStmt*)pStmt->pQuery)->pPartitionByList)) { + if ( (SRealTableNode*)pSelect->pFromTable && ((SRealTableNode*)pSelect->pFromTable)->pMeta + && TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType + && !hasPartitionByTbname(pSelect->pPartitionByList) + && pSelect->pWindow != NULL && pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Stream query on super tables must patitioned by table name"); } From f91d98df2588b1a8b7db8753506d123edfb2cb88 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 5 Dec 2023 14:11:58 +0800 Subject: [PATCH 18/94] fix: error message --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ac92c9a427..88bfcec682 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7393,7 +7393,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm && !hasPartitionByTbname(pSelect->pPartitionByList) && pSelect->pWindow != NULL && pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Stream query on super tables must patitioned by table name"); + "Event window for stream on super table must patitioned by table name"); } if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || !isTimeLineQuery(pStmt->pQuery) || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList || From a6c876bbc15f082bc144306c8eaee03256fdb03e Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 5 Dec 2023 14:16:01 +0800 Subject: [PATCH 19/94] fix: last/last_row crash with dropped column --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 53 ++++++-- source/libs/executor/src/cachescanoperator.c | 6 + tests/system-test/2-query/last_cache_scan.py | 135 +++++++++++++++++++ 3 files changed, 185 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index b6aa791cf0..f668ea5f72 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -22,19 +22,34 @@ #define HASTYPE(_type, _t) (((_type) & (_t)) == (_t)) +static void setFirstLastResColToNull(SColumnInfoData* pCol, int32_t row) { + char *buf = taosMemoryCalloc(1, pCol->info.bytes); + SFirstLastRes* pRes = (SFirstLastRes*)((char*)buf + VARSTR_HEADER_SIZE); + pRes->bytes = 0; + pRes->hasResult = true; + pRes->isNull = true; + varDataSetLen(buf, pCol->info.bytes - VARSTR_HEADER_SIZE); + colDataSetVal(pCol, row, buf, false); + taosMemoryFree(buf); +} + static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* pReader, const int32_t* slotIds, const int32_t* dstSlotIds, void** pRes, const char* idStr) { int32_t numOfRows = pBlock->info.rows; // bool allNullRow = true; if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST)) { - uint64_t ts = 0; - SFirstLastRes* p; - col_id_t colId; + uint64_t ts = TSKEY_MIN; + SFirstLastRes* p = NULL; + col_id_t colId = -1; for (int32_t i = 0; i < pReader->numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]); - int32_t slotId = slotIds[i]; - SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); + if (slotIds[i] == -1) { + setFirstLastResColToNull(pColInfoData, numOfRows); + continue; + } + int32_t slotId = slotIds[i]; + SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); colId = pColVal->colVal.cid; p = (SFirstLastRes*)varDataVal(pRes[i]); @@ -63,10 +78,14 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p for (int32_t idx = 0; idx < taosArrayGetSize(pBlock->pDataBlock); ++idx) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, idx); if (pCol->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID && pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - colDataSetVal(pCol, numOfRows, (const char*)&ts, false); + if (ts == TSKEY_MIN) { + colDataSetNULL(pCol, numOfRows); + } else { + colDataSetVal(pCol, numOfRows, (const char*)&ts, false); + } continue; - } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && pCol->info.colId == colId) { - if (!p->isNull) { + } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && (pCol->info.colId == colId || colId == -1)) { + if (p && !p->isNull) { colDataSetVal(pCol, numOfRows, p->buf, false); } else { colDataSetNULL(pCol, numOfRows); @@ -81,6 +100,10 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]); int32_t slotId = slotIds[i]; + if (slotId == -1) { + colDataSetNULL(pColInfoData, numOfRows); + continue; + } SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); SColVal* pVal = &pColVal->colVal; @@ -300,7 +323,13 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } for (int32_t j = 0; j < pr->numOfCols; ++j) { - pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + pr->pSchema->columns[slotIds[j]].bytes + VARSTR_HEADER_SIZE); + int32_t bytes; + if (slotIds[j] == -1) + bytes = 1; + else + bytes = pr->pSchema->columns[slotIds[j]].bytes; + + pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + bytes + VARSTR_HEADER_SIZE); SFirstLastRes* p = (SFirstLastRes*)varDataVal(pRes[j]); p->ts = INT64_MIN; } @@ -324,6 +353,11 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 for (int32_t i = 0; i < pr->numOfCols; ++i) { int32_t slotId = slotIds[i]; + if (slotId == -1) { + SLastCol p = {.ts = INT64_MIN, .colVal.type = TSDB_DATA_TYPE_BOOL, .colVal.flag = CV_FLAG_NULL}; + taosArrayPush(pLastCols, &p); + continue; + } struct STColumn* pCol = &pr->pSchema->columns[slotId]; SLastCol p = {.ts = INT64_MIN, .colVal.type = pCol->type, .colVal.flag = CV_FLAG_NULL}; @@ -348,6 +382,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 bool hasNotNullRow = true; int64_t singleTableLastTs = INT64_MAX; for (int32_t k = 0; k < pr->numOfCols; ++k) { + if (slotIds[k] == -1) continue; SLastCol* p = taosArrayGet(pLastCols, k); SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, k); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 6d59698855..75fe3c51dd 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -352,6 +352,7 @@ int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTask for (int32_t i = 0; i < numOfCols; ++i) { SColMatchItem* pColMatch = taosArrayGet(pColMatchInfo, i); + bool found = false; for (int32_t j = 0; j < pWrapper->nCols; ++j) { /* if (pColMatch->colId == pWrapper->pSchema[j].colId && pColMatch->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { (*pSlotIds)[pColMatch->dstSlotId] = -1; @@ -361,9 +362,14 @@ int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTask if (pColMatch->colId == pWrapper->pSchema[j].colId) { (*pSlotIds)[i] = j; (*pDstSlotIds)[i] = pColMatch->dstSlotId; + found = true; break; } } + if (!found) { + (*pSlotIds)[i] = -1; + (*pDstSlotIds)[i] = pColMatch->dstSlotId; + } } return TSDB_CODE_SUCCESS; diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 0f0936ebab..01795f6eef 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -1,3 +1,4 @@ +from sqlite3 import ProgrammingError import taos import sys import time @@ -319,10 +320,144 @@ class TDTestCase: tdSql.checkData(0, 0, '2018-11-25 19:30:00.000') tdSql.checkData(0, 1, '2018-11-25 19:30:01.000') + def test_cache_scan_with_drop_and_add_column(self): + tdSql.query("select last(c10) from meters") + tdSql.checkData(0, 0, '2018-11-25 19:30:01') + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c10; alter table test.meters add column c11 int"]) + p.check_returncode() + tdSql.query("select last(c10) from meters", queryTimes=1) + tdSql.checkData(0, 0, None) + tdSql.query('select last(*) from meters', queryTimes=1) + tdSql.checkData(0, 10, None) + tdSql.query('select last(c10), c10, ts from meters', queryTimes=1) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + + def test_cache_scan_with_drop_and_add_column2(self): + tdSql.query("select last(c1) from meters") + tdSql.checkData(0, 0, '999') + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c12 int"]) + p.check_returncode() + tdSql.query("select last(c1) from meters", queryTimes=1) + tdSql.checkData(0, 0, None) + tdSql.query('select last(*) from meters', queryTimes=1) + print(str(tdSql.queryResult)) + tdSql.checkData(0, 1, None) + tdSql.query('select last(c1), c1, ts from meters', queryTimes=1) + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + + try: + tdSql.query('select ts, last(c1), c1, ts, c1 from meters', queryTimes=1) + except Exception as e: + if str(e).count('Invalid column name') == 1: + print('column has been dropped, the cache has been updated: %s' % (str(e))) + return + else: + raise + tdSql.checkRows(1) + tdSql.checkCols(5) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(0, 3, None) + tdSql.checkData(0, 4, None) + + try: + tdSql.query('select last(c1), last(c2), last(c3) from meters', queryTimes=1) + except Exception as e: + if str(e).count('Invalid column name') == 1: + print('column has been dropped, the cache has been updated: %s' % (str(e))) + return + else: + raise + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, None) + + def test_cache_scan_with_drop_column(self): + tdSql.query('select last(*) from meters') + print(str(tdSql.queryResult)) + tdSql.checkCols(11) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c9"]) + p.check_returncode() + tdSql.query('select last(*) from meters') + print(str(tdSql.queryResult)) + tdSql.checkCols(11) + tdSql.checkData(0, 9, None) + + def test_cache_scan_last_row_with_drop_column(self): + tdSql.query('select last_row(*) from meters') + print(str(tdSql.queryResult)) + tdSql.checkCols(11) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c10; alter table test.meters add column c11 int"]) + p.check_returncode() + tdSql.query('select last_row(*) from meters') + print(str(tdSql.queryResult)) + tdSql.checkCols(11) + tdSql.checkData(0, 10, None) + + def test_cache_scan_last_row_with_drop_column2(self): + tdSql.query('select last_row(c1) from meters') + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) + p.check_returncode() + tdSql.query('select last_row(c1) from meters', queryTimes=1) + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + tdSql.checkData(0, 0, None) + + def test_cache_scan_last_row_with_partition_by(self): + tdSql.query('select last(c1) from meters partition by t1') + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + tdSql.checkRows(5) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) + p.check_returncode() + tdSql.query('select last_row(c1) from meters partition by t1', queryTimes=1) + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + + def test_cache_scan_last_row_with_partition_by_tbname(self): + tdSql.query('select last(c1) from meters partition by tbname', queryTimes=1) + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + tdSql.checkRows(10) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) + p.check_returncode() + tdSql.query('select last_row(c1) from meters partition by tbname', queryTimes=1) + print(str(tdSql.queryResult)) + tdSql.checkCols(1) + tdSql.checkRows(10) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + + + def run(self): self.prepareTestEnv() #time.sleep(99999999) self.test_last_cache_scan() + #self.test_cache_scan_with_drop_and_add_column() + self.test_cache_scan_with_drop_and_add_column2() + #self.test_cache_scan_with_drop_column() + #self.test_cache_scan_last_row_with_drop_column() + #self.test_cache_scan_last_row_with_drop_column2() + #self.test_cache_scan_last_row_with_partition_by() + #self.test_cache_scan_last_row_with_partition_by_tbname() def stop(self): tdSql.close() From cb3d9551117f81ab37ff70306938c65307cbb45c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 5 Dec 2023 16:19:56 +0800 Subject: [PATCH 20/94] code coverage --- tests/parallel_test/cases.task | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index cf4559100b..ef222db0f9 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1132,6 +1132,9 @@ e ,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim ,,y,script,./test.sh -f tsim/stream/distributeSession0.sim ,,y,script,./test.sh -f tsim/stream/drop_stream.sim +,,y,script,./test.sh -f tsim/stream/event0.sim +,,y,script,./test.sh -f tsim/stream/event1.sim +,,y,script,./test.sh -f tsim/stream/event2.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic1.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic2.sim ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim From b642f0fc83060c7bb7961d9e6cbadc073242c3cd Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 5 Dec 2023 17:51:48 +0800 Subject: [PATCH 21/94] enh: adjust ENodeType for show statements nodes --- include/common/tmsg.h | 56 +++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b42df78e34..aff1bd55e4 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -295,7 +295,36 @@ typedef enum ENodeType { QUERY_NODE_SYNCDB_STMT, QUERY_NODE_GRANT_STMT, QUERY_NODE_REVOKE_STMT, - QUERY_NODE_SHOW_DNODES_STMT, + // placeholder for [152, 180] + QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181, + QUERY_NODE_SHOW_CREATE_DATABASE_STMT, + QUERY_NODE_SHOW_CREATE_TABLE_STMT, + QUERY_NODE_SHOW_CREATE_STABLE_STMT, + QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT, + QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT, + QUERY_NODE_SHOW_SCORES_STMT, + QUERY_NODE_SHOW_TABLE_TAGS_STMT, + QUERY_NODE_KILL_CONNECTION_STMT, + QUERY_NODE_KILL_QUERY_STMT, + QUERY_NODE_KILL_TRANSACTION_STMT, + QUERY_NODE_DELETE_STMT, + QUERY_NODE_INSERT_STMT, + QUERY_NODE_QUERY, + QUERY_NODE_SHOW_DB_ALIVE_STMT, + QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT, + QUERY_NODE_BALANCE_VGROUP_LEADER_STMT, + QUERY_NODE_RESTORE_DNODE_STMT, + QUERY_NODE_RESTORE_QNODE_STMT, + QUERY_NODE_RESTORE_MNODE_STMT, + QUERY_NODE_RESTORE_VNODE_STMT, + QUERY_NODE_PAUSE_STREAM_STMT, + QUERY_NODE_RESUME_STREAM_STMT, + QUERY_NODE_CREATE_VIEW_STMT, + QUERY_NODE_DROP_VIEW_STMT, + + // show statement nodes + // see 'sysTableShowAdapter', 'SYSTABLE_SHOW_TYPE_OFFSET' + QUERY_NODE_SHOW_DNODES_STMT = 400, QUERY_NODE_SHOW_MNODES_STMT, QUERY_NODE_SHOW_MODULES_STMT, QUERY_NODE_SHOW_QNODES_STMT, @@ -324,31 +353,6 @@ typedef enum ENodeType { QUERY_NODE_SHOW_VNODES_STMT, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT, QUERY_NODE_SHOW_VIEWS_STMT, - QUERY_NODE_SHOW_CREATE_VIEW_STMT, - QUERY_NODE_SHOW_CREATE_DATABASE_STMT, - QUERY_NODE_SHOW_CREATE_TABLE_STMT, - QUERY_NODE_SHOW_CREATE_STABLE_STMT, - QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT, - QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT, - QUERY_NODE_SHOW_SCORES_STMT, - QUERY_NODE_SHOW_TABLE_TAGS_STMT, - QUERY_NODE_KILL_CONNECTION_STMT, - QUERY_NODE_KILL_QUERY_STMT, - QUERY_NODE_KILL_TRANSACTION_STMT, - QUERY_NODE_DELETE_STMT, - QUERY_NODE_INSERT_STMT, - QUERY_NODE_QUERY, - QUERY_NODE_SHOW_DB_ALIVE_STMT, - QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT, - QUERY_NODE_BALANCE_VGROUP_LEADER_STMT, - QUERY_NODE_RESTORE_DNODE_STMT, - QUERY_NODE_RESTORE_QNODE_STMT, - QUERY_NODE_RESTORE_MNODE_STMT, - QUERY_NODE_RESTORE_VNODE_STMT, - QUERY_NODE_PAUSE_STREAM_STMT, - QUERY_NODE_RESUME_STREAM_STMT, - QUERY_NODE_CREATE_VIEW_STMT, - QUERY_NODE_DROP_VIEW_STMT, // logic plan node QUERY_NODE_LOGIC_PLAN_SCAN = 1000, From cc51fd47d0bc7812cc8cc638177c0a23d523ef54 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 5 Dec 2023 19:09:29 +0800 Subject: [PATCH 22/94] fix:offset out of range if poll offset is read from offsetRestored while wal is deleted --- source/dnode/vnode/src/tq/tqUtil.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 8f62928d22..110cf79b4e 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -168,6 +168,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, taosWUnLockLatch(&pTq->lock); } + dataRsp.reqOffset = *pOffset; // reqOffset represents the current date offset, may be changed if wal not exists code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); end : { From 9bb82ac03be31a8b43d5af1395985e18a0433fc9 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Tue, 5 Dec 2023 19:43:00 +0800 Subject: [PATCH 23/94] fix merge key --- source/libs/parser/src/parTranslater.c | 11 ++++++++++- source/libs/planner/src/planSpliter.c | 5 +++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5c30384a6b..f689ebc820 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4622,11 +4622,20 @@ static int32_t addOrderByPrimaryKeyToQueryImpl(STranslateContext* pCxt, SNode* p return TSDB_CODE_OUT_OF_MEMORY; } ((SExprNode*)pOrderByExpr->pExpr)->orderAlias = true; - NODES_DESTORY_LIST(*pOrderByList); + // NODES_DESTORY_LIST(*pOrderByList); return nodesListMakeStrictAppend(pOrderByList, (SNode*)pOrderByExpr); } static int32_t addOrderByPrimaryKeyToQuery(STranslateContext* pCxt, SNode* pPrimaryKeyExpr, SNode* pStmt) { + SNode* pOrderNode = NULL; + SNodeList* pOrederList = ((SSelectStmt*)pStmt)->pOrderByList; + if (pOrederList != NULL) { + FOREACH(pOrderNode, pOrederList) { + if (((SColumnNode*)pPrimaryKeyExpr)->colId == ((SColumnNode*)((SOrderByExprNode*)pOrderNode)->pExpr)->colId) { + return TSDB_CODE_SUCCESS; + } + } + } if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { return addOrderByPrimaryKeyToQueryImpl(pCxt, pPrimaryKeyExpr, &((SSelectStmt*)pStmt)->pOrderByList); } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 43bd8a5589..bb88beba72 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -1062,8 +1062,9 @@ static int32_t stbSplCreateMergeKeys(SNodeList* pSortKeys, SNodeList* pTargets, SNode* pTarget = NULL; bool found = false; FOREACH(pTarget, pTargets) { - if ((QUERY_NODE_COLUMN == nodeType(pSortExpr) && nodesEqualNode((SNode*)pSortExpr, pTarget)) || - (0 == strcmp(pSortExpr->aliasName, ((SColumnNode*)pTarget)->colName))) { + if ((QUERY_NODE_COLUMN == nodeType(pSortExpr) && nodesEqualNode((SNode*)pSortExpr, pTarget)) + // || (0 == strcmp(pSortExpr->aliasName, ((SColumnNode*)pTarget)->colName)) + ) { code = nodesListMakeStrictAppend(&pMergeKeys, stbSplCreateOrderByExpr(pSortKey, pTarget)); if (TSDB_CODE_SUCCESS != code) { break; From 82f54f094a4077b6c7f7f0502e6d598b0d6c544b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 5 Dec 2023 20:08:18 +0800 Subject: [PATCH 24/94] fix:add log for time cost --- source/client/inc/clientSml.h | 1 + source/client/src/clientSml.c | 19 ++++++++++++++----- source/client/src/clientSmlLine.c | 25 +++++++++++++++++-------- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 30d4792279..0a5b91e039 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -199,6 +199,7 @@ typedef struct { STableMeta *currSTableMeta; STableDataCxt *currTableDataCtx; bool needModifySchema; + SArray *parseTimeList; } SSmlHandle; #define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 6ab0bfc563..46e88b4a00 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1261,6 +1261,7 @@ void smlDestroyInfo(SSmlHandle *info) { taosArrayDestroy(info->valueJsonArray); taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); + taosArrayDestroy(info->parseTimeList); if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { @@ -1309,6 +1310,7 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); + info->parseTimeList = taosArrayInit(8, LONG_BYTES); if (NULL == info->pVgHash || NULL == info->childTables || NULL == info->superTables || NULL == info->tableUids) { uError("create SSmlHandle failed"); @@ -1523,16 +1525,23 @@ static int32_t smlInsertData(SSmlHandle *info) { } static void smlPrintStatisticInfo(SSmlHandle *info) { - uDebug( + char parseTimeStr[102400] = {0}; + char* tmp = parseTimeStr; + for(int i = 0; i < taosArrayGetSize(info->parseTimeList); i++){ + int64_t *t = (int64_t *)taosArrayGet(info->parseTimeList, i); + tmp += sprintf(tmp, ":%d", (int32_t)(*t)); + } + uError( "SML:0x%" PRIx64 " smlInsertLines result, code:%d, msg:%s, lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64 - "", + ", parList:%s", info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime, info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime, - info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime); + info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime, parseTimeStr); + } int32_t smlClearForRerun(SSmlHandle *info) { @@ -1574,7 +1583,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { } static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { - uDebug("SML:0x%" PRIx64 " smlParseLine start", info->id); + uError("SML:0x%" PRIx64 " smlParseLine start", info->id); int32_t code = TSDB_CODE_SUCCESS; if (info->protocol == TSDB_SML_JSON_PROTOCOL) { if (lines) { @@ -1647,7 +1656,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } i++; } - uDebug("SML:0x%" PRIx64 " smlParseLine end", info->id); + uError("SML:0x%" PRIx64 " smlParseLine end", info->id); return code; } diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 006475654a..91dd6ea700 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -237,14 +237,21 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } sMeta->tableMeta = pTableMeta; taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); - for (int i = pTableMeta->tableInfo.numOfColumns; - i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { + for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { SSchema *tag = pTableMeta->schema + i; - SSmlKv kv = {.key = tag->name, - .keyLen = strlen(tag->name), - .type = tag->type, - .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE}; - taosArrayPush(sMeta->tags, &kv); + + if(i < pTableMeta->tableInfo.numOfColumns){ + SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type}; + if (tag->type == TSDB_DATA_TYPE_NCHAR) { + kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + } else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY || tag->type == TSDB_DATA_TYPE_VARBINARY) { + kv.length = tag->bytes - VARSTR_HEADER_SIZE; + } + taosArrayPush(sMeta->cols, &kv); + }else{ + SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE}; + taosArrayPush(sMeta->tags, &kv); + } } tmp = &sMeta; } @@ -625,7 +632,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine JUMP_SPACE(sql, sqlEnd) if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA; elements->measure = sql; - + int64_t t1 = taosGetTimestampUs(); // parse measure size_t measureLenEscaped = 0; while (sql < sqlEnd) { @@ -718,6 +725,8 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); return TSDB_CODE_INVALID_TIMESTAMP; } + int64_t t2 = taosGetTimestampUs() - t1; + taosArrayPush(info->parseTimeList, &t2); // add ts to SSmlKv kv = {.key = tsSmlTsDefaultName, .keyLen = strlen(tsSmlTsDefaultName), From d2b2dee256812085a464ba3d170779f106e46efd Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Fri, 1 Dec 2023 23:44:05 +0800 Subject: [PATCH 25/94] opt & add test case --- include/common/tvariant.h | 5 +- source/common/src/tvariant.c | 185 +++++++++++------- source/common/test/commonTests.cpp | 203 +++++++++++++------- source/libs/parser/src/parInsertSql.c | 60 +++--- source/libs/parser/src/parTokenizer.c | 7 +- tests/parallel_test/cases.task | 1 + tests/system-test/1-insert/insert_double.py | 133 +++++++++++++ 7 files changed, 422 insertions(+), 172 deletions(-) create mode 100644 tests/system-test/1-insert/insert_double.py diff --git a/include/common/tvariant.h b/include/common/tvariant.h index 61cf277bb7..66b7302f4e 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -37,8 +37,9 @@ typedef struct SVariant { }; } SVariant; -int32_t toIntegerEx(const char *z, int32_t n, int64_t *value); -int32_t toUIntegerEx(const char *z, int32_t n, uint64_t *value); +int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value); +int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value); +int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double *value); int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value); int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value); diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index adb0964fe6..609bcf1382 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -45,21 +45,6 @@ int32_t parseBinaryUInteger(const char *z, int32_t n, uint64_t *value) { return TSDB_CODE_SUCCESS; } -int32_t parseDecimalUInteger(const char *z, int32_t n, uint64_t *value) { - errno = 0; - char *endPtr = NULL; - while (*z == '0' && n > 1) { - z++; - n--; - } - *value = taosStr2UInt64(z, &endPtr, 10); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - return TSDB_CODE_FAILED; - } - - return TSDB_CODE_SUCCESS; -} - int32_t parseHexUInteger(const char *z, int32_t n, uint64_t *value) { errno = 0; char *endPtr = NULL; @@ -71,10 +56,6 @@ int32_t parseHexUInteger(const char *z, int32_t n, uint64_t *value) { } int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *value) { - *is_neg = false; - if (n < 1) { - return TSDB_CODE_FAILED; - } // parse sign bool has_sign = false; @@ -83,18 +64,18 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v has_sign = true; } else if (z[0] == '+') { has_sign = true; - } else if (z[0] < '0' || z[0] > '9') { + } else if (z[0] != '.' && (z[0] < '0' || z[0] > '9')) { return TSDB_CODE_FAILED; } if (has_sign) { - z++; - n--; - if (n < 1) { + if (n < 2) { return TSDB_CODE_FAILED; } + z++; + n--; } - if (n > 2 && z[0] == '0') { + if (z[0] == '0' && n > 2) { if (z[1] == 'b' || z[1] == 'B') { // paring as binary return parseBinaryUInteger(z, n, value); @@ -106,89 +87,157 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v } } - //rm flag u-unsigned, l-long, f-float(if not in hex str) - char last = tolower(z[n-1]); - if (last == 'u' || last == 'l' || last == 'f') { - n--; - if (n < 1) { - return TSDB_CODE_FAILED; - } - } - - // parsing as decimal - bool parse_int = true; - for (int32_t i = 0; i < n; i++) { - if (z[i] < '0' || z[i] > '9') { - parse_int = false; - break; - } - } - if (parse_int) { - return parseDecimalUInteger(z, n, value); - } - // parsing as double errno = 0; char *endPtr = NULL; double val = taosStr2Double(z, &endPtr); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n || !IS_VALID_UINT64(val)) { + if (errno == ERANGE || errno == EINVAL || endPtr - z != n || val > UINT64_MAX) { return TSDB_CODE_FAILED; } - *value = val; + *value = round(val); return TSDB_CODE_SUCCESS; } int32_t removeSpace(const char **pp, int32_t n) { - // rm blank space from both head and tail + // rm blank space from both head and tail, keep at least one char const char *z = *pp; - while (*z == ' ' && n > 0) { + while (n > 1 && *z == ' ') { z++; n--; } - if (n > 0) { - for (int32_t i = n - 1; i > 0; i--) { - if (z[i] == ' ') { - n--; - } else { - break; - } - } + while (n > 1 && z[n-1] == ' ') { + n--; } *pp = z; return n; } -int32_t toIntegerEx(const char *z, int32_t n, int64_t *value) { - n = removeSpace(&z, n); - if (n < 1) { // fail: all char is space +int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { + if (type != TK_NK_FLOAT) { + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + // rm tail space + while (n > 1 && z[n-1] == ' ') { + n--; + } + } + + errno = 0; + char* endPtr = NULL; + *value = taosStr2Double(z, &endPtr); + + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { return TSDB_CODE_FAILED; } + return TSDB_CODE_SUCCESS; +} + +int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { + errno = 0; + char *endPtr = NULL; + switch (type) + { + case TK_NK_INTEGER: { + *value = taosStr2Int64(z, &endPtr, 10); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + case TK_NK_HEX: { + *value = taosStr2Int64(z, &endPtr, 16); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + case TK_NK_BIN: { + *value = taosStr2Int64(z, &endPtr, 2); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + default: + break; + } + + // parse string + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + n = removeSpace(&z, n); bool is_neg = false; uint64_t uv = 0; int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv); if (code == TSDB_CODE_SUCCESS) { // truncate into int64 - if (uv > INT64_MAX) { - *value = is_neg ? INT64_MIN : INT64_MAX; - return TSDB_CODE_FAILED; + if (is_neg) { + if (uv > 1ull + INT64_MAX) { + *value = INT64_MIN; + return TSDB_CODE_FAILED; + } else { + *value = -uv; + } } else { - *value = is_neg ? -uv : uv; + if (uv > INT64_MAX) { + *value = INT64_MAX; + return TSDB_CODE_FAILED; + } + *value = uv; } } return code; } -int32_t toUIntegerEx(const char *z, int32_t n, uint64_t *value) { - n = removeSpace(&z, n); - if (n < 1) { // fail: all char is space - return TSDB_CODE_FAILED; +int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { + errno = 0; + char *endPtr = NULL; + switch (type) + { + case TK_NK_INTEGER: { + *value = taosStr2UInt64(z, &endPtr, 10); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + case TK_NK_HEX: { + *value = taosStr2UInt64(z, &endPtr, 16); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + case TK_NK_BIN: { + *value = taosStr2UInt64(z, &endPtr, 2); + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + default: + break; } + // parse string + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + n = removeSpace(&z, n); + bool is_neg = false; int32_t code = parseSignAndUInteger(z, n, &is_neg, value); if (is_neg) { + if (TSDB_CODE_SUCCESS == code && 0 == *value) { + return TSDB_CODE_SUCCESS; + } return TSDB_CODE_FAILED; } return code; diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 035bca0e15..291941d9b2 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -14,6 +14,7 @@ #include "tdef.h" #include "tvariant.h" #include "ttime.h" +#include "ttokendef.h" namespace { // @@ -25,125 +26,176 @@ int main(int argc, char** argv) { } +TEST(testCase, toUIntegerEx_test) { + uint64_t val = 0; + + char* s = "123"; + int32_t ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 123); + + s = "1000u"; + ret = toUIntegerEx(s, strlen(s), 0, &val); + ASSERT_EQ(ret, -1); + + s = "0x1f"; + ret = toUIntegerEx(s, strlen(s), TK_NK_HEX, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 31); + + s = "0b110"; + ret = toUIntegerEx(s, strlen(s), 0, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 6); + + s = "2567.4787"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 2567); + + s = "1.869895343e4"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 18699); + + s = "-1"; + ret = toUIntegerEx(s, strlen(s),TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); + + s = "-0b10010"; + ret = toUIntegerEx(s, strlen(s), 0, &val); + ASSERT_EQ(ret, -1); + + s = "-0x40"; + ret = toUIntegerEx(s, strlen(s), TK_NK_HEX, &val); + ASSERT_EQ(ret, -1); + + s = "-80.9999"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, -1); + + s = "-5.2343544534e10"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, -1); + + // INT64_MAX + s = "9223372036854775807"; + ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 9223372036854775807); + + // UINT64_MAX + s = "18446744073709551615"; + ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 18446744073709551615u); + + // out of range + s = "18446744073709551616"; + ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); + + s = "5.23e25"; + ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, -1); +} + TEST(testCase, toIntegerEx_test) { int64_t val = 0; char* s = "123"; - int32_t ret = toIntegerEx(s, strlen(s), &val); + int32_t ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 123); - s = "9223372036854775807"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 9223372036854775807); - - s = "9323372036854775807"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, -1); - - s = "-9323372036854775807"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, -1); - s = "-1"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -1); - s = "-9223372036854775807"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, -9223372036854775807); - s = "1000u"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 1000); - - s = "1000l"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 1000); - - s = "1000.0f"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 1000); + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); s = "0x1f"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_HEX, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 31); s = "-0x40"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_HEX, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -64); s = "0b110"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), 0, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 6); s = "-0b10010"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), 0, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -18); + s = "-80.9999"; + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -81); + + s = "2567.8787"; + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 2568); + s = "-5.2343544534e10"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -52343544534); s = "1.869895343e4"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val); ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 18698); + ASSERT_EQ(val, 18699); + + // INT64_MAX + s = "9223372036854775807"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, 9223372036854775807LL); + + s = "-9223372036854775808"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -9223372036854775808); + + // out of range + s = "9323372036854775807"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); + + s = "-9323372036854775807"; + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); + ASSERT_EQ(ret, -1); // UINT64_MAX s = "18446744073709551615"; - ret = toIntegerEx(s, strlen(s), &val); - ASSERT_EQ(ret, -1); - - s = "18446744073709551616"; - ret = toIntegerEx(s, strlen(s), &val); + ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val); ASSERT_EQ(ret, -1); } TEST(testCase, toInteger_test) { - char* s = "123"; - uint32_t type = 0; - int64_t val = 0; + char* s = "123"; int32_t ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 123); - s = "9223372036854775807"; - ret = toInteger(s, strlen(s), 10, &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 9223372036854775807); - - s = "9323372036854775807"; // out of range, > int64_max - ret = toInteger(s, strlen(s), 10, &val); - ASSERT_EQ(ret, -1); - - s = "-9323372036854775807"; - ret = toInteger(s, strlen(s), 10, &val); - ASSERT_EQ(ret, -1); - s = "-1"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, -1); - s = "-9223372036854775807"; - ret = toInteger(s, strlen(s), 10, &val); - ASSERT_EQ(ret, 0); - ASSERT_EQ(val, -9223372036854775807); - s = "1000u"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, -1); @@ -163,13 +215,22 @@ TEST(testCase, toInteger_test) { ASSERT_EQ(ret, 0); ASSERT_EQ(val, 72); - // 18446744073709551615 UINT64_MAX - s = "18446744073709551615"; + s = "9223372036854775807"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, 0); - ASSERT_EQ(val, 18446744073709551615u); + ASSERT_EQ(val, 9223372036854775807); - s = "18446744073709551616"; + s = "-9223372036854775808"; + ret = toInteger(s, strlen(s), 10, &val); + ASSERT_EQ(ret, 0); + ASSERT_EQ(val, -9223372036854775808); + + // out of range + s = "9323372036854775807"; + ret = toInteger(s, strlen(s), 10, &val); + ASSERT_EQ(ret, -1); + + s = "-9323372036854775807"; ret = toInteger(s, strlen(s), 10, &val); ASSERT_EQ(ret, -1); } @@ -225,7 +286,7 @@ TEST(testCase, Datablock_test) { printf("binary column length:%d\n", *(int32_t*)p1->pData); - ASSERT_EQ(blockDataGetNumOfCols(b), 3); + ASSERT_EQ(blockDataGetNumOfCols(b), 2); ASSERT_EQ(blockDataGetNumOfRows(b), 40); char* pData = colDataGetData(p1, 3); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index c993efb9ae..dd62d05440 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -433,7 +433,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_TINYINT: { - code = toIntegerEx(pToken->z, pToken->n, &iv); + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(iv)) { @@ -445,7 +445,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UTINYINT: { - code = toUIntegerEx(pToken->z, pToken->n, &uv); + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); } else if (uv > UINT8_MAX) { @@ -456,7 +456,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_SMALLINT: { - code = toIntegerEx(pToken->z, pToken->n, &iv); + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(iv)) { @@ -467,7 +467,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_USMALLINT: { - code = toUIntegerEx(pToken->z, pToken->n, &uv); + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); } else if (uv > UINT16_MAX) { @@ -478,7 +478,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_INT: { - code = toIntegerEx(pToken->z, pToken->n, &iv); + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z); } else if (!IS_VALID_INT(iv)) { @@ -489,7 +489,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UINT: { - code = toUIntegerEx(pToken->z, pToken->n, &uv); + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); } else if (uv > UINT32_MAX) { @@ -500,7 +500,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_BIGINT: { - code = toIntegerEx(pToken->z, pToken->n, &iv); + code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); } @@ -509,7 +509,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_UBIGINT: { - code = toUIntegerEx(pToken->z, pToken->n, &uv); + code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); } @@ -519,11 +519,11 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, case TSDB_DATA_TYPE_FLOAT: { double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || - isnan(dv)) { + if (dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } *(float*)(&val->i64) = dv; @@ -532,8 +532,9 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, case TSDB_DATA_TYPE_DOUBLE: { double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); + code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { + return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); @@ -1359,7 +1360,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_TINYINT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(pVal->value.val)) { @@ -1368,7 +1369,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_UTINYINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); } else if (pVal->value.val > UINT8_MAX) { @@ -1377,7 +1378,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_SMALLINT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(pVal->value.val)) { @@ -1386,7 +1387,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_USMALLINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); } else if (pVal->value.val > UINT16_MAX) { @@ -1395,7 +1396,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_INT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); } else if (!IS_VALID_INT(pVal->value.val)) { @@ -1404,7 +1405,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_UINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); } else if (pVal->value.val > UINT32_MAX) { @@ -1413,27 +1414,26 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_BIGINT: { - int32_t code = toIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); } break; } case TSDB_DATA_TYPE_UBIGINT: { - int32_t code = toUIntegerEx(pToken->z, pToken->n, &pVal->value.val); + int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val); if (TSDB_CODE_SUCCESS != code) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); } break; } case TSDB_DATA_TYPE_FLOAT: { - char* endptr = NULL; double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + int32_t code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || - isnan(dv)) { + if (dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } float f = dv; @@ -1441,12 +1441,12 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_DOUBLE: { - char* endptr = NULL; double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); + int32_t code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv); + if (TSDB_CODE_SUCCESS != code) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { + if (isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); } pVal->value.val = *(int64_t*)&dv; diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 5193bdc47f..5988c68e17 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -609,6 +609,11 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) { break; } + // support float with no decimal part after the decimal point + if (z[i] == '.' && seg == 1) { + *tokenId = TK_NK_FLOAT; + i++; + } if ((z[i] == 'e' || z[i] == 'E') && (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) { i += 2; @@ -751,7 +756,7 @@ SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr, bool* pIgnoreC // support parse the -/+number format if ((isPrevOptr) && (t0.type == TK_NK_MINUS || t0.type == TK_NK_PLUS)) { len = tGetToken(&str[*i + t0.n], &type); - if (type == TK_NK_INTEGER || type == TK_NK_FLOAT) { + if (type == TK_NK_INTEGER || type == TK_NK_FLOAT || type == TK_NK_BIN || type == TK_NK_HEX) { t0.type = type; t0.n += len; } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 981a2411eb..15464eb1b3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -249,6 +249,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_double.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py new file mode 100644 index 0000000000..ce482287fb --- /dev/null +++ b/tests/system-test/1-insert/insert_double.py @@ -0,0 +1,133 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * +import random + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + self.database = "db1" + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepare_db(self): + tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database {self.database}") + tdSql.execute(f"use {self.database}") + + def test_value(self, table_name, dtype, bits): + tdSql.execute(f"drop table if exists {table_name}") + tdSql.execute(f"create table {table_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned)") + + tdSql.execute(f"insert into {table_name} values(now, -16, +6)") + tdSql.execute(f"insert into {table_name} values(now, 80.99 , +0042 )") + tdSql.execute(f"insert into {table_name} values(now, -0042 , +80.99 )") + tdSql.execute(f"insert into {table_name} values(now, 52.34354, 18.6)") + tdSql.execute(f"insert into {table_name} values(now, -12., +3.)") + tdSql.execute(f"insert into {table_name} values(now, -.12, +.3)") + tdSql.execute(f"insert into {table_name} values(now, -2.3e1, +2.324e2)") + tdSql.execute(f"insert into {table_name} values(now, -2e1, +2e2)") + tdSql.execute(f"insert into {table_name} values(now, -2.e1, +2.e2)") + tdSql.execute(f"insert into {table_name} values(now, -0x40, +0b10000)") + tdSql.execute(f"insert into {table_name} values(now, -0b10000, +0x40)") + + # str support + tdSql.execute(f"insert into {table_name} values(now, '-16', '+6')") + tdSql.execute(f"insert into {table_name} values(now, ' -80.99 ', ' +0042 ')") + tdSql.execute(f"insert into {table_name} values(now, ' -0042 ', ' +80.99 ')") + tdSql.execute(f"insert into {table_name} values(now, '52.34354', '18.6')") + tdSql.execute(f"insert into {table_name} values(now, '-12.', '+5.')") + tdSql.execute(f"insert into {table_name} values(now, '-.12', '+.5')") + tdSql.execute(f"insert into {table_name} values(now, '-2.e1', '+2.e2')") + tdSql.execute(f"insert into {table_name} values(now, '-2e1', '+2e2')") + tdSql.execute(f"insert into {table_name} values(now, '-2.3e1', '+2.324e2')") + tdSql.execute(f"insert into {table_name} values(now, '-0x40', '+0b10010')") + tdSql.execute(f"insert into {table_name} values(now, '-0b10010', '+0x40')") + + tdSql.query(f"select * from {table_name}") + tdSql.checkRows(22) + + baseval = 2**(bits/2) + negval = -baseval + 1.645 + posval = baseval + 4.323 + bigval = 2**(bits-1) + max_i = bigval - 1 + min_i = -bigval + max_u = 2*bigval - 1 + min_u = 0 + print("val:", baseval, negval, posval, max_i) + + tdSql.execute(f"insert into {table_name} values(now, {negval}, {posval})") + tdSql.execute(f"insert into {table_name} values(now, -{baseval}, {baseval})") + tdSql.execute(f"insert into {table_name} values(now, {max_i}, {max_u})") + tdSql.execute(f"insert into {table_name} values(now, {min_i}, {min_u})") + + tdSql.query(f"select * from {table_name}") + tdSql.checkRows(26) + + # fail + tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})", "error") + tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})", "error") + tdSql.error(f"insert into {table_name} values(now, {max_i+1}, -1)", "error") + + + def test_tags(self, stable_name, dtype, bits): + tdSql.execute(f"create stable {stable_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned) tags(id {dtype})") + + baseval = 2**(bits/2) + negval = -baseval + 1.645 + posval = baseval + 4.323 + bigval = 2**(bits-1) + max_i = bigval - 1 + min_i = -bigval + max_u = 2*bigval - 1 + min_u = 0 + + tdSql.execute(f"insert into {stable_name}_1 using {stable_name} tags('{negval}') values(now, {negval}, {posval})") + tdSql.execute(f"insert into {stable_name}_2 using {stable_name} tags({posval}) values(now, -{baseval} , {baseval})") + tdSql.execute(f"insert into {stable_name}_3 using {stable_name} tags('0x40') values(now, {max_i}, {max_u})") + tdSql.execute(f"insert into {stable_name}_4 using {stable_name} tags(0b10000) values(now, {min_i}, {min_u})") + + tdSql.execute(f"insert into {stable_name}_5 using {stable_name} tags({max_i}) values(now, '{negval}', '{posval}')") + tdSql.execute(f"insert into {stable_name}_6 using {stable_name} tags('{min_i}') values(now, '-{baseval}' , '{baseval}')") + tdSql.execute(f"insert into {stable_name}_7 using {stable_name} tags(-0x40) values(now, '{max_i}', '{max_u}')") + tdSql.execute(f"insert into {stable_name}_8 using {stable_name} tags('-0b10000') values(now, '{min_i}', '{min_u}')") + + tdSql.execute(f"insert into {stable_name}_9 using {stable_name} tags(12.) values(now, {negval}, {posval})") + tdSql.execute(f"insert into {stable_name}_10 using {stable_name} tags('-8.3') values(now, -{baseval} , {baseval})") + tdSql.execute(f"insert into {stable_name}_11 using {stable_name} tags(2.e1) values(now, {max_i}, {max_u})") + tdSql.execute(f"insert into {stable_name}_12 using {stable_name} tags('-2.3e1') values(now, {min_i}, {min_u})") + + tdSql.query(f"select * from {stable_name}") + tdSql.checkRows(12) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare(replica = self.replicaVar) + self.prepare_db() + + self.test_value("t1", "bigint", 64) + self.test_value("t2", "int", 32) + self.test_value("t3", "smallint", 16) + self.test_value("t4", "tinyint", 8) + tdLog.printNoPrefix("==========end case1 run ...............") + + self.test_tags("t_big", "bigint", 64) + self.test_tags("t_int", "int", 32) + self.test_tags("t_small", "smallint", 16) + self.test_tags("t_tiny", "tinyint", 8) + tdLog.printNoPrefix("==========end case2 run ...............") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 9fa884a693ab876cd0c8949870baa5f82cfd6bad Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 6 Dec 2023 01:24:23 +0800 Subject: [PATCH 26/94] toUIntegerEx --- source/common/src/tvariant.c | 122 ++++++++++---------- source/common/test/commonTests.cpp | 2 +- tests/system-test/1-insert/insert_double.py | 4 +- 3 files changed, 67 insertions(+), 61 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 609bcf1382..25f025ad3d 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -45,18 +45,7 @@ int32_t parseBinaryUInteger(const char *z, int32_t n, uint64_t *value) { return TSDB_CODE_SUCCESS; } -int32_t parseHexUInteger(const char *z, int32_t n, uint64_t *value) { - errno = 0; - char *endPtr = NULL; - *value = taosStr2UInt64(z, &endPtr, 16); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; -} - int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *value) { - // parse sign bool has_sign = false; if (z[0] == '-') { @@ -75,6 +64,9 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v n--; } + errno = 0; + char *endPtr = NULL; + bool parsed = false; if (z[0] == '0' && n > 2) { if (z[1] == 'b' || z[1] == 'B') { // paring as binary @@ -83,18 +75,35 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v if (z[1] == 'x' || z[1] == 'X') { // parsing as hex - return parseHexUInteger(z, n, value); + *value = taosStr2UInt64(z, &endPtr, 16); + parsed = true; } } - // parsing as double - errno = 0; - char *endPtr = NULL; - double val = taosStr2Double(z, &endPtr); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n || val > UINT64_MAX) { + if (!parsed) { + bool parse_int = true; + for (int32_t i = 0; i < n; i++) { + if (z[i] < '0' || z[i] > '9') { + parse_int = false; + break; + } + } + if (parse_int) { + // parsing as decimal + *value = taosStr2UInt64(z, &endPtr, 10); + } else { + // parsing as double + double val = taosStr2Double(z, &endPtr); + if (val > UINT64_MAX) { + return TSDB_CODE_FAILED; + } + *value = round(val); + } + } + + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { return TSDB_CODE_FAILED; } - *value = round(val); return TSDB_CODE_SUCCESS; } @@ -113,15 +122,13 @@ int32_t removeSpace(const char **pp, int32_t n) { } int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { - if (type != TK_NK_FLOAT) { - if (n == 0) { - *value = 0; - return TSDB_CODE_SUCCESS; - } - // rm tail space - while (n > 1 && z[n-1] == ' ') { - n--; - } + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + // rm tail space + while (n > 1 && z[n-1] == ' ') { + n--; } errno = 0; @@ -137,33 +144,38 @@ int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { errno = 0; char *endPtr = NULL; + bool parsed = false; switch (type) { case TK_NK_INTEGER: { *value = taosStr2Int64(z, &endPtr, 10); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + parsed = true; + } break; + case TK_NK_FLOAT: { + double val = taosStr2Double(z, &endPtr); + parsed = true; + if (!IS_VALID_INT64(val)) { return TSDB_CODE_FAILED; } - return TSDB_CODE_SUCCESS; - } - case TK_NK_HEX: { - *value = taosStr2Int64(z, &endPtr, 16); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; - } - case TK_NK_BIN: { - *value = taosStr2Int64(z, &endPtr, 2); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; - } + *value = round(val); + } break; default: break; } + if (parsed) { + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } + while (n > 0 && z[n-1] == ' ') { + n--; + } + if (endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + // parse string if (n == 0) { *value = 0; @@ -196,29 +208,23 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { } int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { - errno = 0; - char *endPtr = NULL; + switch (type) { case TK_NK_INTEGER: { - *value = taosStr2UInt64(z, &endPtr, 10); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; + return toUInteger(z, n, 10, value); } - case TK_NK_HEX: { - *value = taosStr2UInt64(z, &endPtr, 16); + case TK_NK_FLOAT: { + errno = 0; + char *endPtr = NULL; + double val = round(taosStr2Double(z, &endPtr)); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { return TSDB_CODE_FAILED; } - return TSDB_CODE_SUCCESS; - } - case TK_NK_BIN: { - *value = taosStr2UInt64(z, &endPtr, 2); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + if (!IS_VALID_UINT64(val)) { return TSDB_CODE_FAILED; } + *value = val; return TSDB_CODE_SUCCESS; } default: diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 291941d9b2..9f7ee165ac 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -128,7 +128,7 @@ TEST(testCase, toIntegerEx_test) { ASSERT_EQ(val, -64); s = "0b110"; - ret = toIntegerEx(s, strlen(s), 0, &val); + ret = toIntegerEx(s, strlen(s), TK_NK_BIN, &val); ASSERT_EQ(ret, 0); ASSERT_EQ(val, 6); diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py index ce482287fb..8a7180325b 100644 --- a/tests/system-test/1-insert/insert_double.py +++ b/tests/system-test/1-insert/insert_double.py @@ -27,8 +27,8 @@ class TDTestCase: tdSql.execute(f"create table {table_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned)") tdSql.execute(f"insert into {table_name} values(now, -16, +6)") - tdSql.execute(f"insert into {table_name} values(now, 80.99 , +0042 )") - tdSql.execute(f"insert into {table_name} values(now, -0042 , +80.99 )") + tdSql.execute(f"insert into {table_name} values(now, 80.99, +0042)") + tdSql.execute(f"insert into {table_name} values(now, -0042, +80.99)") tdSql.execute(f"insert into {table_name} values(now, 52.34354, 18.6)") tdSql.execute(f"insert into {table_name} values(now, -12., +3.)") tdSql.execute(f"insert into {table_name} values(now, -.12, +.3)") From 17866b2b855190dc016f4d316a3dd9376d292421 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 6 Dec 2023 02:44:35 +0800 Subject: [PATCH 27/94] case adjust --- tests/system-test/1-insert/insert_double.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py index 8a7180325b..54df7e0352 100644 --- a/tests/system-test/1-insert/insert_double.py +++ b/tests/system-test/1-insert/insert_double.py @@ -31,7 +31,7 @@ class TDTestCase: tdSql.execute(f"insert into {table_name} values(now, -0042, +80.99)") tdSql.execute(f"insert into {table_name} values(now, 52.34354, 18.6)") tdSql.execute(f"insert into {table_name} values(now, -12., +3.)") - tdSql.execute(f"insert into {table_name} values(now, -.12, +.3)") + tdSql.execute(f"insert into {table_name} values(now, -0.12, +3.0)") tdSql.execute(f"insert into {table_name} values(now, -2.3e1, +2.324e2)") tdSql.execute(f"insert into {table_name} values(now, -2e1, +2e2)") tdSql.execute(f"insert into {table_name} values(now, -2.e1, +2.e2)") From 979abd5c6f48ad82b487442d9abc20ca6f7f924d Mon Sep 17 00:00:00 2001 From: charles Date: Wed, 6 Dec 2023 08:47:05 +0800 Subject: [PATCH 28/94] add geometry test case to case list by charles --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index cf4559100b..39db02f869 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -518,6 +518,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tagFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts_3405_3398_3423.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/geometry.py ,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py From 7bb06cb1cc67a5fc5ebd41f283d1a2d0d59f87b9 Mon Sep 17 00:00:00 2001 From: xsren <285808407@qq.com> Date: Wed, 6 Dec 2023 09:13:18 +0800 Subject: [PATCH 29/94] fix: userAlias --- source/libs/planner/src/planSpliter.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index bb88beba72..15efaae7f4 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -1033,6 +1033,7 @@ static SNode* stbSplCreateColumnNode(SExprNode* pExpr) { strcpy(pCol->colName, pExpr->aliasName); } strcpy(pCol->node.aliasName, pExpr->aliasName); + strcpy(pCol->node.userAlias, pExpr->userAlias); pCol->node.resType = pExpr->resType; return (SNode*)pCol; } From 1ec8bf9ef959782d6976548fc9d41013a5e38bf6 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 6 Dec 2023 10:11:52 +0800 Subject: [PATCH 30/94] add error case --- source/common/src/tvariant.c | 23 ++++++++++++--------- tests/system-test/1-insert/insert_double.py | 13 +++++++----- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 25f025ad3d..d78c76a543 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -126,16 +126,19 @@ int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { *value = 0; return TSDB_CODE_SUCCESS; } - // rm tail space - while (n > 1 && z[n-1] == ' ') { - n--; - } errno = 0; char* endPtr = NULL; *value = taosStr2Double(z, &endPtr); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } + // rm tail space + while (n > 1 && z[n-1] == ' ') { + n--; + } + if (endPtr - z != n) { return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; @@ -152,12 +155,12 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { parsed = true; } break; case TK_NK_FLOAT: { - double val = taosStr2Double(z, &endPtr); + double val = round(taosStr2Double(z, &endPtr)); parsed = true; if (!IS_VALID_INT64(val)) { return TSDB_CODE_FAILED; } - *value = round(val); + *value = val; } break; default: break; @@ -167,7 +170,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { if (errno == ERANGE || errno == EINVAL) { return TSDB_CODE_FAILED; } - while (n > 0 && z[n-1] == ' ') { + while (n > 1 && z[n-1] == ' ') { n--; } if (endPtr - z != n) { @@ -176,7 +179,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { return TSDB_CODE_SUCCESS; } - // parse string + // parse as string if (n == 0) { *value = 0; return TSDB_CODE_SUCCESS; @@ -231,7 +234,7 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { break; } - // parse string + // parse as string if (n == 0) { *value = 0; return TSDB_CODE_SUCCESS; diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py index 54df7e0352..56861f3cd4 100644 --- a/tests/system-test/1-insert/insert_double.py +++ b/tests/system-test/1-insert/insert_double.py @@ -72,11 +72,14 @@ class TDTestCase: tdSql.query(f"select * from {table_name}") tdSql.checkRows(26) - # fail - tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})", "error") - tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})", "error") - tdSql.error(f"insert into {table_name} values(now, {max_i+1}, -1)", "error") - + # error case + tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})") + tdSql.error(f"insert into {table_name} values(now, 0, -1)") + tdSql.error(f"insert into {table_name} values(now, 0, -2.0)") + tdSql.error(f"insert into {table_name} values(now, 0, '-2.0')") + tdSql.error(f"insert into {table_name} values(now, {max_i+1}, 0)") + tdSql.error(f"insert into {table_name} values(now, {min_i-1}, 0)") + tdSql.error(f"insert into {table_name} values(now, '{min_i-1}', 0)") def test_tags(self, stable_name, dtype, bits): tdSql.execute(f"create stable {stable_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned) tags(id {dtype})") From 15f690dd6de0a6671e0a2b6eaa360b7e81c69b7b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 6 Dec 2023 11:44:41 +0800 Subject: [PATCH 31/94] add http fail fast --- source/libs/transport/src/thttp.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index afb982a50a..bbfccf9f70 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -34,6 +34,7 @@ typedef struct SHttpModule { SAsyncPool* asyncPool; TdThread thread; SHashObj* connStatusTable; + int8_t quit; } SHttpModule; typedef struct SHttpMsg { @@ -190,19 +191,37 @@ static void httpDestroyMsg(SHttpMsg* msg) { taosMemoryFree(msg->cont); taosMemoryFree(msg); } + +static void httpMayDiscardMsg(SHttpModule* http, SAsyncItem* item) { + SHttpMsg *msg = NULL, *quitMsg = NULL; + int8_t quit = atomic_load_8(&http->quit); + if (quit == 1) { + while (!QUEUE_IS_EMPTY(&item->qmsg)) { + queue* h = QUEUE_HEAD(&item->qmsg); + QUEUE_REMOVE(h); + msg = QUEUE_DATA(h, SHttpMsg, q); + if (!msg->quit) { + httpDestroyMsg(msg); + } else { + quitMsg = msg; + } + } + QUEUE_PUSH(&item->qmsg, &quitMsg->q); + } +} static void httpAsyncCb(uv_async_t* handle) { SAsyncItem* item = handle->data; SHttpModule* http = item->pThrd; SHttpMsg *msg = NULL, *quitMsg = NULL; - - queue wq; + queue wq; QUEUE_INIT(&wq); static int32_t BATCH_SIZE = 5; int32_t count = 0; taosThreadMutexLock(&item->mtx); + httpMayDiscardMsg(http, item); while (!QUEUE_IS_EMPTY(&item->qmsg) && count++ < BATCH_SIZE) { queue* h = QUEUE_HEAD(&item->qmsg); @@ -526,6 +545,8 @@ void transHttpEnvDestroy() { return; } SHttpModule* load = taosAcquireRef(httpRefMgt, httpRef); + + atomic_store_8(&load->quit, 1); httpSendQuit(); taosThreadJoin(load->thread, NULL); From f48045d9c07de0a78da03ddb39ad8db743950ed1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 6 Dec 2023 11:47:08 +0800 Subject: [PATCH 32/94] refactor code --- include/common/tmsgdef.h | 42 ++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 61b471912f..6c7877b49e 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -26,10 +26,10 @@ #undef TD_NEW_MSG_SEG #undef TD_DEF_MSG_TYPE - #undef TD_CLOSE_MSG_TYPE + #undef TD_CLOSE_MSG_SEG #define TD_NEW_MSG_SEG(TYPE) "null", #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) MSG, MSG "-rsp", - #define TD_CLOSE_MSG_TYPE(TYPE) + #define TD_CLOSE_MSG_SEG(TYPE) char *tMsgInfo[] = { @@ -37,20 +37,20 @@ #undef TD_NEW_MSG_SEG #undef TD_DEF_MSG_TYPE - #undef TD_CLOSE_MSG_TYPE + #undef TD_CLOSE_MSG_SEG #define TD_NEW_MSG_SEG(TYPE) #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) - #define TD_CLOSE_MSG_TYPE(TYPE) TYPE, + #define TD_CLOSE_MSG_SEG(TYPE) TYPE, int32_t tMsgRangeDict[] = { #elif defined(TD_MSG_NUMBER_) #undef TD_NEW_MSG_SEG #undef TD_DEF_MSG_TYPE - #undef TD_CLOSE_MSG_TYPE + #undef TD_CLOSE_MSG_SEG #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM, #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE##_NUM, TYPE##_RSP_NUM, - #define TD_CLOSE_MSG_TYPE(TYPE) + #define TD_CLOSE_MSG_SEG(TYPE) enum { @@ -58,10 +58,10 @@ #undef TD_NEW_MSG_SEG #undef TD_DEF_MSG_TYPE - #undef TD_CLOSE_MSG_TYPE + #undef TD_CLOSE_MSG_SEG #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM, #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) - #define TD_CLOSE_MSG_TYPE(type) + #define TD_CLOSE_MSG_SEG(type) int32_t tMsgDict[] = { @@ -70,10 +70,10 @@ #undef TD_NEW_MSG_SEG #undef TD_DEF_MSG_TYPE - #undef TD_CLOSE_MSG_TYPE + #undef TD_CLOSE_MSG_SEG #define TD_NEW_MSG_SEG(TYPE) TYPE##_SEG_CODE, #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) - #define TD_CLOSE_MSG_TYPE(TYPE) + #define TD_CLOSE_MSG_SEG(TYPE) enum { @@ -82,10 +82,10 @@ #undef TD_NEW_MSG_SEG #undef TD_DEF_MSG_TYPE - #undef TD_CLOSE_MSG_TYPE + #undef TD_CLOSE_MSG_SEG #define TD_NEW_MSG_SEG(TYPE) TYPE = ((TYPE##_SEG_CODE) << 8), #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE, TYPE##_RSP, - #define TD_CLOSE_MSG_TYPE(TYPE) TYPE, + #define TD_CLOSE_MSG_SEG(TYPE) TYPE, enum { // WARN: new msg should be appended to segment tail #endif @@ -109,7 +109,7 @@ TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, "dnode-check-vnode-learner-catchup", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL) - TD_CLOSE_MSG_TYPE(TDMT_END_DND_MSG) + TD_CLOSE_MSG_SEG(TDMT_END_DND_MSG) TD_NEW_MSG_SEG(TDMT_MND_MSG) // 1<<8 TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL) @@ -218,7 +218,7 @@ TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) - TD_CLOSE_MSG_TYPE(TDMT_END_MND_MSG) + TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8 TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp) @@ -268,7 +268,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL) - TD_CLOSE_MSG_TYPE(TDMT_END_VND_MSG) + TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG) TD_NEW_MSG_SEG(TDMT_SCH_MSG) // 3<<8 TD_DEF_MSG_TYPE(TDMT_SCH_QUERY, "query", NULL, NULL) @@ -283,7 +283,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_TASK_NOTIFY, "task-notify", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_MAX_MSG, "sch-max", NULL, NULL) - TD_CLOSE_MSG_TYPE(TDMT_END_SCH_MSG) + TD_CLOSE_MSG_SEG(TDMT_END_SCH_MSG) TD_NEW_MSG_SEG(TDMT_STREAM_MSG) //4 << 8 @@ -301,11 +301,11 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_STOP, "stream-task-stop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_HTASK_DROP, "stream-htask-drop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL) - TD_CLOSE_MSG_TYPE(TDMT_END_STREAM_MSG) + TD_CLOSE_MSG_SEG(TDMT_END_STREAM_MSG) TD_NEW_MSG_SEG(TDMT_MON_MSG) //5 << 8 TD_DEF_MSG_TYPE(TDMT_MON_MAX_MSG, "monitor-max", NULL, NULL) - TD_CLOSE_MSG_TYPE(TDMT_END_MON_MSG) + TD_CLOSE_MSG_SEG(TDMT_END_MON_MSG) TD_NEW_MSG_SEG(TDMT_SYNC_MSG) //6 << 8 TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL) @@ -337,7 +337,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_PREP_SNAPSHOT_REPLY, "sync-prep-snapshot-reply", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL) - TD_CLOSE_MSG_TYPE(TDMT_END_SYNC_MSG) + TD_CLOSE_MSG_SEG(TDMT_END_SYNC_MSG) TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) //7 << 8 @@ -348,7 +348,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) - TD_CLOSE_MSG_TYPE(TDMT_END_VND_STREAM_MSG) + TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG) TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) //8 << 8 TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp) @@ -362,7 +362,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_WALINFO, "vnode-tmq-vg-walinfo", SMqPollReq, SMqDataBlkRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_COMMITTEDINFO, "vnode-tmq-committedinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL) - TD_CLOSE_MSG_TYPE(TDMT_END_TMQ_MSG) + TD_CLOSE_MSG_SEG(TDMT_END_TMQ_MSG) TD_NEW_MSG_SEG(TDMT_MAX_MSG) // msg end mark From 1bb4d1b34d970666a5da654a22a4a7e6a0caa3fe Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 6 Dec 2023 11:51:23 +0800 Subject: [PATCH 33/94] refactor code --- include/common/tmsgdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 6c7877b49e..a2f9a5c475 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -365,7 +365,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_TMQ_MSG) TD_NEW_MSG_SEG(TDMT_MAX_MSG) // msg end mark - + TD_CLOSE_MSG_SEG(TDMT_END_MAX_MSG) From 59c59362f3819c6cb1b666a37ee9c8c34a755e94 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 6 Dec 2023 13:52:45 +0800 Subject: [PATCH 34/94] add http fast quit --- source/libs/transport/src/thttp.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index bbfccf9f70..b36731ab0b 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -194,18 +194,21 @@ static void httpDestroyMsg(SHttpMsg* msg) { static void httpMayDiscardMsg(SHttpModule* http, SAsyncItem* item) { SHttpMsg *msg = NULL, *quitMsg = NULL; - int8_t quit = atomic_load_8(&http->quit); - if (quit == 1) { - while (!QUEUE_IS_EMPTY(&item->qmsg)) { - queue* h = QUEUE_HEAD(&item->qmsg); - QUEUE_REMOVE(h); - msg = QUEUE_DATA(h, SHttpMsg, q); - if (!msg->quit) { - httpDestroyMsg(msg); - } else { - quitMsg = msg; - } + if (atomic_load_8(&http->quit) == 0) { + return; + } + + while (!QUEUE_IS_EMPTY(&item->qmsg)) { + queue* h = QUEUE_HEAD(&item->qmsg); + QUEUE_REMOVE(h); + msg = QUEUE_DATA(h, SHttpMsg, q); + if (!msg->quit) { + httpDestroyMsg(msg); + } else { + quitMsg = msg; } + } + if (quitMsg != NULL) { QUEUE_PUSH(&item->qmsg, &quitMsg->q); } } From a0e356bf849ef3f7b200f1b40df61a1d1d1fe077 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 6 Dec 2023 15:20:05 +0800 Subject: [PATCH 35/94] Keep the last data with the same timestamp when inserting --- source/libs/executor/src/dataInserter.c | 22 ++-- source/libs/parser/src/parTranslater.c | 9 +- tests/system-test/2-query/insert_select.py | 113 ++++++++++++++++++++- 3 files changed, 124 insertions(+), 20 deletions(-) diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index f301ddf4be..00b58263e2 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -189,7 +189,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } int64_t lastTs = TSKEY_MIN; - bool ignoreRow = false; + bool updateLastRow = false; bool disorderTs = false; for (int32_t j = 0; j < rows; ++j) { // iterate by row @@ -249,7 +249,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } else { if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) { if (*(int64_t*)var == lastTs) { - ignoreRow = true; + updateLastRow = true; } else if (*(int64_t*)var < lastTs) { disorderTs = true; } else { @@ -269,15 +269,6 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } break; } - - if (ignoreRow) { - break; - } - } - - if (ignoreRow) { - ignoreRow = false; - continue; } SRow* pRow = NULL; @@ -285,7 +276,14 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); goto _end; } - taosArrayPush(tbData.aRowP, &pRow); + if (updateLastRow) { + updateLastRow = false; + SRow** lastRow = taosArrayPop(tbData.aRowP); + tRowDestroy(*lastRow); + taosArrayPush(tbData.aRowP, &pRow); + } else { + taosArrayPush(tbData.aRowP, &pRow); + } } if (disorderTs) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f689ebc820..9a0a560a1a 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4627,14 +4627,9 @@ static int32_t addOrderByPrimaryKeyToQueryImpl(STranslateContext* pCxt, SNode* p } static int32_t addOrderByPrimaryKeyToQuery(STranslateContext* pCxt, SNode* pPrimaryKeyExpr, SNode* pStmt) { - SNode* pOrderNode = NULL; SNodeList* pOrederList = ((SSelectStmt*)pStmt)->pOrderByList; - if (pOrederList != NULL) { - FOREACH(pOrderNode, pOrederList) { - if (((SColumnNode*)pPrimaryKeyExpr)->colId == ((SColumnNode*)((SOrderByExprNode*)pOrderNode)->pExpr)->colId) { - return TSDB_CODE_SUCCESS; - } - } + if (pOrederList && pOrederList->length > 0) { + return TSDB_CODE_SUCCESS; } if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { return addOrderByPrimaryKeyToQueryImpl(pCxt, pPrimaryKeyExpr, &((SSelectStmt*)pStmt)->pOrderByList); diff --git a/tests/system-test/2-query/insert_select.py b/tests/system-test/2-query/insert_select.py index e74cf7a8d1..cbdcf366d2 100644 --- a/tests/system-test/2-query/insert_select.py +++ b/tests/system-test/2-query/insert_select.py @@ -70,7 +70,116 @@ class TDTestCase: tdSql.error('''insert into %s.tb1 (c8, c9) values(now, 1);'''%(database)) - + def use_select_sort(self,database): + ts = 1604298064000 + + tdSql.execute('''drop database if exists %s ;''' %database) + tdSql.execute('''create database %s keep 36500 ;'''%(database)) + tdSql.execute('''use %s;'''%database) + + tdSql.execute('''create stable %s.st (ts timestamp, val int, vt timestamp) tags (location NCHAR(100));'''%(database)) + tdSql.execute('''create table %s.t1 using %s.st (location) tags ("0001");'''%(database,database)) + tdSql.execute('''create table %s.t2 using %s.st (location) tags ("0002");'''%(database,database)) + tdSql.execute('''create table %s.mt (ts timestamp, val int);'''%(database)) + + + tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) ({ts}, 2, {ts});'''%(database)) + tdSql.query("select ts, val from %s.t1;"%database) + tdSql.checkData(0,1,2) + + ts += 1 + tdSql.execute(f'''insert into %s.t2 values({ts}, 1, {ts}) ({ts}, 5, {ts}) ({ts}, 2, {ts});'''%(database)) + tdSql.query("select ts, val from %s.t2;"%database) + tdSql.checkData(0,1,2) + + tdSql.execute('''delete from %s.t2;'''%(database)) + tdSql.execute('''delete from %s.t1;'''%(database)) + + ts -= 10 + tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) %s.t2 values({ts}, 2, {ts});'''%(database,database)) + ts += 11 + tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) %s.t2 values({ts}, 2, {ts});'''%(database,database)) + ts += 1 + tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) %s.t2 values({ts}, 2, {ts});'''%(database,database)) + + tdSql.query("select count(*) from %s.st;"%database) + tdSql.checkData(0,0,6) + + tdSql.query('''select vt, val from %s.st order by vt, val desc;'''%(database)) + tdSql.checkData(0,1,2) + tdSql.checkData(1,1,1) + tdSql.checkData(2,1,2) + tdSql.checkData(3,1,1) + tdSql.checkData(4,1,2) + tdSql.checkData(5,1,1) + + tdSql.execute('''insert into %s.mt select vt, val from %s.st order by vt, val desc;'''%(database,database)) + tdSql.query("select count(*) from %s.mt;"%database) + tdSql.checkData(0,0,3) + + tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database)) + tdSql.checkData(0,1,1) + tdSql.checkData(1,1,1) + tdSql.checkData(2,1,1) + + tdSql.execute('''delete from %s.mt;'''%(database)) + tdSql.query('''select vt, val from %s.st order by vt, val asc;'''%(database)) + tdSql.checkData(0,1,1) + tdSql.checkData(1,1,2) + tdSql.checkData(2,1,1) + tdSql.checkData(3,1,2) + tdSql.checkData(4,1,1) + tdSql.checkData(5,1,2) + + tdSql.execute('''insert into %s.mt select vt, val from %s.st order by vt, val asc;'''%(database,database)) + tdSql.query("select count(*) from %s.mt;"%database) + tdSql.checkData(0,0,3) + + tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database)) + tdSql.checkData(0,1,2) + tdSql.checkData(1,1,2) + tdSql.checkData(2,1,2) + + tdSql.execute('''delete from %s.mt;'''%(database)) + tdSql.query('''select vt, val from %s.st order by ts, val asc;'''%(database)) + tdSql.checkData(0,1,1) + tdSql.checkData(1,1,2) + tdSql.checkData(2,1,1) + tdSql.checkData(3,1,2) + tdSql.checkData(4,1,1) + tdSql.checkData(5,1,2) + + tdSql.execute('''insert into %s.mt select vt, val from %s.st order by ts, val asc;'''%(database,database)) + tdSql.query("select count(*) from %s.mt;"%database) + tdSql.checkData(0,0,3) + + tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database)) + tdSql.checkData(0,1,2) + tdSql.checkData(1,1,2) + tdSql.checkData(2,1,2) + + tdSql.execute('''delete from %s.mt;'''%(database)) + ts += 1 + tdSql.execute(f'''insert into %s.t1 values({ts}, -1, {ts}) %s.t2 values({ts}, -2, {ts});'''%(database,database)) + tdSql.query('''select vt, val from %s.st order by val asc;'''%(database)) + tdSql.checkData(0,1,-2) + tdSql.checkData(1,1,-1) + tdSql.checkData(2,1,1) + tdSql.checkData(3,1,1) + tdSql.checkData(4,1,1) + tdSql.checkData(5,1,2) + tdSql.checkData(6,1,2) + tdSql.checkData(7,1,2) + + tdSql.execute('''insert into %s.mt select vt, val from %s.st order by val asc;'''%(database,database)) + tdSql.query("select count(*) from %s.mt;"%database) + tdSql.checkData(0,0,4) + + tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database)) + tdSql.checkData(0,1,2) + tdSql.checkData(1,1,2) + tdSql.checkData(2,1,2) + tdSql.checkData(3,1,-1) def run(self): @@ -88,6 +197,8 @@ class TDTestCase: self.users_bug_TD_20592("%s" %self.db) + self.use_select_sort("%s" %self.db) + #taos -f sql print("taos -f sql start!") taos_cmd1 = "taos -f %s/%s.sql" % (self.testcasePath,self.testcaseFilename) From f4b0e7baf520f90e6bf10e744ab3ca6d4e3b118f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 6 Dec 2023 15:46:45 +0800 Subject: [PATCH 36/94] fix:remove debug info --- source/client/inc/clientSml.h | 1 - source/client/src/clientSml.c | 19 +++++-------------- source/client/src/clientSmlLine.c | 3 --- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 0a5b91e039..30d4792279 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -199,7 +199,6 @@ typedef struct { STableMeta *currSTableMeta; STableDataCxt *currTableDataCtx; bool needModifySchema; - SArray *parseTimeList; } SSmlHandle; #define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 46e88b4a00..6bcaae3bf0 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1261,7 +1261,6 @@ void smlDestroyInfo(SSmlHandle *info) { taosArrayDestroy(info->valueJsonArray); taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv); - taosArrayDestroy(info->parseTimeList); if (!info->dataFormat) { for (int i = 0; i < info->lineNum; i++) { @@ -1310,7 +1309,6 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); - info->parseTimeList = taosArrayInit(8, LONG_BYTES); if (NULL == info->pVgHash || NULL == info->childTables || NULL == info->superTables || NULL == info->tableUids) { uError("create SSmlHandle failed"); @@ -1525,22 +1523,15 @@ static int32_t smlInsertData(SSmlHandle *info) { } static void smlPrintStatisticInfo(SSmlHandle *info) { - char parseTimeStr[102400] = {0}; - char* tmp = parseTimeStr; - for(int i = 0; i < taosArrayGetSize(info->parseTimeList); i++){ - int64_t *t = (int64_t *)taosArrayGet(info->parseTimeList, i); - tmp += sprintf(tmp, ":%d", (int32_t)(*t)); - } - uError( + uDebug( "SML:0x%" PRIx64 " smlInsertLines result, code:%d, msg:%s, lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ - parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64 - ", parList:%s", + parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64, info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime, info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime, - info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime, parseTimeStr); + info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime); } @@ -1583,7 +1574,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { } static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { - uError("SML:0x%" PRIx64 " smlParseLine start", info->id); + uDebug("SML:0x%" PRIx64 " smlParseLine start", info->id); int32_t code = TSDB_CODE_SUCCESS; if (info->protocol == TSDB_SML_JSON_PROTOCOL) { if (lines) { @@ -1656,7 +1647,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } i++; } - uError("SML:0x%" PRIx64 " smlParseLine end", info->id); + uDebug("SML:0x%" PRIx64 " smlParseLine end", info->id); return code; } diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 91dd6ea700..7ad43d90c7 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -632,7 +632,6 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine JUMP_SPACE(sql, sqlEnd) if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA; elements->measure = sql; - int64_t t1 = taosGetTimestampUs(); // parse measure size_t measureLenEscaped = 0; while (sql < sqlEnd) { @@ -725,8 +724,6 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); return TSDB_CODE_INVALID_TIMESTAMP; } - int64_t t2 = taosGetTimestampUs() - t1; - taosArrayPush(info->parseTimeList, &t2); // add ts to SSmlKv kv = {.key = tsSmlTsDefaultName, .keyLen = strlen(tsSmlTsDefaultName), From 9c909fd7a2960644c9a8a707f06affb13724b56a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 6 Dec 2023 15:49:21 +0800 Subject: [PATCH 37/94] refactor code --- source/libs/transport/src/thttp.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index b36731ab0b..33d1a2565a 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -167,7 +167,8 @@ _OVER: static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) { uint32_t ip = taosGetIpv4FromFqdn(server); if (ip == 0xffffffff) { - tError("http-report failed to get http server:%s since %s", server, errno == 0 ? "invalid http server" : terrstr()); + tError("http-report failed to get http server:%s since %s", server, + (terrno == 0 || errno == 0) ? "invalid http server" : terrstr()); return -1; } char buf[128] = {0}; @@ -519,9 +520,10 @@ static void transHttpDestroyHandle(void* handle) { taosMemoryFree(handle); } static void transHttpEnvInit() { httpRefMgt = taosOpenRef(1, transHttpDestroyHandle); - SHttpModule* http = taosMemoryMalloc(sizeof(SHttpModule)); + SHttpModule* http = taosMemoryCalloc(1, sizeof(SHttpModule)); http->loop = taosMemoryMalloc(sizeof(uv_loop_t)); http->connStatusTable = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + http->quit = 0; uv_loop_init(http->loop); From 90588fac6988d672d853a44842015a8a915a2b08 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 6 Dec 2023 16:06:41 +0800 Subject: [PATCH 38/94] refactor code --- source/libs/transport/src/thttp.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 33d1a2565a..96537a950e 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -167,8 +167,7 @@ _OVER: static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) { uint32_t ip = taosGetIpv4FromFqdn(server); if (ip == 0xffffffff) { - tError("http-report failed to get http server:%s since %s", server, - (terrno == 0 || errno == 0) ? "invalid http server" : terrstr()); + tError("http-report failed to resolving domain names: %s", server); return -1; } char buf[128] = {0}; From a9309cb295c6b65d538994353ac5461145f84394 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 6 Dec 2023 17:52:15 +0800 Subject: [PATCH 39/94] fix(tsdb/cache): del from mem with time range --- source/dnode/vnode/src/tsdb/tsdbCache.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 4b26697464..b1424c1944 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1167,26 +1167,33 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE // taosThreadMutexLock(&pTsdb->lruMutex); + bool erase = false; LRUHandle *h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[i], klen); if (h) { SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h); - if (pLastCol->dirty) { + if (pLastCol->dirty && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { pLastCol->dirty = 0; + erase = true; } - taosLRUCacheRelease(pTsdb->lruCache, h, true); + taosLRUCacheRelease(pTsdb->lruCache, h, erase); + } + if (erase) { + taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen); } - taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen); + erase = false; h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[num_keys + i], klen); if (h) { SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h); - if (pLastCol->dirty) { + if (pLastCol->dirty && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { pLastCol->dirty = 0; + erase = true; } - taosLRUCacheRelease(pTsdb->lruCache, h, true); + taosLRUCacheRelease(pTsdb->lruCache, h, erase); + } + if (erase) { + taosLRUCacheErase(pTsdb->lruCache, keys_list[num_keys + i], klen); } - taosLRUCacheErase(pTsdb->lruCache, keys_list[num_keys + i], klen); - // taosThreadMutexUnlock(&pTsdb->lruMutex); } for (int i = 0; i < num_keys; ++i) { From 9e36b0dee4593247f5ca62753781cd46a9f55ff4 Mon Sep 17 00:00:00 2001 From: charles Date: Wed, 6 Dec 2023 18:04:26 +0800 Subject: [PATCH 40/94] add test case to create/drop same name db multiple times --- .../0-others/test_create_same_name_db.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/system-test/0-others/test_create_same_name_db.py diff --git a/tests/system-test/0-others/test_create_same_name_db.py b/tests/system-test/0-others/test_create_same_name_db.py new file mode 100644 index 0000000000..2b2c63af53 --- /dev/null +++ b/tests/system-test/0-others/test_create_same_name_db.py @@ -0,0 +1,38 @@ +import time +import os +import platform +import taos +import threading +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + + +class TDTestCase: + """This test case is used to veirfy TD-25762 + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.db_name = "db" + + def run(self): + try: + # create same name database multiple times + for i in range(100): + tdLog.debug(f"round {str(i+1)} create database {self.db_name}") + tdSql.execute(f"create database {self.db_name}") + tdLog.debug(f"round {str(i+1)} drop database {self.db_name}") + tdSql.execute(f"drop database {self.db_name}") + except Exception as ex: + tdLog.exit(str(ex)) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 241515726b462d2c5af891ed8e8894db9fad0077 Mon Sep 17 00:00:00 2001 From: "chao.feng" Date: Wed, 6 Dec 2023 18:27:36 +0800 Subject: [PATCH 41/94] modify cmake for true of DSIMD_SUPPORT by hjliao --- cmake/cmake.define | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index 7db6baafab..73f9497809 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -181,17 +181,17 @@ ELSE () ENDIF() MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED") - IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi") - MESSAGE(STATUS "avx512f/avx512bmi supported by compiler") - ENDIF() - - IF (COMPILER_SUPPORT_AVX512VL) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl") - MESSAGE(STATUS "avx512vl supported by compiler") - ENDIF() +# IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI) +# SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi") +# SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi") +# MESSAGE(STATUS "avx512f/avx512bmi supported by compiler") +# ENDIF() +# +# IF (COMPILER_SUPPORT_AVX512VL) +# SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl") +# SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl") +# MESSAGE(STATUS "avx512vl supported by compiler") +# ENDIF() ENDIF() # build mode From bfc12f03cb3e50018b35df3caab0527d5e079d71 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Wed, 6 Dec 2023 21:26:29 +0800 Subject: [PATCH 42/94] optimize & perf test --- source/common/src/tvariant.c | 126 ++++++++++++++-------- tests/system-test/1-insert/insert_perf.py | 118 ++++++++++++++++++++ 2 files changed, 198 insertions(+), 46 deletions(-) create mode 100644 tests/system-test/1-insert/insert_perf.py diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index d78c76a543..384df73342 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -81,24 +81,12 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v } if (!parsed) { - bool parse_int = true; - for (int32_t i = 0; i < n; i++) { - if (z[i] < '0' || z[i] > '9') { - parse_int = false; - break; - } - } - if (parse_int) { - // parsing as decimal - *value = taosStr2UInt64(z, &endPtr, 10); - } else { - // parsing as double - double val = taosStr2Double(z, &endPtr); - if (val > UINT64_MAX) { - return TSDB_CODE_FAILED; - } - *value = round(val); + // parsing as double + double val = taosStr2Double(z, &endPtr); + if (val > UINT64_MAX) { + return TSDB_CODE_FAILED; } + *value = round(val); } if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { @@ -145,6 +133,11 @@ int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { } int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { + if (n == 0) { + *value = 0; + return TSDB_CODE_SUCCESS; + } + errno = 0; char *endPtr = NULL; bool parsed = false; @@ -179,13 +172,21 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { return TSDB_CODE_SUCCESS; } - // parse as string - if (n == 0) { - *value = 0; + // rm tail space + while (n > 1 && z[n-1] == ' ') { + n--; + } + + // 1. try to parse as integer + *value = taosStr2Int64(z, &endPtr, 10); + if (endPtr - z == n) { + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } return TSDB_CODE_SUCCESS; } - n = removeSpace(&z, n); + // 2. parse as other bool is_neg = false; uint64_t uv = 0; int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv); @@ -211,36 +212,69 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { } int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { - - switch (type) - { - case TK_NK_INTEGER: { - return toUInteger(z, n, 10, value); - } - case TK_NK_FLOAT: { - errno = 0; - char *endPtr = NULL; - double val = round(taosStr2Double(z, &endPtr)); - if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { - return TSDB_CODE_FAILED; - } - if (!IS_VALID_UINT64(val)) { - return TSDB_CODE_FAILED; - } - *value = val; - return TSDB_CODE_SUCCESS; - } - default: - break; - } - - // parse as string if (n == 0) { *value = 0; return TSDB_CODE_SUCCESS; } - n = removeSpace(&z, n); + errno = 0; + const char *p = z; + char *endPtr = NULL; + bool parsed = false; + while (*p == ' ') { + p++; + } + switch (type) { + case TK_NK_INTEGER: { + *value = taosStr2UInt64(p, &endPtr, 10); + if (*p == '-' && *value) { + return TSDB_CODE_FAILED; + } + parsed = true; + } break; + case TK_NK_FLOAT: { + double val = round(taosStr2Double(p, &endPtr)); + if (!IS_VALID_UINT64(val)) { + return TSDB_CODE_FAILED; + } + *value = val; + parsed = true; + } break; + default: + break; + } + + if (parsed) { + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } + while (n > 1 && z[n-1] == ' ') { + n--; + } + if (endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + + // rm tail space + while (n > 1 && z[n-1] == ' ') { + n--; + } + + // 1. parse as integer + *value = taosStr2UInt64(p, &endPtr, 10); + if (endPtr - z == n) { + if (*p == '-' && *value) { + return TSDB_CODE_FAILED; + } + if (errno == ERANGE || errno == EINVAL) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; + } + + // 2. parse as other bool is_neg = false; int32_t code = parseSignAndUInteger(z, n, &is_neg, value); if (is_neg) { diff --git a/tests/system-test/1-insert/insert_perf.py b/tests/system-test/1-insert/insert_perf.py new file mode 100644 index 0000000000..bf6fb6799d --- /dev/null +++ b/tests/system-test/1-insert/insert_perf.py @@ -0,0 +1,118 @@ +import taos +import sys +import random +import time +import csv + +from datetime import datetime + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + + self.testcasePath = os.path.split(__file__)[0] + self.testcasefilename = os.path.split(__file__)[-1] + self.file1 = f"{self.testcasePath}/int.csv" + self.file2 = f"{self.testcasePath}/double.csv" + self.ts = 1700638570000 # 2023-11-22T07:36:10.000Z + self.database = "db1" + self.tb1 = "t1" + self.tb2 = "t2" + self.stable1 = "st1" + self.stable2 = "st2" + self.tag1 = f'using {self.stable1}(groupId) tags(1)' + self.tag2 = f'using {self.stable2}(groupId) tags(2)' + self.once = 1000 + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepare_db(self): + #tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database if not exists {self.database}") + tdSql.execute(f"use {self.database}") + tdSql.execute(f"create stable if not exists {self.stable1} (ts timestamp, q_bigint bigint , i1 int, i2 int, i3 int, i4 int, i5 int, i6 int, i7 int, i8 int, i9 int) tags (location binary(64), groupId int);") + tdSql.execute(f"create stable if not exists {self.stable2} (ts timestamp, q_double double, f1 double, f2 double, f3 double, f4 double, f5 double, f6 double, f7 double, f8 double, f9 double) tags (location binary(64), groupId int);") + tdSql.execute(f"drop table if exists {self.tb1};") + tdSql.execute(f"drop table if exists {self.tb2};") + tdSql.execute(f"create table {self.tb1} {self.tag1};") + tdSql.execute(f"create table {self.tb2} {self.tag2};") + + def make_csv(self, filepath, once, isint): + f = open(filepath, 'w') + with f: + writer = csv.writer(f) + rows = [] + for i in range(once): + r = [] + if isint: + for k in range(10): + r.append(random.randint(-2147483648, 2147483647)) + else: + for k in range(10): + r.append(random.randint(-2147483648, 2147483647) + random.random()) + rows.append(r) + writer.writerows(rows) + f.close() + print(f"{filepath} ready!") + + def test_insert(self, tbname, qtime, startts, filepath, isint): + f = open(filepath, 'r') + rows = [] + with f: + reader = csv.reader(f, delimiter=',', quotechar='|') + for row in reader: + rows.append(row) + f.close() + self.once = len(rows) + + sum = 0 + for j in range(qtime): + offset = j * self.once + ts = startts + offset + sql = f"insert into {self.database}.{tbname} values" + for i in range(self.once): + r = rows[i] + sql +=f"({ts + i},'{r[0]}','{r[1]}','{r[2]}','{r[3]}','{r[4]}','{r[5]}','{r[6]}','{r[7]}','{r[8]}','{r[9]}')" + + t1 = time.time() + tdSql.execute(f"{sql};", 1) + t2 = time.time() + #print(f"{t2} insert test {j}.") + #print(sql) + sum += t2 - t1 + + sum = sum + tbtype = "10 double col/per row" + if isint: + tbtype = "10 int col/per row" + print(f" insert {self.once} * {qtime} rows: {sum} s, {tbtype}") + + # tdSql.query(f"select count(*) from {self.database}.{tbname};") + # tdSql.checkData(0, 0, once*qtime) + + def run(self): + tdSql.prepare(replica = self.replicaVar) + # self.make_csv(self.file1, self.once, True) + # self.make_csv(self.file2, self.once, False) + self.prepare_db() + self.test_insert(self.tb1, 10000, self.ts, self.file1, True) + self.test_insert(self.tb2, 10000, self.ts, self.file2, False) + self.test_insert(self.tb2, 10000, self.ts, self.file1, False) + + self.test_insert(self.tb1, 10000, self.ts, self.file2, True) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 8f5df0b6a3aba15831f8372051315f4005c582a1 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 7 Dec 2023 01:51:23 +0800 Subject: [PATCH 43/94] opt double -> int --- source/common/src/tvariant.c | 68 +++++++++++++----- tests/system-test/1-insert/insert_perf.py | 86 +++++++++++++++-------- 2 files changed, 108 insertions(+), 46 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 384df73342..5c7567777f 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -95,19 +95,6 @@ int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *v return TSDB_CODE_SUCCESS; } -int32_t removeSpace(const char **pp, int32_t n) { - // rm blank space from both head and tail, keep at least one char - const char *z = *pp; - while (n > 1 && *z == ' ') { - z++; - n--; - } - while (n > 1 && z[n-1] == ' ') { - n--; - } - *pp = z; - return n; -} int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { if (n == 0) { @@ -184,6 +171,36 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; + } else if (errno == 0 && *endPtr == '.') { + // pure decimal part + const char *s = endPtr + 1; + const char *end = z + n; + bool pure = true; + while (s < end) { + if (*s < '0' || *s > '9') { + pure = false; + break; + } + s++; + } + if (pure) { + if (endPtr+1 < end && endPtr[1] > '4') { + const char *p = z; + while (*p == ' ') { + p++; + } + if (*p == '-') { + if ( *value > INT64_MIN) { + (*value)--; + } + } else { + if ( *value < INT64_MAX) { + (*value)++; + } + } + } + return TSDB_CODE_SUCCESS; + } } // 2. parse as other @@ -218,9 +235,9 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { } errno = 0; - const char *p = z; char *endPtr = NULL; bool parsed = false; + const char *p = z; while (*p == ' ') { p++; } @@ -264,14 +281,31 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { // 1. parse as integer *value = taosStr2UInt64(p, &endPtr, 10); + if (*p == '-' && *value) { + return TSDB_CODE_FAILED; + } if (endPtr - z == n) { - if (*p == '-' && *value) { - return TSDB_CODE_FAILED; - } if (errno == ERANGE || errno == EINVAL) { return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; + } else if (errno == 0 && *endPtr == '.') { + const char *s = endPtr + 1; + const char *end = z + n; + bool pure = true; + while (s < end) { + if (*s < '0' || *s > '9') { + pure = false; + break; + } + s++; + } + if (pure) { + if (endPtr + 1 < end && endPtr[1] > '4' && *value < UINT64_MAX) { + (*value)++; + } + return TSDB_CODE_SUCCESS; + } } // 2. parse as other diff --git a/tests/system-test/1-insert/insert_perf.py b/tests/system-test/1-insert/insert_perf.py index bf6fb6799d..d96a031458 100644 --- a/tests/system-test/1-insert/insert_perf.py +++ b/tests/system-test/1-insert/insert_perf.py @@ -21,48 +21,70 @@ class TDTestCase: self.testcasefilename = os.path.split(__file__)[-1] self.file1 = f"{self.testcasePath}/int.csv" self.file2 = f"{self.testcasePath}/double.csv" + self.file3 = f"{self.testcasePath}/d+.csv" + self.file4 = f"{self.testcasePath}/uint.csv" self.ts = 1700638570000 # 2023-11-22T07:36:10.000Z self.database = "db1" self.tb1 = "t1" self.tb2 = "t2" - self.stable1 = "st1" - self.stable2 = "st2" - self.tag1 = f'using {self.stable1}(groupId) tags(1)' - self.tag2 = f'using {self.stable2}(groupId) tags(2)' + self.tb3 = "t3" self.once = 1000 tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) def prepare_db(self): - #tdSql.execute(f"drop database if exists {self.database}") - tdSql.execute(f"create database if not exists {self.database}") + tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database {self.database}") tdSql.execute(f"use {self.database}") - tdSql.execute(f"create stable if not exists {self.stable1} (ts timestamp, q_bigint bigint , i1 int, i2 int, i3 int, i4 int, i5 int, i6 int, i7 int, i8 int, i9 int) tags (location binary(64), groupId int);") - tdSql.execute(f"create stable if not exists {self.stable2} (ts timestamp, q_double double, f1 double, f2 double, f3 double, f4 double, f5 double, f6 double, f7 double, f8 double, f9 double) tags (location binary(64), groupId int);") - tdSql.execute(f"drop table if exists {self.tb1};") - tdSql.execute(f"drop table if exists {self.tb2};") - tdSql.execute(f"create table {self.tb1} {self.tag1};") - tdSql.execute(f"create table {self.tb2} {self.tag2};") - - def make_csv(self, filepath, once, isint): + tdSql.execute(f"create table {self.tb1} (ts timestamp, i0 bigint , i1 bigint, i2 bigint, i3 bigint, i4 bigint, i5 bigint, i6 bigint, i7 bigint, i8 bigint, i9 bigint)") + tdSql.execute(f"create table {self.tb2} (ts timestamp, f0 double, f1 double, f2 double, f3 double, f4 double, f5 double, f6 double, f7 double, f8 double, f9 double)") + tdSql.execute(f"create table {self.tb3} (ts timestamp, i0 int unsigned , i1 int unsigned, i2 int unsigned, i3 int unsigned, i4 int unsigned, i5 int unsigned, i6 int unsigned, i7 int unsigned, i8 int unsigned, i9 int unsigned)") + + def make_csv(self, once, intype): + filepath = self.file1 + if intype == 2: + filepath = self.file2 + elif intype == 3: + filepath = self.file3 + elif intype == 4: + filepath = self.file4 + f = open(filepath, 'w') with f: writer = csv.writer(f) rows = [] for i in range(once): r = [] - if isint: + if intype == 1: for k in range(10): r.append(random.randint(-2147483648, 2147483647)) + elif intype == 2: + for k in range(10): + r.append(random.randint(-2147483648, 2147483646) + random.random()) + elif intype == 3: + for k in range(10): + r.append(random.randint(0, 4294967294) + random.random()) else: for k in range(10): - r.append(random.randint(-2147483648, 2147483647) + random.random()) + r.append(random.randint(0, 4294967295)) rows.append(r) writer.writerows(rows) f.close() print(f"{filepath} ready!") - def test_insert(self, tbname, qtime, startts, filepath, isint): + def test_insert(self, tbname, qtime, startts, intype, outtype): + filepath = self.file1 + dinfo = "int" + if intype == 2: + filepath = self.file2 + dinfo = "double" + elif intype == 3: + filepath = self.file3 + dinfo = "+double" + elif intype == 4: + filepath = self.file4 + dinfo = "uint" + f = open(filepath, 'r') rows = [] with f: @@ -89,25 +111,31 @@ class TDTestCase: sum += t2 - t1 sum = sum - tbtype = "10 double col/per row" - if isint: - tbtype = "10 int col/per row" - print(f" insert {self.once} * {qtime} rows: {sum} s, {tbtype}") + tbinfo = "10 bigint col/per row" + if outtype == 2: + tbinfo = "10 double col/per row" + elif outtype == 3: + tbinfo = "10 uint col/per row" + print(f" insert {self.once} * {qtime} rows: {sum} s, {dinfo} -> {tbinfo}") # tdSql.query(f"select count(*) from {self.database}.{tbname};") # tdSql.checkData(0, 0, once*qtime) def run(self): tdSql.prepare(replica = self.replicaVar) - # self.make_csv(self.file1, self.once, True) - # self.make_csv(self.file2, self.once, False) + # self.make_csv(self.once, 1) + # self.make_csv(self.once, 2) + # self.make_csv(self.once, 3) + # self.make_csv(self.once, 4) + self.prepare_db() - self.test_insert(self.tb1, 10000, self.ts, self.file1, True) - self.test_insert(self.tb2, 10000, self.ts, self.file2, False) - self.test_insert(self.tb2, 10000, self.ts, self.file1, False) - - self.test_insert(self.tb1, 10000, self.ts, self.file2, True) - + self.test_insert(self.tb1, 1000, self.ts-10000000, 1, 1) + self.test_insert(self.tb2, 1000, self.ts-10000000, 2, 2) + self.test_insert(self.tb3, 1000, self.ts-10000000, 4, 3) + self.test_insert(self.tb2, 1000, self.ts, 1, 2) + + self.test_insert(self.tb1, 1000, self.ts, 2, 1) + self.test_insert(self.tb3, 1000, self.ts, 3, 3) def stop(self): tdSql.close() From aa1445742bb5ae27c1563d84eef3e1c91801dc5d Mon Sep 17 00:00:00 2001 From: Charles Date: Thu, 7 Dec 2023 08:06:45 +0800 Subject: [PATCH 44/94] Update cases.task --- tests/parallel_test/cases.task | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 39db02f869..cf4559100b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -518,7 +518,6 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tagFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts_3405_3398_3423.py -N 3 -n 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/geometry.py ,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py From 515d671e00621620dc7940fda4cd04552d10c617 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 7 Dec 2023 10:04:47 +0800 Subject: [PATCH 45/94] test:modify the keep parameter of db in python.sh cases --- docs/examples/python/bind_param_example.py | 2 +- docs/examples/python/conn_websocket_pandas.py | 2 +- docs/examples/python/connect_rest_examples.py | 2 +- docs/examples/python/connect_rest_with_req_id_examples.py | 2 +- docs/examples/python/connect_websocket_examples.py | 6 +++--- .../python/connect_websocket_with_req_id_examples.py | 2 +- docs/examples/python/connection_usage_native_reference.py | 2 +- .../python/connection_usage_native_reference_with_req_id.py | 2 +- docs/examples/python/cursor_usage_native_reference.py | 2 +- .../python/cursor_usage_native_reference_with_req_id.py | 2 +- docs/examples/python/fast_write_example.py | 2 +- docs/examples/python/json_protocol_example.py | 2 +- docs/examples/python/kafka_example_common.py | 2 +- docs/examples/python/line_protocol_example.py | 2 +- docs/examples/python/multi_bind_example.py | 2 +- docs/examples/python/native_insert_example.py | 2 +- docs/examples/python/result_set_examples.py | 2 +- docs/examples/python/result_set_with_req_id_examples.py | 2 +- docs/examples/python/schemaless_insert.py | 2 +- docs/examples/python/schemaless_insert_raw.py | 4 ++-- docs/examples/python/schemaless_insert_raw_req_id.py | 4 ++-- docs/examples/python/schemaless_insert_raw_ttl.py | 4 ++-- docs/examples/python/schemaless_insert_req_id.py | 2 +- docs/examples/python/schemaless_insert_ttl.py | 2 +- docs/examples/python/sql_writer.py | 2 +- docs/examples/python/stmt_example.py | 2 +- docs/examples/python/stmt_websocket_example.py | 2 +- docs/examples/python/telnet_line_protocol_example.py | 2 +- docs/examples/python/tmq_assignment_example.py | 2 +- docs/examples/python/tmq_example.py | 2 +- docs/examples/python/tmq_websocket_assgnment_example.py | 2 +- tests/docs-examples-test/python.sh | 2 +- 32 files changed, 37 insertions(+), 37 deletions(-) diff --git a/docs/examples/python/bind_param_example.py b/docs/examples/python/bind_param_example.py index 6a67434f87..e3df9f7d25 100644 --- a/docs/examples/python/bind_param_example.py +++ b/docs/examples/python/bind_param_example.py @@ -20,7 +20,7 @@ def get_ts(ts: str): def create_stable(): conn = taos.connect() try: - conn.execute("CREATE DATABASE power") + conn.execute("CREATE DATABASE power keep 36500") conn.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " "TAGS (location BINARY(64), groupId INT)") finally: diff --git a/docs/examples/python/conn_websocket_pandas.py b/docs/examples/python/conn_websocket_pandas.py index 5cad5384b2..2986aace9f 100644 --- a/docs/examples/python/conn_websocket_pandas.py +++ b/docs/examples/python/conn_websocket_pandas.py @@ -4,7 +4,7 @@ import taos taos_conn = taos.connect() taos_conn.execute('drop database if exists power') -taos_conn.execute('create database if not exists power wal_retention_period 3600') +taos_conn.execute('create database if not exists power wal_retention_period 3600 keep 36500 ') taos_conn.execute("use power") taos_conn.execute( "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") diff --git a/docs/examples/python/connect_rest_examples.py b/docs/examples/python/connect_rest_examples.py index 1c432dcc65..c8a9292547 100644 --- a/docs/examples/python/connect_rest_examples.py +++ b/docs/examples/python/connect_rest_examples.py @@ -11,7 +11,7 @@ conn = connect(url="http://localhost:6041", # create STable cursor = conn.cursor() cursor.execute("DROP DATABASE IF EXISTS power") -cursor.execute("CREATE DATABASE power") +cursor.execute("CREATE DATABASE power keep 36500 ") cursor.execute( "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") diff --git a/docs/examples/python/connect_rest_with_req_id_examples.py b/docs/examples/python/connect_rest_with_req_id_examples.py index f1b5915ea3..568cbea168 100644 --- a/docs/examples/python/connect_rest_with_req_id_examples.py +++ b/docs/examples/python/connect_rest_with_req_id_examples.py @@ -11,7 +11,7 @@ conn = connect(url="http://localhost:6041", # create STable cursor = conn.cursor() cursor.execute("DROP DATABASE IF EXISTS power", req_id=1) -cursor.execute("CREATE DATABASE power", req_id=2) +cursor.execute("CREATE DATABASE power keep 36500", req_id=2) cursor.execute( "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)", req_id=3) diff --git a/docs/examples/python/connect_websocket_examples.py b/docs/examples/python/connect_websocket_examples.py index 29452bbf9d..75e7422a90 100644 --- a/docs/examples/python/connect_websocket_examples.py +++ b/docs/examples/python/connect_websocket_examples.py @@ -6,13 +6,13 @@ conn = taosws.connect("taosws://root:taosdata@localhost:6041") # ANCHOR: basic conn.execute("drop database if exists connwspy") -conn.execute("create database if not exists connwspy wal_retention_period 3600") +conn.execute("create database if not exists connwspy wal_retention_period 3600 keep 36500 ") conn.execute("use connwspy") conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)") conn.execute("create table if not exists tb1 using stb tags (1)") conn.execute("insert into tb1 values (now, 1)") -conn.execute("insert into tb1 values (now, 2)") -conn.execute("insert into tb1 values (now, 3)") +conn.execute("insert into tb1 values (now+1s, 2)") +conn.execute("insert into tb1 values (now+2s, 3)") r = conn.execute("select * from stb") result = conn.query("select * from stb") diff --git a/docs/examples/python/connect_websocket_with_req_id_examples.py b/docs/examples/python/connect_websocket_with_req_id_examples.py index f5f76c8446..3588b8e41f 100644 --- a/docs/examples/python/connect_websocket_with_req_id_examples.py +++ b/docs/examples/python/connect_websocket_with_req_id_examples.py @@ -6,7 +6,7 @@ conn = taosws.connect("taosws://root:taosdata@localhost:6041") # ANCHOR: basic conn.execute("drop database if exists connwspy", req_id=1) -conn.execute("create database if not exists connwspy", req_id=2) +conn.execute("create database if not exists connwspy keep 36500", req_id=2) conn.execute("use connwspy", req_id=3) conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)", req_id=4) conn.execute("create table if not exists tb1 using stb tags (1)", req_id=5) diff --git a/docs/examples/python/connection_usage_native_reference.py b/docs/examples/python/connection_usage_native_reference.py index 0a23c5f95b..3610087e7f 100644 --- a/docs/examples/python/connection_usage_native_reference.py +++ b/docs/examples/python/connection_usage_native_reference.py @@ -4,7 +4,7 @@ import taos conn = taos.connect() # Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement. conn.execute("DROP DATABASE IF EXISTS test") -conn.execute("CREATE DATABASE test") +conn.execute("CREATE DATABASE test keep 36500") # change database. same as execute "USE db" conn.select_db("test") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") diff --git a/docs/examples/python/connection_usage_native_reference_with_req_id.py b/docs/examples/python/connection_usage_native_reference_with_req_id.py index 24d0914ad5..3d568a1e1e 100644 --- a/docs/examples/python/connection_usage_native_reference_with_req_id.py +++ b/docs/examples/python/connection_usage_native_reference_with_req_id.py @@ -4,7 +4,7 @@ import taos conn = taos.connect() # Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement. conn.execute("DROP DATABASE IF EXISTS test", req_id=1) -conn.execute("CREATE DATABASE test", req_id=2) +conn.execute("CREATE DATABASE test keep 36500", req_id=2) # change database. same as execute "USE db" conn.select_db("test") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3) diff --git a/docs/examples/python/cursor_usage_native_reference.py b/docs/examples/python/cursor_usage_native_reference.py index a5103810f0..32ee51354d 100644 --- a/docs/examples/python/cursor_usage_native_reference.py +++ b/docs/examples/python/cursor_usage_native_reference.py @@ -4,7 +4,7 @@ conn = taos.connect() cursor = conn.cursor() cursor.execute("DROP DATABASE IF EXISTS test") -cursor.execute("CREATE DATABASE test") +cursor.execute("CREATE DATABASE test keep 36500") cursor.execute("USE test") cursor.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") diff --git a/docs/examples/python/cursor_usage_native_reference_with_req_id.py b/docs/examples/python/cursor_usage_native_reference_with_req_id.py index 15207ee6bc..345a804923 100644 --- a/docs/examples/python/cursor_usage_native_reference_with_req_id.py +++ b/docs/examples/python/cursor_usage_native_reference_with_req_id.py @@ -4,7 +4,7 @@ conn = taos.connect() cursor = conn.cursor() cursor.execute("DROP DATABASE IF EXISTS test", req_id=1) -cursor.execute("CREATE DATABASE test", req_id=2) +cursor.execute("CREATE DATABASE test keep 36500", req_id=2) cursor.execute("USE test", req_id=3) cursor.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=4) diff --git a/docs/examples/python/fast_write_example.py b/docs/examples/python/fast_write_example.py index 626e3310b1..76e84e97ac 100644 --- a/docs/examples/python/fast_write_example.py +++ b/docs/examples/python/fast_write_example.py @@ -160,7 +160,7 @@ def main(infinity): conn = get_connection() conn.execute("DROP DATABASE IF EXISTS test") - conn.execute("CREATE DATABASE IF NOT EXISTS test") + conn.execute("CREATE DATABASE IF NOT EXISTS test keep 36500") conn.execute("CREATE STABLE IF NOT EXISTS test.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " "TAGS (location BINARY(64), groupId INT)") conn.close() diff --git a/docs/examples/python/json_protocol_example.py b/docs/examples/python/json_protocol_example.py index 58b38f3ff6..a38dcbf0ad 100644 --- a/docs/examples/python/json_protocol_example.py +++ b/docs/examples/python/json_protocol_example.py @@ -16,7 +16,7 @@ def get_connection(): def create_database(conn): - conn.execute("CREATE DATABASE test") + conn.execute("CREATE DATABASE test keep 36500") conn.execute("USE test") diff --git a/docs/examples/python/kafka_example_common.py b/docs/examples/python/kafka_example_common.py index 1c735abfc0..ed0540574f 100644 --- a/docs/examples/python/kafka_example_common.py +++ b/docs/examples/python/kafka_example_common.py @@ -5,7 +5,7 @@ LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanD 'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale', 'California.SantaClara', 'California.Cupertino'] -CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1 wal_retention_period 3600' +CREATE_DATABASE_SQL = 'create database if not exists {} keep 36500 duration 10 buffer 16 wal_level 1 wal_retention_period 3600' USE_DATABASE_SQL = 'use {}' DROP_TABLE_SQL = 'drop table if exists meters' DROP_DATABASE_SQL = 'drop database if exists {}' diff --git a/docs/examples/python/line_protocol_example.py b/docs/examples/python/line_protocol_example.py index 735e8e7eb8..6482032e6e 100644 --- a/docs/examples/python/line_protocol_example.py +++ b/docs/examples/python/line_protocol_example.py @@ -15,7 +15,7 @@ def get_connection(): def create_database(conn): # the default precision is ms (microsecond), but we use us(microsecond) here. - conn.execute("CREATE DATABASE test precision 'us'") + conn.execute("CREATE DATABASE test precision 'us' keep 36500") conn.execute("USE test") diff --git a/docs/examples/python/multi_bind_example.py b/docs/examples/python/multi_bind_example.py index 205ba69fb2..b29e1cd17e 100644 --- a/docs/examples/python/multi_bind_example.py +++ b/docs/examples/python/multi_bind_example.py @@ -71,7 +71,7 @@ def insert_data(): def create_stable(): conn = taos.connect() try: - conn.execute("CREATE DATABASE power") + conn.execute("CREATE DATABASE power keep 36500") conn.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " "TAGS (location BINARY(64), groupId INT)") finally: diff --git a/docs/examples/python/native_insert_example.py b/docs/examples/python/native_insert_example.py index cdde7d23d2..0fba375678 100644 --- a/docs/examples/python/native_insert_example.py +++ b/docs/examples/python/native_insert_example.py @@ -18,7 +18,7 @@ def get_connection() -> taos.TaosConnection: def create_stable(conn: taos.TaosConnection): - conn.execute("CREATE DATABASE power") + conn.execute("CREATE DATABASE power keep 36500") conn.execute("USE power") conn.execute("CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) " "TAGS (location BINARY(64), groupId INT)") diff --git a/docs/examples/python/result_set_examples.py b/docs/examples/python/result_set_examples.py index 6cba0d3f73..234c624a4d 100644 --- a/docs/examples/python/result_set_examples.py +++ b/docs/examples/python/result_set_examples.py @@ -2,7 +2,7 @@ import taos conn = taos.connect() conn.execute("DROP DATABASE IF EXISTS test") -conn.execute("CREATE DATABASE test") +conn.execute("CREATE DATABASE test keep 36500") conn.select_db("test") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") # prepare data diff --git a/docs/examples/python/result_set_with_req_id_examples.py b/docs/examples/python/result_set_with_req_id_examples.py index 90ae2f4f26..f46a3697b3 100644 --- a/docs/examples/python/result_set_with_req_id_examples.py +++ b/docs/examples/python/result_set_with_req_id_examples.py @@ -2,7 +2,7 @@ import taos conn = taos.connect() conn.execute("DROP DATABASE IF EXISTS test", req_id=1) -conn.execute("CREATE DATABASE test", req_id=2) +conn.execute("CREATE DATABASE test keep 36500", req_id=2) conn.select_db("test") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3) # prepare data diff --git a/docs/examples/python/schemaless_insert.py b/docs/examples/python/schemaless_insert.py index 334a4b728f..74ab4b15fe 100644 --- a/docs/examples/python/schemaless_insert.py +++ b/docs/examples/python/schemaless_insert.py @@ -3,7 +3,7 @@ import taos conn = taos.connect() dbname = "pytest_line" conn.execute("drop database if exists %s" % dbname) -conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname) conn.select_db(dbname) lines = [ diff --git a/docs/examples/python/schemaless_insert_raw.py b/docs/examples/python/schemaless_insert_raw.py index 0fda7dc505..b5ef5833a6 100644 --- a/docs/examples/python/schemaless_insert_raw.py +++ b/docs/examples/python/schemaless_insert_raw.py @@ -10,9 +10,9 @@ try: conn.execute("drop database if exists %s" % dbname) if taos.IS_V3: - conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname) else: - conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname) conn.select_db(dbname) diff --git a/docs/examples/python/schemaless_insert_raw_req_id.py b/docs/examples/python/schemaless_insert_raw_req_id.py index 606e510986..5488e2ddc0 100644 --- a/docs/examples/python/schemaless_insert_raw_req_id.py +++ b/docs/examples/python/schemaless_insert_raw_req_id.py @@ -10,9 +10,9 @@ try: conn.execute("drop database if exists %s" % dbname) if taos.IS_V3: - conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname) else: - conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname) conn.select_db(dbname) diff --git a/docs/examples/python/schemaless_insert_raw_ttl.py b/docs/examples/python/schemaless_insert_raw_ttl.py index cf57792534..73909e4290 100644 --- a/docs/examples/python/schemaless_insert_raw_ttl.py +++ b/docs/examples/python/schemaless_insert_raw_ttl.py @@ -10,9 +10,9 @@ try: conn.execute("drop database if exists %s" % dbname) if taos.IS_V3: - conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname) else: - conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname) conn.select_db(dbname) diff --git a/docs/examples/python/schemaless_insert_req_id.py b/docs/examples/python/schemaless_insert_req_id.py index ee1472db69..a5091d80a8 100644 --- a/docs/examples/python/schemaless_insert_req_id.py +++ b/docs/examples/python/schemaless_insert_req_id.py @@ -4,7 +4,7 @@ from taos import SmlProtocol, SmlPrecision conn = taos.connect() dbname = "pytest_line" conn.execute("drop database if exists %s" % dbname) -conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname) conn.select_db(dbname) lines = [ diff --git a/docs/examples/python/schemaless_insert_ttl.py b/docs/examples/python/schemaless_insert_ttl.py index 85050439f2..6ad134fae1 100644 --- a/docs/examples/python/schemaless_insert_ttl.py +++ b/docs/examples/python/schemaless_insert_ttl.py @@ -4,7 +4,7 @@ from taos import SmlProtocol, SmlPrecision conn = taos.connect() dbname = "pytest_line" conn.execute("drop database if exists %s" % dbname) -conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname) conn.select_db(dbname) lines = [ diff --git a/docs/examples/python/sql_writer.py b/docs/examples/python/sql_writer.py index 3456981a7b..d62f4c8a8d 100644 --- a/docs/examples/python/sql_writer.py +++ b/docs/examples/python/sql_writer.py @@ -10,7 +10,7 @@ class SQLWriter: self._tb_tags = {} self._conn = get_connection_func() self._max_sql_length = self.get_max_sql_length() - self._conn.execute("create database if not exists test") + self._conn.execute("create database if not exists test keep 36500") self._conn.execute("USE test") def get_max_sql_length(self): diff --git a/docs/examples/python/stmt_example.py b/docs/examples/python/stmt_example.py index 83197a777a..cfdd81c90c 100644 --- a/docs/examples/python/stmt_example.py +++ b/docs/examples/python/stmt_example.py @@ -10,7 +10,7 @@ db_name = 'test_ws_stmt' def before(): taos_conn = taos.connect() taos_conn.execute("drop database if exists %s" % db_name) - taos_conn.execute("create database %s" % db_name) + taos_conn.execute("create database %s keep 36500" % db_name) taos_conn.select_db(db_name) taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))") taos_conn.execute( diff --git a/docs/examples/python/stmt_websocket_example.py b/docs/examples/python/stmt_websocket_example.py index d0824cfa9f..2acab188f3 100644 --- a/docs/examples/python/stmt_websocket_example.py +++ b/docs/examples/python/stmt_websocket_example.py @@ -9,7 +9,7 @@ import taos def before_test(db_name): taos_conn = taos.connect() taos_conn.execute("drop database if exists %s" % db_name) - taos_conn.execute("create database %s" % db_name) + taos_conn.execute("create database %s keep 36500" % db_name) taos_conn.select_db(db_name) taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))") taos_conn.execute( diff --git a/docs/examples/python/telnet_line_protocol_example.py b/docs/examples/python/telnet_line_protocol_example.py index d812e186af..0d524e8d4a 100644 --- a/docs/examples/python/telnet_line_protocol_example.py +++ b/docs/examples/python/telnet_line_protocol_example.py @@ -19,7 +19,7 @@ def get_connection(): def create_database(conn): - conn.execute("CREATE DATABASE test") + conn.execute("CREATE DATABASE test keep 36500") conn.execute("USE test") diff --git a/docs/examples/python/tmq_assignment_example.py b/docs/examples/python/tmq_assignment_example.py index c370db47a5..c063768af4 100644 --- a/docs/examples/python/tmq_assignment_example.py +++ b/docs/examples/python/tmq_assignment_example.py @@ -7,7 +7,7 @@ def prepare(): conn = taos.connect() conn.execute("drop topic if exists tmq_assignment_demo_topic") conn.execute("drop database if exists tmq_assignment_demo_db") - conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600") + conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600 keep 36500") conn.select_db("tmq_assignment_demo_db") conn.execute( "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)") diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py index 5b462fa153..e0bddd9911 100644 --- a/docs/examples/python/tmq_example.py +++ b/docs/examples/python/tmq_example.py @@ -6,7 +6,7 @@ def init_tmq_env(db, topic): conn = taos.connect() conn.execute("drop topic if exists {}".format(topic)) conn.execute("drop database if exists {}".format(db)) - conn.execute("create database if not exists {} wal_retention_period 3600".format(db)) + conn.execute("create database if not exists {} wal_retention_period 3600 keep 36500".format(db)) conn.select_db(db) conn.execute( "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))") diff --git a/docs/examples/python/tmq_websocket_assgnment_example.py b/docs/examples/python/tmq_websocket_assgnment_example.py index a180ef840e..ca50015162 100644 --- a/docs/examples/python/tmq_websocket_assgnment_example.py +++ b/docs/examples/python/tmq_websocket_assgnment_example.py @@ -6,7 +6,7 @@ def prepare(): conn = taos.connect() conn.execute("drop topic if exists tmq_assignment_demo_topic") conn.execute("drop database if exists tmq_assignment_demo_db") - conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600") + conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600 keep 36500") conn.select_db("tmq_assignment_demo_db") conn.execute( "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)") diff --git a/tests/docs-examples-test/python.sh b/tests/docs-examples-test/python.sh index 5de7261e02..84f0771ec5 100644 --- a/tests/docs-examples-test/python.sh +++ b/tests/docs-examples-test/python.sh @@ -86,7 +86,7 @@ pip3 install kafka-python python3 kafka_example_consumer.py # 21 -pip3 install taos-ws-py==0.2.6 +pip3 install taos-ws-py==0.3.1 python3 conn_websocket_pandas.py # 22 From d3f3ebfe0def31fa04d06bb4287c3dd0f49dae93 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 7 Dec 2023 12:01:59 +0800 Subject: [PATCH 46/94] fix:memory access out of bounds in doGeomFromTextFunc --- source/libs/geometry/src/geomFunc.c | 19 +++++++++++-------- tests/system-test/pytest.sh | 5 +++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/source/libs/geometry/src/geomFunc.c b/source/libs/geometry/src/geomFunc.c index 3588bf8b7d..2ac1761737 100644 --- a/source/libs/geometry/src/geomFunc.c +++ b/source/libs/geometry/src/geomFunc.c @@ -67,15 +67,19 @@ int32_t doGeomFromTextFunc(const char *input, unsigned char **output) { return TSDB_CODE_SUCCESS; } - // make input as a zero ending string - char *end = varDataVal(input) + varDataLen(input); - char endValue = *end; - *end = 0; - + char *inputGeom = NULL; unsigned char *outputGeom = NULL; size_t size = 0; - code = doGeomFromText(varDataVal(input), &outputGeom, &size); + // make a zero ending string + inputGeom = taosMemoryCalloc(1, varDataLen(input) + 1); + if (inputGeom == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(inputGeom, varDataVal(input), varDataLen(input)); + + code = doGeomFromText(inputGeom, &outputGeom, &size); if (code != TSDB_CODE_SUCCESS) { goto _exit; } @@ -92,8 +96,7 @@ int32_t doGeomFromTextFunc(const char *input, unsigned char **output) { _exit: geosFreeBuffer(outputGeom); - - *end = endValue; //recover the input string + geosFreeBuffer(inputGeom); return code; } diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index dfdfc32ed0..2c95516d10 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -79,7 +79,8 @@ else unset LD_PRELOAD #export LD_PRELOAD=libasan.so.5 - export LD_PRELOAD=$(gcc -print-file-name=libasan.so) + #export LD_PRELOAD=$(gcc -print-file-name=libasan.so) + export LD_PRELOAD="$(realpath "$(gcc -print-file-name=libasan.so)") $(realpath "$(gcc -print-file-name=libstdc++.so)")" echo "Preload AsanSo:" $? $* -a 2>$AsanFile @@ -104,4 +105,4 @@ else echo "Execute script failure" exit 1 fi -fi \ No newline at end of file +fi From 07af4e9a08e80935bcdefd5a36ac36bcaf62ebf0 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 7 Dec 2023 12:02:04 +0800 Subject: [PATCH 47/94] adjust --- source/common/src/tvariant.c | 56 +++++++++++++----------------------- 1 file changed, 20 insertions(+), 36 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 5c7567777f..40de0428ff 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -120,42 +120,34 @@ int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { } int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { - if (n == 0) { - *value = 0; - return TSDB_CODE_SUCCESS; - } - errno = 0; char *endPtr = NULL; - bool parsed = false; switch (type) { case TK_NK_INTEGER: { *value = taosStr2Int64(z, &endPtr, 10); - parsed = true; + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; } break; case TK_NK_FLOAT: { double val = round(taosStr2Double(z, &endPtr)); - parsed = true; if (!IS_VALID_INT64(val)) { return TSDB_CODE_FAILED; } + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } *value = val; + return TSDB_CODE_SUCCESS; } break; default: break; } - if (parsed) { - if (errno == ERANGE || errno == EINVAL) { - return TSDB_CODE_FAILED; - } - while (n > 1 && z[n-1] == ' ') { - n--; - } - if (endPtr - z != n) { - return TSDB_CODE_FAILED; - } + if (n == 0) { + *value = 0; return TSDB_CODE_SUCCESS; } @@ -229,14 +221,8 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { } int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { - if (n == 0) { - *value = 0; - return TSDB_CODE_SUCCESS; - } - errno = 0; char *endPtr = NULL; - bool parsed = false; const char *p = z; while (*p == ' ') { p++; @@ -247,30 +233,28 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { if (*p == '-' && *value) { return TSDB_CODE_FAILED; } - parsed = true; + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; } break; case TK_NK_FLOAT: { double val = round(taosStr2Double(p, &endPtr)); if (!IS_VALID_UINT64(val)) { return TSDB_CODE_FAILED; } + if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { + return TSDB_CODE_FAILED; + } *value = val; - parsed = true; + return TSDB_CODE_SUCCESS; } break; default: break; } - if (parsed) { - if (errno == ERANGE || errno == EINVAL) { - return TSDB_CODE_FAILED; - } - while (n > 1 && z[n-1] == ' ') { - n--; - } - if (endPtr - z != n) { - return TSDB_CODE_FAILED; - } + if (n == 0) { + *value = 0; return TSDB_CODE_SUCCESS; } From 4737b00879433b5bc7892077ea4a6bf87643c658 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 7 Dec 2023 14:00:50 +0800 Subject: [PATCH 48/94] cancel rm tail space --- source/common/src/tvariant.c | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 40de0428ff..dd5e8240b9 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -109,10 +109,7 @@ int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) { if (errno == ERANGE || errno == EINVAL) { return TSDB_CODE_FAILED; } - // rm tail space - while (n > 1 && z[n-1] == ' ') { - n--; - } + if (endPtr - z != n) { return TSDB_CODE_FAILED; } @@ -151,11 +148,6 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { return TSDB_CODE_SUCCESS; } - // rm tail space - while (n > 1 && z[n-1] == ' ') { - n--; - } - // 1. try to parse as integer *value = taosStr2Int64(z, &endPtr, 10); if (endPtr - z == n) { @@ -258,11 +250,6 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { return TSDB_CODE_SUCCESS; } - // rm tail space - while (n > 1 && z[n-1] == ' ') { - n--; - } - // 1. parse as integer *value = taosStr2UInt64(p, &endPtr, 10); if (*p == '-' && *value) { From 26855848f61cd350d7a413fcbceeec8899bb5eba Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 7 Dec 2023 14:07:09 +0800 Subject: [PATCH 49/94] opti:schemaless logic --- source/client/inc/clientSml.h | 85 ++--- source/client/src/clientSml.c | 560 ++++++++++++++++------------ source/client/src/clientSmlJson.c | 416 +++------------------ source/client/src/clientSmlLine.c | 311 +++------------ source/client/src/clientSmlTelnet.c | 204 +++------- 5 files changed, 522 insertions(+), 1054 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 30d4792279..26c0a38c8a 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -65,14 +65,32 @@ extern "C" { #define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) #define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) -//#define TS "_ts" -//#define TS_LEN 3 #define VALUE "_value" -#define VALUE_LEN 6 +#define VALUE_LEN (sizeof(VALUE)-1) #define OTD_JSON_FIELDS_NUM 4 #define MAX_RETRY_TIMES 10 -typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; + +#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0) + +#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0) + +#define IS_SAME_KEY (maxKV->type == kv->type && maxKV->keyLen == kv->keyLen && memcmp(maxKV->key, kv->key, kv->keyLen) == 0) + +#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE)) + +#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) + +#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) { \ + MOVE_FORWARD_ONE(key + i, keyLen - i); \ + keyLen--; \ + } \ + } typedef enum { SCHEMA_ACTION_NULL, @@ -83,18 +101,6 @@ typedef enum { SCHEMA_ACTION_CHANGE_TAG_SIZE, } ESchemaAction; -typedef struct { - const void *key; - int32_t keyLen; - void *value; - bool used; -}Node; - -typedef struct NodeList{ - Node data; - struct NodeList* next; -}NodeList; - typedef struct { char *measure; char *tags; @@ -117,7 +123,6 @@ typedef struct { int32_t sTableNameLen; char childTableName[TSDB_TABLE_NAME_LEN]; uint64_t uid; -// void *key; // for openTsdb SArray *tags; @@ -161,7 +166,8 @@ typedef struct { typedef struct { int64_t id; - SMLProtocolType protocol; + TSDB_SML_PROTOCOL_TYPE protocol; + int8_t precision; bool reRun; bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) @@ -201,29 +207,8 @@ typedef struct { bool needModifySchema; } SSmlHandle; -#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ -&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0) - -#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \ -&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0) - -#define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0) - -#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ - (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE)) - -#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) - -#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \ - for (int i = 1; i < keyLen; ++i) { \ - if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) { \ - MOVE_FORWARD_ONE(key + i, keyLen - i); \ - keyLen--; \ - } \ - } - -extern int64_t smlFactorNS[3]; -extern int64_t smlFactorS[3]; +extern int64_t smlFactorNS[]; +extern int64_t smlFactorS[]; typedef int32_t (*_equal_fn_sml)(const void *, const void *); @@ -231,11 +216,7 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos); void smlDestroyInfo(SSmlHandle *info); int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); -//SArray *smlJsonParseTags(char *start, char *end); bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); -//void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn); -//int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn); -//int nodeListSize(NodeList* list); bool smlDoubleToInt64OverFlow(double num); int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); @@ -253,12 +234,22 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg); uint8_t smlGetTimestampLen(int64_t num); void smlDestroyTableInfo(void *para); -void freeSSmlKv(void* data); +void freeSSmlKv(void* data); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseJSON(SSmlHandle *info, char *payload); - void smlStrReplace(char* src, int32_t len); + +SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement); +bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv); +bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv); +int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements); +int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements); +int32_t smlJoinMeasureTag(SSmlLineInfo *elements); +void smlBuildTsKv(SSmlKv *kv, int64_t ts); +int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); +int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs); + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 6bcaae3bf0..aff8f4f43f 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -20,75 +20,93 @@ #include "clientSml.h" -int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL}; -int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; -int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL}; +#define RETURN_FALSE \ + smlBuildInvalidDataMsg(msg, "invalid data", pVal); \ + return false; -//void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) { -// NodeList *tmp = list; -// while (tmp) { -// if (fn == NULL) { -// if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { -// return tmp->data.value; -// } -// } else { -// if (tmp->data.used && fn(tmp->data.key, key) == 0) { -// return tmp->data.value; -// } -// } -// -// tmp = tmp->next; -// } -// return NULL; -//} -// -//int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) { -// NodeList *tmp = *list; -// while (tmp) { -// if (!tmp->data.used) break; -// if (fn == NULL) { -// if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { -// return -1; -// } -// } else { -// if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) { -// return -1; -// } -// } -// -// tmp = tmp->next; -// } -// if (tmp) { -// tmp->data.key = key; -// tmp->data.keyLen = len; -// tmp->data.value = value; -// tmp->data.used = true; -// } else { -// NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList)); -// if (newNode == NULL) { -// return -1; -// } -// newNode->data.key = key; -// newNode->data.keyLen = len; -// newNode->data.value = value; -// newNode->data.used = true; -// newNode->next = *list; -// *list = newNode; -// } -// return 0; -//} -// -//int nodeListSize(NodeList *list) { -// int cnt = 0; -// while (list) { -// if (list->data.used) -// cnt++; -// else -// break; -// list = list->next; -// } -// return cnt; -//} +#define SET_DOUBLE \ + kvVal->type = TSDB_DATA_TYPE_DOUBLE; \ + kvVal->d = result; + +#define SET_FLOAT \ + if (!IS_VALID_FLOAT(result)) { \ + smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_FLOAT; \ + kvVal->f = (float)result; + +#define SET_BIGINT \ + errno = 0; \ + int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ + if (errno == ERANGE) { \ + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_BIGINT; \ + kvVal->i = tmp; + +#define SET_INT \ + if (!IS_VALID_INT(result)) { \ + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_INT; \ + kvVal->i = result; + +#define SET_SMALL_INT \ + if (!IS_VALID_SMALLINT(result)) { \ + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_SMALLINT; \ + kvVal->i = result; + +#define SET_UBIGINT \ + errno = 0; \ + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ + if (errno == ERANGE || result < 0) { \ + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ + kvVal->u = tmp; + +#define SET_UINT \ + if (!IS_VALID_UINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UINT; \ + kvVal->u = result; + +#define SET_USMALL_INT \ + if (!IS_VALID_USMALLINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_USMALLINT; \ + kvVal->u = result; + +#define SET_TINYINT \ + if (!IS_VALID_TINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_TINYINT; \ + kvVal->i = result; + +#define SET_UTINYINT \ + if (!IS_VALID_UTINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UTINYINT; \ + kvVal->u = result; + +int64_t smlToMilli[] = {3600000LL, 60000LL, 1000LL}; +int64_t smlFactorNS[] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; +int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL}; static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo* conn, const char* pTabName, AUTH_TYPE type){ SUserAuthInfo pAuth = {0}; @@ -114,7 +132,7 @@ inline bool smlDoubleToInt64OverFlow(double num) { return false; } -void smlStrReplace(char* src, int32_t len){ +inline void smlStrReplace(char* src, int32_t len){ if (!tsSmlDot2Underline) return; for(int i = 0; i < len; i++){ if(src[i] == '.'){ @@ -155,7 +173,7 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u return convertTimePrecision(tsInt64, fromPrecision, toPrecision); } -int8_t smlGetTsTypeByLen(int32_t len) { +inline int8_t smlGetTsTypeByLen(int32_t len) { if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { return TSDB_TIME_PRECISION_SECONDS; } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { @@ -180,11 +198,6 @@ SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measu goto cleanup; } - // tag->tags = taosArrayInit(16, sizeof(SSmlKv)); - // if (tag->tags == NULL) { - // uError("SML:smlBuildTableInfo failed to allocate memory"); - // goto cleanup; - // } return tag; cleanup: @@ -192,6 +205,242 @@ cleanup: return NULL; } +void smlBuildTsKv(SSmlKv *kv, int64_t ts){ + kv->key = tsSmlTsDefaultName; + kv->keyLen = strlen(tsSmlTsDefaultName); + kv->type = TSDB_DATA_TYPE_TIMESTAMP; + kv->i = ts; + kv->length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; +} + +SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement){ + SSmlSTableMeta* sMeta = NULL; + char *measure = currElement->measure; + int measureLen = currElement->measureLen; + if (currElement->measureEscaped) { + measure = (char *)taosMemoryMalloc(measureLen); + memcpy(measure, currElement->measure, measureLen); + PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); + smlStrReplace(measure, measureLen); + } + STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); + if (currElement->measureEscaped) { + taosMemoryFree(measure); + } + if (pTableMeta == NULL) { + info->dataFormat = false; + info->reRun = true; + terrno = TSDB_CODE_SUCCESS; + return sMeta; + } + sMeta = smlBuildSTableMeta(info->dataFormat); + if(sMeta == NULL){ + taosMemoryFreeClear(pTableMeta); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return sMeta; + } + sMeta->tableMeta = pTableMeta; + taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); + for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { + SSchema *col = pTableMeta->schema + i; + SSmlKv kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type}; + if (col->type == TSDB_DATA_TYPE_NCHAR) { + kv.length = (col->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + } else if (col->type == TSDB_DATA_TYPE_BINARY || col->type == TSDB_DATA_TYPE_GEOMETRY || col->type == TSDB_DATA_TYPE_VARBINARY) { + kv.length = col->bytes - VARSTR_HEADER_SIZE; + } else{ + kv.length = col->bytes; + } + + if(i < pTableMeta->tableInfo.numOfColumns){ + taosArrayPush(sMeta->cols, &kv); + }else{ + taosArrayPush(sMeta->tags, &kv); + } + } + return sMeta; +} + +bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv){ + // cnt begin 0, add ts so + 2 + if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { + info->dataFormat = false; + info->reRun = true; + return false; + } + // bind data + int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uDebug("smlBuildCol error, retry"); + info->dataFormat = false; + info->reRun = true; + return false; + } + if (cnt >= taosArrayGetSize(info->maxColKVs)) { + info->dataFormat = false; + info->reRun = true; + return false; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt); + + if (unlikely(!IS_SAME_KEY)) { + info->dataFormat = false; + info->reRun = true; + return false; + } + + if (unlikely(IS_VAR_DATA_TYPE(kv->type) && kv->length > maxKV->length)) { + maxKV->length = kv->length; + info->needModifySchema = true; + } + return true; +} + +bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv){ + if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { + goto END; + } + + if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) { + goto END; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); + + if (unlikely(!IS_SAME_KEY)) { + goto END; + } + + if (unlikely(kv->length > maxKV->length)) { + maxKV->length = kv->length; + info->needModifySchema = true; + } + return true; + +END: + info->dataFormat = false; + info->reRun = true; + return false; +} + +int32_t smlJoinMeasureTag(SSmlLineInfo *elements){ + elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen); + if(elements->measureTag == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(elements->measureTag, elements->measure, elements->measureLen); + memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); + elements->measureTagsLen = elements->measureLen + elements->tagsLen; + return TSDB_CODE_SUCCESS; +} + +int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { + bool isSameMeasure = IS_SAME_SUPER_TABLE; + if(isSameMeasure) { + return 0; + } + SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); + + SSmlSTableMeta *sMeta = NULL; + if (unlikely(tmp == NULL)) { + sMeta = smlBuildSuperTableInfo(info, elements); + if(sMeta == NULL) return -1; + }else{ + sMeta = *tmp; + } + ASSERT(sMeta != NULL); + info->currSTableMeta = sMeta->tableMeta; + info->maxTagKVs = sMeta->tags; + info->maxColKVs = sMeta->cols; + return 0; +} + +int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements){ + SSmlTableInfo **oneTable = + (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); + SSmlTableInfo *tinfo = NULL; + if (unlikely(oneTable == NULL)) { + tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); + if (unlikely(!tinfo)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES); + + tinfo->tags = taosArrayDup(info->preLineTagKV, NULL); + for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i); + if (kv->keyEscaped) kv->key = NULL; + if (kv->valueEscaped) kv->value = NULL; + } + + smlSetCTableName(tinfo); + getTableUid(info, elements, tinfo); + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + smlDestroyTableInfo(&tinfo); + return TSDB_CODE_SML_INVALID_DATA; + } + } + }else{ + tinfo = *oneTable; + } + ASSERT(tinfo != NULL); + if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv){ + if (info->dataFormat) { + uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i); + int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1); + } + if (ret == TSDB_CODE_SUCCESS) { + ret = smlBuildRow(info->currTableDataCtx); + } + clearColValArraySml(info->currTableDataCtx->pValues); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + } else { + uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i); + if (elements->colArray == NULL) { + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, kvTs); + taosArrayPush(elements->colArray, kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs){ + if (info->dataFormat) { + uDebug("SML:0x%" PRIx64 " smlParseEndLine format true, ts:%" PRId64, info->id, kvTs->i); + int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlBuildRow(info->currTableDataCtx); + } + + clearColValArraySml(info->currTableDataCtx->pValues); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + } else { + uDebug("SML:0x%" PRIx64 " smlParseEndLine format false, ts:%" PRId64, info->id, kvTs->i); + taosArraySet(elements->colArray, 0, &kvTs); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} + static int32_t smlParseTableName(SArray *tags, char *childTableName) { bool autoChildName = false; size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); @@ -328,98 +577,6 @@ cleanup: return NULL; } -// uint16_t smlCalTypeSum(char* endptr, int32_t left){ -// uint16_t sum = 0; -// for(int i = 0; i < left; i++){ -// sum += endptr[i]; -// } -// return sum; -// } - -#define RETURN_FALSE \ - smlBuildInvalidDataMsg(msg, "invalid data", pVal); \ - return false; - -#define SET_DOUBLE \ - kvVal->type = TSDB_DATA_TYPE_DOUBLE; \ - kvVal->d = result; - -#define SET_FLOAT \ - if (!IS_VALID_FLOAT(result)) { \ - smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_FLOAT; \ - kvVal->f = (float)result; - -#define SET_BIGINT \ - errno = 0; \ - int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ - if (errno == ERANGE) { \ - smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_BIGINT; \ - kvVal->i = tmp; - -#define SET_INT \ - if (!IS_VALID_INT(result)) { \ - smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_INT; \ - kvVal->i = result; - -#define SET_SMALL_INT \ - if (!IS_VALID_SMALLINT(result)) { \ - smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_SMALLINT; \ - kvVal->i = result; - -#define SET_UBIGINT \ - errno = 0; \ - uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ - if (errno == ERANGE || result < 0) { \ - smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ - kvVal->u = tmp; - -#define SET_UINT \ - if (!IS_VALID_UINT(result)) { \ - smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_UINT; \ - kvVal->u = result; - -#define SET_USMALL_INT \ - if (!IS_VALID_USMALLINT(result)) { \ - smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_USMALLINT; \ - kvVal->u = result; - -#define SET_TINYINT \ - if (!IS_VALID_TINYINT(result)) { \ - smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_TINYINT; \ - kvVal->i = result; - -#define SET_UTINYINT \ - if (!IS_VALID_UTINYINT(result)) { \ - smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_UTINYINT; \ - kvVal->u = result; - bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->length; @@ -765,8 +922,6 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO return TSDB_CODE_SUCCESS; } -// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData, -// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, ESchemaAction action) { SRequestObj *pRequest = NULL; @@ -1121,35 +1276,6 @@ end: return code; } -/* -static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){ - for(int i = 0; i < taosArrayGetSize(tags); i++) { - SSmlKv *tag = taosArrayGet(tags, i); - if (smlCheckDuplicateKey(tag->key, tag->keyLen, dumplicateKey)) { - smlBuildInvalidDataMsg(msg, "dumplicate key", tag->key); - return TSDB_CODE_TSC_DUP_NAMES; - } - } - return TSDB_CODE_SUCCESS; -} - -static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) { - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - int ret = smlCheckDupUnit(dumplicateKey, cols, msg); - if(ret != TSDB_CODE_SUCCESS){ - goto end; - } - ret = smlCheckDupUnit(dumplicateKey, tags, msg); - if(ret != TSDB_CODE_SUCCESS){ - goto end; - } - - end: - taosHashCleanup(dumplicateKey); - return ret; -} -*/ - static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); @@ -1206,11 +1332,6 @@ void smlDestroyTableInfo(void *para) { taosHashCleanup(kvHash); } - // if (info->parseJsonByLib) { - // SSmlLineInfo *key = (SSmlLineInfo *)(tag->key); - // if (key != NULL) taosMemoryFree(key->tags); - // } - // taosMemoryFree(tag->key); taosArrayDestroy(tag->cols); taosArrayDestroyEx(tag->tags, freeSSmlKv); taosMemoryFree(tag); @@ -1228,21 +1349,6 @@ void smlDestroyInfo(SSmlHandle *info) { if (!info) return; qDestroyQuery(info->pQuery); - // destroy info->childTables -// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); -// while (oneTable) { -// smlDestroyTableInfo(oneTable); -// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); -// } - - // destroy info->superTables -// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); -// while (oneSTable) { -// smlDestroySTableMeta(*oneSTable); -// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); -// } - - // destroy info->pVgHash taosHashCleanup(info->pVgHash); taosHashCleanup(info->childTables); taosHashCleanup(info->superTables); @@ -1399,11 +1505,6 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { return ret; } } else { - // ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf); - // if (ret != TSDB_CODE_SUCCESS) { - // uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); - // return ret; - // } uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); @@ -1537,19 +1638,6 @@ static void smlPrintStatisticInfo(SSmlHandle *info) { int32_t smlClearForRerun(SSmlHandle *info) { info->reRun = false; - // clear info->childTables -// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); -// while (oneTable) { -// smlDestroyTableInfo(info, *oneTable); -// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); -// } - - // clear info->superTables -// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); -// while (oneSTable) { -// smlDestroySTableMeta(*oneSTable); -// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); -// } taosHashClear(info->childTables); taosHashClear(info->superTables); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 5e656c71a7..bf53d3c2ab 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -29,199 +29,6 @@ (start)++; \ } -// SArray *smlJsonParseTags(char *start, char *end){ -// SArray *tags = taosArrayInit(4, sizeof(SSmlKv)); -// while(start < end){ -// SSmlKv kv = {0}; -// kv.type = TSDB_DATA_TYPE_NCHAR; -// bool isInQuote = false; -// while(start < end){ -// if(unlikely(!isInQuote && *start == '"')){ -// start++; -// kv.key = start; -// isInQuote = true; -// continue; -// } -// if(unlikely(isInQuote && *start == '"')){ -// kv.keyLen = start - kv.key; -// start++; -// break; -// } -// start++; -// } -// bool hasColon = false; -// while(start < end){ -// if(unlikely(!hasColon && *start == ':')){ -// start++; -// hasColon = true; -// continue; -// } -// if(unlikely(hasColon && kv.value == NULL && (*start > 32 && *start != '"'))){ -// kv.value = start; -// start++; -// continue; -// } -// -// if(unlikely(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}'))){ -// kv.length = start - kv.value; -// taosArrayPush(tags, &kv); -// start++; -// break; -// } -// start++; -// } -// } -// return tags; -// } - -// static int32_t smlParseTagsFromJSON(SSmlHandle *info, SSmlLineInfo *elements) { -// int32_t ret = TSDB_CODE_SUCCESS; -// -// if(is_same_child_table_telnet(elements, &info->preLine) == 0){ -// return TSDB_CODE_SUCCESS; -// } -// -// bool isSameMeasure = IS_SAME_SUPER_TABLE; -// -// int cnt = 0; -// SArray *preLineKV = info->preLineTagKV; -// bool isSuperKVInit = true; -// SArray *superKV = NULL; -// if(info->dataFormat){ -// if(unlikely(!isSameMeasure)){ -// SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, -// elements->measureLen, NULL); -// -// if(unlikely(sMeta == NULL)){ -// sMeta = smlBuildSTableMeta(info->dataFormat); -// STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); -// sMeta->tableMeta = pTableMeta; -// if(pTableMeta == NULL){ -// info->dataFormat = false; -// info->reRun = true; -// return TSDB_CODE_SUCCESS; -// } -// nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); -// } -// info->currSTableMeta = sMeta->tableMeta; -// superKV = sMeta->tags; -// -// if(unlikely(taosArrayGetSize(superKV) == 0)){ -// isSuperKVInit = false; -// } -// taosArraySetSize(preLineKV, 0); -// } -// }else{ -// taosArraySetSize(preLineKV, 0); -// } -// -// SArray *tags = smlJsonParseTags(elements->tags, elements->tags + elements->tagsLen); -// int32_t tagNum = taosArrayGetSize(tags); -// if (tagNum == 0) { -// uError("SML:tag is empty:%s", elements->tags) -// taosArrayDestroy(tags); -// return TSDB_CODE_SML_INVALID_DATA; -// } -// for (int32_t i = 0; i < tagNum; ++i) { -// SSmlKv kv = *(SSmlKv*)taosArrayGet(tags, i); -// -// if(info->dataFormat){ -// if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ -// info->dataFormat = false; -// info->reRun = true; -// taosArrayDestroy(tags); -// return TSDB_CODE_SUCCESS; -// } -// -// if(isSameMeasure){ -// if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { -// info->dataFormat = false; -// info->reRun = true; -// taosArrayDestroy(tags); -// return TSDB_CODE_SUCCESS; -// } -// SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); -// if(unlikely(kv.length > preKV->length)){ -// preKV->length = kv.length; -// SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, -// elements->measureLen, NULL); -// if(unlikely(NULL == tableMeta)){ -// uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); -// return TSDB_CODE_SML_INTERNAL_ERROR; -// } -// -// SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); -// oldKV->length = kv.length; -// info->needModifySchema = true; -// } -// if(unlikely(!IS_SAME_KEY)){ -// info->dataFormat = false; -// info->reRun = true; -// taosArrayDestroy(tags); -// return TSDB_CODE_SUCCESS; -// } -// }else{ -// if(isSuperKVInit){ -// if(unlikely(cnt >= taosArrayGetSize(superKV))) { -// info->dataFormat = false; -// info->reRun = true; -// taosArrayDestroy(tags); -// return TSDB_CODE_SUCCESS; -// } -// SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt); -// if(unlikely(kv.length > preKV->length)) { -// preKV->length = kv.length; -// }else{ -// kv.length = preKV->length; -// } -// info->needModifySchema = true; -// -// if(unlikely(!IS_SAME_KEY)){ -// info->dataFormat = false; -// info->reRun = true; -// taosArrayDestroy(tags); -// return TSDB_CODE_SUCCESS; -// } -// }else{ -// taosArrayPush(superKV, &kv); -// } -// taosArrayPush(preLineKV, &kv); -// } -// }else{ -// taosArrayPush(preLineKV, &kv); -// } -// cnt++; -// } -// taosArrayDestroy(tags); -// -// SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, -// is_same_child_table_telnet); if (unlikely(tinfo == NULL)) { -// tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); -// if (unlikely(!tinfo)) { -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// tinfo->tags = taosArrayDup(preLineKV, NULL); -// -// smlSetCTableName(tinfo); -// if (info->dataFormat) { -// info->currSTableMeta->uid = tinfo->uid; -// tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); -// if (tinfo->tableDataCtx == NULL) { -// smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); -// return TSDB_CODE_SML_INVALID_DATA; -// } -// } -// -// SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); -// *key = *elements; -// tinfo->key = key; -// nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); -// } -// if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; -// -// return ret; -// } - static char *smlJsonGetObj(char *payload) { int leftBracketCnt = 0; bool isInQuote = false; @@ -659,12 +466,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { break; } case cJSON_String: { - /* set default JSON type to binary/nchar according to - * user configured parameter tsDefaultJSONStrType - */ - - char *tsDefaultJSONStrType = "binary"; // todo - smlConvertJSONString(kv, tsDefaultJSONStrType, root); + smlConvertJSONString(kv, "binary", root); break; } case cJSON_Object: { @@ -682,138 +484,71 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { return TSDB_CODE_SUCCESS; } -static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { - int32_t ret = TSDB_CODE_SUCCESS; - - bool isSameMeasure = IS_SAME_SUPER_TABLE; - - int cnt = 0; +static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ SArray *preLineKV = info->preLineTagKV; - if (info->dataFormat) { - if (unlikely(!isSameMeasure)) { - SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); - SSmlSTableMeta *sMeta = NULL; - if (unlikely(tmp == NULL)) { - STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); - if (pTableMeta == NULL) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - sMeta = smlBuildSTableMeta(info->dataFormat); - if(sMeta == NULL){ - taosMemoryFreeClear(pTableMeta); - return TSDB_CODE_OUT_OF_MEMORY; - } - sMeta->tableMeta = pTableMeta; - taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); - for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ - SSchema *tag = pTableMeta->schema + i; - SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE }; - taosArrayPush(sMeta->tags, &kv); - } - tmp = &sMeta; - } - info->currSTableMeta = (*tmp)->tableMeta; - info->maxTagKVs = (*tmp)->tags; - } - } - taosArrayClear(preLineKV); + taosArrayClearEx(preLineKV, freeSSmlKv); + int cnt = 0; int32_t tagNum = cJSON_GetArraySize(tags); if (unlikely(tagNum == 0)) { uError("SML:Tag should not be empty"); - return TSDB_CODE_TSC_INVALID_JSON; + terrno = TSDB_CODE_TSC_INVALID_JSON; + return -1; } for (int32_t i = 0; i < tagNum; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); if (unlikely(tag == NULL)) { - return TSDB_CODE_TSC_INVALID_JSON; + terrno = TSDB_CODE_TSC_INVALID_JSON; + return -1; } - // if(unlikely(tag == cMeasure)) continue; size_t keyLen = strlen(tag->string); if (unlikely(IS_INVALID_COL_LEN(keyLen))) { uError("OTD:Tag key length is 0 or too large than 64"); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + return -1; } // add kv to SSmlKv - SSmlKv kv = {.key = tag->string, .keyLen = keyLen}; - // value - ret = smlParseValueFromJSON(tag, &kv); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - return ret; - } - - if (info->dataFormat) { - if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - - if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - if (unlikely(kv.length > maxKV->length)) { - maxKV->length = kv.length; - info->needModifySchema = true; - } - } + SSmlKv kv = {0}; + kv.key = tag->string; + kv.keyLen = keyLen; taosArrayPush(preLineKV, &kv); + + // value + int32_t ret = smlParseValueFromJSON(tag, &kv); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + terrno = ret; + return -1; + } + + + if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { + terrno = TSDB_CODE_SUCCESS; + return -1; + } + cnt++; } + return 0; +} - elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen); - memcpy(elements->measureTag, elements->measure, elements->measureLen); - memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); - elements->measureTagsLen = elements->measureLen + elements->tagsLen; - - SSmlTableInfo **tmp = - (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen); - SSmlTableInfo *tinfo = NULL; - if (unlikely(tmp == NULL)) { - tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); - if (unlikely(!tinfo)) { - return TSDB_CODE_OUT_OF_MEMORY; +static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { + int32_t ret = 0; + if(info->dataFormat){ + ret = smlProcessSuperTable(info, elements); + if(ret != 0){ + return terrno; } - tinfo->tags = taosArrayDup(preLineKV, NULL); - - smlSetCTableName(tinfo); - getTableUid(info, elements, tinfo); - if (info->dataFormat) { - info->currSTableMeta->uid = tinfo->uid; - tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); - if (tinfo->tableDataCtx == NULL) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(&tinfo); - return TSDB_CODE_SML_INVALID_DATA; - } - } - - // SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); - // *key = *elements; - // if(info->parseJsonByLib){ - // key->tags = taosMemoryMalloc(elements->tagsLen + 1); - // memcpy(key->tags, elements->tags, elements->tagsLen); - // key->tags[elements->tagsLen] = 0; - // } - // tinfo->key = key; - taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo, - POINTER_BYTES); - tmp = &tinfo; } - if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx; - - return ret; + ret = smlProcessTagJson(info, tags); + if(ret != 0){ + return terrno; + } + ret = smlJoinMeasureTag(elements); + if(ret != 0){ + return ret; + } + return smlProcessChildTable(info, elements); } static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) { @@ -998,35 +733,10 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); return TSDB_CODE_INVALID_TIMESTAMP; } - SSmlKv kvTs = {.key = tsSmlTsDefaultName, - .keyLen = strlen(tsSmlTsDefaultName), - .type = TSDB_DATA_TYPE_TIMESTAMP, - .i = ts, - .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + SSmlKv kvTs = {0}; + smlBuildTsKv(&kvTs, ts); - if (info->dataFormat) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); - } - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildRow(info->currTableDataCtx); - } - clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - return ret; - } - } else { - if (elements->colArray == NULL) { - elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); - } - taosArrayPush(elements->colArray, &kvTs); - taosArrayPush(elements->colArray, &kv); - } - info->preLine = *elements; - - return TSDB_CODE_SUCCESS; + return smlParseEndTelnetJson(info, elements, &kvTs, &kv); } static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { @@ -1054,7 +764,6 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { return TSDB_CODE_TSC_INVALID_JSON; } - if (unlikely(info->lines != NULL)) { for (int i = 0; i < info->lineNum; i++) { taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv); @@ -1202,35 +911,10 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * return TSDB_CODE_INVALID_TIMESTAMP; } } - SSmlKv kvTs = {.key = tsSmlTsDefaultName, - .keyLen = strlen(tsSmlTsDefaultName), - .type = TSDB_DATA_TYPE_TIMESTAMP, - .i = ts, - .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + SSmlKv kvTs = {0}; + smlBuildTsKv(&kvTs, ts); - if (info->dataFormat) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); - } - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildRow(info->currTableDataCtx); - } - clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - return ret; - } - } else { - if (elements->colArray == NULL) { - elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); - } - taosArrayPush(elements->colArray, &kvTs); - taosArrayPush(elements->colArray, &kv); - } - info->preLine = *elements; - - return TSDB_CODE_SUCCESS; + return smlParseEndTelnetJson(info, elements, &kvTs, &kv); } int32_t smlParseJSON(SSmlHandle *info, char *payload) { diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 7ad43d90c7..0c610a4611 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -20,15 +20,9 @@ #include "clientSml.h" -// comma , #define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) -// space #define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) -// equal = #define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) -// quote " -// #define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) -// SLASH #define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH)) @@ -51,10 +45,10 @@ } \ } -#define BINARY_ADD_LEN 2 // "binary" 2 means " " -#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " +#define BINARY_ADD_LEN (sizeof("\"\"")-1) // "binary" 2 means length of ("") +#define NCHAR_ADD_LEN (sizeof("L\"\"")-1) // L"nchar" 3 means length of (L"") -uint8_t smlPrecisionConvert[7] = {TSDB_TIME_PRECISION_NANO, TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES, +uint8_t smlPrecisionConvert[] = {TSDB_TIME_PRECISION_NANO, TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES, TSDB_TIME_PRECISION_SECONDS, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO}; @@ -198,68 +192,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { return TSDB_CODE_TSC_INVALID_VALUE; } -static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure, - bool isSameCTable) { - if (isSameCTable) { - return TSDB_CODE_SUCCESS; - } - - int cnt = 0; +static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ SArray *preLineKV = info->preLineTagKV; - if (info->dataFormat) { - if (unlikely(!isSameMeasure)) { - SSmlSTableMeta **tmp = - (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); - - SSmlSTableMeta *sMeta = NULL; - if (unlikely(tmp == NULL)) { - char *measure = currElement->measure; - int measureLen = currElement->measureLen; - if (currElement->measureEscaped) { - measure = (char *)taosMemoryMalloc(currElement->measureLen); - memcpy(measure, currElement->measure, currElement->measureLen); - PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); - smlStrReplace(measure, measureLen); - } - STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); - if (currElement->measureEscaped) { - taosMemoryFree(measure); - } - if (pTableMeta == NULL) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - sMeta = smlBuildSTableMeta(info->dataFormat); - if(sMeta == NULL){ - taosMemoryFreeClear(pTableMeta); - return TSDB_CODE_OUT_OF_MEMORY; - } - sMeta->tableMeta = pTableMeta; - taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); - for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) { - SSchema *tag = pTableMeta->schema + i; - - if(i < pTableMeta->tableInfo.numOfColumns){ - SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type}; - if (tag->type == TSDB_DATA_TYPE_NCHAR) { - kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; - } else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY || tag->type == TSDB_DATA_TYPE_VARBINARY) { - kv.length = tag->bytes - VARSTR_HEADER_SIZE; - } - taosArrayPush(sMeta->cols, &kv); - }else{ - SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE}; - taosArrayPush(sMeta->tags, &kv); - } - } - tmp = &sMeta; - } - info->currSTableMeta = (*tmp)->tableMeta; - info->maxTagKVs = (*tmp)->tags; - } - } taosArrayClearEx(preLineKV, freeSSmlKv); + int cnt = 0; while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql))) { @@ -274,7 +210,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - return TSDB_CODE_SML_INVALID_DATA; + terrno = TSDB_CODE_SML_INVALID_DATA; + return -1; } if (unlikely(IS_EQUAL(*sql))) { keyLen = *sql - key; @@ -290,7 +227,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + return -1; } // parse value @@ -304,7 +242,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin break; } else if (unlikely(IS_EQUAL(*sql))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); - return TSDB_CODE_SML_INVALID_DATA; + terrno = TSDB_CODE_SML_INVALID_DATA; + return -1; } if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { @@ -318,11 +257,13 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin if (unlikely(valueLen == 0)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); - return TSDB_CODE_SML_INVALID_DATA; + terrno = TSDB_CODE_SML_INVALID_DATA; + return -1; } if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + return -1; } if (keyEscaped) { @@ -338,37 +279,17 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin value = tmp; } SSmlKv kv = {.key = key, - .keyLen = keyLen, - .type = TSDB_DATA_TYPE_NCHAR, - .value = value, - .length = valueLen, - .keyEscaped = keyEscaped, - .valueEscaped = valueEscaped}; + .keyLen = keyLen, + .type = TSDB_DATA_TYPE_NCHAR, + .value = value, + .length = valueLen, + .keyEscaped = keyEscaped, + .valueEscaped = valueEscaped}; taosArrayPush(preLineKV, &kv); - if (info->dataFormat) { - if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); - - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - - if (unlikely(kv.length > maxKV->length)) { - maxKV->length = kv.length; - info->needModifySchema = true; - } + if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { + terrno = TSDB_CODE_SUCCESS; + return -1; } cnt++; @@ -377,99 +298,33 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } (*sql)++; } + return 0; +} - void *oneTable = taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen); - if ((oneTable != NULL)) { +static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) { + bool isSameCTable = IS_SAME_CHILD_TABLE; + if(isSameCTable){ return TSDB_CODE_SUCCESS; } - SSmlTableInfo *tinfo = smlBuildTableInfo(1, currElement->measure, currElement->measureLen); - if (unlikely(!tinfo)) { - return TSDB_CODE_OUT_OF_MEMORY; - } - tinfo->tags = taosArrayDup(preLineKV, NULL); - for (size_t i = 0; i < taosArrayGetSize(preLineKV); i++) { - SSmlKv *kv = (SSmlKv *)taosArrayGet(preLineKV, i); - if (kv->keyEscaped) kv->key = NULL; - if (kv->valueEscaped) kv->value = NULL; - } - - smlSetCTableName(tinfo); - getTableUid(info, currElement, tinfo); - if (info->dataFormat) { - info->currSTableMeta->uid = tinfo->uid; - tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); - if (tinfo->tableDataCtx == NULL) { - smlDestroyTableInfo(&tinfo); - smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - return TSDB_CODE_SML_INVALID_DATA; + int32_t ret = 0; + if(info->dataFormat){ + ret = smlProcessSuperTable(info, elements); + if(ret != 0){ + return terrno; } } - taosHashPut(info->childTables, currElement->measure, currElement->measureTagsLen, &tinfo, POINTER_BYTES); + ret = smlProcessTagLine(info, sql, sqlEnd); + if(ret != 0){ + return terrno; + } - return TSDB_CODE_SUCCESS; + return smlProcessChildTable(info, elements); } -static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure, - bool isSameCTable) { +static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) { int cnt = 0; - if (info->dataFormat) { - if (unlikely(!isSameCTable)) { - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen); - if (unlikely(oneTable == NULL)) { - smlBuildInvalidDataMsg(&info->msgBuf, "child table should inside", currElement->measure); - return TSDB_CODE_SML_INVALID_DATA; - } - info->currTableDataCtx = (*oneTable)->tableDataCtx; - } - - if (unlikely(!isSameMeasure)) { - SSmlSTableMeta **tmp = - (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); - if (unlikely(tmp == NULL)) { - char *measure = currElement->measure; - int measureLen = currElement->measureLen; - if (currElement->measureEscaped) { - measure = (char *)taosMemoryMalloc(currElement->measureLen); - memcpy(measure, currElement->measure, currElement->measureLen); - PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen); - smlStrReplace(measure, measureLen); - } - STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen); - if (currElement->measureEscaped) { - taosMemoryFree(measure); - } - if (pTableMeta == NULL) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - *tmp = smlBuildSTableMeta(info->dataFormat); - if(*tmp == NULL){ - taosMemoryFreeClear(pTableMeta); - return TSDB_CODE_OUT_OF_MEMORY; - } - (*tmp)->tableMeta = pTableMeta; - taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES); - - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema *tag = pTableMeta->schema + i; - SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type}; - if (tag->type == TSDB_DATA_TYPE_NCHAR) { - kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; - } else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY || tag->type == TSDB_DATA_TYPE_VARBINARY) { - kv.length = tag->bytes - VARSTR_HEADER_SIZE; - } - taosArrayPush((*tmp)->cols, &kv); - } - } - info->currSTableMeta = (*tmp)->tableMeta; - info->maxColKVs = (*tmp)->cols; - } - } - while (*sql < sqlEnd) { if (unlikely(IS_SPACE(*sql))) { break; @@ -569,47 +424,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin } if (info->dataFormat) { - // cnt begin 0, add ts so + 2 - if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { - info->dataFormat = false; - info->reRun = true; - freeSSmlKv(&kv); - return TSDB_CODE_SUCCESS; - } - // bind data - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, cnt + 1); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - uDebug("smlBuildCol error, retry"); - info->dataFormat = false; - info->reRun = true; - freeSSmlKv(&kv); - return TSDB_CODE_SUCCESS; - } - if (cnt >= taosArrayGetSize(info->maxColKVs)) { - info->dataFormat = false; - info->reRun = true; - freeSSmlKv(&kv); - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt); - if (kv.type != maxKV->type) { - info->dataFormat = false; - info->reRun = true; - freeSSmlKv(&kv); - return TSDB_CODE_SUCCESS; - } - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - freeSSmlKv(&kv); - return TSDB_CODE_SUCCESS; - } - - if (unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)) { - maxKV->length = kv.length; - info->needModifySchema = true; - } + bool isAligned = isSmlColAligned(info, cnt, &kv); freeSSmlKv(&kv); + if(!isAligned){ + return TSDB_CODE_SUCCESS; + } } else { if (currElement->colArray == NULL) { currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); @@ -665,20 +484,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine tmp++; } elements->measureTagsLen = tmp - elements->measure; - - bool isSameCTable = false; - bool isSameMeasure = false; - if (IS_SAME_CHILD_TABLE) { - isSameCTable = true; - isSameMeasure = true; - } else if (info->dataFormat) { - isSameMeasure = IS_SAME_SUPER_TABLE; - } + elements->measureTag = elements->measure; // parse tag if (*sql == COMMA) sql++; elements->tags = sql; - int ret = smlParseTagKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + int ret = smlParseTagLine(info, &sql, sqlEnd, elements); if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } @@ -693,7 +504,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine JUMP_SPACE(sql, sqlEnd) elements->cols = sql; - ret = smlParseColKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + ret = smlParseColLine(info, &sql, sqlEnd, elements); if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } @@ -724,31 +535,9 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); return TSDB_CODE_INVALID_TIMESTAMP; } - // add ts to - SSmlKv kv = {.key = tsSmlTsDefaultName, - .keyLen = strlen(tsSmlTsDefaultName), - .type = TSDB_DATA_TYPE_TIMESTAMP, - .i = ts, - .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, - .keyEscaped = false, - .valueEscaped = false}; - if (info->dataFormat) { - uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts); - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildRow(info->currTableDataCtx); - } - clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - return ret; - } - } else { - uDebug("SML:0x%" PRIx64 " smlParseInfluxString format false, ts:%" PRId64, info->id, ts); - taosArraySet(elements->colArray, 0, &kv); - } - info->preLine = *elements; + SSmlKv kvTs = {0}; + smlBuildTsKv(&kvTs, ts); - return ret; + return smlParseEndLine(info, elements, &kvTs); } diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index f9f8602c5a..f715f32556 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -23,8 +23,6 @@ int32_t is_same_child_table_telnet(const void *a, const void *b) { SSmlLineInfo *t1 = (SSmlLineInfo *)a; SSmlLineInfo *t2 = (SSmlLineInfo *)b; - // uError("is_same_child_table_telnet len:%d,%d %s,%s @@@ len:%d,%d %s,%s", t1->measureLen, t2->measureLen, - // t1->measure, t2->measure, t1->tagsLen, t2->tagsLen, t1->tags, t2->tags); if (t1 == NULL || t2 == NULL || t1->measure == NULL || t2->measure == NULL || t1->tags == NULL || t2->tags == NULL) return 1; return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) && @@ -69,47 +67,11 @@ static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t } } -static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { - if (is_same_child_table_telnet(elements, &info->preLine) == 0) { - elements->measureTag = info->preLine.measureTag; - return TSDB_CODE_SUCCESS; - } - - bool isSameMeasure = IS_SAME_SUPER_TABLE; - - int cnt = 0; +static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){ SArray *preLineKV = info->preLineTagKV; - if (info->dataFormat) { - if (!isSameMeasure) { - SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); - SSmlSTableMeta *sMeta = NULL; - if (unlikely(tmp == NULL)) { - STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); - if (pTableMeta == NULL) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - sMeta = smlBuildSTableMeta(info->dataFormat); - if(sMeta == NULL){ - taosMemoryFreeClear(pTableMeta); - return TSDB_CODE_OUT_OF_MEMORY; - } - sMeta->tableMeta = pTableMeta; - taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); - for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){ - SSchema *tag = pTableMeta->schema + i; - SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE }; - taosArrayPush(sMeta->tags, &kv); - } - tmp = &sMeta; - } - info->currSTableMeta = (*tmp)->tableMeta; - info->maxTagKVs = (*tmp)->tags; - } - } + taosArrayClearEx(preLineKV, freeSSmlKv); + int cnt = 0; - taosArrayClear(preLineKV); const char *sql = data; while (sql < sqlEnd) { JUMP_SPACE(sql, sqlEnd) @@ -121,8 +83,9 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS // parse key while (sql < sqlEnd) { if (unlikely(*sql == SPACE)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); + terrno = TSDB_CODE_SML_INVALID_DATA; + return -1; } if (unlikely(*sql == EQUAL)) { keyLen = sql - key; @@ -133,13 +96,10 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS } if (unlikely(IS_INVALID_COL_LEN(keyLen))) { - smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + return -1; } - // if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { - // smlBuildInvalidDataMsg(msg, "dumplicate key", key); - // return TSDB_CODE_TSC_DUP_NAMES; - // } // parse value const char *value = sql; @@ -150,87 +110,67 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS break; } if (unlikely(*sql == EQUAL)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); + terrno = TSDB_CODE_SML_INVALID_DATA; + return -1; } sql++; } valueLen = sql - value; if (unlikely(valueLen == 0)) { - smlBuildInvalidDataMsg(msg, "invalid value", value); - return TSDB_CODE_TSC_INVALID_VALUE; + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + terrno = TSDB_CODE_TSC_INVALID_VALUE; + return -1; } if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + return -1; } - SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; - - if (info->dataFormat) { - if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt); - if (unlikely(!IS_SAME_KEY)) { - info->dataFormat = false; - info->reRun = true; - return TSDB_CODE_SUCCESS; - } - if (unlikely(kv.length > maxKV->length)) { - maxKV->length = kv.length; - info->needModifySchema = true; - } - } + SSmlKv kv = {.key = key, + .keyLen = keyLen, + .type = TSDB_DATA_TYPE_NCHAR, + .value = value, + .length = valueLen, + .keyEscaped = false, + .valueEscaped = false}; taosArrayPush(preLineKV, &kv); + if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { + terrno = TSDB_CODE_SUCCESS; + return -1; + } cnt++; } + return 0; +} - elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen); - memcpy(elements->measureTag, elements->measure, elements->measureLen); - memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); - elements->measureTagsLen = elements->measureLen + elements->tagsLen; - - SSmlTableInfo **tmp = - (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen); - SSmlTableInfo *tinfo = NULL; - if (unlikely(tmp == NULL)) { - tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); - if (!tinfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - tinfo->tags = taosArrayDup(preLineKV, NULL); - - smlSetCTableName(tinfo); - getTableUid(info, elements, tinfo); - if (info->dataFormat) { - info->currSTableMeta->uid = tinfo->uid; - tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); - if (tinfo->tableDataCtx == NULL) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); - smlDestroyTableInfo(&tinfo); - return TSDB_CODE_SML_INVALID_DATA; - } - } - - // SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); - // *key = *elements; - // tinfo->key = key; - taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo, - POINTER_BYTES); - tmp = &tinfo; +static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) { + if (is_same_child_table_telnet(elements, &info->preLine) == 0) { + elements->measureTag = info->preLine.measureTag; + return TSDB_CODE_SUCCESS; } - if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx; - return TSDB_CODE_SUCCESS; + int32_t ret = 0; + if(info->dataFormat){ + ret = smlProcessSuperTable(info, elements); + if(ret != 0){ + return terrno; + } + } + + ret = smlProcessTagTelnet(info, data, sqlEnd); + if(ret != 0){ + return terrno; + } + + ret = smlJoinMeasureTag(elements); + if(ret != 0){ + return ret; + } + + return smlProcessChildTable(info, elements); } // format: =[ =] @@ -251,7 +191,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine return TSDB_CODE_SML_INVALID_DATA; } - bool needConverTime = false; // get TS before parse tag(get meta), so need conver time + bool needConverTime = false; // get TS before parse tag(get meta), so need convert time if (info->dataFormat && info->currSTableMeta == NULL) { needConverTime = true; } @@ -260,11 +200,6 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); return TSDB_CODE_INVALID_TIMESTAMP; } - SSmlKv kvTs = {.key = tsSmlTsDefaultName, - .keyLen = strlen(tsSmlTsDefaultName), - .type = TSDB_DATA_TYPE_TIMESTAMP, - .i = ts, - .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; // parse value smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen); @@ -287,7 +222,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine return TSDB_CODE_TSC_INVALID_VALUE; } - int ret = smlParseTelnetTags(info, sql, sqlEnd, elements, &info->msgBuf); + int ret = smlParseTelnetTags(info, sql, sqlEnd, elements); if (unlikely(ret != TSDB_CODE_SUCCESS)) { return ret; } @@ -296,30 +231,11 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine return TSDB_CODE_SUCCESS; } - if (info->dataFormat && info->currSTableMeta != NULL) { - if (needConverTime) { - kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); - } - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); - } - if (ret == TSDB_CODE_SUCCESS) { - ret = smlBuildRow(info->currTableDataCtx); - } - clearColValArraySml(info->currTableDataCtx->pValues); - if (unlikely(ret != TSDB_CODE_SUCCESS)) { - smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); - return ret; - } - } else { - if (elements->colArray == NULL) { - elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); - } - taosArrayPush(elements->colArray, &kvTs); - taosArrayPush(elements->colArray, &kv); + SSmlKv kvTs = {0}; + smlBuildTsKv(&kvTs, ts); + if (needConverTime) { + kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); } - info->preLine = *elements; - return TSDB_CODE_SUCCESS; + return smlParseEndTelnetJson(info, elements, &kvTs, &kv); } \ No newline at end of file From 8488f25d204d2a35a458d939672921326381fd0a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 7 Dec 2023 15:01:38 +0800 Subject: [PATCH 50/94] fix: add write_raw_block api check ret error message --- .../7-tmq/raw_block_interface_test.py | 16 ++- utils/test/c/write_raw_block_test.c | 114 +++++++++++++++--- 2 files changed, 114 insertions(+), 16 deletions(-) diff --git a/tests/system-test/7-tmq/raw_block_interface_test.py b/tests/system-test/7-tmq/raw_block_interface_test.py index 1e89de1cce..1c9798421d 100644 --- a/tests/system-test/7-tmq/raw_block_interface_test.py +++ b/tests/system-test/7-tmq/raw_block_interface_test.py @@ -36,7 +36,21 @@ class TDTestCase: buildPath = tdCom.getBuildPath() cmdStr = '%s/build/bin/write_raw_block_test'%(buildPath) tdLog.info(cmdStr) - os.system(cmdStr) + retCode = os.system(cmdStr) + # run program code from system return , 0 is success + runCode = retCode & 0xFF + # program retur code from main function + progCode = retCode >> 8 + + tdLog.info(f"{cmdStr} ret={retCode} runCode={runCode} progCode={progCode}") + + if runCode != 0: + tdLog.exit(f"run {cmdStr} failed, have system error.") + return + + if progCode != 0: + tdLog.exit(f"{cmdStr} found problem, return code = {progCode}.") + return self.checkData() diff --git a/utils/test/c/write_raw_block_test.c b/utils/test/c/write_raw_block_test.c index ee2594af7a..b2b56c19dd 100644 --- a/utils/test/c/write_raw_block_test.c +++ b/utils/test/c/write_raw_block_test.c @@ -19,8 +19,8 @@ #include "taos.h" #include "types.h" -int buildStable(TAOS* pConn, TAOS_RES* pRes) { - pRes = taos_query(pConn, +int buildStable(TAOS* pConn) { + TAOS_RES* pRes = taos_query(pConn, "CREATE STABLE `meters` (`ts` TIMESTAMP, `current` INT, `voltage` INT, `phase` FLOAT) TAGS " "(`groupid` INT, `location` VARCHAR(16))"); if (taos_errno(pRes) != 0) { @@ -57,6 +57,34 @@ int buildStable(TAOS* pConn, TAOS_RES* pRes) { } taos_free_result(pRes); + pRes = taos_query(pConn, "create table ntba(ts timestamp, addr binary(32))"); + if (taos_errno(pRes) != 0) { + printf("failed to create ntba, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table ntbb(ts timestamp, addr binary(8))"); + if (taos_errno(pRes) != 0) { + printf("failed to create ntbb, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ntba values(now,'123456789abcdefg123456789')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert table ntba, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ntba values(now,'hello')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert table ntba, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + return 0; } @@ -65,40 +93,43 @@ int32_t init_env() { if (pConn == NULL) { return -1; } + int32_t ret = -1; TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_raw"); if (taos_errno(pRes) != 0) { printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; + goto END; } taos_free_result(pRes); pRes = taos_query(pConn, "create database if not exists db_raw vgroups 2"); if (taos_errno(pRes) != 0) { printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; + goto END; } taos_free_result(pRes); pRes = taos_query(pConn, "use db_raw"); if (taos_errno(pRes) != 0) { printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; + goto END; } taos_free_result(pRes); - buildStable(pConn, pRes); + buildStable(pConn); pRes = taos_query(pConn, "select * from d0"); if (taos_errno(pRes) != 0) { printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; + goto END; } void *data = NULL; int32_t numOfRows = 0; int error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); - ASSERT(error_code == 0); - ASSERT(numOfRows == 1); + if(error_code !=0 ){ + printf("error fetch raw block, reason:%s\n", taos_errstr(pRes)); + goto END; + } taos_write_raw_block(pConn, numOfRows, data, "d1"); taos_free_result(pRes); @@ -106,23 +137,76 @@ int32_t init_env() { pRes = taos_query(pConn, "select ts,phase from d0"); if (taos_errno(pRes) != 0) { printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); - return -1; + goto END; } error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); - ASSERT(error_code == 0); - ASSERT(numOfRows == 1); + if(error_code !=0 ){ + printf("error fetch raw block, reason:%s\n", taos_errstr(pRes)); + goto END; + } int numFields = taos_num_fields(pRes); TAOS_FIELD *fields = taos_fetch_fields(pRes); taos_write_raw_block_with_fields(pConn, numOfRows, data, "d2", fields, numFields); taos_free_result(pRes); - taos_close(pConn); - return 0; + // check error msg + pRes = taos_query(pConn, "select * from ntba"); + if (taos_errno(pRes) != 0) { + printf("error select * from ntba, reason:%s\n", taos_errstr(pRes)); + goto END; + } + + data = NULL; + numOfRows = 0; + error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); + if(error_code !=0 ){ + printf("error fetch select * from ntba, reason:%s\n", taos_errstr(pRes)); + goto END; + } + error_code = taos_write_raw_block(pConn, numOfRows, data, "ntbb"); + if(error_code == 0) { + printf(" taos_write_raw_block to ntbb expect failed , but success!\n"); + goto END; + } + + // pass NULL return last error code describe + const char* err = taos_errstr(NULL); + printf("write_raw_block return code =0x%x err=%s\n", error_code, err); + if(strcmp(err, "success") == 0) { + printf("expect failed , but error string is success! err=%s\n", err); + goto END; + } + + // no exist table + error_code = taos_write_raw_block(pConn, numOfRows, data, "no-exist-table"); + if(error_code == 0) { + printf(" taos_write_raw_block to no-exist-table expect failed , but success!\n"); + goto END; + } + + err = taos_errstr(NULL); + printf("write_raw_block no exist table return code =0x%x err=%s\n", error_code, err); + if(strcmp(err, "success") == 0) { + printf("expect failed write no exist table, but error string is success! err=%s\n", err); + goto END; + } + + // success + ret = 0; + +END: + // free + if(pRes) taos_free_result(pRes); + if(pConn) taos_close(pConn); + return ret; } int main(int argc, char* argv[]) { + printf("test write_raw_block...\n"); if (init_env() < 0) { + printf("test write_raw_block failed.\n"); return -1; } -} + printf("test write_raw_block ok.\n"); +} \ No newline at end of file From 5143def1778349423cb34dccae4c106aef6b0fce Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 7 Dec 2023 15:05:56 +0800 Subject: [PATCH 51/94] fix: add write_raw_block api check ret error message --- utils/test/c/write_raw_block_test.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils/test/c/write_raw_block_test.c b/utils/test/c/write_raw_block_test.c index b2b56c19dd..162ecd229c 100644 --- a/utils/test/c/write_raw_block_test.c +++ b/utils/test/c/write_raw_block_test.c @@ -204,9 +204,11 @@ END: int main(int argc, char* argv[]) { printf("test write_raw_block...\n"); - if (init_env() < 0) { + int ret = init_env(); + if (ret < 0) { printf("test write_raw_block failed.\n"); - return -1; + return ret; } printf("test write_raw_block ok.\n"); + return 0; } \ No newline at end of file From 60500f4a81e3f41f086cf77a6a0cb2fc811d5fa8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 7 Dec 2023 15:17:15 +0800 Subject: [PATCH 52/94] fix:error --- source/client/src/clientSml.c | 8 ++- source/client/src/clientSmlJson.c | 3 +- tests/system-test/2-query/sml.py | 4 +- utils/test/c/sml_test.c | 101 ++++++++++++++++++++++++++++-- 4 files changed, 105 insertions(+), 11 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index aff8f4f43f..a066bd8014 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -422,7 +422,7 @@ int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv * int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs){ if (info->dataFormat) { uDebug("SML:0x%" PRIx64 " smlParseEndLine format true, ts:%" PRId64, info->id, kvTs->i); - int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); if (ret == TSDB_CODE_SUCCESS) { ret = smlBuildRow(info->currTableDataCtx); } @@ -434,7 +434,7 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs){ } } else { uDebug("SML:0x%" PRIx64 " smlParseEndLine format false, ts:%" PRId64, info->id, kvTs->i); - taosArraySet(elements->colArray, 0, &kvTs); + taosArraySet(elements->colArray, 0, kvTs); } info->preLine = *elements; @@ -1374,7 +1374,9 @@ void smlDestroyInfo(SSmlHandle *info) { if (info->parseJsonByLib) { taosMemoryFree(info->lines[i].tags); } - if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag); + if (info->lines[i].measureTagsLen != 0 && info->protocol != TSDB_SML_LINE_PROTOCOL) { + taosMemoryFree(info->lines[i].measureTag); + } } taosMemoryFree(info->lines); } diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index bf53d3c2ab..845884d5ac 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -512,7 +512,6 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ SSmlKv kv = {0}; kv.key = tag->string; kv.keyLen = keyLen; - taosArrayPush(preLineKV, &kv); // value int32_t ret = smlParseValueFromJSON(tag, &kv); @@ -520,7 +519,7 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){ terrno = ret; return -1; } - + taosArrayPush(preLineKV, &kv); if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { terrno = TSDB_CODE_SUCCESS; diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index 0369f45723..e28f3b1edd 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -80,12 +80,12 @@ class TDTestCase: tdSql.checkData(0, 1, 13.000000000) tdSql.checkData(0, 2, "web01") tdSql.checkData(0, 3, None) - tdSql.checkData(0, 4, "lga") + tdSql.checkData(0, 4, 1) tdSql.checkData(1, 1, 9.000000000) tdSql.checkData(1, 2, "web02") tdSql.checkData(3, 3, "t1") - tdSql.checkData(0, 4, "lga") + tdSql.checkData(2, 4, 4) tdSql.query(f"select * from {dbname}.macylr") tdSql.checkRows(2) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 9153706d23..2c334eb67b 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -116,8 +116,7 @@ int smlProcess_json1_Test() { const char *sql[] = { "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":" - "\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":" - "\"lga\"}}]"}; + "34}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}}]"}; char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { @@ -142,8 +141,8 @@ int smlProcess_json1_Test() { const char *sql2[] = { - "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":\"lga\"}" - "},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}" + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":1}" + "},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":4}" "}]", }; @@ -270,6 +269,98 @@ int smlProcess_json3_Test() { return code; } +int smlProcess_json_tag_not_same_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "[{\"metric\":\"jstable\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":" + "\"lga\"}},{\"metric\":\"jstable\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":" + "\"lga\"}}]"}; + + char *sql1[1] = {0}; + for (int i = 0; i < 1; i++) { + sql1[i] = taosMemoryCalloc(1, 1024); + strncpy(sql1[i], sql[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + int code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql1[i]); + } + ASSERT(code == 0); + + + const char *sql2[] = { + "[{\"metric\":\"jstable\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":6,\"dc\":\"lga\"}}]", + }; + + char *sql3[1] = {0}; + for (int i = 0; i < 1; i++) { + sql3[i] = taosMemoryCalloc(1, 1024); + strncpy(sql3[i], sql2[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql3[i]); + } + + ASSERT(code != 0); + + const char *sql4[] = { + "[{\"metric\":\"jstable\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":6,\"dc\":\"lga\"}}," + "{\"metric\":\"jstable\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":false,\"dc\":\"lga\"}}]", + }; + char *sql5[1] = {0}; + for (int i = 0; i < 1; i++) { + sql5[i] = taosMemoryCalloc(1, 1024); + strncpy(sql5[i], sql4[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } + taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql5[i]); + } + ASSERT(code != 0); + + taos_close(taos); + + return code; +} + int sml_TD15662_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1729,6 +1820,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_19221_Test(); ASSERT(!ret); + ret = smlProcess_json_tag_not_same_Test(); + ASSERT(ret); ret = sml_ts3724_Test(); ASSERT(!ret); From e429526789f4d79477508ad41b156795dd85e875 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 13 Nov 2023 12:05:55 +0800 Subject: [PATCH 53/94] refact: rename STSnapRange to STFileSetRange --- source/dnode/vnode/src/inc/tsdb.h | 14 +++++------ source/dnode/vnode/src/tsdb/tsdbFS2.c | 16 ++++++------- source/dnode/vnode/src/tsdb/tsdbFS2.h | 8 +++---- source/dnode/vnode/src/tsdb/tsdbFSet2.c | 4 ++-- source/dnode/vnode/src/tsdb/tsdbFSet2.h | 6 ++--- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 28 +++++++++++----------- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 26 ++++++++++---------- 7 files changed, 51 insertions(+), 51 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 79f9caab33..9cc6cd1132 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -681,14 +681,14 @@ struct SDelFWriter { typedef struct STFileSet STFileSet; typedef TARRAY2(STFileSet *) TFileSetArray; -typedef struct STSnapRange STSnapRange; -typedef TARRAY2(STSnapRange *) TSnapRangeArray; // disjoint snap ranges +typedef struct STFileSetRange STFileSetRange; +typedef TARRAY2(STFileSetRange *) TFileSetRangeArray; // disjoint ranges // util -int32_t tSerializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); -int32_t tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); -void tsdbSnapRangeArrayDestroy(TSnapRangeArray **ppSnap); -SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges); +int32_t tSerializeSnapRangeArray(void *buf, int32_t bufLen, TFileSetRangeArray *pSnapR); +int32_t tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TFileSetRangeArray *pSnapR); +void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray **ppSnap); +SHashObj *tsdbGetSnapRangeHash(TFileSetRangeArray *pRanges); // snap partition list typedef TARRAY2(SVersionRange) SVerRangeList; @@ -699,7 +699,7 @@ STsdbSnapPartList *tsdbSnapPartListCreate(); void tsdbSnapPartListDestroy(STsdbSnapPartList **ppList); int32_t tSerializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList); int32_t tDeserializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList); -int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList *pList, TSnapRangeArray **ppRanges); +int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList *pList, TFileSetRangeArray **ppRanges); enum { TSDB_SNAP_RANGE_TYP_HEAD = 0, diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 635c53bbed..df4df18dc3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -1072,7 +1072,7 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) { return 0; } -int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRanges, TFileSetArray **fsetArr, +int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pRanges, TFileSetArray **fsetArr, TFileOpArray *fopArr) { int32_t code = 0; STFileSet *fset; @@ -1096,7 +1096,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange int64_t ever = VERSION_MAX; if (pHash) { int32_t fid = fset->fid; - STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid)); + STFileSetRange *u = taosHashGet(pHash, &fid, sizeof(fid)); if (u) { ever = u->sver - 1; } @@ -1123,7 +1123,7 @@ _out: return code; } -SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges) { +SHashObj *tsdbGetSnapRangeHash(TFileSetRangeArray *pRanges) { int32_t capacity = TARRAY2_SIZE(pRanges) * 2; SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (pHash == NULL) { @@ -1132,7 +1132,7 @@ SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges) { } for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) { - STSnapRange *u = TARRAY2_GET(pRanges, i); + STFileSetRange *u = TARRAY2_GET(pRanges, i); int32_t fid = u->fid; int32_t code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u)); ASSERT(code == 0); @@ -1141,11 +1141,11 @@ SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges) { return pHash; } -int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges, - TSnapRangeArray **fsrArr) { +int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges, + TFileSetRangeArray **fsrArr) { int32_t code = 0; STFileSet *fset; - STSnapRange *fsr1 = NULL; + STFileSetRange *fsr1 = NULL; SHashObj *pHash = NULL; fsrArr[0] = taosMemoryCalloc(1, sizeof(*fsrArr[0])); @@ -1170,7 +1170,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev if (pHash) { int32_t fid = fset->fid; - STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid)); + STFileSetRange *u = taosHashGet(pHash, &fid, sizeof(fid)); if (u) { sver1 = u->sver; tsdbDebug("range hash get fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 74453126cf..8fdce9e690 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -44,12 +44,12 @@ int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr); int32_t tsdbFSCreateRefSnapshotWithoutLock(STFileSystem *fs, TFileSetArray **fsetArr); int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr); -int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pExclude, TFileSetArray **fsetArr, +int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pExclude, TFileSetArray **fsetArr, TFileOpArray *fopArr); int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr, TFileOpArray *fopArr); -int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges, - TSnapRangeArray **fsrArr); -int32_t tsdbFSDestroyRefRangedSnapshot(TSnapRangeArray **fsrArr); +int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges, + TFileSetRangeArray **fsrArr); +int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr); // txn int64_t tsdbFSAllocEid(STFileSystem *fs); int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 1bf886e3b0..7673299e4b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -533,7 +533,7 @@ int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_ return 0; } -int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr) { +int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STFileSetRange **fsr) { fsr[0] = taosMemoryCalloc(1, sizeof(*fsr[0])); if (fsr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; fsr[0]->fid = fset1->fid; @@ -575,7 +575,7 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs return 0; } -int32_t tsdbTSnapRangeClear(STSnapRange **fsr) { +int32_t tsdbTSnapRangeClear(STFileSetRange **fsr) { if (!fsr[0]) return 0; tsdbTFileSetClear(&fsr[0]->fset); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index 83f5b1e83c..3a6427a42c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -49,8 +49,8 @@ int32_t tsdbTFileSetRemove(STFileSet *fset); int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset, TFileOpArray *fopArr); -int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr); -int32_t tsdbTSnapRangeClear(STSnapRange **fsr); +int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STFileSetRange **fsr); +int32_t tsdbTSnapRangeClear(STFileSetRange **fsr); // to/from json int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json); @@ -101,7 +101,7 @@ struct STFileSet { bool blockCommit; }; -struct STSnapRange { +struct STFileSetRange { int32_t fid; int64_t sver; int64_t ever; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index e757daa0af..9d3efcfdae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -32,12 +32,12 @@ struct STsdbSnapReader { uint8_t* aBuf[5]; SSkmInfo skmTb[1]; - TSnapRangeArray* fsrArr; + TFileSetRangeArray* fsrArr; // context struct { int32_t fsrArrIdx; - STSnapRange* fsr; + STFileSetRange* fsr; bool isDataDone; bool isTombDone; } ctx[1]; @@ -437,14 +437,14 @@ int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, reader[0]->ever = ever; reader[0]->type = type; - code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TSnapRangeArray*)pRanges, &reader[0]->fsrArr); + code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TFileSetRangeArray*)pRanges, &reader[0]->fsrArr); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code), sver, ever, type); - tsdbSnapRangeArrayDestroy(&reader[0]->fsrArr); + tsdbFileSetRangeArrayDestroy(&reader[0]->fsrArr); taosMemoryFree(reader[0]); reader[0] = NULL; } else { @@ -472,7 +472,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) { TARRAY2_DESTROY(reader[0]->sttReaderArr, tsdbSttFileReaderClose); tsdbDataFileReaderClose(&reader[0]->dataReader); - tsdbSnapRangeArrayDestroy(&reader[0]->fsrArr); + tsdbFileSetRangeArrayDestroy(&reader[0]->fsrArr); tDestroyTSchema(reader[0]->skmTb->pTSchema); for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->aBuf); ++i) { @@ -1061,7 +1061,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRang writer[0]->compactVersion = INT64_MAX; writer[0]->now = taosGetTimestampMs(); - code = tsdbFSCreateCopyRangedSnapshot(pTsdb->pFS, (TSnapRangeArray*)pRanges, &writer[0]->fsetArr, writer[0]->fopArr); + code = tsdbFSCreateCopyRangedSnapshot(pTsdb->pFS, (TFileSetRangeArray*)pRanges, &writer[0]->fsetArr, writer[0]->fopArr); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -1168,7 +1168,7 @@ _exit: return code; } -// snap part +// STsdbSnapPartition ===================================== static int32_t tsdbSnapPartCmprFn(STsdbSnapPartition* x, STsdbSnapPartition* y) { if (x->fid < y->fid) return -1; if (x->fid > y->fid) return 1; @@ -1183,7 +1183,7 @@ static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) { return 0; } -static int32_t tsdbSnapRangeCmprFn(STSnapRange* x, STSnapRange* y) { +static int32_t tsdbFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) { if (x->fid < y->fid) return -1; if (x->fid > y->fid) return 1; return 0; @@ -1462,8 +1462,8 @@ _err: return -1; } -int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TSnapRangeArray** ppRanges) { - TSnapRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TSnapRangeArray)); +int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TFileSetRangeArray** ppRanges) { + TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); if (pDiff == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -1472,7 +1472,7 @@ int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TSnapRangeArray** STsdbSnapPartition* part; TARRAY2_FOREACH(pList, part) { - STSnapRange* r = taosMemoryCalloc(1, sizeof(STSnapRange)); + STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); if (r == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -1493,7 +1493,7 @@ int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TSnapRangeArray** r->sver = maxVerValid + 1; r->ever = VERSION_MAX; tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever); - int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbSnapRangeCmprFn); + int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbFileSetRangeCmprFn); ASSERT(code == 0); } ppRanges[0] = pDiff; @@ -1503,12 +1503,12 @@ int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TSnapRangeArray** _err: if (pDiff) { - tsdbSnapRangeArrayDestroy(&pDiff); + tsdbFileSetRangeArrayDestroy(&pDiff); } return -1; } -void tsdbSnapRangeArrayDestroy(TSnapRangeArray** ppSnap) { +void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray** ppSnap) { if (ppSnap && ppSnap[0]) { TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear); taosMemoryFree(ppSnap[0]); diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 34b508388f..f65d9085fd 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -29,7 +29,7 @@ struct SVSnapReader { SMetaSnapReader *pMetaReader; // tsdb int8_t tsdbDone; - TSnapRangeArray *pRanges; + TFileSetRangeArray *pRanges; STsdbSnapReader *pTsdbReader; // tq int8_t tqHandleDone; @@ -45,11 +45,11 @@ struct SVSnapReader { SStreamStateReader *pStreamStateReader; // rsma int8_t rsmaDone; - TSnapRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; + TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; SRSmaSnapReader *pRsmaReader; }; -static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TSnapRangeArray **ppRanges) { +static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) { int32_t code = -1; STsdbSnapPartList *pList = tsdbSnapPartListCreate(); if (pList == NULL) { @@ -69,7 +69,7 @@ _out: return code; } -static TSnapRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) { +static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) { ASSERTS(sizeof(pReader->pRsmaRanges) / sizeof(pReader->pRsmaRanges[0]) == 2, "Unexpected array size"); switch (tsdbTyp) { case SNAP_DATA_TSDB: @@ -94,7 +94,7 @@ static int32_t vnodeSnapReaderDoSnapInfo(SVSnapReader *pReader, SSnapshotParam * goto _out; } - TSnapRangeArray **ppRanges = NULL; + TFileSetRangeArray **ppRanges = NULL; int32_t offset = 0; while (offset + sizeof(SSyncTLV) < datHead->len) { @@ -152,9 +152,9 @@ _err: static void vnodeSnapReaderDestroyTsdbRanges(SVSnapReader *pReader) { int32_t tsdbTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) { - TSnapRangeArray **ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, tsdbTyps[j]); + TFileSetRangeArray **ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, tsdbTyps[j]); if (ppRanges == NULL) continue; - tsdbSnapRangeArrayDestroy(ppRanges); + tsdbFileSetRangeArrayDestroy(ppRanges); } } @@ -455,7 +455,7 @@ struct SVSnapWriter { // meta SMetaSnapWriter *pMetaSnapWriter; // tsdb - TSnapRangeArray *pRanges; + TFileSetRangeArray *pRanges; STsdbSnapWriter *pTsdbSnapWriter; // tq STqSnapWriter *pTqSnapWriter; @@ -465,11 +465,11 @@ struct SVSnapWriter { SStreamTaskWriter *pStreamTaskWriter; SStreamStateWriter *pStreamStateWriter; // rsma - TSnapRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; + TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; SRSmaSnapWriter *pRsmaSnapWriter; }; -TSnapRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) { +TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) { ASSERTS(sizeof(pWriter->pRsmaRanges) / sizeof(pWriter->pRsmaRanges[0]) == 2, "Unexpected array size"); switch (tsdbTyp) { case SNAP_DATA_TSDB: @@ -494,7 +494,7 @@ static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam * goto _out; } - TSnapRangeArray **ppRanges = NULL; + TFileSetRangeArray **ppRanges = NULL; int32_t offset = 0; while (offset + sizeof(SSyncTLV) < datHead->len) { @@ -576,9 +576,9 @@ _err: static void vnodeSnapWriterDestroyTsdbRanges(SVSnapWriter *pWriter) { int32_t tsdbTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) { - TSnapRangeArray **ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, tsdbTyps[j]); + TFileSetRangeArray **ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, tsdbTyps[j]); if (ppRanges == NULL) continue; - tsdbSnapRangeArrayDestroy(ppRanges); + tsdbFileSetRangeArrayDestroy(ppRanges); } } From ddcabcfa4afc3f5a46b3ab6c1e40c5647ba17f51 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 13 Nov 2023 14:39:28 +0800 Subject: [PATCH 54/94] refact: remove unused fn tSerializeSnapRangeArray --- source/dnode/vnode/src/inc/tsdb.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 9cc6cd1132..99b92ace3a 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -685,8 +685,6 @@ typedef struct STFileSetRange STFileSetRange; typedef TARRAY2(STFileSetRange *) TFileSetRangeArray; // disjoint ranges // util -int32_t tSerializeSnapRangeArray(void *buf, int32_t bufLen, TFileSetRangeArray *pSnapR); -int32_t tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TFileSetRangeArray *pSnapR); void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray **ppSnap); SHashObj *tsdbGetSnapRangeHash(TFileSetRangeArray *pRanges); From 3ad68bdfea9c14acf1a886a85af65e25c2f42771 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 30 Nov 2023 17:35:28 +0800 Subject: [PATCH 55/94] enh: add tsdbSnapInfo.c --- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbSnapInfo.c diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c new file mode 100644 index 0000000000..e69de29bb2 From b8b53a4b0e7f847dbb65c2e9f93aa4192b4047ef Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 30 Nov 2023 17:53:16 +0800 Subject: [PATCH 56/94] refact: put source code of snap info into tsdbSnapInfo.c --- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 456 +++++++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 436 -------------------- 2 files changed, 456 insertions(+), 436 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index e69de29bb2..e70d0eaabc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -0,0 +1,456 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" +#include "tsdbDataFileRW.h" +#include "tsdbFS2.h" +#include "tsdbFSetRW.h" +#include "tsdbIter.h" +#include "tsdbSttFileRW.h" + +// STsdbSnapPartition ===================================== +static int32_t tsdbSnapPartCmprFn(STsdbSnapPartition* x, STsdbSnapPartition* y) { + if (x->fid < y->fid) return -1; + if (x->fid > y->fid) return 1; + return 0; +} + +static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) { + if (x->minVer < y->minVer) return -1; + if (x->minVer > y->minVer) return 1; + if (x->maxVer < y->maxVer) return -1; + if (x->maxVer > y->maxVer) return 1; + return 0; +} + +static int32_t tsdbFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) { + if (x->fid < y->fid) return -1; + if (x->fid > y->fid) return 1; + return 0; +} + +STsdbSnapPartition* tsdbSnapPartitionCreate() { + STsdbSnapPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbSnapPartition)); + if (pSP == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { + TARRAY2_INIT(&pSP->verRanges[i]); + } + return pSP; +} + +void tsdbSnapPartitionClear(STsdbSnapPartition** ppSP) { + if (ppSP == NULL || ppSP[0] == NULL) { + return; + } + for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { + TARRAY2_DESTROY(&ppSP[0]->verRanges[i], NULL); + } + taosMemoryFree(ppSP[0]); + ppSP[0] = NULL; +} + +static int32_t tsdbFTypeToSRangeTyp(tsdb_ftype_t ftype) { + switch (ftype) { + case TSDB_FTYPE_HEAD: + return TSDB_SNAP_RANGE_TYP_HEAD; + case TSDB_FTYPE_DATA: + return TSDB_SNAP_RANGE_TYP_DATA; + case TSDB_FTYPE_SMA: + return TSDB_SNAP_RANGE_TYP_SMA; + case TSDB_FTYPE_TOMB: + return TSDB_SNAP_RANGE_TYP_TOMB; + case TSDB_FTYPE_STT: + return TSDB_SNAP_RANGE_TYP_STT; + } + return TSDB_SNAP_RANGE_TYP_MAX; +} + +static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP) { + STsdbSnapPartition* p = tsdbSnapPartitionCreate(); + if (p == NULL) { + goto _err; + } + + p->fid = fset->fid; + + int32_t code = 0; + int32_t typ = 0; + int32_t corrupt = false; + int32_t count = 0; + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset->farr[ftype] == NULL) continue; + typ = tsdbFTypeToSRangeTyp(ftype); + ASSERT(typ < TSDB_SNAP_RANGE_TYP_MAX); + STFile* f = fset->farr[ftype]->f; + if (f->maxVer > fset->maxVerValid) { + corrupt = true; + tsdbError("skip incomplete data file: fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 + ", ftype: %d", + fset->fid, fset->maxVerValid, f->minVer, f->maxVer, ftype); + continue; + } + count++; + SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; + code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); + ASSERT(code == 0); + } + + typ = TSDB_SNAP_RANGE_TYP_STT; + const SSttLvl* lvl; + TARRAY2_FOREACH(fset->lvlArr, lvl) { + STFileObj* fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + STFile* f = fobj->f; + if (f->maxVer > fset->maxVerValid) { + corrupt = true; + tsdbError("skip incomplete stt file.fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 + ", ftype: %d", + fset->fid, fset->maxVerValid, f->minVer, f->maxVer, typ); + continue; + } + count++; + SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; + code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); + ASSERT(code == 0); + } + } + if (corrupt && count == 0) { + SVersionRange vr = {.minVer = VERSION_MIN, .maxVer = fset->maxVerValid}; + code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); + ASSERT(code == 0); + } + ppSP[0] = p; + return 0; + +_err: + tsdbSnapPartitionClear(&p); + return -1; +} + +STsdbSnapPartList* tsdbSnapPartListCreate() { + STsdbSnapPartList* pList = taosMemoryCalloc(1, sizeof(STsdbSnapPartList)); + if (pList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + TARRAY2_INIT(pList); + return pList; +} + +static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) { + STsdbSnapPartList* pList = tsdbSnapPartListCreate(); + if (pList == NULL) { + return NULL; + } + + int32_t code = 0; + taosThreadMutexLock(&fs->tsdb->mutex); + STFileSet* fset; + TARRAY2_FOREACH(fs->fSetArr, fset) { + STsdbSnapPartition* pItem = NULL; + if (tsdbTFileSetToSnapPart(fset, &pItem) < 0) { + code = -1; + break; + } + ASSERT(pItem != NULL); + code = TARRAY2_SORT_INSERT(pList, pItem, tsdbSnapPartCmprFn); + ASSERT(code == 0); + } + taosThreadMutexUnlock(&fs->tsdb->mutex); + + if (code) { + TARRAY2_DESTROY(pList, tsdbSnapPartitionClear); + taosMemoryFree(pList); + pList = NULL; + } + return pList; +} + +int32_t tTsdbSnapPartListDataLenCalc(STsdbSnapPartList* pList) { + int32_t hdrLen = sizeof(int32_t); + int32_t datLen = 0; + + int8_t msgVer = 1; + int32_t len = TARRAY2_SIZE(pList); + hdrLen += sizeof(msgVer); + hdrLen += sizeof(len); + datLen += hdrLen; + + for (int32_t u = 0; u < len; u++) { + STsdbSnapPartition* p = TARRAY2_GET(pList, u); + int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; + int32_t uItem = 0; + uItem += sizeof(STsdbSnapPartition); + uItem += sizeof(typMax); + + for (int32_t i = 0; i < typMax; i++) { + int32_t iLen = TARRAY2_SIZE(&p->verRanges[i]); + int32_t jItem = 0; + jItem += sizeof(SVersionRange); + jItem += sizeof(int64_t); + uItem += sizeof(iLen) + jItem * iLen; + } + datLen += uItem; + } + return datLen; +} + +int32_t tSerializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + int8_t reserved8 = 0; + int16_t reserved16 = 0; + int64_t reserved64 = 0; + + int8_t msgVer = 1; + int32_t len = TARRAY2_SIZE(pList); + + if (tStartEncode(&encoder) < 0) goto _err; + if (tEncodeI8(&encoder, msgVer) < 0) goto _err; + if (tEncodeI32(&encoder, len) < 0) goto _err; + + for (int32_t u = 0; u < len; u++) { + STsdbSnapPartition* p = TARRAY2_GET(pList, u); + if (tEncodeI64(&encoder, p->fid) < 0) goto _err; + if (tEncodeI8(&encoder, p->stat) < 0) goto _err; + if (tEncodeI8(&encoder, reserved8) < 0) goto _err; + if (tEncodeI16(&encoder, reserved16) < 0) goto _err; + + int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; + if (tEncodeI32(&encoder, typMax) < 0) goto _err; + + for (int32_t i = 0; i < typMax; i++) { + SVerRangeList* iList = &p->verRanges[i]; + int32_t iLen = TARRAY2_SIZE(iList); + + if (tEncodeI32(&encoder, iLen) < 0) goto _err; + for (int32_t j = 0; j < iLen; j++) { + SVersionRange r = TARRAY2_GET(iList, j); + if (tEncodeI64(&encoder, r.minVer) < 0) goto _err; + if (tEncodeI64(&encoder, r.maxVer) < 0) goto _err; + if (tEncodeI64(&encoder, reserved64) < 0) goto _err; + } + } + } + + tEndEncode(&encoder); + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; + +_err: + tEncoderClear(&encoder); + return -1; +} + +int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + int8_t reserved8 = 0; + int16_t reserved16 = 0; + int64_t reserved64 = 0; + + STsdbSnapPartition* p = NULL; + + int8_t msgVer = 0; + int32_t len = 0; + if (tStartDecode(&decoder) < 0) goto _err; + if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; + if (tDecodeI32(&decoder, &len) < 0) goto _err; + + for (int32_t u = 0; u < len; u++) { + p = tsdbSnapPartitionCreate(); + if (p == NULL) goto _err; + if (tDecodeI64(&decoder, &p->fid) < 0) goto _err; + if (tDecodeI8(&decoder, &p->stat) < 0) goto _err; + if (tDecodeI8(&decoder, &reserved8) < 0) goto _err; + if (tDecodeI16(&decoder, &reserved16) < 0) goto _err; + + int32_t typMax = 0; + if (tDecodeI32(&decoder, &typMax) < 0) goto _err; + + for (int32_t i = 0; i < typMax; i++) { + SVerRangeList* iList = &p->verRanges[i]; + int32_t iLen = 0; + if (tDecodeI32(&decoder, &iLen) < 0) goto _err; + for (int32_t j = 0; j < iLen; j++) { + SVersionRange r = {0}; + if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err; + if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err; + if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; + TARRAY2_APPEND(iList, r); + } + } + TARRAY2_APPEND(pList, p); + p = NULL; + } + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; + +_err: + if (p) { + tsdbSnapPartitionClear(&p); + } + tDecoderClear(&decoder); + return -1; +} + +int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TFileSetRangeArray** ppRanges) { + TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); + if (pDiff == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + TARRAY2_INIT(pDiff); + + STsdbSnapPartition* part; + TARRAY2_FOREACH(pList, part) { + STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); + if (r == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + int64_t maxVerValid = -1; + int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; + for (int32_t i = 0; i < typMax; i++) { + SVerRangeList* iList = &part->verRanges[i]; + SVersionRange vr = {0}; + TARRAY2_FOREACH(iList, vr) { + if (vr.maxVer < vr.minVer) { + continue; + } + maxVerValid = TMAX(maxVerValid, vr.maxVer); + } + } + r->fid = part->fid; + r->sver = maxVerValid + 1; + r->ever = VERSION_MAX; + tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever); + int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbFileSetRangeCmprFn); + ASSERT(code == 0); + } + ppRanges[0] = pDiff; + + tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff)); + return 0; + +_err: + if (pDiff) { + tsdbFileSetRangeArrayDestroy(&pDiff); + } + return -1; +} + +void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray** ppSnap) { + if (ppSnap && ppSnap[0]) { + TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear); + taosMemoryFree(ppSnap[0]); + ppSnap[0] = NULL; + } +} + +void tsdbSnapPartListDestroy(STsdbSnapPartList** ppList) { + if (ppList == NULL || ppList[0] == NULL) return; + + TARRAY2_DESTROY(ppList[0], tsdbSnapPartitionClear); + taosMemoryFree(ppList[0]); + ppList[0] = NULL; +} + +ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) { + if (!VND_IS_RSMA(pVnode)) { + return pVnode->pTsdb->pFS->fsstate; + } + for (int32_t lvl = 0; lvl < TSDB_RETENTION_MAX; ++lvl) { + STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, lvl); + if (pTsdb && pTsdb->pFS->fsstate != TSDB_FS_STATE_NORMAL) { + return TSDB_FS_STATE_INCOMPLETE; + } + } + return TSDB_FS_STATE_NORMAL; +} + +int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { + int code = -1; + int32_t tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX); + int32_t subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; + STsdbSnapPartList* pLists[TSDB_RETENTION_MAX] = {0}; + + // get part list + for (int32_t j = 0; j < tsdbMaxCnt; ++j) { + STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j); + pLists[j] = tsdbGetSnapPartList(pTsdb->pFS); + if (pLists[j] == NULL) goto _out; + } + + // estimate bufLen and prepare + int32_t bufLen = sizeof(SSyncTLV); // typ: TDMT_SYNC_PREP_SNAPSHOT or TDMT_SYNC_PREP_SNAPSOT_REPLY + for (int32_t j = 0; j < tsdbMaxCnt; ++j) { + bufLen += sizeof(SSyncTLV); // subTyps[j] + bufLen += tTsdbSnapPartListDataLenCalc(pLists[j]); + } + + tsdbInfo("vgId:%d, allocate %d bytes for data of snapshot info.", TD_VID(pVnode), bufLen); + + void* data = taosMemoryRealloc(pSnap->data, bufLen); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tsdbError("vgId:%d, failed to realloc memory for data of snapshot info. bytes:%d", TD_VID(pVnode), bufLen); + goto _out; + } + pSnap->data = data; + + // header + SSyncTLV* head = data; + head->len = 0; + head->typ = pSnap->type; + int32_t offset = sizeof(SSyncTLV); + int32_t tlen = 0; + + // fill snapshot info + for (int32_t j = 0; j < tsdbMaxCnt; ++j) { + // subHead + SSyncTLV* subHead = (void*)((char*)data + offset); + subHead->typ = subTyps[j]; + ASSERT(subHead->val == (char*)data + offset + sizeof(SSyncTLV)); + + if ((tlen = tSerializeTsdbSnapPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) { + tsdbError("vgId:%d, failed to serialize snap partition list of tsdb %d since %s", TD_VID(pVnode), j, terrstr()); + goto _out; + } + subHead->len = tlen; + offset += sizeof(SSyncTLV) + tlen; + } + + // total length of subfields + head->len = offset - sizeof(SSyncTLV); + ASSERT(offset <= bufLen); + code = 0; + +_out: + for (int32_t j = 0; j < tsdbMaxCnt; ++j) { + if (pLists[j] == NULL) continue; + tsdbSnapPartListDestroy(&pLists[j]); + } + + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 9d3efcfdae..48872404ed 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1167,439 +1167,3 @@ _exit: } return code; } - -// STsdbSnapPartition ===================================== -static int32_t tsdbSnapPartCmprFn(STsdbSnapPartition* x, STsdbSnapPartition* y) { - if (x->fid < y->fid) return -1; - if (x->fid > y->fid) return 1; - return 0; -} - -static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) { - if (x->minVer < y->minVer) return -1; - if (x->minVer > y->minVer) return 1; - if (x->maxVer < y->maxVer) return -1; - if (x->maxVer > y->maxVer) return 1; - return 0; -} - -static int32_t tsdbFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) { - if (x->fid < y->fid) return -1; - if (x->fid > y->fid) return 1; - return 0; -} - -STsdbSnapPartition* tsdbSnapPartitionCreate() { - STsdbSnapPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbSnapPartition)); - if (pSP == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { - TARRAY2_INIT(&pSP->verRanges[i]); - } - return pSP; -} - -void tsdbSnapPartitionClear(STsdbSnapPartition** ppSP) { - if (ppSP == NULL || ppSP[0] == NULL) { - return; - } - for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { - TARRAY2_DESTROY(&ppSP[0]->verRanges[i], NULL); - } - taosMemoryFree(ppSP[0]); - ppSP[0] = NULL; -} - -static int32_t tsdbFTypeToSRangeTyp(tsdb_ftype_t ftype) { - switch (ftype) { - case TSDB_FTYPE_HEAD: - return TSDB_SNAP_RANGE_TYP_HEAD; - case TSDB_FTYPE_DATA: - return TSDB_SNAP_RANGE_TYP_DATA; - case TSDB_FTYPE_SMA: - return TSDB_SNAP_RANGE_TYP_SMA; - case TSDB_FTYPE_TOMB: - return TSDB_SNAP_RANGE_TYP_TOMB; - case TSDB_FTYPE_STT: - return TSDB_SNAP_RANGE_TYP_STT; - } - return TSDB_SNAP_RANGE_TYP_MAX; -} - -static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP) { - STsdbSnapPartition* p = tsdbSnapPartitionCreate(); - if (p == NULL) { - goto _err; - } - - p->fid = fset->fid; - - int32_t code = 0; - int32_t typ = 0; - int32_t corrupt = false; - int32_t count = 0; - for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { - if (fset->farr[ftype] == NULL) continue; - typ = tsdbFTypeToSRangeTyp(ftype); - ASSERT(typ < TSDB_SNAP_RANGE_TYP_MAX); - STFile* f = fset->farr[ftype]->f; - if (f->maxVer > fset->maxVerValid) { - corrupt = true; - tsdbError("skip incomplete data file: fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 - ", ftype: %d", - fset->fid, fset->maxVerValid, f->minVer, f->maxVer, ftype); - continue; - } - count++; - SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; - code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); - ASSERT(code == 0); - } - - typ = TSDB_SNAP_RANGE_TYP_STT; - const SSttLvl* lvl; - TARRAY2_FOREACH(fset->lvlArr, lvl) { - STFileObj* fobj; - TARRAY2_FOREACH(lvl->fobjArr, fobj) { - STFile* f = fobj->f; - if (f->maxVer > fset->maxVerValid) { - corrupt = true; - tsdbError("skip incomplete stt file.fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 - ", ftype: %d", - fset->fid, fset->maxVerValid, f->minVer, f->maxVer, typ); - continue; - } - count++; - SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer}; - code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); - ASSERT(code == 0); - } - } - if (corrupt && count == 0) { - SVersionRange vr = {.minVer = VERSION_MIN, .maxVer = fset->maxVerValid}; - code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn); - ASSERT(code == 0); - } - ppSP[0] = p; - return 0; - -_err: - tsdbSnapPartitionClear(&p); - return -1; -} - -STsdbSnapPartList* tsdbSnapPartListCreate() { - STsdbSnapPartList* pList = taosMemoryCalloc(1, sizeof(STsdbSnapPartList)); - if (pList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - TARRAY2_INIT(pList); - return pList; -} - -static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) { - STsdbSnapPartList* pList = tsdbSnapPartListCreate(); - if (pList == NULL) { - return NULL; - } - - int32_t code = 0; - taosThreadMutexLock(&fs->tsdb->mutex); - STFileSet* fset; - TARRAY2_FOREACH(fs->fSetArr, fset) { - STsdbSnapPartition* pItem = NULL; - if (tsdbTFileSetToSnapPart(fset, &pItem) < 0) { - code = -1; - break; - } - ASSERT(pItem != NULL); - code = TARRAY2_SORT_INSERT(pList, pItem, tsdbSnapPartCmprFn); - ASSERT(code == 0); - } - taosThreadMutexUnlock(&fs->tsdb->mutex); - - if (code) { - TARRAY2_DESTROY(pList, tsdbSnapPartitionClear); - taosMemoryFree(pList); - pList = NULL; - } - return pList; -} - -int32_t tTsdbSnapPartListDataLenCalc(STsdbSnapPartList* pList) { - int32_t hdrLen = sizeof(int32_t); - int32_t datLen = 0; - - int8_t msgVer = 1; - int32_t len = TARRAY2_SIZE(pList); - hdrLen += sizeof(msgVer); - hdrLen += sizeof(len); - datLen += hdrLen; - - for (int32_t u = 0; u < len; u++) { - STsdbSnapPartition* p = TARRAY2_GET(pList, u); - int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; - int32_t uItem = 0; - uItem += sizeof(STsdbSnapPartition); - uItem += sizeof(typMax); - - for (int32_t i = 0; i < typMax; i++) { - int32_t iLen = TARRAY2_SIZE(&p->verRanges[i]); - int32_t jItem = 0; - jItem += sizeof(SVersionRange); - jItem += sizeof(int64_t); - uItem += sizeof(iLen) + jItem * iLen; - } - datLen += uItem; - } - return datLen; -} - -int32_t tSerializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - int8_t reserved8 = 0; - int16_t reserved16 = 0; - int64_t reserved64 = 0; - - int8_t msgVer = 1; - int32_t len = TARRAY2_SIZE(pList); - - if (tStartEncode(&encoder) < 0) goto _err; - if (tEncodeI8(&encoder, msgVer) < 0) goto _err; - if (tEncodeI32(&encoder, len) < 0) goto _err; - - for (int32_t u = 0; u < len; u++) { - STsdbSnapPartition* p = TARRAY2_GET(pList, u); - if (tEncodeI64(&encoder, p->fid) < 0) goto _err; - if (tEncodeI8(&encoder, p->stat) < 0) goto _err; - if (tEncodeI8(&encoder, reserved8) < 0) goto _err; - if (tEncodeI16(&encoder, reserved16) < 0) goto _err; - - int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; - if (tEncodeI32(&encoder, typMax) < 0) goto _err; - - for (int32_t i = 0; i < typMax; i++) { - SVerRangeList* iList = &p->verRanges[i]; - int32_t iLen = TARRAY2_SIZE(iList); - - if (tEncodeI32(&encoder, iLen) < 0) goto _err; - for (int32_t j = 0; j < iLen; j++) { - SVersionRange r = TARRAY2_GET(iList, j); - if (tEncodeI64(&encoder, r.minVer) < 0) goto _err; - if (tEncodeI64(&encoder, r.maxVer) < 0) goto _err; - if (tEncodeI64(&encoder, reserved64) < 0) goto _err; - } - } - } - - tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; - -_err: - tEncoderClear(&encoder); - return -1; -} - -int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - int8_t reserved8 = 0; - int16_t reserved16 = 0; - int64_t reserved64 = 0; - - STsdbSnapPartition* p = NULL; - - int8_t msgVer = 0; - int32_t len = 0; - if (tStartDecode(&decoder) < 0) goto _err; - if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; - if (tDecodeI32(&decoder, &len) < 0) goto _err; - - for (int32_t u = 0; u < len; u++) { - p = tsdbSnapPartitionCreate(); - if (p == NULL) goto _err; - if (tDecodeI64(&decoder, &p->fid) < 0) goto _err; - if (tDecodeI8(&decoder, &p->stat) < 0) goto _err; - if (tDecodeI8(&decoder, &reserved8) < 0) goto _err; - if (tDecodeI16(&decoder, &reserved16) < 0) goto _err; - - int32_t typMax = 0; - if (tDecodeI32(&decoder, &typMax) < 0) goto _err; - - for (int32_t i = 0; i < typMax; i++) { - SVerRangeList* iList = &p->verRanges[i]; - int32_t iLen = 0; - if (tDecodeI32(&decoder, &iLen) < 0) goto _err; - for (int32_t j = 0; j < iLen; j++) { - SVersionRange r = {0}; - if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err; - if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err; - if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; - TARRAY2_APPEND(iList, r); - } - } - TARRAY2_APPEND(pList, p); - p = NULL; - } - - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; - -_err: - if (p) { - tsdbSnapPartitionClear(&p); - } - tDecoderClear(&decoder); - return -1; -} - -int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TFileSetRangeArray** ppRanges) { - TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); - if (pDiff == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - TARRAY2_INIT(pDiff); - - STsdbSnapPartition* part; - TARRAY2_FOREACH(pList, part) { - STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); - if (r == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - int64_t maxVerValid = -1; - int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; - for (int32_t i = 0; i < typMax; i++) { - SVerRangeList* iList = &part->verRanges[i]; - SVersionRange vr = {0}; - TARRAY2_FOREACH(iList, vr) { - if (vr.maxVer < vr.minVer) { - continue; - } - maxVerValid = TMAX(maxVerValid, vr.maxVer); - } - } - r->fid = part->fid; - r->sver = maxVerValid + 1; - r->ever = VERSION_MAX; - tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever); - int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbFileSetRangeCmprFn); - ASSERT(code == 0); - } - ppRanges[0] = pDiff; - - tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff)); - return 0; - -_err: - if (pDiff) { - tsdbFileSetRangeArrayDestroy(&pDiff); - } - return -1; -} - -void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray** ppSnap) { - if (ppSnap && ppSnap[0]) { - TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear); - taosMemoryFree(ppSnap[0]); - ppSnap[0] = NULL; - } -} - -void tsdbSnapPartListDestroy(STsdbSnapPartList** ppList) { - if (ppList == NULL || ppList[0] == NULL) return; - - TARRAY2_DESTROY(ppList[0], tsdbSnapPartitionClear); - taosMemoryFree(ppList[0]); - ppList[0] = NULL; -} - -ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) { - if (!VND_IS_RSMA(pVnode)) { - return pVnode->pTsdb->pFS->fsstate; - } - for (int32_t lvl = 0; lvl < TSDB_RETENTION_MAX; ++lvl) { - STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, lvl); - if (pTsdb && pTsdb->pFS->fsstate != TSDB_FS_STATE_NORMAL) { - return TSDB_FS_STATE_INCOMPLETE; - } - } - return TSDB_FS_STATE_NORMAL; -} - -int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { - int code = -1; - int32_t tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX); - int32_t subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; - STsdbSnapPartList* pLists[TSDB_RETENTION_MAX] = {0}; - - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j); - pLists[j] = tsdbGetSnapPartList(pTsdb->pFS); - if (pLists[j] == NULL) goto _out; - } - - // estimate bufLen and prepare - int32_t bufLen = sizeof(SSyncTLV); // typ: TDMT_SYNC_PREP_SNAPSHOT or TDMT_SYNC_PREP_SNAPSOT_REPLY - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - bufLen += sizeof(SSyncTLV); // subTyps[j] - bufLen += tTsdbSnapPartListDataLenCalc(pLists[j]); - } - - tsdbInfo("vgId:%d, allocate %d bytes for data of snapshot info.", TD_VID(pVnode), bufLen); - - void* data = taosMemoryRealloc(pSnap->data, bufLen); - if (data == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tsdbError("vgId:%d, failed to realloc memory for data of snapshot info. bytes:%d", TD_VID(pVnode), bufLen); - goto _out; - } - pSnap->data = data; - - // header - SSyncTLV* head = data; - head->len = 0; - head->typ = pSnap->type; - int32_t offset = sizeof(SSyncTLV); - int32_t tlen = 0; - - // fill snapshot info - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { - } - - // subHead - SSyncTLV* subHead = (void*)((char*)data + offset); - subHead->typ = subTyps[j]; - ASSERT(subHead->val == (char*)data + offset + sizeof(SSyncTLV)); - - if ((tlen = tSerializeTsdbSnapPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) { - tsdbError("vgId:%d, failed to serialize snap partition list of tsdb %d since %s", TD_VID(pVnode), j, terrstr()); - goto _out; - } - subHead->len = tlen; - offset += sizeof(SSyncTLV) + tlen; - } - - head->len = offset - sizeof(SSyncTLV); - ASSERT(offset <= bufLen); - code = 0; - -_out: - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - if (pLists[j] == NULL) continue; - tsdbSnapPartListDestroy(&pLists[j]); - } - - return code; -} \ No newline at end of file From 4487eeacb58eebdb2cc897a32e9c7dafab653e6a Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 30 Nov 2023 19:09:32 +0800 Subject: [PATCH 57/94] refact: improve funcs dealing with SnapInfo for vnode snap reader and writer --- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 100 ++++++++++++--------- 1 file changed, 60 insertions(+), 40 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index f65d9085fd..941660f776 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -16,6 +16,26 @@ #include "tsdb.h" #include "vnd.h" +static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) { + int32_t code = -1; + STsdbSnapPartList *pList = tsdbSnapPartListCreate(); + if (pList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _out; + } + if (tDeserializeTsdbSnapPartList(buf, bufLen, pList) < 0) { + terrno = TSDB_CODE_INVALID_DATA_FMT; + goto _out; + } + if (tsdbSnapPartListToRangeDiff(pList, ppRanges) < 0) { + goto _out; + } + code = 0; +_out: + tsdbSnapPartListDestroy(&pList); + return code; +} + // SVSnapReader ======================================================== struct SVSnapReader { SVnode *pVnode; @@ -49,26 +69,6 @@ struct SVSnapReader { SRSmaSnapReader *pRsmaReader; }; -static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) { - int32_t code = -1; - STsdbSnapPartList *pList = tsdbSnapPartListCreate(); - if (pList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _out; - } - if (tDeserializeTsdbSnapPartList(buf, bufLen, pList) < 0) { - terrno = TSDB_CODE_INVALID_DATA_FMT; - goto _out; - } - if (tsdbSnapPartListToRangeDiff(pList, ppRanges) < 0) { - goto _out; - } - code = 0; -_out: - tsdbSnapPartListDestroy(&pList); - return code; -} - static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) { ASSERTS(sizeof(pReader->pRsmaRanges) / sizeof(pReader->pRsmaRanges[0]) == 2, "Unexpected array size"); switch (tsdbTyp) { @@ -83,7 +83,7 @@ static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, } } -static int32_t vnodeSnapReaderDoSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) { +static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) { SVnode *pVnode = pReader->pVnode; int32_t code = -1; @@ -102,14 +102,24 @@ static int32_t vnodeSnapReaderDoSnapInfo(SVSnapReader *pReader, SSnapshotParam * offset += sizeof(SSyncTLV) + subField->len; void *buf = subField->val; int32_t bufLen = subField->len; - ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ); - if (ppRanges == NULL) { - vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); - goto _out; - } - if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { - vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); - goto _out; + + switch (subField->typ) { + case SNAP_DATA_TSDB: + case SNAP_DATA_RSMA1: + case SNAP_DATA_RSMA2: { + ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ); + if (ppRanges == NULL) { + vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); + goto _out; + } + if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { + vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } break; + default: + vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ); + goto _out; } } } @@ -135,7 +145,7 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader pReader->ever = ever; // snapshot info - if (vnodeSnapReaderDoSnapInfo(pReader, pParam) < 0) { + if (vnodeSnapReaderDealWithSnapInfo(pReader, pParam) < 0) { goto _err; } @@ -483,7 +493,7 @@ TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t } } -static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) { +static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) { SVnode *pVnode = pWriter->pVnode; int32_t code = -1; @@ -502,14 +512,24 @@ static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam * offset += sizeof(SSyncTLV) + subField->len; void *buf = subField->val; int32_t bufLen = subField->len; - ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ); - if (ppRanges == NULL) { - vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); - goto _out; - } - if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { - vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); - goto _out; + + switch (subField->typ) { + case SNAP_DATA_TSDB: + case SNAP_DATA_RSMA1: + case SNAP_DATA_RSMA2: { + ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ); + if (ppRanges == NULL) { + vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); + goto _out; + } + if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { + vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } break; + default: + vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ); + goto _out; } } } @@ -558,7 +578,7 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter pWriter->commitID = ++pVnode->state.commitID; // snapshot info - if (vnodeSnapWriterDoSnapInfo(pWriter, pParam) < 0) { + if (vnodeSnapWriterDealWithSnapInfo(pWriter, pParam) < 0) { goto _err; } From 20b5cf8d49e98ff15c69d849fdfca07c69f53595 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 30 Nov 2023 20:30:59 +0800 Subject: [PATCH 58/94] refact: remove redundent includes from tsdbSnapInfo.c --- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index e70d0eaabc..5aaa88511a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -14,11 +14,7 @@ */ #include "tsdb.h" -#include "tsdbDataFileRW.h" #include "tsdbFS2.h" -#include "tsdbFSetRW.h" -#include "tsdbIter.h" -#include "tsdbSttFileRW.h" // STsdbSnapPartition ===================================== static int32_t tsdbSnapPartCmprFn(STsdbSnapPartition* x, STsdbSnapPartition* y) { From ed916b1a80385223491f25e53a033e51bfd7e3fc Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 30 Nov 2023 21:02:26 +0800 Subject: [PATCH 59/94] refact: rename tsdb snap partition to fset partition --- source/dnode/vnode/src/inc/tsdb.h | 16 +++--- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 65 +++++++++++----------- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 8 +-- 3 files changed, 45 insertions(+), 44 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 99b92ace3a..374753fcde 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -690,14 +690,14 @@ SHashObj *tsdbGetSnapRangeHash(TFileSetRangeArray *pRanges); // snap partition list typedef TARRAY2(SVersionRange) SVerRangeList; -typedef struct STsdbSnapPartition STsdbSnapPartition; -typedef TARRAY2(STsdbSnapPartition *) STsdbSnapPartList; +typedef struct STsdbFSetPartition STsdbFSetPartition; +typedef TARRAY2(STsdbFSetPartition *) STsdbFSetPartList; // util -STsdbSnapPartList *tsdbSnapPartListCreate(); -void tsdbSnapPartListDestroy(STsdbSnapPartList **ppList); -int32_t tSerializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList); -int32_t tDeserializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList); -int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList *pList, TFileSetRangeArray **ppRanges); +STsdbFSetPartList *tsdbFSetPartListCreate(); +void tsdbFSetPartListDestroy(STsdbFSetPartList **ppList); +int32_t tSerializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); +int32_t tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); +int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges); enum { TSDB_SNAP_RANGE_TYP_HEAD = 0, @@ -708,7 +708,7 @@ enum { TSDB_SNAP_RANGE_TYP_MAX, }; -struct STsdbSnapPartition { +struct STsdbFSetPartition { int64_t fid; int8_t stat; SVerRangeList verRanges[TSDB_SNAP_RANGE_TYP_MAX]; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index 5aaa88511a..c73d75030d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -16,8 +16,8 @@ #include "tsdb.h" #include "tsdbFS2.h" -// STsdbSnapPartition ===================================== -static int32_t tsdbSnapPartCmprFn(STsdbSnapPartition* x, STsdbSnapPartition* y) { +// STsdbFSetPartition ===================================== +static int32_t tsdbFSetPartCmprFn(STsdbFSetPartition* x, STsdbFSetPartition* y) { if (x->fid < y->fid) return -1; if (x->fid > y->fid) return 1; return 0; @@ -37,8 +37,8 @@ static int32_t tsdbFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) { return 0; } -STsdbSnapPartition* tsdbSnapPartitionCreate() { - STsdbSnapPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbSnapPartition)); +STsdbFSetPartition* tsdbFSetPartitionCreate() { + STsdbFSetPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbFSetPartition)); if (pSP == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -49,7 +49,7 @@ STsdbSnapPartition* tsdbSnapPartitionCreate() { return pSP; } -void tsdbSnapPartitionClear(STsdbSnapPartition** ppSP) { +void tsdbFSetPartitionClear(STsdbFSetPartition** ppSP) { if (ppSP == NULL || ppSP[0] == NULL) { return; } @@ -76,8 +76,8 @@ static int32_t tsdbFTypeToSRangeTyp(tsdb_ftype_t ftype) { return TSDB_SNAP_RANGE_TYP_MAX; } -static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP) { - STsdbSnapPartition* p = tsdbSnapPartitionCreate(); +static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbFSetPartition** ppSP) { + STsdbFSetPartition* p = tsdbFSetPartitionCreate(); if (p == NULL) { goto _err; } @@ -134,12 +134,12 @@ static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP return 0; _err: - tsdbSnapPartitionClear(&p); + tsdbFSetPartitionClear(&p); return -1; } -STsdbSnapPartList* tsdbSnapPartListCreate() { - STsdbSnapPartList* pList = taosMemoryCalloc(1, sizeof(STsdbSnapPartList)); +STsdbFSetPartList* tsdbFSetPartListCreate() { + STsdbFSetPartList* pList = taosMemoryCalloc(1, sizeof(STsdbFSetPartList)); if (pList == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -148,8 +148,8 @@ STsdbSnapPartList* tsdbSnapPartListCreate() { return pList; } -static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) { - STsdbSnapPartList* pList = tsdbSnapPartListCreate(); +static STsdbFSetPartList* tsdbGetSnapPartList(STFileSystem* fs) { + STsdbFSetPartList* pList = tsdbFSetPartListCreate(); if (pList == NULL) { return NULL; } @@ -158,26 +158,26 @@ static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) { taosThreadMutexLock(&fs->tsdb->mutex); STFileSet* fset; TARRAY2_FOREACH(fs->fSetArr, fset) { - STsdbSnapPartition* pItem = NULL; + STsdbFSetPartition* pItem = NULL; if (tsdbTFileSetToSnapPart(fset, &pItem) < 0) { code = -1; break; } ASSERT(pItem != NULL); - code = TARRAY2_SORT_INSERT(pList, pItem, tsdbSnapPartCmprFn); + code = TARRAY2_SORT_INSERT(pList, pItem, tsdbFSetPartCmprFn); ASSERT(code == 0); } taosThreadMutexUnlock(&fs->tsdb->mutex); if (code) { - TARRAY2_DESTROY(pList, tsdbSnapPartitionClear); + TARRAY2_DESTROY(pList, tsdbFSetPartitionClear); taosMemoryFree(pList); pList = NULL; } return pList; } -int32_t tTsdbSnapPartListDataLenCalc(STsdbSnapPartList* pList) { +int32_t tTsdbFSetPartListDataLenCalc(STsdbFSetPartList* pList) { int32_t hdrLen = sizeof(int32_t); int32_t datLen = 0; @@ -188,10 +188,10 @@ int32_t tTsdbSnapPartListDataLenCalc(STsdbSnapPartList* pList) { datLen += hdrLen; for (int32_t u = 0; u < len; u++) { - STsdbSnapPartition* p = TARRAY2_GET(pList, u); + STsdbFSetPartition* p = TARRAY2_GET(pList, u); int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; int32_t uItem = 0; - uItem += sizeof(STsdbSnapPartition); + uItem += sizeof(STsdbFSetPartition); uItem += sizeof(typMax); for (int32_t i = 0; i < typMax; i++) { @@ -206,7 +206,7 @@ int32_t tTsdbSnapPartListDataLenCalc(STsdbSnapPartList* pList) { return datLen; } -int32_t tSerializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { +int32_t tSerializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -222,7 +222,7 @@ int32_t tSerializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* if (tEncodeI32(&encoder, len) < 0) goto _err; for (int32_t u = 0; u < len; u++) { - STsdbSnapPartition* p = TARRAY2_GET(pList, u); + STsdbFSetPartition* p = TARRAY2_GET(pList, u); if (tEncodeI64(&encoder, p->fid) < 0) goto _err; if (tEncodeI8(&encoder, p->stat) < 0) goto _err; if (tEncodeI8(&encoder, reserved8) < 0) goto _err; @@ -255,7 +255,7 @@ _err: return -1; } -int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) { +int32_t tDeserializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -263,7 +263,7 @@ int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartLis int16_t reserved16 = 0; int64_t reserved64 = 0; - STsdbSnapPartition* p = NULL; + STsdbFSetPartition* p = NULL; int8_t msgVer = 0; int32_t len = 0; @@ -272,7 +272,7 @@ int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartLis if (tDecodeI32(&decoder, &len) < 0) goto _err; for (int32_t u = 0; u < len; u++) { - p = tsdbSnapPartitionCreate(); + p = tsdbFSetPartitionCreate(); if (p == NULL) goto _err; if (tDecodeI64(&decoder, &p->fid) < 0) goto _err; if (tDecodeI8(&decoder, &p->stat) < 0) goto _err; @@ -304,13 +304,13 @@ int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartLis _err: if (p) { - tsdbSnapPartitionClear(&p); + tsdbFSetPartitionClear(&p); } tDecoderClear(&decoder); return -1; } -int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TFileSetRangeArray** ppRanges) { +int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray** ppRanges) { TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); if (pDiff == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -318,7 +318,7 @@ int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TFileSetRangeArray } TARRAY2_INIT(pDiff); - STsdbSnapPartition* part; + STsdbFSetPartition* part; TARRAY2_FOREACH(pList, part) { STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); if (r == NULL) { @@ -364,10 +364,10 @@ void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray** ppSnap) { } } -void tsdbSnapPartListDestroy(STsdbSnapPartList** ppList) { +void tsdbFSetPartListDestroy(STsdbFSetPartList** ppList) { if (ppList == NULL || ppList[0] == NULL) return; - TARRAY2_DESTROY(ppList[0], tsdbSnapPartitionClear); + TARRAY2_DESTROY(ppList[0], tsdbFSetPartitionClear); taosMemoryFree(ppList[0]); ppList[0] = NULL; } @@ -389,7 +389,7 @@ int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { int code = -1; int32_t tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX); int32_t subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; - STsdbSnapPartList* pLists[TSDB_RETENTION_MAX] = {0}; + STsdbFSetPartList* pLists[TSDB_RETENTION_MAX] = {0}; // get part list for (int32_t j = 0; j < tsdbMaxCnt; ++j) { @@ -402,7 +402,7 @@ int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { int32_t bufLen = sizeof(SSyncTLV); // typ: TDMT_SYNC_PREP_SNAPSHOT or TDMT_SYNC_PREP_SNAPSOT_REPLY for (int32_t j = 0; j < tsdbMaxCnt; ++j) { bufLen += sizeof(SSyncTLV); // subTyps[j] - bufLen += tTsdbSnapPartListDataLenCalc(pLists[j]); + bufLen += tTsdbFSetPartListDataLenCalc(pLists[j]); } tsdbInfo("vgId:%d, allocate %d bytes for data of snapshot info.", TD_VID(pVnode), bufLen); @@ -429,7 +429,7 @@ int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { subHead->typ = subTyps[j]; ASSERT(subHead->val == (char*)data + offset + sizeof(SSyncTLV)); - if ((tlen = tSerializeTsdbSnapPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) { + if ((tlen = tSerializeTsdbFSetPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) { tsdbError("vgId:%d, failed to serialize snap partition list of tsdb %d since %s", TD_VID(pVnode), j, terrstr()); goto _out; } @@ -445,8 +445,9 @@ int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { _out: for (int32_t j = 0; j < tsdbMaxCnt; ++j) { if (pLists[j] == NULL) continue; - tsdbSnapPartListDestroy(&pLists[j]); + tsdbFSetPartListDestroy(&pLists[j]); } return code; } + diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 941660f776..bb3bd59971 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -18,21 +18,21 @@ static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) { int32_t code = -1; - STsdbSnapPartList *pList = tsdbSnapPartListCreate(); + STsdbFSetPartList *pList = tsdbFSetPartListCreate(); if (pList == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _out; } - if (tDeserializeTsdbSnapPartList(buf, bufLen, pList) < 0) { + if (tDeserializeTsdbFSetPartList(buf, bufLen, pList) < 0) { terrno = TSDB_CODE_INVALID_DATA_FMT; goto _out; } - if (tsdbSnapPartListToRangeDiff(pList, ppRanges) < 0) { + if (tsdbFSetPartListToRangeDiff(pList, ppRanges) < 0) { goto _out; } code = 0; _out: - tsdbSnapPartListDestroy(&pList); + tsdbFSetPartListDestroy(&pList); return code; } From 962febef02bea1c4c0ed1635e3d0c6cdc1ade0a4 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 1 Dec 2023 16:05:14 +0800 Subject: [PATCH 60/94] refact: rename tsdb snap range to tsdb fset range --- source/dnode/vnode/src/inc/tsdb.h | 29 ++++++------- source/dnode/vnode/src/tsdb/tsdbFS2.c | 48 +++++++++++----------- source/dnode/vnode/src/tsdb/tsdbFS2.h | 2 +- source/dnode/vnode/src/tsdb/tsdbFSet2.c | 14 ++++++- source/dnode/vnode/src/tsdb/tsdbFSet2.h | 4 +- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 13 ++---- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 +- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 4 +- 8 files changed, 61 insertions(+), 57 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 374753fcde..b0cf6ecf5b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -681,24 +681,14 @@ struct SDelFWriter { typedef struct STFileSet STFileSet; typedef TARRAY2(STFileSet *) TFileSetArray; +// fset range typedef struct STFileSetRange STFileSetRange; typedef TARRAY2(STFileSetRange *) TFileSetRangeArray; // disjoint ranges -// util -void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray **ppSnap); -SHashObj *tsdbGetSnapRangeHash(TFileSetRangeArray *pRanges); - -// snap partition list -typedef TARRAY2(SVersionRange) SVerRangeList; -typedef struct STsdbFSetPartition STsdbFSetPartition; -typedef TARRAY2(STsdbFSetPartition *) STsdbFSetPartList; -// util -STsdbFSetPartList *tsdbFSetPartListCreate(); -void tsdbFSetPartListDestroy(STsdbFSetPartList **ppList); -int32_t tSerializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); -int32_t tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); -int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges); +int32_t tsdbTFileSetRangeClear(STFileSetRange **fsr); +int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr); +// fset partition enum { TSDB_SNAP_RANGE_TYP_HEAD = 0, TSDB_SNAP_RANGE_TYP_DATA, @@ -708,12 +698,23 @@ enum { TSDB_SNAP_RANGE_TYP_MAX, }; +typedef TARRAY2(SVersionRange) SVerRangeList; + struct STsdbFSetPartition { int64_t fid; int8_t stat; SVerRangeList verRanges[TSDB_SNAP_RANGE_TYP_MAX]; }; +typedef struct STsdbFSetPartition STsdbFSetPartition; +typedef TARRAY2(STsdbFSetPartition *) STsdbFSetPartList; + +STsdbFSetPartList *tsdbFSetPartListCreate(); +void tsdbFSetPartListDestroy(STsdbFSetPartList **ppList); +int32_t tSerializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); +int32_t tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); +int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges); + // snap read struct STsdbReadSnap { SMemTable *pMem; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index df4df18dc3..ab52f5799d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -1072,6 +1072,24 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) { return 0; } +static SHashObj *tsdbFSetRangeArrayToHash(TFileSetRangeArray *pRanges) { + int32_t capacity = TARRAY2_SIZE(pRanges) * 2; + SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); + if (pHash == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) { + STFileSetRange *u = TARRAY2_GET(pRanges, i); + int32_t fid = u->fid; + int32_t code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u)); + ASSERT(code == 0); + tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever); + } + return pHash; +} + int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pRanges, TFileSetArray **fsetArr, TFileOpArray *fopArr) { int32_t code = 0; @@ -1084,7 +1102,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pRa TARRAY2_INIT(fsetArr[0]); if (pRanges) { - pHash = tsdbGetSnapRangeHash(pRanges); + pHash = tsdbFSetRangeArrayToHash(pRanges); if (pHash == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _out; @@ -1123,24 +1141,6 @@ _out: return code; } -SHashObj *tsdbGetSnapRangeHash(TFileSetRangeArray *pRanges) { - int32_t capacity = TARRAY2_SIZE(pRanges) * 2; - SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); - if (pHash == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) { - STFileSetRange *u = TARRAY2_GET(pRanges, i); - int32_t fid = u->fid; - int32_t code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u)); - ASSERT(code == 0); - tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever); - } - return pHash; -} - int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges, TFileSetRangeArray **fsrArr) { int32_t code = 0; @@ -1156,7 +1156,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev tsdbInfo("pRanges size:%d", (pRanges == NULL ? 0 : TARRAY2_SIZE(pRanges))); if (pRanges) { - pHash = tsdbGetSnapRangeHash(pRanges); + pHash = tsdbFSetRangeArrayToHash(pRanges); if (pHash == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _out; @@ -1184,7 +1184,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev tsdbDebug("fsrArr:%p, fid:%d, sver:%" PRId64 ", ever:%" PRId64, fsrArr, fset->fid, sver1, ever1); - code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1); + code = tsdbTFileSetRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1); if (code) break; code = TARRAY2_APPEND(fsrArr[0], fsr1); @@ -1195,8 +1195,8 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev taosThreadMutexUnlock(&fs->tsdb->mutex); if (code) { - tsdbTSnapRangeClear(&fsr1); - TARRAY2_DESTROY(fsrArr[0], tsdbTSnapRangeClear); + tsdbTFileSetRangeClear(&fsr1); + TARRAY2_DESTROY(fsrArr[0], tsdbTFileSetRangeClear); fsrArr[0] = NULL; } @@ -1206,4 +1206,4 @@ _out: pHash = NULL; } return code; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 8fdce9e690..3960ca908a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -50,7 +50,7 @@ int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr, TFileOpArray *f int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges, TFileSetRangeArray **fsrArr); int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr); -// txn +// txn int64_t tsdbFSAllocEid(STFileSystem *fs); int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype); int32_t tsdbFSEditCommit(STFileSystem *fs); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 7673299e4b..e088f54930 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -533,7 +533,8 @@ int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_ return 0; } -int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STFileSetRange **fsr) { +int32_t tsdbTFileSetRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, + STFileSetRange **fsr) { fsr[0] = taosMemoryCalloc(1, sizeof(*fsr[0])); if (fsr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; fsr[0]->fid = fset1->fid; @@ -575,7 +576,7 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs return 0; } -int32_t tsdbTSnapRangeClear(STFileSetRange **fsr) { +int32_t tsdbTFileSetRangeClear(STFileSetRange **fsr) { if (!fsr[0]) return 0; tsdbTFileSetClear(&fsr[0]->fset); @@ -584,6 +585,15 @@ int32_t tsdbTSnapRangeClear(STFileSetRange **fsr) { return 0; } +int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray** ppArr) { + if (ppArr && ppArr[0]) { + TARRAY2_DESTROY(ppArr[0], tsdbTFileSetRangeClear); + taosMemoryFree(ppArr[0]); + ppArr[0] = NULL; + } + return 0; +} + int32_t tsdbTFileSetClear(STFileSet **fset) { if (!fset[0]) return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index 3a6427a42c..0951a28f4e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -49,8 +49,8 @@ int32_t tsdbTFileSetRemove(STFileSet *fset); int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset, TFileOpArray *fopArr); -int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STFileSetRange **fsr); -int32_t tsdbTSnapRangeClear(STFileSetRange **fsr); +int32_t tsdbTFileSetRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, + STFileSetRange **fsr); // to/from json int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index c73d75030d..9d741ab2d3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -16,7 +16,7 @@ #include "tsdb.h" #include "tsdbFS2.h" -// STsdbFSetPartition ===================================== +// fset partition static int32_t tsdbFSetPartCmprFn(STsdbFSetPartition* x, STsdbFSetPartition* y) { if (x->fid < y->fid) return -1; if (x->fid > y->fid) return 1; @@ -138,6 +138,7 @@ _err: return -1; } +// fset partition list STsdbFSetPartList* tsdbFSetPartListCreate() { STsdbFSetPartList* pList = taosMemoryCalloc(1, sizeof(STsdbFSetPartList)); if (pList == NULL) { @@ -351,19 +352,11 @@ int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray _err: if (pDiff) { - tsdbFileSetRangeArrayDestroy(&pDiff); + tsdbTFileSetRangeArrayDestroy(&pDiff); } return -1; } -void tsdbFileSetRangeArrayDestroy(TFileSetRangeArray** ppSnap) { - if (ppSnap && ppSnap[0]) { - TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear); - taosMemoryFree(ppSnap[0]); - ppSnap[0] = NULL; - } -} - void tsdbFSetPartListDestroy(STsdbFSetPartList** ppList) { if (ppList == NULL || ppList[0] == NULL) return; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 48872404ed..f890353320 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -444,7 +444,7 @@ _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code), sver, ever, type); - tsdbFileSetRangeArrayDestroy(&reader[0]->fsrArr); + tsdbTFileSetRangeArrayDestroy(&reader[0]->fsrArr); taosMemoryFree(reader[0]); reader[0] = NULL; } else { @@ -472,7 +472,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) { TARRAY2_DESTROY(reader[0]->sttReaderArr, tsdbSttFileReaderClose); tsdbDataFileReaderClose(&reader[0]->dataReader); - tsdbFileSetRangeArrayDestroy(&reader[0]->fsrArr); + tsdbTFileSetRangeArrayDestroy(&reader[0]->fsrArr); tDestroyTSchema(reader[0]->skmTb->pTSchema); for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->aBuf); ++i) { diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index bb3bd59971..21c858709b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -164,7 +164,7 @@ static void vnodeSnapReaderDestroyTsdbRanges(SVSnapReader *pReader) { for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) { TFileSetRangeArray **ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, tsdbTyps[j]); if (ppRanges == NULL) continue; - tsdbFileSetRangeArrayDestroy(ppRanges); + tsdbTFileSetRangeArrayDestroy(ppRanges); } } @@ -598,7 +598,7 @@ static void vnodeSnapWriterDestroyTsdbRanges(SVSnapWriter *pWriter) { for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) { TFileSetRangeArray **ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, tsdbTyps[j]); if (ppRanges == NULL) continue; - tsdbFileSetRangeArrayDestroy(ppRanges); + tsdbTFileSetRangeArrayDestroy(ppRanges); } } From 3550347f0c9b1283701e7230c4d84c60a21097b3 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 1 Dec 2023 16:29:48 +0800 Subject: [PATCH 61/94] refact: use tsdbFSDestroyCopyRangedSnapshot and tsdbFSDestroyRefRangedSnapshot --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 4 ++++ source/dnode/vnode/src/tsdb/tsdbFS2.h | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index ab52f5799d..e933b3a7dc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -1141,6 +1141,8 @@ _out: return code; } +int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr) { return tsdbFSDestroyCopySnapshot(fsetArr); } + int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges, TFileSetRangeArray **fsrArr) { int32_t code = 0; @@ -1207,3 +1209,5 @@ _out: } return code; } + +int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr) { return tsdbTFileSetRangeArrayDestroy(fsrArr); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 3960ca908a..714bf5bf16 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -46,7 +46,7 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr); int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pExclude, TFileSetArray **fsetArr, TFileOpArray *fopArr); -int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr, TFileOpArray *fopArr); +int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr); int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges, TFileSetRangeArray **fsrArr); int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index f890353320..8f5394a9bc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -472,7 +472,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) { TARRAY2_DESTROY(reader[0]->sttReaderArr, tsdbSttFileReaderClose); tsdbDataFileReaderClose(&reader[0]->dataReader); - tsdbTFileSetRangeArrayDestroy(&reader[0]->fsrArr); + tsdbFSDestroyRefRangedSnapshot(&reader[0]->fsrArr); tDestroyTSchema(reader[0]->skmTb->pTSchema); for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->aBuf); ++i) { @@ -1125,7 +1125,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** writer, int8_t rollback) { tsdbDataFileReaderClose(&writer[0]->ctx->dataReader); TARRAY2_DESTROY(writer[0]->fopArr, NULL); - tsdbFSDestroyCopySnapshot(&writer[0]->fsetArr); + tsdbFSDestroyCopyRangedSnapshot(&writer[0]->fsetArr); for (int32_t i = 0; i < ARRAY_SIZE(writer[0]->aBuf); ++i) { tFree(writer[0]->aBuf[i]); From ed39c9a57253be3ee7e5ffb5f8278414feb8dc64 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 1 Dec 2023 17:21:27 +0800 Subject: [PATCH 62/94] refact: rename snap range to fset range in tsdbSnapInfo.c --- source/dnode/vnode/src/inc/tsdb.h | 14 +- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 179 +++++++++++---------- 2 files changed, 98 insertions(+), 95 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index b0cf6ecf5b..a5203353db 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -690,12 +690,12 @@ int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr); // fset partition enum { - TSDB_SNAP_RANGE_TYP_HEAD = 0, - TSDB_SNAP_RANGE_TYP_DATA, - TSDB_SNAP_RANGE_TYP_SMA, - TSDB_SNAP_RANGE_TYP_TOMB, - TSDB_SNAP_RANGE_TYP_STT, - TSDB_SNAP_RANGE_TYP_MAX, + TSDB_FSET_RANGE_TYP_HEAD = 0, + TSDB_FSET_RANGE_TYP_DATA, + TSDB_FSET_RANGE_TYP_SMA, + TSDB_FSET_RANGE_TYP_TOMB, + TSDB_FSET_RANGE_TYP_STT, + TSDB_FSET_RANGE_TYP_MAX, }; typedef TARRAY2(SVersionRange) SVerRangeList; @@ -703,7 +703,7 @@ typedef TARRAY2(SVersionRange) SVerRangeList; struct STsdbFSetPartition { int64_t fid; int8_t stat; - SVerRangeList verRanges[TSDB_SNAP_RANGE_TYP_MAX]; + SVerRangeList verRanges[TSDB_FSET_RANGE_TYP_MAX]; }; typedef struct STsdbFSetPartition STsdbFSetPartition; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index 9d741ab2d3..8afbe187a4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -31,7 +31,7 @@ static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) { return 0; } -static int32_t tsdbFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) { +static int32_t tsdbTFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) { if (x->fid < y->fid) return -1; if (x->fid > y->fid) return 1; return 0; @@ -43,7 +43,7 @@ STsdbFSetPartition* tsdbFSetPartitionCreate() { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { + for (int32_t i = 0; i < TSDB_FSET_RANGE_TYP_MAX; i++) { TARRAY2_INIT(&pSP->verRanges[i]); } return pSP; @@ -53,30 +53,30 @@ void tsdbFSetPartitionClear(STsdbFSetPartition** ppSP) { if (ppSP == NULL || ppSP[0] == NULL) { return; } - for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) { + for (int32_t i = 0; i < TSDB_FSET_RANGE_TYP_MAX; i++) { TARRAY2_DESTROY(&ppSP[0]->verRanges[i], NULL); } taosMemoryFree(ppSP[0]); ppSP[0] = NULL; } -static int32_t tsdbFTypeToSRangeTyp(tsdb_ftype_t ftype) { +static int32_t tsdbFTypeToFRangeType(tsdb_ftype_t ftype) { switch (ftype) { case TSDB_FTYPE_HEAD: - return TSDB_SNAP_RANGE_TYP_HEAD; + return TSDB_FSET_RANGE_TYP_HEAD; case TSDB_FTYPE_DATA: - return TSDB_SNAP_RANGE_TYP_DATA; + return TSDB_FSET_RANGE_TYP_DATA; case TSDB_FTYPE_SMA: - return TSDB_SNAP_RANGE_TYP_SMA; + return TSDB_FSET_RANGE_TYP_SMA; case TSDB_FTYPE_TOMB: - return TSDB_SNAP_RANGE_TYP_TOMB; + return TSDB_FSET_RANGE_TYP_TOMB; case TSDB_FTYPE_STT: - return TSDB_SNAP_RANGE_TYP_STT; + return TSDB_FSET_RANGE_TYP_STT; } - return TSDB_SNAP_RANGE_TYP_MAX; + return TSDB_FSET_RANGE_TYP_MAX; } -static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbFSetPartition** ppSP) { +static int32_t tsdbTFileSetToFSetPartition(STFileSet* fset, STsdbFSetPartition** ppSP) { STsdbFSetPartition* p = tsdbFSetPartitionCreate(); if (p == NULL) { goto _err; @@ -90,8 +90,8 @@ static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbFSetPartition** ppSP int32_t count = 0; for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { if (fset->farr[ftype] == NULL) continue; - typ = tsdbFTypeToSRangeTyp(ftype); - ASSERT(typ < TSDB_SNAP_RANGE_TYP_MAX); + typ = tsdbFTypeToFRangeType(ftype); + ASSERT(typ < TSDB_FSET_RANGE_TYP_MAX); STFile* f = fset->farr[ftype]->f; if (f->maxVer > fset->maxVerValid) { corrupt = true; @@ -106,7 +106,7 @@ static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbFSetPartition** ppSP ASSERT(code == 0); } - typ = TSDB_SNAP_RANGE_TYP_STT; + typ = TSDB_FSET_RANGE_TYP_STT; const SSttLvl* lvl; TARRAY2_FOREACH(fset->lvlArr, lvl) { STFileObj* fobj; @@ -149,35 +149,61 @@ STsdbFSetPartList* tsdbFSetPartListCreate() { return pList; } -static STsdbFSetPartList* tsdbGetSnapPartList(STFileSystem* fs) { - STsdbFSetPartList* pList = tsdbFSetPartListCreate(); - if (pList == NULL) { - return NULL; - } +void tsdbFSetPartListDestroy(STsdbFSetPartList** ppList) { + if (ppList == NULL || ppList[0] == NULL) return; - int32_t code = 0; - taosThreadMutexLock(&fs->tsdb->mutex); - STFileSet* fset; - TARRAY2_FOREACH(fs->fSetArr, fset) { - STsdbFSetPartition* pItem = NULL; - if (tsdbTFileSetToSnapPart(fset, &pItem) < 0) { - code = -1; - break; - } - ASSERT(pItem != NULL); - code = TARRAY2_SORT_INSERT(pList, pItem, tsdbFSetPartCmprFn); - ASSERT(code == 0); - } - taosThreadMutexUnlock(&fs->tsdb->mutex); - - if (code) { - TARRAY2_DESTROY(pList, tsdbFSetPartitionClear); - taosMemoryFree(pList); - pList = NULL; - } - return pList; + TARRAY2_DESTROY(ppList[0], tsdbFSetPartitionClear); + taosMemoryFree(ppList[0]); + ppList[0] = NULL; } +int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray** ppRanges) { + TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); + if (pDiff == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + TARRAY2_INIT(pDiff); + + STsdbFSetPartition* part; + TARRAY2_FOREACH(pList, part) { + STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); + if (r == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + int64_t maxVerValid = -1; + int32_t typMax = TSDB_FSET_RANGE_TYP_MAX; + for (int32_t i = 0; i < typMax; i++) { + SVerRangeList* iList = &part->verRanges[i]; + SVersionRange vr = {0}; + TARRAY2_FOREACH(iList, vr) { + if (vr.maxVer < vr.minVer) { + continue; + } + maxVerValid = TMAX(maxVerValid, vr.maxVer); + } + } + r->fid = part->fid; + r->sver = maxVerValid + 1; + r->ever = VERSION_MAX; + tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever); + int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbTFileSetRangeCmprFn); + ASSERT(code == 0); + } + ppRanges[0] = pDiff; + + tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff)); + return 0; + +_err: + if (pDiff) { + tsdbTFileSetRangeArrayDestroy(&pDiff); + } + return -1; +} + +// serialization int32_t tTsdbFSetPartListDataLenCalc(STsdbFSetPartList* pList) { int32_t hdrLen = sizeof(int32_t); int32_t datLen = 0; @@ -190,7 +216,7 @@ int32_t tTsdbFSetPartListDataLenCalc(STsdbFSetPartList* pList) { for (int32_t u = 0; u < len; u++) { STsdbFSetPartition* p = TARRAY2_GET(pList, u); - int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; + int32_t typMax = TSDB_FSET_RANGE_TYP_MAX; int32_t uItem = 0; uItem += sizeof(STsdbFSetPartition); uItem += sizeof(typMax); @@ -229,7 +255,7 @@ int32_t tSerializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* if (tEncodeI8(&encoder, reserved8) < 0) goto _err; if (tEncodeI16(&encoder, reserved16) < 0) goto _err; - int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; + int32_t typMax = TSDB_FSET_RANGE_TYP_MAX; if (tEncodeI32(&encoder, typMax) < 0) goto _err; for (int32_t i = 0; i < typMax; i++) { @@ -311,58 +337,34 @@ _err: return -1; } -int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray** ppRanges) { - TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray)); - if (pDiff == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; +// fs state +static STsdbFSetPartList* tsdbSnapGetFSetPartList(STFileSystem* fs) { + STsdbFSetPartList* pList = tsdbFSetPartListCreate(); + if (pList == NULL) { + return NULL; } - TARRAY2_INIT(pDiff); - STsdbFSetPartition* part; - TARRAY2_FOREACH(pList, part) { - STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange)); - if (r == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + int32_t code = 0; + taosThreadMutexLock(&fs->tsdb->mutex); + STFileSet* fset; + TARRAY2_FOREACH(fs->fSetArr, fset) { + STsdbFSetPartition* pItem = NULL; + if (tsdbTFileSetToFSetPartition(fset, &pItem) < 0) { + code = -1; + break; } - int64_t maxVerValid = -1; - int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX; - for (int32_t i = 0; i < typMax; i++) { - SVerRangeList* iList = &part->verRanges[i]; - SVersionRange vr = {0}; - TARRAY2_FOREACH(iList, vr) { - if (vr.maxVer < vr.minVer) { - continue; - } - maxVerValid = TMAX(maxVerValid, vr.maxVer); - } - } - r->fid = part->fid; - r->sver = maxVerValid + 1; - r->ever = VERSION_MAX; - tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever); - int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbFileSetRangeCmprFn); + ASSERT(pItem != NULL); + code = TARRAY2_SORT_INSERT(pList, pItem, tsdbFSetPartCmprFn); ASSERT(code == 0); } - ppRanges[0] = pDiff; + taosThreadMutexUnlock(&fs->tsdb->mutex); - tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff)); - return 0; - -_err: - if (pDiff) { - tsdbTFileSetRangeArrayDestroy(&pDiff); + if (code) { + TARRAY2_DESTROY(pList, tsdbFSetPartitionClear); + taosMemoryFree(pList); + pList = NULL; } - return -1; -} - -void tsdbFSetPartListDestroy(STsdbFSetPartList** ppList) { - if (ppList == NULL || ppList[0] == NULL) return; - - TARRAY2_DESTROY(ppList[0], tsdbFSetPartitionClear); - taosMemoryFree(ppList[0]); - ppList[0] = NULL; + return pList; } ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) { @@ -378,6 +380,7 @@ ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) { return TSDB_FS_STATE_NORMAL; } +// description int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { int code = -1; int32_t tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX); @@ -387,7 +390,7 @@ int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { // get part list for (int32_t j = 0; j < tsdbMaxCnt; ++j) { STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j); - pLists[j] = tsdbGetSnapPartList(pTsdb->pFS); + pLists[j] = tsdbSnapGetFSetPartList(pTsdb->pFS); if (pLists[j] == NULL) goto _out; } From f68804322c2e6ea54d7843102169e5d30153e0c0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 4 Dec 2023 16:56:20 +0800 Subject: [PATCH 63/94] refact: improve code of tsdbSnapGetDetails as tsdbSnapPrepDescription --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 133 ++++++++++++++------- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- 3 files changed, 89 insertions(+), 48 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a5203353db..dc3aa418b4 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -1041,7 +1041,7 @@ typedef enum { // utils ETsdbFsState tsdbSnapGetFsState(SVnode *pVnode); -int32_t tsdbSnapGetDetails(SVnode *pVnode, SSnapshot *pSnap); +int32_t tsdbSnapPrepDescription(SVnode *pVnode, SSnapshot *pSnap); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index 8afbe187a4..573ba48774 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -381,69 +381,110 @@ ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) { } // description -int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) { - int code = -1; - int32_t tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX); - int32_t subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; - STsdbFSetPartList* pLists[TSDB_RETENTION_MAX] = {0}; +typedef struct STsdbPartitionInfo { + int32_t vgId; + int32_t tsdbMaxCnt; + int32_t subTyps[TSDB_RETENTION_MAX]; + STsdbFSetPartList* pLists[TSDB_RETENTION_MAX]; +} STsdbPartitionInfo; - // get part list - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { +static int32_t tsdbPartitionInfoInit(SVnode* pVnode, STsdbPartitionInfo* pInfo) { + int32_t subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2}; + pInfo->vgId = TD_VID(pVnode); + pInfo->tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX); + + ASSERT(sizeof(pInfo->subTyps) == sizeof(subTyps)); + memcpy(pInfo->subTyps, (char*)subTyps, sizeof(subTyps)); + + // fset partition list + memset(pInfo->pLists, 0, sizeof(pInfo->pLists[0]) * TSDB_RETENTION_MAX); + for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j); - pLists[j] = tsdbSnapGetFSetPartList(pTsdb->pFS); - if (pLists[j] == NULL) goto _out; + pInfo->pLists[j] = tsdbSnapGetFSetPartList(pTsdb->pFS); + if (pInfo->pLists[j] == NULL) return -1; } + return 0; +} - // estimate bufLen and prepare - int32_t bufLen = sizeof(SSyncTLV); // typ: TDMT_SYNC_PREP_SNAPSHOT or TDMT_SYNC_PREP_SNAPSOT_REPLY - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - bufLen += sizeof(SSyncTLV); // subTyps[j] - bufLen += tTsdbFSetPartListDataLenCalc(pLists[j]); +static void tsdbPartitionInfoClear(STsdbPartitionInfo* pInfo) { + for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { + if (pInfo->pLists[j] == NULL) continue; + tsdbFSetPartListDestroy(&pInfo->pLists[j]); } +} - tsdbInfo("vgId:%d, allocate %d bytes for data of snapshot info.", TD_VID(pVnode), bufLen); +static int32_t tsdbPartitionInfoEstSize(STsdbPartitionInfo* pInfo) { + int32_t dataLen = 0; + for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { + dataLen += sizeof(SSyncTLV); // subTyps[j] + dataLen += tTsdbFSetPartListDataLenCalc(pInfo->pLists[j]); + } + return dataLen; +} - void* data = taosMemoryRealloc(pSnap->data, bufLen); +static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* buf, int32_t bufLen, int32_t* offset) { + int32_t tlen = 0; + for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { + SSyncTLV* pSubHead = (void*)((char*)buf + offset[0]); + int32_t valOffset = offset[0] + sizeof(*pSubHead); + ASSERT(pSubHead->val == (char*)buf + valOffset); + if ((tlen = tSerializeTsdbFSetPartList(pSubHead->val, bufLen - valOffset, pInfo->pLists[j])) < 0) { + tsdbError("vgId:%d, failed to serialize fset partition list of tsdb %d since %s", pInfo->vgId, j, terrstr()); + return -1; + } + pSubHead->typ = pInfo->subTyps[j]; + pSubHead->len = tlen; + offset[0] += sizeof(*pSubHead) + tlen; + } + return 0; +} + +int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size) { + void* data = taosMemoryRealloc(pSnap->data, size); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pSnap->data = data; + return 0; +} + +int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { + ASSERT(pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY); + STsdbPartitionInfo partitionInfo = {0}; + int code = -1; + STsdbPartitionInfo* pInfo = &partitionInfo; + + if (tsdbPartitionInfoInit(pVnode, pInfo) != 0) { + goto _out; + } + + // info data realloc + int32_t bufLen = sizeof(SSyncTLV); + bufLen += tsdbPartitionInfoEstSize(pInfo); + if (syncSnapInfoDataRealloc(pSnap, bufLen) != 0) { tsdbError("vgId:%d, failed to realloc memory for data of snapshot info. bytes:%d", TD_VID(pVnode), bufLen); goto _out; } - pSnap->data = data; - // header - SSyncTLV* head = data; - head->len = 0; - head->typ = pSnap->type; - int32_t offset = sizeof(SSyncTLV); - int32_t tlen = 0; + // serialization + SSyncTLV* pHead = pSnap->data; + pHead->typ = pSnap->type; - // fill snapshot info - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - // subHead - SSyncTLV* subHead = (void*)((char*)data + offset); - subHead->typ = subTyps[j]; - ASSERT(subHead->val == (char*)data + offset + sizeof(SSyncTLV)); - - if ((tlen = tSerializeTsdbFSetPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) { - tsdbError("vgId:%d, failed to serialize snap partition list of tsdb %d since %s", TD_VID(pVnode), j, terrstr()); - goto _out; - } - subHead->len = tlen; - offset += sizeof(SSyncTLV) + tlen; + int32_t offset = 0; + if (tsdbPartitionInfoSerialize(pInfo, pHead->val, bufLen - sizeof(*pHead), &offset) != 0) { + tsdbError("vgId:%d, failed to serialize tsdb partition info since %s", TD_VID(pVnode), terrstr()); + goto _out; } - // total length of subfields - head->len = offset - sizeof(SSyncTLV); - ASSERT(offset <= bufLen); + // set header of info data + ASSERT(sizeof(*pHead) + offset <= bufLen); + pHead->len = offset; + + tsdbInfo("vgId:%d, tsdb snap info prepared. type:%s, val length:%d", TD_VID(pVnode), TMSG_INFO(pHead->typ), + pHead->len); code = 0; - _out: - for (int32_t j = 0; j < tsdbMaxCnt; ++j) { - if (pLists[j] == NULL) continue; - tsdbFSetPartListDestroy(&pLists[j]); - } - + tsdbPartitionInfoClear(pInfo); return code; } - diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 817d5124a2..5871a60c9e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -804,7 +804,7 @@ int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) { } if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { - code = tsdbSnapGetDetails(pVnode, pSnap); + code = tsdbSnapPrepDescription(pVnode, pSnap); } return code; } From 1a9b08fa0874c39ddadb4285e1f54b32a53c2400 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 4 Dec 2023 17:05:25 +0800 Subject: [PATCH 64/94] refact: relocate func syncSnapInfoDataRealloc --- include/libs/sync/sync.h | 3 +++ source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 10 ---------- source/libs/sync/src/syncUtil.c | 10 ++++++++++ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index ece1e40585..a428a9ae6a 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -289,6 +289,9 @@ const char* syncStr(ESyncState state); int32_t syncNodeGetConfig(int64_t rid, SSyncCfg *cfg); +// util +int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index 573ba48774..1662567247 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -439,16 +439,6 @@ static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* bu return 0; } -int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size) { - void* data = taosMemoryRealloc(pSnap->data, size); - if (data == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pSnap->data = data; - return 0; -} - int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { ASSERT(pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY); STsdbPartitionInfo partitionInfo = {0}; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 06847c081c..2ce56af946 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -487,3 +487,13 @@ void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteRepl sNInfo(pSyncNode, "send sync-request-vote-reply to dnode:%d {term:%" PRId64 ", grant:%d}, %s", DID(&pMsg->destId), pMsg->term, pMsg->voteGranted, s); } + +int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size) { + void* data = taosMemoryRealloc(pSnap->data, size); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pSnap->data = data; + return 0; +} From de9c9d4f205f8f30e29c1d47b4be5f6aa4474772 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 4 Dec 2023 21:05:18 +0800 Subject: [PATCH 65/94] refact: improve code of tsdbPartitionInfoSerialize --- source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 36 +++++++++++++--------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index 1662567247..65ee1a7db3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -16,6 +16,8 @@ #include "tsdb.h" #include "tsdbFS2.h" +#define TSDB_SNAP_MSG_VER 1 + // fset partition static int32_t tsdbFSetPartCmprFn(STsdbFSetPartition* x, STsdbFSetPartition* y) { if (x->fid < y->fid) return -1; @@ -241,7 +243,7 @@ int32_t tSerializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* int16_t reserved16 = 0; int64_t reserved64 = 0; - int8_t msgVer = 1; + int8_t msgVer = TSDB_SNAP_MSG_VER; int32_t len = TARRAY2_SIZE(pList); if (tStartEncode(&encoder) < 0) goto _err; @@ -296,6 +298,7 @@ int32_t tDeserializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartLis int32_t len = 0; if (tStartDecode(&decoder) < 0) goto _err; if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; + if (msgVer != TSDB_SNAP_MSG_VER) goto _err; if (tDecodeI32(&decoder, &len) < 0) goto _err; for (int32_t u = 0; u < len; u++) { @@ -422,11 +425,12 @@ static int32_t tsdbPartitionInfoEstSize(STsdbPartitionInfo* pInfo) { return dataLen; } -static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* buf, int32_t bufLen, int32_t* offset) { +static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* buf, int32_t bufLen) { int32_t tlen = 0; + int32_t offset = 0; for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) { - SSyncTLV* pSubHead = (void*)((char*)buf + offset[0]); - int32_t valOffset = offset[0] + sizeof(*pSubHead); + SSyncTLV* pSubHead = (void*)((char*)buf + offset); + int32_t valOffset = offset + sizeof(*pSubHead); ASSERT(pSubHead->val == (char*)buf + valOffset); if ((tlen = tSerializeTsdbFSetPartList(pSubHead->val, bufLen - valOffset, pInfo->pLists[j])) < 0) { tsdbError("vgId:%d, failed to serialize fset partition list of tsdb %d since %s", pInfo->vgId, j, terrstr()); @@ -434,9 +438,9 @@ static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* bu } pSubHead->typ = pInfo->subTyps[j]; pSubHead->len = tlen; - offset[0] += sizeof(*pSubHead) + tlen; + offset += sizeof(*pSubHead) + tlen; } - return 0; + return offset; } int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { @@ -450,26 +454,30 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { } // info data realloc - int32_t bufLen = sizeof(SSyncTLV); + const int32_t headLen = sizeof(SSyncTLV); + int32_t bufLen = headLen; bufLen += tsdbPartitionInfoEstSize(pInfo); if (syncSnapInfoDataRealloc(pSnap, bufLen) != 0) { - tsdbError("vgId:%d, failed to realloc memory for data of snapshot info. bytes:%d", TD_VID(pVnode), bufLen); + tsdbError("vgId:%d, failed to realloc memory for data of snap info. bytes:%d", TD_VID(pVnode), bufLen); goto _out; } // serialization - SSyncTLV* pHead = pSnap->data; - pHead->typ = pSnap->type; + char* buf = (void*)pSnap->data; + int32_t offset = headLen; + int32_t tlen = 0; - int32_t offset = 0; - if (tsdbPartitionInfoSerialize(pInfo, pHead->val, bufLen - sizeof(*pHead), &offset) != 0) { + if ((tlen = tsdbPartitionInfoSerialize(pInfo, buf + offset, bufLen - offset)) < 0) { tsdbError("vgId:%d, failed to serialize tsdb partition info since %s", TD_VID(pVnode), terrstr()); goto _out; } + offset += tlen; + ASSERT(offset <= bufLen); // set header of info data - ASSERT(sizeof(*pHead) + offset <= bufLen); - pHead->len = offset; + SSyncTLV* pHead = pSnap->data; + pHead->typ = pSnap->type; + pHead->len = offset - headLen; tsdbInfo("vgId:%d, tsdb snap info prepared. type:%s, val length:%d", TD_VID(pVnode), TMSG_INFO(pHead->typ), pHead->len); From d4add073cc676d9ab52de5d24422d7e88a2f9704 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 3 Nov 2023 16:42:41 +0800 Subject: [PATCH 66/94] refact: improve code with syncSnapSendRsp --- source/libs/sync/inc/syncReplication.h | 4 ++ source/libs/sync/src/syncSnapshot.c | 70 ++++++++++---------------- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 04456b2454..ecd2b5163e 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -56,6 +56,10 @@ int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode); int32_t syncNodeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg); +int32_t syncSnapSendMsg(SSyncSnapshotSender* pSender, int32_t seq, void* pBlock, int32_t len, int32_t typ); +int32_t syncSnapSendRsp(SSyncSnapshotReceiver* pReceiver, SyncSnapshotSend* pMsg, void* pBlock, int32_t len, + int32_t typ, int32_t code); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 95952c960e..e1471ad3d9 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -23,8 +23,6 @@ #include "syncReplication.h" #include "syncUtil.h" -int32_t syncSnapSendMsg(SSyncSnapshotSender *pSender, int32_t seq, void *pBlock, int32_t len, int32_t typ); - static void syncSnapBufferReset(SSyncSnapBuffer *pBuf) { taosThreadMutexLock(&pBuf->mutex); for (int64_t i = pBuf->start; i < pBuf->end; ++i) { @@ -153,7 +151,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { pSender->lastSendTime = taosGetTimestampMs(); pSender->finish = false; - // Get full snapshot info + // Get snapshot info SSyncNode *pSyncNode = pSender->pSyncNode; SSnapshot snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT}; if (pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo) != 0) { @@ -161,11 +159,10 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { goto _out; } - int dataLen = 0; void *pData = snapInfo.data; - int32_t type = 0; + int32_t type = (pData) ? snapInfo.type : 0; + int32_t dataLen = 0; if (pData) { - type = snapInfo.type; SSyncTLV *datHead = pData; if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT) { sSError(pSender, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ); @@ -688,24 +685,23 @@ _START_RECEIVER: snapshotReceiverStart(pReceiver, pMsg); // set start-time same with sender -_SEND_REPLY: - // build msg - ; // make complier happy +_SEND_REPLY:; SSnapshot snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT_REPLY}; int32_t dataLen = 0; - if (pMsg->dataLen > 0) { + if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT) { void *data = taosMemoryCalloc(1, pMsg->dataLen); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = terrno; goto _out; } - memcpy(data, pMsg->data, pMsg->dataLen); snapInfo.data = data; data = NULL; - pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo); + memcpy(snapInfo.data, pMsg->data, pMsg->dataLen); + // exchange snap info + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo); SSyncTLV *datHead = snapInfo.data; if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { sRError(pReceiver, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ); @@ -715,29 +711,16 @@ _SEND_REPLY: dataLen = sizeof(SSyncTLV) + datHead->len; } - SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSendRsp(&rpcMsg, dataLen, pSyncNode->vgId) != 0) { - sRError(pReceiver, "snapshot receiver failed to build resp since %s", terrstr()); + // send response + void *pData = snapInfo.data; + int32_t type = (pData) ? snapInfo.type : 0; + + if (syncSnapSendRsp(pReceiver, pMsg, pData, dataLen, type, code) != 0) { code = terrno; goto _out; } - SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; - pRspMsg->srcId = pSyncNode->myRaftId; - pRspMsg->destId = pMsg->srcId; - pRspMsg->term = raftStoreGetTerm(pSyncNode); - pRspMsg->lastIndex = pMsg->lastIndex; - pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->startTime = pMsg->startTime; - pRspMsg->ack = pMsg->seq; // receiver maybe already closed - pRspMsg->code = code; - pRspMsg->snapBeginIndex = syncNodeGetSnapBeginIndex(pSyncNode); - - if (snapInfo.data) { - pRspMsg->payloadType = snapInfo.type; - memcpy(pRspMsg->data, snapInfo.data, dataLen); - - // save snapshot info + if (pData) { SSnapshotParam *pParam = &pReceiver->snapshotParam; void *data = taosMemoryRealloc(pParam->data, dataLen); if (data == NULL) { @@ -748,15 +731,10 @@ _SEND_REPLY: goto _out; } pParam->data = data; + data = NULL; memcpy(pParam->data, snapInfo.data, dataLen); } - // send msg - if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { - sRError(pReceiver, "failed to send resp since %s", terrstr()); - code = terrno; - } - _out: if (snapInfo.data) { taosMemoryFree(snapInfo.data); @@ -820,11 +798,12 @@ _SEND_REPLY: return code; } -static int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, int32_t code) { +int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, void *pBlock, int32_t blockLen, + int32_t type, int32_t code) { SSyncNode *pSyncNode = pReceiver->pSyncNode; // build msg SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId)) { + if (syncBuildSnapshotSendRsp(&rpcMsg, blockLen, pSyncNode->vgId)) { sRError(pReceiver, "failed to build snapshot receiver resp since %s", terrstr()); return -1; } @@ -832,13 +811,18 @@ static int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSen SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; pRspMsg->srcId = pSyncNode->myRaftId; pRspMsg->destId = pMsg->srcId; - pRspMsg->term = raftStoreGetTerm(pSyncNode); + pRspMsg->term = pMsg->term; pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->startTime = pMsg->startTime; pRspMsg->ack = pMsg->seq; pRspMsg->code = code; pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; + pRspMsg->payloadType = type; + + if (pBlock != NULL && blockLen > 0) { + memcpy(pRspMsg->data, pBlock, blockLen); + } // send msg if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { @@ -872,7 +856,7 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot ppMsg[0] = NULL; pRcvBuf->end = TMAX(pMsg->seq + 1, pRcvBuf->end); } else if (pMsg->seq < pRcvBuf->start) { - syncSnapSendRsp(pReceiver, pMsg, code); + syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, code); goto _out; } @@ -892,7 +876,7 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot } } pRcvBuf->start = seq + 1; - syncSnapSendRsp(pReceiver, pRcvBuf->entries[seq % pRcvBuf->size], code); + syncSnapSendRsp(pReceiver, pRcvBuf->entries[seq % pRcvBuf->size], NULL, 0, 0, code); pRcvBuf->entryDeleteCb(pRcvBuf->entries[seq % pRcvBuf->size]); pRcvBuf->entries[seq % pRcvBuf->size] = NULL; if (code) goto _out; @@ -915,7 +899,7 @@ static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend if (snapshotReceiverSignatureCmp(pReceiver, pMsg) != 0) { terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; sRError(pReceiver, "failed to receive snapshot data since %s.", terrstr()); - return syncSnapSendRsp(pReceiver, pMsg, terrno); + return syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, terrno); } return syncSnapBufferRecv(pReceiver, ppMsg); From 41fe39de3edbbda0eb3c276e176dd447c817e579 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 3 Nov 2023 17:19:17 +0800 Subject: [PATCH 67/94] refact: improve code with syncNodeExchangeSnapInfo --- source/libs/sync/src/syncSnapshot.c | 83 +++++++++++++++++------------ 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index e1471ad3d9..a315b91791 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -641,6 +641,50 @@ SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) { return snapStart; } +static int32_t syncNodeExchangeSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, + SSnapshot *pInfo) { + ASSERT(pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT); + int32_t code = 0; + + // copy snap info from leader + void *data = taosMemoryCalloc(1, pMsg->dataLen); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; + goto _out; + } + pInfo->data = data; + data = NULL; + memcpy(pInfo->data, pMsg->data, pMsg->dataLen); + + // exchange snap info + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, pInfo); + SSyncTLV *datHead = pInfo->data; + if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + sRError(pReceiver, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ); + code = TSDB_CODE_INVALID_DATA_FMT; + goto _out; + } + int32_t dataLen = sizeof(SSyncTLV) + datHead->len; + + // save exchanged snap info + SSnapshotParam *pParam = &pReceiver->snapshotParam; + data = taosMemoryRealloc(pParam->data, dataLen); + if (data == NULL) { + sError("vgId:%d, failed to realloc memory for snapshot prep due to %s. dataLen:%d", pSyncNode->vgId, + strerror(errno), dataLen); + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; + goto _out; + } + pParam->data = data; + data = NULL; + memcpy(pParam->data, pInfo->data, dataLen); + +_out: + return code; +} + static int32_t syncNodeOnSnapshotPrep(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; int64_t timeNow = taosGetTimestampMs(); @@ -683,58 +727,27 @@ _START_RECEIVER: snapshotReceiverStop(pReceiver); } - snapshotReceiverStart(pReceiver, pMsg); // set start-time same with sender + snapshotReceiverStart(pReceiver, pMsg); _SEND_REPLY:; SSnapshot snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT_REPLY}; int32_t dataLen = 0; if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT) { - void *data = taosMemoryCalloc(1, pMsg->dataLen); - if (data == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = terrno; + if (syncNodeExchangeSnapInfo(pSyncNode, pReceiver, pMsg, &snapInfo) != 0) { goto _out; } - snapInfo.data = data; - data = NULL; - memcpy(snapInfo.data, pMsg->data, pMsg->dataLen); - - // exchange snap info - pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo); SSyncTLV *datHead = snapInfo.data; - if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { - sRError(pReceiver, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ); - code = TSDB_CODE_INVALID_DATA_FMT; - goto _out; - } dataLen = sizeof(SSyncTLV) + datHead->len; } // send response - void *pData = snapInfo.data; - int32_t type = (pData) ? snapInfo.type : 0; - - if (syncSnapSendRsp(pReceiver, pMsg, pData, dataLen, type, code) != 0) { + int32_t type = (snapInfo.data) ? snapInfo.type : 0; + if (syncSnapSendRsp(pReceiver, pMsg, snapInfo.data, dataLen, type, code) != 0) { code = terrno; goto _out; } - if (pData) { - SSnapshotParam *pParam = &pReceiver->snapshotParam; - void *data = taosMemoryRealloc(pParam->data, dataLen); - if (data == NULL) { - sError("vgId:%d, failed to realloc memory for snapshot prep due to %s. dataLen:%d", pSyncNode->vgId, - strerror(errno), dataLen); - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = terrno; - goto _out; - } - pParam->data = data; - data = NULL; - memcpy(pParam->data, snapInfo.data, dataLen); - } - _out: if (snapInfo.data) { taosMemoryFree(snapInfo.data); From 030f3db4d604ae89a151ba5e270a4d38c806bd67 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 3 Nov 2023 17:38:33 +0800 Subject: [PATCH 68/94] refact: improve code of syncNodeOnSnapshotBegin with syncSnapSendRsp --- source/libs/sync/src/syncSnapshot.c | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index a315b91791..25b620854e 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -784,27 +784,8 @@ _SEND_REPLY: code = terrno; } - // build msg - SRpcMsg rpcMsg = {0}; - if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId) != 0) { - sRError(pReceiver, "failed to build snapshot receiver resp since %s", terrstr()); - return -1; - } - - SyncSnapshotRsp *pRspMsg = rpcMsg.pCont; - pRspMsg->srcId = pSyncNode->myRaftId; - pRspMsg->destId = pMsg->srcId; - pRspMsg->term = raftStoreGetTerm(pSyncNode); - pRspMsg->lastIndex = pMsg->lastIndex; - pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->startTime = pMsg->startTime; - pRspMsg->ack = pReceiver->ack; // receiver maybe already closed - pRspMsg->code = code; - pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start; - - // send msg - if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) { - sRError(pReceiver, "failed to send snapshot receiver resp since %s", terrstr()); + // send response + if (syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, code) != 0) { return -1; } From 081c83710ebde03067d5dd496b90a21fe299086d Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 9 Nov 2023 18:21:15 +0800 Subject: [PATCH 69/94] enh: save a copy of snapshot info in syncNodeOnSnapshotPrepRsp --- source/libs/sync/src/syncSnapshot.c | 79 +++++++++++++---------------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 25b620854e..1e3614857e 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -121,6 +121,11 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { if (pSender->pSndBuf) { syncSnapBufferDestroy(&pSender->pSndBuf); } + + if (pSender->snapshotParam.data) { + taosMemoryFree(pSender->snapshotParam.data); + pSender->snapshotParam.data = NULL; + } // free sender taosMemoryFree(pSender); } @@ -344,9 +349,6 @@ _out:; return code; } -// return 0, start ok -// return 1, last snapshot finish ok -// return -1, error int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, pDestId); if (pSender == NULL) { @@ -377,6 +379,7 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { return 0; } +// receiver SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId) { bool condition = (pSyncNode->pFsm->FpSnapshotStartWrite != NULL) && (pSyncNode->pFsm->FpSnapshotStopWrite != NULL) && (pSyncNode->pFsm->FpSnapshotDoWrite != NULL); @@ -506,8 +509,6 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *p sRInfo(pReceiver, "snapshot receiver start, from dnode:%d.", DID(&pReceiver->fromId)); } -// just set start = false -// FpSnapshotStopWrite should not be called void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { sRDebug(pReceiver, "snapshot receiver stop, not apply, writer:%p", pReceiver->pWriter); @@ -528,7 +529,6 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { syncSnapBufferReset(pReceiver->pRcvBuf); } -// when recv last snapshot block, apply data into snapshot static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) { int32_t code = 0; if (pReceiver->pWriter != NULL) { @@ -587,8 +587,6 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap return 0; } -// apply data block -// update progress static int32_t snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) { if (pMsg->seq != pReceiver->ack + 1) { sRError(pReceiver, "snapshot receiver invalid seq, ack:%d seq:%d", pReceiver->ack, pMsg->seq); @@ -641,8 +639,8 @@ SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) { return snapStart; } -static int32_t syncNodeExchangeSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, - SSnapshot *pInfo) { +static int32_t syncSnapReceiverExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotReceiver *pReceiver, + SyncSnapshotSend *pMsg, SSnapshot *pInfo) { ASSERT(pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT); int32_t code = 0; @@ -734,7 +732,7 @@ _SEND_REPLY:; SSnapshot snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT_REPLY}; int32_t dataLen = 0; if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT) { - if (syncNodeExchangeSnapInfo(pSyncNode, pReceiver, pMsg, &snapInfo) != 0) { + if (syncSnapReceiverExchgSnapInfo(pSyncNode, pReceiver, pMsg, &snapInfo) != 0) { goto _out; } SSyncTLV *datHead = snapInfo.data; @@ -949,26 +947,6 @@ _SEND_REPLY:; return code; } -// receiver on message -// -// condition 1, recv SYNC_SNAPSHOT_SEQ_PREP -// if receiver already start -// if sender.start-time > receiver.start-time, restart receiver(reply snapshot start) -// if sender.start-time = receiver.start-time, maybe duplicate msg -// if sender.start-time < receiver.start-time, ignore -// else -// waiting for clock match -// start receiver(reply snapshot start) -// -// condition 2, recv SYNC_SNAPSHOT_SEQ_BEGIN -// a. create writer with -// -// condition 3, recv SYNC_SNAPSHOT_SEQ_END, finish receiver(apply snapshot data, update commit index, maybe reconfig) -// -// condition 4, recv SYNC_SNAPSHOT_SEQ_FORCE_CLOSE, force close -// -// condition 5, got data, update ack -// int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { SyncSnapshotSend **ppMsg = (SyncSnapshotSend **)&pRpcMsg->pCont; SyncSnapshotSend *pMsg = ppMsg[0]; @@ -1052,6 +1030,32 @@ _out:; return code; } +static int32_t syncSnapSenderExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { + ASSERT(pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY); + + SSyncTLV *datHead = (void *)pMsg->data; + if (datHead->typ != pMsg->payloadType) { + sSError(pSender, "unexpected data type in data of SyncSnapshotRsp. typ: %d", datHead->typ); + terrno = TSDB_CODE_INVALID_DATA_FMT; + return -1; + } + int32_t dataLen = sizeof(SSyncTLV) + datHead->len; + + SSnapshotParam *pParam = &pSender->snapshotParam; + void *data = taosMemoryRealloc(pParam->data, dataLen); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + memcpy(data, pMsg->data, dataLen); + + pParam->data = data; + data = NULL; + sSInfo(pSender, "data of snapshot param. len: %d", datHead->len); + return 0; +} + +// sender static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { SSnapshot snapshot = {0}; pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); @@ -1068,14 +1072,9 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend // start reader if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { - SSyncTLV *datHead = (void *)pMsg->data; - if (datHead->typ != pMsg->payloadType) { - sSError(pSender, "unexpected data type in data of SyncSnapshotRsp. typ: %d", datHead->typ); - terrno = TSDB_CODE_INVALID_DATA_FMT; + if (syncSnapSenderExchgSnapInfo(pSyncNode, pSender, pMsg) != 0) { return -1; } - pSender->snapshotParam.data = (void *)pMsg->data; - sSInfo(pSender, "data of snapshot param. len: %d", datHead->len); } int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader); @@ -1160,12 +1159,6 @@ _out: return code; } -// sender on message -// -// condition 1 sender receives SYNC_SNAPSHOT_SEQ_END, close sender -// condition 2 sender receives ack, set seq = ack + 1, send msg from seq -// condition 3 sender receives error msg, just print error log -// int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { SyncSnapshotRsp **ppMsg = (SyncSnapshotRsp **)&pRpcMsg->pCont; SyncSnapshotRsp *pMsg = ppMsg[0]; From fe258e226d67f6c65cda967ead94cd530fd3aadf Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 9 Nov 2023 18:27:07 +0800 Subject: [PATCH 70/94] feat: add data type of SNAP_DATA_RAW for snap replication --- source/dnode/vnode/src/inc/vnodeInt.h | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 7ed0b5103f..2c051ea642 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -531,6 +531,7 @@ enum { SNAP_DATA_STREAM_STATE = 11, SNAP_DATA_STREAM_STATE_BACKEND = 12, SNAP_DATA_TQ_CHECKINFO = 13, + SNAP_DATA_RAW = 14, }; struct SSnapDataHdr { From e05914119ed8515df37af53eca1d6a2a1747310f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 10 Nov 2023 15:27:06 +0800 Subject: [PATCH 71/94] feat: add file tsdbSnapshotRAW.c --- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c new file mode 100644 index 0000000000..e69de29bb2 From 61fbf089cc6f8a2929ed31c1decddb172e740679 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 10 Nov 2023 15:28:30 +0800 Subject: [PATCH 72/94] enh: add cscope.files into .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 704b2e7415..08e3d57717 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ CMakeSettings.json cmake-build-debug/ cmake-build-release/ cscope.out +cscope.files .DS_Store debug/ release/ From da646b5a5b158839d9f445eb6cc12d457d8c252c Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 13 Nov 2023 16:00:37 +0800 Subject: [PATCH 73/94] feat: add file tsdbDataFileRAW.c --- source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c | 0 source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c create mode 100644 source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h new file mode 100644 index 0000000000..e69de29bb2 From a4c504169d1a64589ae6b29a8df79592911038d4 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 15 Nov 2023 15:49:11 +0800 Subject: [PATCH 74/94] feat: add file tsdbFSetRAW.c --- source/dnode/vnode/src/tsdb/tsdbFSetRAW.c | 0 source/dnode/vnode/src/tsdb/tsdbFSetRAW.h | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbFSetRAW.c create mode 100644 source/dnode/vnode/src/tsdb/tsdbFSetRAW.h diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h new file mode 100644 index 0000000000..e69de29bb2 From 6c419423de9c2b7f3319da17e8d70994e851fbd2 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 20 Nov 2023 20:19:46 +0800 Subject: [PATCH 75/94] feat: impl tsdb snapshot reader and writer for raw files --- source/dnode/vnode/src/inc/vnodeInt.h | 11 + source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c | 212 +++++++ source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h | 127 ++++ source/dnode/vnode/src/tsdb/tsdbFSetRAW.c | 173 ++++++ source/dnode/vnode/src/tsdb/tsdbFSetRAW.h | 45 ++ source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 586 ++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeSnapshot.c | 17 + 7 files changed, 1171 insertions(+) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 2c051ea642..50a28357e5 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -65,6 +65,8 @@ typedef struct SMetaSnapReader SMetaSnapReader; typedef struct SMetaSnapWriter SMetaSnapWriter; typedef struct STsdbSnapReader STsdbSnapReader; typedef struct STsdbSnapWriter STsdbSnapWriter; +typedef struct STsdbSnapRAWReader STsdbSnapRAWReader; +typedef struct STsdbSnapRAWWriter STsdbSnapRAWWriter; typedef struct STqSnapReader STqSnapReader; typedef struct STqSnapWriter STqSnapWriter; typedef struct STqOffsetReader STqOffsetReader; @@ -313,6 +315,15 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRang int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, SSnapDataHdr* pHdr); int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* pWriter); int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback); +// STsdbSnapRAWReader ======================================== +int32_t tsdbSnapRAWReaderOpen(STsdb* pTsdb, int64_t ever, int8_t type, STsdbSnapRAWReader** ppReader); +int32_t tsdbSnapRAWReaderClose(STsdbSnapRAWReader** ppReader); +int32_t tsdbSnapRAWRead(STsdbSnapRAWReader* pReader, uint8_t** ppData); +// STsdbSnapRAWWriter ======================================== +int32_t tsdbSnapRAWWriterOpen(STsdb* pTsdb, int64_t ever, STsdbSnapRAWWriter** ppWriter); +int32_t tsdbSnapRAWWrite(STsdbSnapRAWWriter* pWriter, SSnapDataHdr* pHdr); +int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* pWriter); +int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** ppWriter, int8_t rollback); // STqSnapshotReader == int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader); int32_t tqSnapReaderClose(STqSnapReader** ppReader); diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c index e69de29bb2..0c3d890966 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDataFileRAW.h" + +// SDataFileRAWReader ============================================= +int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderConfig *config, + SDataFileRAWReader **reader) { + int32_t code = 0; + int32_t lino = 0; + + reader[0] = taosMemoryCalloc(1, sizeof(SDataFileRAWReader)); + if (reader[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + reader[0]->config[0] = config[0]; + + if (fname) { + if (fname) { + code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + char fname1[TSDB_FILENAME_LEN]; + tsdbTFileName(config->tsdb, &config->file, fname1); + code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader) { + if (reader[0] == NULL) return 0; + + if (reader[0]->fd) { + tsdbCloseFile(&reader[0]->fd); + } + + taosMemoryFree(reader[0]); + reader[0] = NULL; + return 0; +} + +int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *bHdr) { + int32_t code = 0; + int32_t lino = 0; + + bHdr->file.type = reader->config->file.type; + bHdr->file.fid = reader->config->file.fid; + bHdr->file.cid = reader->config->file.cid; + bHdr->file.size = reader->config->file.size; + bHdr->file.minVer = reader->config->file.minVer; + bHdr->file.maxVer = reader->config->file.maxVer; + bHdr->file.stt->level = reader->config->file.stt->level; + + int64_t size = TMIN(bHdr->dataLength, reader->config->file.size - reader->ctx->offset); + ASSERT(size > 0); + bHdr->dataLength = 0; + bHdr->offset = reader->ctx->offset; + + code = tsdbReadFile(reader->fd, bHdr->offset, bHdr->data, size); + TSDB_CHECK_CODE(code, lino, _exit); + + bHdr->dataLength = size; +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +// SDataFileRAWWriter ============================================= +int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **writer) { + writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); + if (!writer[0]) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->config[0] = config[0]; + return 0; +} + +static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) { + ASSERT(0); + return 0; +} + +static int32_t tsdbDataFileRAWWriterDoClose(SDataFileRAWWriter *writer) { return 0; } + +int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + writer->file = writer->config->file; + writer->ctx->offset = 0; + + writer->ctx->opened = true; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFileOpArray *opArr) { + int32_t code = 0; + int32_t lino = 0; + STFileOp op; + + op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = writer->config->fid, + .nf = writer->file, + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + + if (writer->fd) { + code = tsdbFsyncFile(writer->fd); + TSDB_CHECK_CODE(code, lino, _exit); + tsdbCloseFile(&writer->fd); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileRAWWriterOpenDataFD(SDataFileRAWWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + char fname[TSDB_FILENAME_LEN]; + int32_t flag = TD_FILE_READ | TD_FILE_WRITE; + + if (writer->file.size == 0) { + flag |= (TD_FILE_CREATE | TD_FILE_TRUNC); + } + + tsdbTFileName(writer->config->tsdb, &writer->file, fname); + code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr) { + if (writer[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + if (writer[0]->ctx->opened) { + if (abort) { + code = tsdbDataFileRAWWriterCloseAbort(writer[0]); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbDataFileRAWWriterCloseCommit(writer[0], opArr); + TSDB_CHECK_CODE(code, lino, _exit); + } + tsdbDataFileRAWWriterDoClose(writer[0]); + } + taosMemoryFree(writer[0]); + writer[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->file.size += pDataBlock->dataLength; + writer->ctx->offset += pDataBlock->dataLength; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h index e69de29bb2..49f80b0be5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tarray2.h" +#include "tsdbDef.h" +#include "tsdbFSet2.h" +#include "tsdbFile2.h" +#include "tsdbUtil2.h" + +#ifndef _TSDB_DATA_FILE_RAW_H +#define _TSDB_DATA_FILE_RAW_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define TSDB_SNAP_RAW_PAYLOAD_SIZE (4096 * 1024) +#if 0 +struct SDataRAWBlock { + int8_t *data; + int64_t size; +}; + +int32_t tsdbDataRAWBlockReset(SDataRAWBlock *pBlock); +int32_t tsdbDataRAWBlockAlloc(SDataRawBlock *pBlock); +void tsdbDataRAWBlockFree(SDataRAWBlock *pBlock); +#endif + +// STsdbDataRAWBlockHeader ======================================= +typedef struct STsdbDataRAWBlockHeader { + struct { + int32_t type; + int64_t fid; + int64_t cid; + int64_t size; + int64_t minVer; + int64_t maxVer; + union { + struct { + int32_t level; + } stt[1]; + }; + } file; + + int64_t offset; + int64_t dataLength; + uint8_t data[0]; +} STsdbDataRAWBlockHeader; + +// SDataFileRAWReader ============================================= +typedef struct SDataFileRAWReaderConfig { + STsdb *tsdb; + int32_t szPage; + + STFile file; +} SDataFileRAWReaderConfig; + +typedef struct SDataFileRAWReader { + SDataFileRAWReaderConfig config[1]; + + struct { + bool opened; + int64_t offset; + } ctx[1]; + + STsdbFD *fd; +} SDataFileRAWReader; + +typedef TARRAY2(SDataFileRAWReader *) SDataFileRAWReaderArray; + +int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderConfig *config, + SDataFileRAWReader **reader); +int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader); + +int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *bHdr); + +// SDataFileRAWWriter ============================================= +typedef struct SDataFileRAWWriterConfig { + STsdb *tsdb; + int32_t szPage; + + SDiskID did; + int64_t fid; + int64_t cid; + int32_t level; + + STFile file; +} SDataFileRAWWriterConfig; + +typedef struct SDataFileRAWWriter { + SDataFileRAWWriterConfig config[1]; + + struct { + bool opened; + int64_t offset; + } ctx[1]; + + STFile file; + STsdbFD *fd; +} SDataFileRAWWriter; + +typedef struct SDataFileRAWWriter SDataFileRAWWriter; + +int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **writer); +int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr); + +int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer); +int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *bHdr); +int32_t tsdbDataFileRAWFlush(SDataFileRAWWriter *writer); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_DATA_FILE_RAW_H*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c index e69de29bb2..d9cd419ef9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFSetRAW.h" + +// SFSetRAWWriter ================================================== +typedef struct SFSetRAWWriter { + SFSetRAWWriterConfig config[1]; + + struct { + TFileOpArray fopArr[1]; + STFile file; + int64_t offset; + } ctx[1]; + + // writer + SDataFileRAWWriter *dataWriter; +} SFSetRAWWriter; + +int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer) { + int32_t code = 0; + int32_t lino = 0; + + writer[0] = taosMemoryCalloc(1, sizeof(SFSetRAWWriter)); + if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->config[0] = config[0]; + + TARRAY2_INIT(writer[0]->ctx->fopArr); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbFSetRAWWriterFinish(SFSetRAWWriter *writer, TFileOpArray *fopArr) { + int32_t code = 0; + int32_t lino = 0; + + STsdb *tsdb = writer->config->tsdb; + + STFileOp op; + TARRAY2_FOREACH(writer->ctx->fopArr, op) { + code = TARRAY2_APPEND(fopArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + + TARRAY2_CLEAR(writer->ctx->fopArr, NULL); +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr) { + int32_t code = 0; + int32_t lino = 0; + + SDataFileRAWWriterConfig config = { + .tsdb = writer->config->tsdb, + .szPage = writer->config->szPage, + .did = writer->config->did, + .cid = writer->config->cid, + .level = writer->config->level, + + .file = + { + .type = bHdr->file.type, + .did = writer->config->did, + .cid = writer->config->cid, + .size = bHdr->file.size, + .minVer = bHdr->file.minVer, + .maxVer = bHdr->file.maxVer, + .stt = {{ + .level = bHdr->file.stt->level, + }}, + }, + }; + + code = tsdbDataFileRAWWriterOpen(&config, &writer->dataWriter); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbFSetRAWWriteFileDataEnd(SFSetRAWWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbDataFileRAWWriterClose(&writer->dataWriter, false, writer->ctx->fopArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr) { + if (writer[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + STsdb *tsdb = writer[0]->config->tsdb; + + // end + code = tsdbFSetRAWWriteFileDataEnd(writer[0]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileRAWWriterClose(&writer[0]->dataWriter, abort, writer[0]->ctx->fopArr); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFSetRAWWriterFinish(writer[0], fopArr); + TSDB_CHECK_CODE(code, lino, _exit); + // free + TARRAY2_DESTROY(writer[0]->ctx->fopArr, NULL); + taosMemoryFree(writer[0]); + writer[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(writer->ctx->offset >= 0 && writer->ctx->offset <= writer->ctx->file.size); + + if (writer->ctx->offset == writer->ctx->file.size) { + code = tsdbFSetRAWWriteFileDataEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFSetRAWWriteFileDataBegin(writer, bHdr); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDataFileRAWWriteBlockData(writer->dataWriter, bHdr); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->offset += bHdr->dataLength; + ASSERT(writer->ctx->offset == writer->dataWriter->ctx->offset); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h index e69de29bb2..205c785e99 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDataFileRAW.h" + +#ifndef _TSDB_FSET_RAW_H +#define _TSDB_FSET_RAW_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SFSetRAWWriterConfig { + STsdb *tsdb; + int32_t szPage; + + SDiskID did; + int64_t fid; + int64_t cid; + int32_t level; +} SFSetRAWWriterConfig; + +typedef struct SFSetRAWWriter SFSetRAWWriter; + +int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer); +int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr); +int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_FSET_RAW_H*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index e69de29bb2..2bcd00bfec 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -0,0 +1,586 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" +#include "tsdbDataFileRAW.h" +#include "tsdbFS2.h" +#include "tsdbFSetRAW.h" + +// reader + +typedef struct SDataFileRAWReaderIter { + int32_t count; + int32_t idx; + int64_t offset; + int64_t size; +} SDataFileRAWReaderIter; + +typedef struct STsdbSnapRAWReader { + STsdb* tsdb; + int64_t ever; + int8_t type; + + TFileSetArray* fsetArr; + + // context + struct { + int32_t fsetArrIdx; + STFileSet* fset; + bool isDataDone; + } ctx[1]; + + // reader + SDataFileRAWReaderArray dataReaderArr[1]; + + // iter + SDataFileRAWReaderIter dataIter[1]; +} STsdbSnapRAWReader; + +int32_t tsdbSnapRAWReaderOpen(STsdb* tsdb, int64_t ever, int8_t type, STsdbSnapRAWReader** reader) { + int32_t code = 0; + int32_t lino = 0; + + reader[0] = taosMemoryCalloc(1, sizeof(STsdbSnapRAWReader)); + if (reader[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + reader[0]->tsdb = tsdb; + reader[0]->ever = ever; + reader[0]->type = type; + + code = tsdbFSCreateRefSnapshot(tsdb->pFS, &reader[0]->fsetArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), + __func__, lino, tstrerror(code), 0, ever, type); + tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); + taosMemoryFree(reader[0]); + reader[0] = NULL; + } else { + tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), 0, + ever, type); + } + return code; +} + +int32_t tsdbSnapRAWReaderClose(STsdbSnapRAWReader** reader) { + if (reader[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + STsdb* tsdb = reader[0]->tsdb; + + tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); + taosMemoryFree(reader[0]); + reader[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); + } + return code; +} + +static int32_t tsdbSnapRAWReadFileSetOpenReader(STsdbSnapRAWReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + // data + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ftype++) { + if (reader->ctx->fset->farr[ftype] == NULL) { + continue; + } + STFileObj* fobj = reader->ctx->fset->farr[ftype]; + SDataFileRAWReader* dataReader; + SDataFileRAWReaderConfig config = { + .tsdb = reader->tsdb, + .szPage = reader->tsdb->pVnode->config.tsdbPageSize, + .file = fobj->f[0], + }; + code = tsdbDataFileRAWReaderOpen(NULL, &config, &dataReader); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(reader->dataReaderArr, dataReader); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // stt + SSttLvl* lvl; + TARRAY2_FOREACH(reader->ctx->fset->lvlArr, lvl) { + STFileObj* fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + SDataFileRAWReader* dataReader; + SDataFileRAWReaderConfig config = { + .tsdb = reader->tsdb, + .szPage = reader->tsdb->pVnode->config.tsdbPageSize, + .file = fobj->f[0], + }; + code = tsdbDataFileRAWReaderOpen(NULL, &config, &dataReader); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(reader->dataReaderArr, dataReader); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadFileSetCloseReader(STsdbSnapRAWReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + TARRAY2_CLEAR(reader->dataReaderArr, tsdbDataFileRAWReaderClose); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadFileSetOpenIter(STsdbSnapRAWReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + reader->dataIter->count = TARRAY2_SIZE(reader->dataReaderArr); + reader->dataIter->idx = -1; + reader->dataIter->offset = 0; + reader->dataIter->size = 0; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadFileSetCloseIter(STsdbSnapRAWReader* reader) { + reader->dataIter->count = 0; + reader->dataIter->idx = 0; + reader->dataIter->offset = 0; + reader->dataIter->size = 0; + return 0; +} + +static int32_t tsdbSnapRAWReadNext(STsdbSnapRAWReader* reader, STsdbDataRAWBlockHeader* bHdr) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(reader->dataIter->offset <= reader->dataIter->size); + ASSERT(reader->dataIter->idx <= reader->dataIter->count); + + if (reader->dataIter->offset == reader->dataIter->size && reader->dataIter->idx < reader->dataIter->count) { + reader->dataIter->idx++; + } + if (reader->dataIter->idx == reader->dataIter->count) { + return 0; + } + + SDataFileRAWReader* dataReader = TARRAY2_GET(reader->dataReaderArr, reader->dataIter->idx); + code = tsdbDataFileRAWReadBlockData(dataReader, bHdr); + TSDB_CHECK_CODE(code, lino, _exit); + + reader->dataIter->offset += bHdr->dataLength; +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadData(STsdbSnapRAWReader* reader, SSnapDataHdr** data) { + int32_t code = 0; + int32_t lino = 0; + + void* pBuf = taosMemoryCalloc(1, sizeof(SSnapDataHdr) + sizeof(STsdbDataRAWBlockHeader) + TSDB_SNAP_RAW_PAYLOAD_SIZE); + if (pBuf == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + SSnapDataHdr* pHdr = pBuf; + pHdr->type = reader->type; + STsdbDataRAWBlockHeader* pData = (void*)pHdr->data; + pData->dataLength = TSDB_SNAP_RAW_PAYLOAD_SIZE; + + code = tsdbSnapRAWReadNext(reader, pData); + TSDB_CHECK_CODE(code, lino, _exit); + + ASSERT(pData->dataLength > 0 && pData->dataLength <= TSDB_SNAP_RAW_PAYLOAD_SIZE); + pHdr->size = sizeof(STsdbDataRAWBlockHeader) + pData->dataLength; + +_exit: + if (code) { + taosMemoryFree(pBuf); + pBuf = NULL; + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + data[0] = pBuf; + return code; +} + +static int32_t tsdbSnapRAWReadBegin(STsdbSnapRAWReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(reader->ctx->fset == NULL); + + if (reader->ctx->fsetArrIdx < TARRAY2_SIZE(reader->fsetArr)) { + reader->ctx->fset = TARRAY2_GET(reader->fsetArr, reader->ctx->fsetArrIdx++); + reader->ctx->isDataDone = false; + + code = tsdbSnapRAWReadFileSetOpenReader(reader); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSnapRAWReadFileSetOpenIter(reader); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapRAWReadEnd(STsdbSnapRAWReader* reader) { + tsdbSnapRAWReadFileSetCloseIter(reader); + tsdbSnapRAWReadFileSetCloseReader(reader); + reader->ctx->fset = NULL; + return 0; +} + +int32_t tsdbSnapRAWRead(STsdbSnapRAWReader* reader, uint8_t** data) { + int32_t code = 0; + int32_t lino = 0; + + data[0] = NULL; + + for (;;) { + if (reader->ctx->fset == NULL) { + code = tsdbSnapRAWReadBegin(reader); + TSDB_CHECK_CODE(code, lino, _exit); + + if (reader->ctx->fset == NULL) { + break; + } + } + + if (!reader->ctx->isDataDone) { + code = tsdbSnapRAWReadData(reader, (SSnapDataHdr**)data); + TSDB_CHECK_CODE(code, lino, _exit); + if (data[0]) { + goto _exit; + } else { + reader->ctx->isDataDone = true; + } + } + + code = tsdbSnapRAWReadEnd(reader); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } else { + tsdbDebug("vgId:%d %s done", TD_VID(reader->tsdb->pVnode), __func__); + } + return code; +} + +// writer +struct STsdbSnapRAWWriter { + STsdb* tsdb; + int64_t sver; + int64_t ever; + int32_t minutes; + int8_t precision; + int32_t minRow; + int32_t maxRow; + int8_t cmprAlg; + int64_t commitID; + int32_t szPage; + int64_t compactVersion; + int64_t now; + + TFileSetArray* fsetArr; + TFileOpArray fopArr[1]; + + struct { + bool fsetWriteBegin; + int32_t fid; + STFileSet* fset; + SDiskID did; + int64_t cid; + int64_t level; + + // writer + SFSetRAWWriter* fsetWriter; + } ctx[1]; +}; + +int32_t tsdbSnapRAWWriterOpen(STsdb* pTsdb, int64_t ever, STsdbSnapRAWWriter** writer) { + int32_t code = 0; + int32_t lino = 0; + + // disable background tasks + tsdbFSDisableBgTask(pTsdb->pFS); + + // start to write + writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); + if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->tsdb = pTsdb; + writer[0]->ever = ever; + writer[0]->minutes = pTsdb->keepCfg.days; + writer[0]->precision = pTsdb->keepCfg.precision; + writer[0]->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; + writer[0]->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; + writer[0]->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; + writer[0]->commitID = tsdbFSAllocEid(pTsdb->pFS); + writer[0]->szPage = pTsdb->pVnode->config.tsdbPageSize; + writer[0]->compactVersion = INT64_MAX; + writer[0]->now = taosGetTimestampMs(); + + code = tsdbFSCreateCopySnapshot(pTsdb->pFS, &writer[0]->fsetArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbInfo("vgId:%d %s done, sver:%" PRId64 " ever:%" PRId64, TD_VID(pTsdb->pVnode), __func__, 0, ever); + } + return code; +} + +static int32_t tsdbSnapRAWWriteFileSetOpenIter(STsdbSnapRAWWriter* writer) { + int32_t code = 0; + int32_t lino = 0; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSnapRAWWriteFileSetCloseIter(STsdbSnapRAWWriter* writer) { return 0; } + +static int32_t tsdbSnapRAWWriteFileSetOpenWriter(STsdbSnapRAWWriter* writer) { + int32_t code = 0; + int32_t lino = 0; + + SFSetRAWWriterConfig config = { + .tsdb = writer->tsdb, + .szPage = writer->szPage, + .fid = writer->ctx->fid, + .cid = writer->commitID, + .did = writer->ctx->did, + .level = writer->ctx->level, + }; + + code = tsdbFSetRAWWriterOpen(&config, &writer->ctx->fsetWriter); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSnapRAWWriteFileSetCloseWriter(STsdbSnapRAWWriter* writer) { + return tsdbFSetRAWWriterClose(&writer->ctx->fsetWriter, 0, writer->fopArr); +} + +static int32_t tsdbSnapRAWWriteFileSetBegin(STsdbSnapRAWWriter* writer, int32_t fid) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(writer->ctx->fsetWriteBegin == false); + + STFileSet* fset = &(STFileSet){.fid = fid}; + + writer->ctx->fid = fid; + STFileSet** fsetPtr = TARRAY2_SEARCH(writer->fsetArr, &fset, tsdbTFileSetCmprFn, TD_EQ); + writer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr; + + int32_t level = tsdbFidLevel(fid, &writer->tsdb->keepCfg, taosGetTimestampSec()); + if (tfsAllocDisk(writer->tsdb->pVnode->pTfs, level, &writer->ctx->did)) { + code = TSDB_CODE_NO_AVAIL_DISK; + TSDB_CHECK_CODE(code, lino, _exit); + } + tfsMkdirRecurAt(writer->tsdb->pVnode->pTfs, writer->tsdb->path, writer->ctx->did); + + code = tsdbSnapRAWWriteFileSetOpenWriter(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->level = level; + writer->ctx->fsetWriteBegin = true; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSnapRAWWriteFileSetEnd(STsdbSnapRAWWriter* writer) { + if (!writer->ctx->fsetWriteBegin) return 0; + + int32_t code = 0; + int32_t lino = 0; + + // close write + code = tsdbSnapRAWWriteFileSetCloseWriter(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->fsetWriteBegin = false; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* writer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbSnapRAWWriteFileSetEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFSEditBegin(writer->tsdb->pFS, writer->fopArr, TSDB_FEDIT_COMMIT); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done", TD_VID(writer->tsdb->pVnode), __func__); + } + return code; +} + +int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** writer, int8_t rollback) { + if (writer[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + STsdb* tsdb = writer[0]->tsdb; + + if (rollback) { + code = tsdbFSEditAbort(writer[0]->tsdb->pFS); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + taosThreadMutexLock(&writer[0]->tsdb->mutex); + + code = tsdbFSEditCommit(writer[0]->tsdb->pFS); + if (code) { + taosThreadMutexUnlock(&writer[0]->tsdb->mutex); + TSDB_CHECK_CODE(code, lino, _exit); + } + + writer[0]->tsdb->pFS->fsstate = TSDB_FS_STATE_NORMAL; + + taosThreadMutexUnlock(&writer[0]->tsdb->mutex); + } + tsdbFSEnableBgTask(tsdb->pFS); + + TARRAY2_DESTROY(writer[0]->fopArr, NULL); + tsdbFSDestroyCopySnapshot(&writer[0]->fsetArr); + + taosMemoryFree(writer[0]); + writer[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else { + tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); + } + return code; +} + +static int32_t tsdbSnapRAWWriteTimeSeriesData(STsdbSnapRAWWriter* writer, STsdbDataRAWBlockHeader* bHdr) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSnapRAWWriteData(STsdbSnapRAWWriter* writer, SSnapDataHdr* hdr) { + int32_t code = 0; + int32_t lino = 0; + + STsdbDataRAWBlockHeader* bHdr = (void*)hdr->data; + int32_t fid = bHdr->file.fid; + if (!writer->ctx->fsetWriteBegin || fid != writer->ctx->fid) { + code = tsdbSnapRAWWriteFileSetEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSnapRAWWriteFileSetBegin(writer, fid); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbSnapRAWWriteTimeSeriesData(writer, bHdr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSnapRAWWrite(STsdbSnapRAWWriter* writer, SSnapDataHdr* hdr) { + ASSERT(hdr->type == SNAP_DATA_RAW); + + int32_t code = 0; + int32_t lino = 0; + + code = tsdbSnapRAWWriteData(writer, hdr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, type:%d index:%" PRId64 " size:%" PRId64, + TD_VID(writer->tsdb->pVnode), __func__, lino, tstrerror(code), hdr->type, hdr->index, hdr->size); + } else { + tsdbDebug("vgId:%d %s done, type:%d index:%" PRId64 " size:%" PRId64, TD_VID(writer->tsdb->pVnode), __func__, + hdr->type, hdr->index, hdr->size); + } + return code; +} diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 21c858709b..1a6b1a7a6c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -51,6 +51,10 @@ struct SVSnapReader { int8_t tsdbDone; TFileSetRangeArray *pRanges; STsdbSnapReader *pTsdbReader; + // tsdb raw + int8_t tsdbRawDone; + STsdbSnapRAWReader *pTsdbRawReader; + // tq int8_t tqHandleDone; STqSnapReader *pTqSnapReader; @@ -467,6 +471,8 @@ struct SVSnapWriter { // tsdb TFileSetRangeArray *pRanges; STsdbSnapWriter *pTsdbSnapWriter; + // tsdb raw + STsdbSnapRAWWriter *pTsdbSnapRAWWriter; // tq STqSnapWriter *pTqSnapWriter; STqOffsetWriter *pTqOffsetWriter; @@ -772,6 +778,17 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pHdr); if (code) goto _err; } break; + case SNAP_DATA_RAW: { + // tsdb + if (pWriter->pTsdbSnapRAWWriter == NULL) { + ASSERT(pWriter->sver == 0); + code = tsdbSnapRAWWriterOpen(pVnode->pTsdb, pWriter->ever, &pWriter->pTsdbSnapRAWWriter); + if (code) goto _err; + } + + code = tsdbSnapRAWWrite(pWriter->pTsdbSnapRAWWriter, pHdr); + if (code) goto _err; + } break; case SNAP_DATA_TQ_HANDLE: { // tq handle if (pWriter->pTqSnapWriter == NULL) { From a504007ade887f559c3ad2486d7a82de09651b5f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 24 Nov 2023 16:13:53 +0800 Subject: [PATCH 76/94] refact: alloc just enough memory for snap data in tsdbSnapRAWReadData --- source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c | 24 +++---- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 65 ++++++++++++------- 2 files changed, 49 insertions(+), 40 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c index 0c3d890966..aaaaa56b0e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c @@ -60,27 +60,21 @@ int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader) { return 0; } -int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *bHdr) { +int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *pBlock) { int32_t code = 0; int32_t lino = 0; - bHdr->file.type = reader->config->file.type; - bHdr->file.fid = reader->config->file.fid; - bHdr->file.cid = reader->config->file.cid; - bHdr->file.size = reader->config->file.size; - bHdr->file.minVer = reader->config->file.minVer; - bHdr->file.maxVer = reader->config->file.maxVer; - bHdr->file.stt->level = reader->config->file.stt->level; + pBlock->file.type = reader->config->file.type; + pBlock->file.fid = reader->config->file.fid; + pBlock->file.cid = reader->config->file.cid; + pBlock->file.size = reader->config->file.size; + pBlock->file.minVer = reader->config->file.minVer; + pBlock->file.maxVer = reader->config->file.maxVer; + pBlock->file.stt->level = reader->config->file.stt->level; - int64_t size = TMIN(bHdr->dataLength, reader->config->file.size - reader->ctx->offset); - ASSERT(size > 0); - bHdr->dataLength = 0; - bHdr->offset = reader->ctx->offset; - - code = tsdbReadFile(reader->fd, bHdr->offset, bHdr->data, size); + code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength); TSDB_CHECK_CODE(code, lino, _exit); - bHdr->dataLength = size; _exit: if (code) { TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index 2bcd00bfec..14c52c0bfb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -183,52 +183,54 @@ static int32_t tsdbSnapRAWReadFileSetCloseIter(STsdbSnapRAWReader* reader) { return 0; } -static int32_t tsdbSnapRAWReadNext(STsdbSnapRAWReader* reader, STsdbDataRAWBlockHeader* bHdr) { +static int64_t tsdbSnapRAWReadPeek(SDataFileRAWReader* reader) { + int64_t size = TMIN(reader->config->file.size - reader->ctx->offset, TSDB_SNAP_RAW_PAYLOAD_SIZE); + return size; +} + +static int32_t tsdbSnapRAWReadNext(STsdbSnapRAWReader* reader, SSnapDataHdr** ppData) { int32_t code = 0; int32_t lino = 0; + ppData[0] = NULL; ASSERT(reader->dataIter->offset <= reader->dataIter->size); ASSERT(reader->dataIter->idx <= reader->dataIter->count); + // dataReader if (reader->dataIter->offset == reader->dataIter->size && reader->dataIter->idx < reader->dataIter->count) { reader->dataIter->idx++; } if (reader->dataIter->idx == reader->dataIter->count) { return 0; } - + int8_t type = reader->type; SDataFileRAWReader* dataReader = TARRAY2_GET(reader->dataReaderArr, reader->dataIter->idx); - code = tsdbDataFileRAWReadBlockData(dataReader, bHdr); - TSDB_CHECK_CODE(code, lino, _exit); - reader->dataIter->offset += bHdr->dataLength; -_exit: - if (code) { - TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); - } - return code; -} + // prepare + int64_t dataLength = tsdbSnapRAWReadPeek(dataReader); + ASSERT(dataLength > 0); -static int32_t tsdbSnapRAWReadData(STsdbSnapRAWReader* reader, SSnapDataHdr** data) { - int32_t code = 0; - int32_t lino = 0; - - void* pBuf = taosMemoryCalloc(1, sizeof(SSnapDataHdr) + sizeof(STsdbDataRAWBlockHeader) + TSDB_SNAP_RAW_PAYLOAD_SIZE); + void* pBuf = taosMemoryCalloc(1, sizeof(SSnapDataHdr) + sizeof(STsdbDataRAWBlockHeader) + dataLength); if (pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); } - SSnapDataHdr* pHdr = pBuf; - pHdr->type = reader->type; - STsdbDataRAWBlockHeader* pData = (void*)pHdr->data; - pData->dataLength = TSDB_SNAP_RAW_PAYLOAD_SIZE; + pHdr->type = type; + pHdr->size = sizeof(STsdbDataRAWBlockHeader) + dataLength; - code = tsdbSnapRAWReadNext(reader, pData); + STsdbDataRAWBlockHeader* pBlock = (void*)pHdr->data; + pBlock->offset = dataReader->ctx->offset; + pBlock->dataLength = dataLength; + + // read + code = tsdbDataFileRAWReadBlockData(dataReader, pBlock); TSDB_CHECK_CODE(code, lino, _exit); - ASSERT(pData->dataLength > 0 && pData->dataLength <= TSDB_SNAP_RAW_PAYLOAD_SIZE); - pHdr->size = sizeof(STsdbDataRAWBlockHeader) + pData->dataLength; + // finish + reader->dataIter->offset += pBlock->dataLength; + ppData[0] = pBuf; + ASSERT(reader->dataIter->offset <= reader->dataIter->size); _exit: if (code) { @@ -236,7 +238,20 @@ _exit: pBuf = NULL; TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } - data[0] = pBuf; + return code; +} + +static int32_t tsdbSnapRAWReadData(STsdbSnapRAWReader* reader, uint8_t** ppData) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbSnapRAWReadNext(reader, (SSnapDataHdr**)ppData); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } return code; } @@ -288,7 +303,7 @@ int32_t tsdbSnapRAWRead(STsdbSnapRAWReader* reader, uint8_t** data) { } if (!reader->ctx->isDataDone) { - code = tsdbSnapRAWReadData(reader, (SSnapDataHdr**)data); + code = tsdbSnapRAWReadData(reader, data); TSDB_CHECK_CODE(code, lino, _exit); if (data[0]) { goto _exit; From ae60e1f810e411273b59a0c8b0fad324cc37404a Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 24 Nov 2023 16:15:33 +0800 Subject: [PATCH 77/94] refact: add tsdbSnapRAWReaderIterNext --- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index 14c52c0bfb..7010e6ad27 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -23,8 +23,6 @@ typedef struct SDataFileRAWReaderIter { int32_t count; int32_t idx; - int64_t offset; - int64_t size; } SDataFileRAWReaderIter; typedef struct STsdbSnapRAWReader { @@ -164,9 +162,7 @@ static int32_t tsdbSnapRAWReadFileSetOpenIter(STsdbSnapRAWReader* reader) { int32_t lino = 0; reader->dataIter->count = TARRAY2_SIZE(reader->dataReaderArr); - reader->dataIter->idx = -1; - reader->dataIter->offset = 0; - reader->dataIter->size = 0; + reader->dataIter->idx = 0; _exit: if (code) { @@ -178,8 +174,6 @@ _exit: static int32_t tsdbSnapRAWReadFileSetCloseIter(STsdbSnapRAWReader* reader) { reader->dataIter->count = 0; reader->dataIter->idx = 0; - reader->dataIter->offset = 0; - reader->dataIter->size = 0; return 0; } @@ -188,23 +182,30 @@ static int64_t tsdbSnapRAWReadPeek(SDataFileRAWReader* reader) { return size; } +static SDataFileRAWReader* tsdbSnapRAWReaderIterNext(STsdbSnapRAWReader* reader) { + ASSERT(reader->dataIter->idx <= reader->dataIter->count); + + while (reader->dataIter->idx < reader->dataIter->count) { + SDataFileRAWReader* dataReader = TARRAY2_GET(reader->dataReaderArr, reader->dataIter->idx); + ASSERT(dataReader); + if (dataReader->ctx->offset < dataReader->config->file.size) { + return dataReader; + } + reader->dataIter->idx++; + } + return NULL; +} + static int32_t tsdbSnapRAWReadNext(STsdbSnapRAWReader* reader, SSnapDataHdr** ppData) { int32_t code = 0; int32_t lino = 0; + int8_t type = reader->type; ppData[0] = NULL; - ASSERT(reader->dataIter->offset <= reader->dataIter->size); - ASSERT(reader->dataIter->idx <= reader->dataIter->count); - - // dataReader - if (reader->dataIter->offset == reader->dataIter->size && reader->dataIter->idx < reader->dataIter->count) { - reader->dataIter->idx++; - } - if (reader->dataIter->idx == reader->dataIter->count) { + SDataFileRAWReader* dataReader = tsdbSnapRAWReaderIterNext(reader); + if (dataReader == NULL) { return 0; } - int8_t type = reader->type; - SDataFileRAWReader* dataReader = TARRAY2_GET(reader->dataReaderArr, reader->dataIter->idx); // prepare int64_t dataLength = tsdbSnapRAWReadPeek(dataReader); @@ -219,18 +220,18 @@ static int32_t tsdbSnapRAWReadNext(STsdbSnapRAWReader* reader, SSnapDataHdr** pp pHdr->type = type; pHdr->size = sizeof(STsdbDataRAWBlockHeader) + dataLength; + // read STsdbDataRAWBlockHeader* pBlock = (void*)pHdr->data; pBlock->offset = dataReader->ctx->offset; pBlock->dataLength = dataLength; - // read code = tsdbDataFileRAWReadBlockData(dataReader, pBlock); TSDB_CHECK_CODE(code, lino, _exit); // finish - reader->dataIter->offset += pBlock->dataLength; + dataReader->ctx->offset += pBlock->dataLength; + ASSERT(dataReader->ctx->offset <= dataReader->config->file.size); ppData[0] = pBuf; - ASSERT(reader->dataIter->offset <= reader->dataIter->size); _exit: if (code) { From 1dc9019baa66e30d88497ab993c7948a0b1e70b0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 24 Nov 2023 16:28:39 +0800 Subject: [PATCH 78/94] feat: add tsdbSnapRAWRead into vnodeSnapRead --- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 26 ++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 1a6b1a7a6c..cb3d346df5 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -52,8 +52,8 @@ struct SVSnapReader { TFileSetRangeArray *pRanges; STsdbSnapReader *pTsdbReader; // tsdb raw - int8_t tsdbRawDone; - STsdbSnapRAWReader *pTsdbRawReader; + int8_t tsdbRAWDone; + STsdbSnapRAWReader *pTsdbRAWReader; // tq int8_t tqHandleDone; @@ -299,6 +299,28 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) } } + if (!pReader->tsdbRAWDone) { + // open if not + if (pReader->pTsdbRAWReader == NULL) { + ASSERT(pReader->sver == 0); + code = tsdbSnapRAWReaderOpen(pReader->pVnode->pTsdb, pReader->ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader); + if (code) goto _err; + } + + code = tsdbSnapRAWRead(pReader->pTsdbRAWReader, ppData); + if (code) { + goto _err; + } else { + if (*ppData) { + goto _exit; + } else { + pReader->tsdbRAWDone = 1; + code = tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader); + if (code) goto _err; + } + } + } + // TQ ================ vInfo("vgId:%d tq transform start", vgId); if (!pReader->tqHandleDone) { From f1362669722690e5a70eaf3672b44f06f896c85b Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 24 Nov 2023 20:04:39 +0800 Subject: [PATCH 79/94] feat: support snap replication by file blocks --- source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c | 66 ++++++++++++------- source/dnode/vnode/src/tsdb/tsdbFSetRAW.c | 5 ++ source/dnode/vnode/src/vnd/vnodeSnapshot.c | 11 ++++ 3 files changed, 57 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c index aaaaa56b0e..4a72fcfda8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c @@ -83,12 +83,26 @@ _exit: } // SDataFileRAWWriter ============================================= -int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **writer) { - writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); - if (!writer[0]) return TSDB_CODE_OUT_OF_MEMORY; +int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **ppWriter) { + int32_t code = 0; + int32_t lino = 0; - writer[0]->config[0] = config[0]; - return 0; + SDataFileRAWWriter *writer = taosMemoryCalloc(1, sizeof(SDataFileRAWWriter)); + if (!writer) return TSDB_CODE_OUT_OF_MEMORY; + + writer->config[0] = config[0]; + + code = tsdbDataFileRAWWriterDoOpen(writer); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + taosMemoryFree(writer); + writer = NULL; + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + ppWriter[0] = writer; + return code; } static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) { @@ -98,28 +112,13 @@ static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) { static int32_t tsdbDataFileRAWWriterDoClose(SDataFileRAWWriter *writer) { return 0; } -int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer) { - int32_t code = 0; - int32_t lino = 0; - - writer->file = writer->config->file; - writer->ctx->offset = 0; - - writer->ctx->opened = true; - -_exit: - if (code) { - TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); - } - return code; -} - static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFileOpArray *opArr) { int32_t code = 0; int32_t lino = 0; - STFileOp op; + ASSERT(writer->ctx->offset == writer->file.size); + ASSERT(writer->config->fid == writer->file.fid); - op = (STFileOp){ + STFileOp op = (STFileOp){ .optype = TSDB_FOP_CREATE, .fid = writer->config->fid, .nf = writer->file, @@ -147,7 +146,7 @@ static int32_t tsdbDataFileRAWWriterOpenDataFD(SDataFileRAWWriter *writer) { char fname[TSDB_FILENAME_LEN]; int32_t flag = TD_FILE_READ | TD_FILE_WRITE; - if (writer->file.size == 0) { + if (writer->ctx->offset == 0) { flag |= (TD_FILE_CREATE | TD_FILE_TRUNC); } @@ -162,6 +161,24 @@ _exit: return code; } +int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + writer->file = writer->config->file; + writer->ctx->offset = 0; + + code = tsdbDataFileRAWWriterOpenDataFD(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->opened = true; +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr) { if (writer[0] == NULL) return 0; @@ -195,7 +212,6 @@ int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDat code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength); TSDB_CHECK_CODE(code, lino, _exit); - writer->file.size += pDataBlock->dataLength; writer->ctx->offset += pDataBlock->dataLength; _exit: diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c index d9cd419ef9..03c12502d5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c @@ -74,6 +74,7 @@ static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRA SDataFileRAWWriterConfig config = { .tsdb = writer->config->tsdb, .szPage = writer->config->szPage, + .fid = bHdr->file.fid, .did = writer->config->did, .cid = writer->config->cid, .level = writer->config->level, @@ -81,6 +82,7 @@ static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRA .file = { .type = bHdr->file.type, + .fid = bHdr->file.fid, .did = writer->config->did, .cid = writer->config->cid, .size = bHdr->file.size, @@ -92,6 +94,9 @@ static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRA }, }; + writer->ctx->offset = 0; + writer->ctx->file = config.file; + code = tsdbDataFileRAWWriterOpen(&config, &writer->dataWriter); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index cb3d346df5..2a8484bcd2 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -277,6 +277,8 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) } // TSDB ============== + pReader->tsdbDone = true; + if (!pReader->tsdbDone) { // open if not if (pReader->pTsdbReader == NULL) { @@ -641,6 +643,10 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * tsdbSnapWriterPrepareClose(pWriter->pTsdbSnapWriter); } + if (pWriter->pTsdbSnapRAWWriter) { + tsdbSnapRAWWriterPrepareClose(pWriter->pTsdbSnapRAWWriter); + } + if (pWriter->pRsmaSnapWriter) { rsmaSnapWriterPrepareClose(pWriter->pRsmaSnapWriter); } @@ -677,6 +683,11 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * if (code) goto _exit; } + if (pWriter->pTsdbSnapRAWWriter) { + code = tsdbSnapRAWWriterClose(&pWriter->pTsdbSnapRAWWriter, rollback); + if (code) goto _exit; + } + if (pWriter->pTqSnapWriter) { code = tqSnapWriterClose(&pWriter->pTqSnapWriter, rollback); if (code) goto _exit; From 6c4c0242fd4c7a9565ecf5d4dbf29a3cc96746e4 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 27 Nov 2023 11:16:09 +0800 Subject: [PATCH 80/94] refact: adjust logging format in tsdbSnapRAWReaderOpen --- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index 7010e6ad27..ff121a3d30 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -62,14 +62,13 @@ int32_t tsdbSnapRAWReaderOpen(STsdb* tsdb, int64_t ever, int8_t type, STsdbSnapR _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), - __func__, lino, tstrerror(code), 0, ever, type); + tsdbError("vgId:%d %s failed at line %d since %s, sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, + lino, tstrerror(code), ever, type); tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); taosMemoryFree(reader[0]); reader[0] = NULL; } else { - tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), 0, - ever, type); + tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), ever, type); } return code; } @@ -387,7 +386,7 @@ _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } else { - tsdbInfo("vgId:%d %s done, sver:%" PRId64 " ever:%" PRId64, TD_VID(pTsdb->pVnode), __func__, 0, ever); + tsdbInfo("vgId:%d %s done, sver:0, ever:%" PRId64, TD_VID(pTsdb->pVnode), __func__, ever); } return code; } From ef34176e37f76202931a918cc4c1de7d95a3011f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 30 Nov 2023 16:54:17 +0800 Subject: [PATCH 81/94] fix: close data file readers in tsbSnapRAWReaderClose --- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index ff121a3d30..c3503a0cd6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -18,6 +18,8 @@ #include "tsdbFS2.h" #include "tsdbFSetRAW.h" +static int32_t tsdbSnapRAWReadFileSetCloseReader(STsdbSnapRAWReader* reader); + // reader typedef struct SDataFileRAWReaderIter { @@ -81,6 +83,7 @@ int32_t tsdbSnapRAWReaderClose(STsdbSnapRAWReader** reader) { STsdb* tsdb = reader[0]->tsdb; + tsdbSnapRAWReadFileSetCloseReader(reader[0]); tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); taosMemoryFree(reader[0]); reader[0] = NULL; From 52672657c17f56c90d0e1fccbf35347baea1c16c Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 5 Dec 2023 11:36:58 +0800 Subject: [PATCH 82/94] feat: toggle tsdb snap replication mode by snap info handshake ahead of time --- source/dnode/vnode/src/inc/tsdb.h | 14 +++ source/dnode/vnode/src/tsdb/tsdbSnapInfo.c | 139 +++++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeSnapshot.c | 34 ++++- 3 files changed, 183 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index dc3aa418b4..95982abfbe 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -715,6 +715,20 @@ int32_t tSerializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFS int32_t tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList); int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges); +// snap rep format +typedef enum ETsdbRepFmt { + TSDB_SNAP_REP_FMT_DEFAULT = 0, + TSDB_SNAP_REP_FMT_RAW, + TSDB_SNAP_REP_FMT_HYBRID, +} ETsdbRepFmt; + +typedef struct STsdbRepOpts { + ETsdbRepFmt format; +} STsdbRepOpts; + +int32_t tSerializeTsdbRepOpts(void *buf, int32_t bufLen, STsdbRepOpts *pInfo); +int32_t tDeserializeTsdbRepOpts(void *buf, int32_t bufLen, STsdbRepOpts *pInfo); + // snap read struct STsdbReadSnap { SMemTable *pMem; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index 65ee1a7db3..9dae9bdd36 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -443,6 +443,126 @@ static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* bu return offset; } +// tsdb replication opts +static int32_t tTsdbRepOptsDataLenCalc(STsdbRepOpts* pInfo) { + int32_t hdrLen = sizeof(int32_t); + int32_t datLen = 0; + + int8_t msgVer = 0; + int64_t reserved64 = 0; + int16_t format = 0; + hdrLen += sizeof(msgVer); + datLen += hdrLen; + datLen += sizeof(format); + datLen += sizeof(reserved64); + datLen += sizeof(*pInfo); + return datLen; +} + +int32_t tSerializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + int64_t reserved64 = 0; + int8_t msgVer = TSDB_SNAP_MSG_VER; + + if (tStartEncode(&encoder) < 0) goto _err; + if (tEncodeI8(&encoder, msgVer) < 0) goto _err; + int16_t format = pOpts->format; + if (tEncodeI16(&encoder, format) < 0) goto _err; + if (tEncodeI64(&encoder, reserved64) < 0) goto _err; + + tEndEncode(&encoder); + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; + +_err: + tEncoderClear(&encoder); + return -1; +} + +int32_t tDeserializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + int64_t reserved64 = 0; + int8_t msgVer = 0; + + if (tStartDecode(&decoder) < 0) goto _err; + if (tDecodeI8(&decoder, &msgVer) < 0) goto _err; + if (msgVer != TSDB_SNAP_MSG_VER) goto _err; + int16_t format = 0; + if (tDecodeI16(&decoder, &format) < 0) goto _err; + pOpts->format = format; + if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; + +_err: + tDecoderClear(&decoder); + return -1; +} + +static int32_t tsdbRepOptsEstSize(STsdbRepOpts* pOpts) { + int32_t dataLen = 0; + dataLen += sizeof(SSyncTLV); + dataLen += tTsdbRepOptsDataLenCalc(pOpts); + return dataLen; +} + +static int32_t tsdbRepOptsSerialize(STsdbRepOpts* pOpts, void* buf, int32_t bufLen) { + SSyncTLV* pSubHead = buf; + int32_t offset = 0; + int32_t tlen = 0; + if ((tlen = tSerializeTsdbRepOpts(pSubHead->val, bufLen, pOpts)) < 0) { + return -1; + } + pSubHead->typ = SNAP_DATA_RAW; + pSubHead->len = tlen; + offset += sizeof(*pSubHead) + tlen; + return offset; +} + +// snap info +static int32_t tsdbSnapPrepDealWithSnapInfo(SVnode* pVnode, SSnapshot* pSnap, STsdbRepOpts* pInfo) { + if (!pSnap->data) return 0; + int32_t code = -1; + + SSyncTLV* pHead = (void*)pSnap->data; + int32_t offset = 0; + + while (offset + sizeof(*pHead) < pHead->len) { + SSyncTLV* pField = (void*)(pHead->val + offset); + offset += sizeof(*pField) + pField->len; + void* buf = pField->val; + int32_t bufLen = pField->len; + + switch (pField->typ) { + case SNAP_DATA_TSDB: + case SNAP_DATA_RSMA1: + case SNAP_DATA_RSMA2: { + } break; + case SNAP_DATA_RAW: { + if (tDeserializeTsdbRepOpts(buf, bufLen, pInfo) < 0) { + terrno = TSDB_CODE_INVALID_DATA_FMT; + tsdbError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } break; + default: + tsdbError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), pField->typ); + goto _out; + } + } + + code = 0; +_out: + return code; +} + int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { ASSERT(pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY); STsdbPartitionInfo partitionInfo = {0}; @@ -453,10 +573,22 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { goto _out; } + // deal with snap info for reply + STsdbRepOpts opts = {.format = TSDB_SNAP_REP_FMT_RAW}; + if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { + STsdbRepOpts leaderOpts = {0}; + if (tsdbSnapPrepDealWithSnapInfo(pVnode, pSnap, &leaderOpts) < 0) { + tsdbError("vgId:%d, failed to deal with snap info for reply since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + opts.format = TMIN(opts.format, leaderOpts.format); + } + // info data realloc const int32_t headLen = sizeof(SSyncTLV); int32_t bufLen = headLen; bufLen += tsdbPartitionInfoEstSize(pInfo); + bufLen += tsdbRepOptsEstSize(&opts); if (syncSnapInfoDataRealloc(pSnap, bufLen) != 0) { tsdbError("vgId:%d, failed to realloc memory for data of snap info. bytes:%d", TD_VID(pVnode), bufLen); goto _out; @@ -474,6 +606,13 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) { offset += tlen; ASSERT(offset <= bufLen); + if ((tlen = tsdbRepOptsSerialize(&opts, buf + offset, bufLen - offset)) < 0) { + tsdbError("vgId:%d, failed to serialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + offset += tlen; + ASSERT(offset <= bufLen); + // set header of info data SSyncTLV* pHead = pSnap->data; pHead->typ = pSnap->type; diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 2a8484bcd2..438fa35713 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -92,14 +92,16 @@ static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotP int32_t code = -1; if (pParam->data) { + // decode SSyncTLV *datHead = (void *)pParam->data; if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { terrno = TSDB_CODE_INVALID_DATA_FMT; goto _out; } + STsdbRepOpts tsdbOpts = {0}; TFileSetRangeArray **ppRanges = NULL; - int32_t offset = 0; + int32_t offset = 0; while (offset + sizeof(SSyncTLV) < datHead->len) { SSyncTLV *subField = (void *)(datHead->val + offset); @@ -121,13 +123,30 @@ static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotP goto _out; } } break; + case SNAP_DATA_RAW: { + if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) { + vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } break; default: vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ); goto _out; } } - } + // toggle snap replication mode + vInfo("vgId:%d, vnode snap reader supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format); + if (pReader->sver == 0 && tsdbOpts.format == TSDB_SNAP_REP_FMT_RAW) { + pReader->tsdbDone = true; + } else { + pReader->tsdbRAWDone = true; + } + + ASSERT(pReader->tsdbDone != pReader->tsdbRAWDone); + vInfo("vgId:%d, vnode snap writer enabled replication mode: %s", TD_VID(pVnode), + (pReader->tsdbDone ? "raw" : "normal")); + } code = 0; _out: return code; @@ -277,8 +296,6 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) } // TSDB ============== - pReader->tsdbDone = true; - if (!pReader->tsdbDone) { // open if not if (pReader->pTsdbReader == NULL) { @@ -534,6 +551,7 @@ static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotP goto _out; } + STsdbRepOpts tsdbOpts = {0}; TFileSetRangeArray **ppRanges = NULL; int32_t offset = 0; @@ -557,11 +575,19 @@ static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotP goto _out; } } break; + case SNAP_DATA_RAW: { + if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) { + vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); + goto _out; + } + } break; default: vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ); goto _out; } } + + vInfo("vgId:%d, vnode snap writer supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format); } code = 0; From 28e1d836628a71de0e23f731f03e7f649d079bce Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 6 Dec 2023 16:38:09 +0800 Subject: [PATCH 83/94] fix: adjust tsdbReadFile call in tsdbDataFileRAWReadBlockData --- source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c index 4a72fcfda8..3f448379c9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c @@ -72,7 +72,7 @@ int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlo pBlock->file.maxVer = reader->config->file.maxVer; pBlock->file.stt->level = reader->config->file.stt->level; - code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength); + code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0); TSDB_CHECK_CODE(code, lino, _exit); _exit: From 5cdf2b0b041755c3316512ee40be667c9a734634 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 6 Dec 2023 16:40:04 +0800 Subject: [PATCH 84/94] fixup: remove call to obsolete funcs of BgTask in tsdbSnapRAWWriter open and close --- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index c3503a0cd6..6b61fcc324 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -363,9 +363,6 @@ int32_t tsdbSnapRAWWriterOpen(STsdb* pTsdb, int64_t ever, STsdbSnapRAWWriter** w int32_t code = 0; int32_t lino = 0; - // disable background tasks - tsdbFSDisableBgTask(pTsdb->pFS); - // start to write writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; @@ -528,7 +525,6 @@ int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** writer, int8_t rollback) { taosThreadMutexUnlock(&writer[0]->tsdb->mutex); } - tsdbFSEnableBgTask(tsdb->pFS); TARRAY2_DESTROY(writer[0]->fopArr, NULL); tsdbFSDestroyCopySnapshot(&writer[0]->fsetArr); From ebd3b697451e4eaa40e70e8084438ffaedbcfa29 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 6 Dec 2023 17:51:40 +0800 Subject: [PATCH 85/94] enh: control playload size of tsdb snap replication with TSDB_SNAP_DATA_PAYLOAD_SIZE --- include/util/tdef.h | 1 + source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h | 12 ------------ source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 3 +-- 4 files changed, 3 insertions(+), 15 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 1a440c7268..51b0b63da2 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -288,6 +288,7 @@ typedef enum ELogicConditionType { #define TSDB_CONN_ACTIVE_KEY_LEN 255 #define TSDB_DEFAULT_PKT_SIZE 65480 // same as RPC_MAX_UDP_SIZE +#define TSDB_SNAP_DATA_PAYLOAD_SIZE (1 * 1024 * 1024) #define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE #define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h index 49f80b0be5..d765671698 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h @@ -26,18 +26,6 @@ extern "C" { #endif -#define TSDB_SNAP_RAW_PAYLOAD_SIZE (4096 * 1024) -#if 0 -struct SDataRAWBlock { - int8_t *data; - int64_t size; -}; - -int32_t tsdbDataRAWBlockReset(SDataRAWBlock *pBlock); -int32_t tsdbDataRAWBlockAlloc(SDataRawBlock *pBlock); -void tsdbDataRAWBlockFree(SDataRAWBlock *pBlock); -#endif - // STsdbDataRAWBlockHeader ======================================= typedef struct STsdbDataRAWBlockHeader { struct { diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 8f5394a9bc..6aff1c2930 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -331,7 +331,7 @@ static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** dat if (!(reader->blockData->nRow % 16)) { int64_t nData = tBlockDataSize(reader->blockData); - if (nData >= 1 * 1024 * 1024) { + if (nData >= TSDB_SNAP_DATA_PAYLOAD_SIZE) { break; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index 6b61fcc324..462afcbec3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -21,7 +21,6 @@ static int32_t tsdbSnapRAWReadFileSetCloseReader(STsdbSnapRAWReader* reader); // reader - typedef struct SDataFileRAWReaderIter { int32_t count; int32_t idx; @@ -180,7 +179,7 @@ static int32_t tsdbSnapRAWReadFileSetCloseIter(STsdbSnapRAWReader* reader) { } static int64_t tsdbSnapRAWReadPeek(SDataFileRAWReader* reader) { - int64_t size = TMIN(reader->config->file.size - reader->ctx->offset, TSDB_SNAP_RAW_PAYLOAD_SIZE); + int64_t size = TMIN(reader->config->file.size - reader->ctx->offset, TSDB_SNAP_DATA_PAYLOAD_SIZE); return size; } From 1255b46469b274682c5dc216d4a40b65897521e6 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 7 Dec 2023 14:49:31 +0800 Subject: [PATCH 86/94] fix: destroy dataReaderArr properly in tsdbSnapRAWReaderClose --- source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c | 2 +- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 4 ++++ source/libs/sync/src/syncSnapshot.c | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index 462afcbec3..b7c22aa0e9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -82,7 +82,7 @@ int32_t tsdbSnapRAWReaderClose(STsdbSnapRAWReader** reader) { STsdb* tsdb = reader[0]->tsdb; - tsdbSnapRAWReadFileSetCloseReader(reader[0]); + TARRAY2_DESTROY(reader[0]->dataReaderArr, tsdbDataFileRAWReaderClose); tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); taosMemoryFree(reader[0]); reader[0] = NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 438fa35713..ed1dcc64c9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -203,6 +203,10 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) { tsdbSnapReaderClose(&pReader->pTsdbReader); } + if (pReader->pTsdbRAWReader) { + tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader); + } + if (pReader->pMetaReader) { metaSnapReaderClose(&pReader->pMetaReader); } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 1e3614857e..93e81fd8e2 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -1108,7 +1108,7 @@ static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp goto _out; } - if (pSender->pReader == NULL || pSender->finish) { + if (pSender->pReader == NULL || pSender->finish || !snapshotSenderIsStart(pSender)) { code = terrno = TSDB_CODE_SYN_INTERNAL_ERROR; goto _out; } From f23171d980e798468d97e4bf8152ccccc0f33341 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Thu, 7 Dec 2023 16:06:09 +0800 Subject: [PATCH 87/94] adjust case --- tests/system-test/1-insert/insert_double.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py index 56861f3cd4..b47b22ab44 100644 --- a/tests/system-test/1-insert/insert_double.py +++ b/tests/system-test/1-insert/insert_double.py @@ -40,8 +40,8 @@ class TDTestCase: # str support tdSql.execute(f"insert into {table_name} values(now, '-16', '+6')") - tdSql.execute(f"insert into {table_name} values(now, ' -80.99 ', ' +0042 ')") - tdSql.execute(f"insert into {table_name} values(now, ' -0042 ', ' +80.99 ')") + tdSql.execute(f"insert into {table_name} values(now, ' -80.99', ' +0042')") + tdSql.execute(f"insert into {table_name} values(now, ' -0042', ' +80.99')") tdSql.execute(f"insert into {table_name} values(now, '52.34354', '18.6')") tdSql.execute(f"insert into {table_name} values(now, '-12.', '+5.')") tdSql.execute(f"insert into {table_name} values(now, '-.12', '+.5')") From ff23e9afae6d0dce3d78197cc519c2247f4239c6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 7 Dec 2023 17:03:48 +0800 Subject: [PATCH 88/94] fix:inline error --- source/client/inc/clientSml.h | 27 ++++++++++++++++++++++++--- source/client/src/clientSml.c | 23 ----------------------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 26c0a38c8a..9ae28dd55e 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -217,11 +217,9 @@ void smlDestroyInfo(SSmlHandle *info); int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); -bool smlDoubleToInt64OverFlow(double num); int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); -int8_t smlGetTsTypeByLen(int32_t len); SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen); SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat); int32_t smlSetCTableName(SSmlTableInfo *oneTable); @@ -238,7 +236,6 @@ void freeSSmlKv(void* data); int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseJSON(SSmlHandle *info, char *payload); -void smlStrReplace(char* src, int32_t len); SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement); bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv); @@ -250,6 +247,30 @@ void smlBuildTsKv(SSmlKv *kv, int64_t ts); int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv); int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs); +static inline bool smlDoubleToInt64OverFlow(double num) { + if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; + return false; +} + +static inline void smlStrReplace(char* src, int32_t len){ + if (!tsSmlDot2Underline) return; + for(int i = 0; i < len; i++){ + if(src[i] == '.'){ + src[i] = '_'; + } + } +} + +static inline int8_t smlGetTsTypeByLen(int32_t len) { + if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { + return TSDB_TIME_PRECISION_SECONDS; + } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { + return TSDB_TIME_PRECISION_MILLI; + } else { + return -1; + } +} + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index a066bd8014..67b23792ad 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -127,19 +127,6 @@ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo* conn, const cha return (code == TSDB_CODE_SUCCESS) ? (authRes.pass[AUTH_RES_BASIC] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code; } -inline bool smlDoubleToInt64OverFlow(double num) { - if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; - return false; -} - -inline void smlStrReplace(char* src, int32_t len){ - if (!tsSmlDot2Underline) return; - for(int i = 0; i < len; i++){ - if(src[i] == '.'){ - src[i] = '_'; - } - } -} int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { if (pBuf->buf) { @@ -173,16 +160,6 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u return convertTimePrecision(tsInt64, fromPrecision, toPrecision); } -inline int8_t smlGetTsTypeByLen(int32_t len) { - if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { - return TSDB_TIME_PRECISION_SECONDS; - } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { - return TSDB_TIME_PRECISION_MILLI; - } else { - return -1; - } -} - SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) { SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); if (!tag) { From 4cc47a2edf7fc5976e5546a43293932f861c525b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 7 Dec 2023 17:19:10 +0800 Subject: [PATCH 89/94] fix:test case error --- tests/system-test/2-query/sml_TS-3724.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/sml_TS-3724.py b/tests/system-test/2-query/sml_TS-3724.py index 410e266f10..b537ad9b9a 100644 --- a/tests/system-test/2-query/sml_TS-3724.py +++ b/tests/system-test/2-query/sml_TS-3724.py @@ -85,12 +85,12 @@ class TDTestCase: tdSql.checkData(0, 1, 13.000000000) tdSql.checkData(0, 2, "web01") tdSql.checkData(0, 3, None) - tdSql.checkData(0, 4, "lga") + tdSql.checkData(0, 4, 1) tdSql.checkData(1, 1, 9.000000000) tdSql.checkData(1, 2, "web02") tdSql.checkData(3, 3, "t1") - tdSql.checkData(0, 4, "lga") + tdSql.checkData(2, 4, 4) tdSql.query(f"select * from {dbname}.macylr") tdSql.checkRows(2) From bd3fd91a50c591d6583dba7eac7a86ddb1ac7c0f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 7 Dec 2023 19:24:19 +0800 Subject: [PATCH 90/94] fix: too long multiple command line --- tools/shell/src/shellEngine.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 115abdcd36..e8a5b04178 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1091,7 +1091,12 @@ void shellSourceFile(const char *file) { char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1); while ((read_len = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) { - if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; + if ( cmd_len + read_len >= TSDB_MAX_ALLOWED_SQL_LEN) { + printf("read command line too long over 1M, ignore this line. cmd_len = %d read_len=%d \n", (int32_t)cmd_len, read_len); + cmd_len = 0; + memset(line, 0, TSDB_MAX_ALLOWED_SQL_LEN + 1); + continue; + } line[--read_len] = '\0'; if (read_len == 0 || shellIsCommentLine(line)) { // line starts with # From 2528fff8a887df9f14124abf475f7fc8c4773c19 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 8 Dec 2023 10:52:52 +0800 Subject: [PATCH 91/94] rebuild stream event window --- source/libs/executor/src/streameventwindowoperator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 99b198f35a..55029ac036 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -195,6 +195,10 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW pWinInfo->pWinFlag->endFlag = ends[i]; } else if (pWin->ekey == pTsData[i]) { pWinInfo->pWinFlag->endFlag |= ends[i]; + } else { + *pRebuild = true; + pWinInfo->pWinFlag->endFlag |= ends[i]; + return i + 1 - start; } memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey)); From b9d056c6a8b6bec29605e03a4811edff24c1bd4e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 8 Dec 2023 14:11:36 +0800 Subject: [PATCH 92/94] fix: little fix --- source/dnode/vnode/src/vnd/vnodeAsync.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeAsync.c b/source/dnode/vnode/src/vnd/vnodeAsync.c index c95d2324aa..c6bf60fa76 100644 --- a/source/dnode/vnode/src/vnd/vnodeAsync.c +++ b/source/dnode/vnode/src/vnd/vnodeAsync.c @@ -177,12 +177,15 @@ static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) { } static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) { - for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) { - while (async->queue[i].next != &async->queue[i]) { - SVATask *task = async->queue[i].next; - task->prev->next = task->next; - task->next->prev = task->prev; - vnodeAsyncTaskDone(async, task); + while (async->queue[0].next != &async->queue[0] || async->queue[1].next != &async->queue[1] || + async->queue[2].next != &async->queue[2]) { + for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) { + while (async->queue[i].next != &async->queue[i]) { + SVATask *task = async->queue[i].next; + task->prev->next = task->next; + task->next->prev = task->prev; + vnodeAsyncTaskDone(async, task); + } } } return 0; From 13193e29b746e8565ec8fcd811aa9dc081fd5a4c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 8 Dec 2023 14:28:40 +0800 Subject: [PATCH 93/94] delete invalid event window --- source/libs/executor/src/streameventwindowoperator.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 55029ac036..914b95ba83 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -323,6 +323,9 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin); releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); + SSessionKey tmpSeInfo = {0}; + getSessionHashKey(&curWin.winInfo.sessionWin, &tmpSeInfo); + tSimpleHashPut(pStDeleted, &tmpSeInfo, sizeof(SSessionKey), NULL, 0); continue; } code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, From a0487529d029fb58db753c905222edbe6dbfb237 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 8 Dec 2023 14:37:21 +0800 Subject: [PATCH 94/94] more fix --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index a974eb27bf..dc76aa61b2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -435,7 +435,7 @@ _exit: tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode), __func__, committer->ctx->fid, committer->ctx->minKey, committer->ctx->maxKey, committer->ctx->expLevel); } - return 0; + return code; } static int32_t tsdbCommitFileSetEnd(SCommitter2 *committer) {