From 6e3a62803c22ea7d71be75e8a8b2d9d6c08f449a Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 23 Nov 2023 14:08:31 +0800 Subject: [PATCH 1/9] fix: add check in mnd when alter table col/tag --- source/dnode/mnode/impl/src/mndStb.c | 34 ++++++ tests/script/api/alterTableTest.c | 175 +++++++++++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 tests/script/api/alterTableTest.c diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index eb2f9dcbe3..1ddd2f34e6 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1014,6 +1014,20 @@ static int32_t mndFindSuperTableColumnIndex(const SStbObj *pStb, const char *col return -1; } +static bool mndValidateSchema(SSchema *pSchemas, int32_t nSchema, SArray *pFields, int32_t maxLen) { + int32_t rowLen = 0; + for (int32_t i = 0; i < nSchema; ++i) { + rowLen += (pSchemas + i)->bytes; + } + + int32_t nField = taosArrayGetSize(pFields); + for (int32_t i = 0; i < nField; ++i) { + rowLen += ((SField *)TARRAY_GET_ELEM(pFields, i))->bytes; + } + + return rowLen <= maxLen; +} + static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq *createReq) { taosRLockLatch(&pStb->lock); memcpy(pDst, pStb, sizeof(SStbObj)); @@ -1269,6 +1283,11 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p return -1; } + if (!mndValidateSchema(pOld->pTags, pOld->numOfTags, pFields, TSDB_MAX_TAGS_LEN)) { + terrno = TSDB_CODE_PAR_INVALID_TAGS_LENGTH; + return -1; + } + pNew->numOfTags = pNew->numOfTags + ntags; if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; @@ -1558,6 +1577,16 @@ static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj return -1; } + uint32_t nLen = 0; + for (int32_t i = 0; i < pOld->numOfTags; ++i) { + nLen += (pOld->pTags[i].colId == colId) ? pField->bytes : pOld->pTags[i].bytes; + } + + if (nLen > TSDB_MAX_TAGS_LEN) { + terrno = TSDB_CODE_PAR_INVALID_TAGS_LENGTH; + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; } @@ -1592,6 +1621,11 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray return -1; } + if (!mndValidateSchema(pOld->pColumns, pOld->numOfColumns, pFields, TSDB_MAX_BYTES_PER_ROW)) { + terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; + return -1; + } + pNew->numOfColumns = pNew->numOfColumns + ncols; if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; diff --git a/tests/script/api/alterTableTest.c b/tests/script/api/alterTableTest.c new file mode 100644 index 0000000000..14605f4b8b --- /dev/null +++ b/tests/script/api/alterTableTest.c @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o demo demo.c -ltaos + +/** + * passwdTest.c + * - Run the test case in clear TDengine environment with default root passwd 'taosdata' + */ + +#include +#include +#include +#include +#include +#include "taos.h" // TAOS header file + +#define nDup 3 +#define USER_LEN 24 +#define BUF_LEN 1024 +#define DB "d0" +#define DB_BUFFER 32 +#define STB "stb" +#define CTB "ctb" +#define COL "c" + +#if 0 +#define STB_NUM 125 +#define CTB_NUM 960 +#define COL_NUM 52 +#define COL_NCHAR_LEN 320 +#else +#define STB_NUM 5 +#define CTB_NUM 1 +#define COL_NUM 505 +#define COL_NCHAR_LEN 32 +#endif +typedef uint16_t VarDataLenT; + +int32_t isDropDb = 0; + +#define TSDB_NCHAR_SIZE sizeof(int32_t) +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + +#define GET_FLOAT_VAL(x) (*(float *)(x)) +#define GET_DOUBLE_VAL(x) (*(double *)(x)) + +#define varDataLen(v) ((VarDataLenT *)(v))[0] + +static int32_t queryDB(TAOS *taos, char *command, bool skipError) { + int i; + TAOS_RES *pSql = NULL; + int32_t code = -1; + + for (i = 0; i < nDup; ++i) { + if (NULL != pSql) { + taos_free_result(pSql); + pSql = NULL; + } + + pSql = taos_query(taos, command); + code = taos_errno(pSql); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "failed to run: %s, reason: %s\n", command, taos_errstr(pSql)); + if (!skipError) { + taos_free_result(pSql); + taos_close(taos); + exit(EXIT_FAILURE); + } + } else { + fprintf(stderr, "success to run: %s\n", command); + } + + taos_free_result(pSql); +} + +static void createDatabaseAlterStbColumns(TAOS *taos, const char *host, char *qstr, int32_t addColMode) { + if (isDropDb) { + sprintf(qstr, "drop database if exists %s", DB); + queryDB(taos, qstr, false); + sprintf(qstr, "create database if not exists %s vgroups 2 buffer %d", DB, DB_BUFFER); + queryDB(taos, qstr, false); + } + sprintf(qstr, "use %s", DB); + queryDB(taos, qstr, false); + + // create stb + if (isDropDb) { + for (int i = 0; i < STB_NUM; ++i) { + sprintf(qstr, "CREATE table if not exists %s_%d (ts timestamp, %s_%d NCHAR(32)) tags(t0 nchar(16));", STB, i, COL, + 0); + queryDB(taos, qstr, false); + // create ctb + for (int j = 0; j < CTB_NUM; ++j) { + sprintf(qstr, "CREATE table %s_%d_%s_%d using %s_%d tags('%d_%d');", STB, i, CTB, j, STB, i, i, j); + queryDB(taos, qstr, false); + } + } + } + + if (isDropDb) { + printf("sleep 86400s to wait other terminal\n"); + sleep(86400); + } + + // alter stb cols + if (addColMode == 0) { + for (int i = 0; i < STB_NUM; ++i) { + for (int c = 1; c < COL_NUM; ++c) { + sprintf(qstr, "alter table %s_%d add column c_%d NCHAR(%d);", STB, i, c, COL_NCHAR_LEN); + queryDB(taos, qstr, true); + } + sprintf(qstr, "desc %s_%d;", STB, i); + queryDB(taos, qstr, false); + } + } else if (addColMode == 1) { + for (int c = 1; c < COL_NUM; ++c) { + for (int i = 0; i < STB_NUM; ++i) { + sprintf(qstr, "alter table %s_%d add column c_%d NCHAR(%d);", STB, i, c, COL_NCHAR_LEN); + queryDB(taos, qstr, true); + } + } + } + + // check + for (int i = 0; i < STB_NUM; ++i) { + sprintf(qstr, "desc %s_%d;", STB, i); + queryDB(taos, qstr, false); + } +} + +int main(int argc, char *argv[]) { + char qstr[1024]; + + // connect to server + if (argc < 2) { + printf("please input server-ip \n"); + return 0; + } + + if (argc < 3) { + printf("please specify if drop DB\n"); + return 0; + } + + isDropDb = atoi(argv[2]); + + TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/); + exit(1); + } + createDatabaseAlterStbColumns(taos, argv[1], qstr, 0); + + taos_close(taos); + taos_cleanup(); +} \ No newline at end of file From f7ac2d9d3922489c762d6d568d7b441d7ebf4b6b Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 23 Nov 2023 19:40:02 +0800 Subject: [PATCH 2/9] test: add alter table test --- tests/script/api/alterTableTest.c | 264 +++++++++++++++++++++++------- tests/script/api/makefile | 2 + 2 files changed, 209 insertions(+), 57 deletions(-) diff --git a/tests/script/api/alterTableTest.c b/tests/script/api/alterTableTest.c index 14605f4b8b..402cdb347d 100644 --- a/tests/script/api/alterTableTest.c +++ b/tests/script/api/alterTableTest.c @@ -17,8 +17,15 @@ // to compile: gcc -o demo demo.c -ltaos /** - * passwdTest.c + * alterTableTest.c + * - for JIRA: PI-23 * - Run the test case in clear TDengine environment with default root passwd 'taosdata' + * + * Usage Example: check add column for stable + * step 1) Open terminal 1, execute: "./alterTableTest localhost 1 0" to prepare db/stables. + * step 2) Open terminal 2 and 3, execute: "./alterTableTest localhost 0 0" to add columns simultaneously. + * + * Check Result: If reproduced, "Invalid value in client" error appears during checking "desc tables ..." */ #include @@ -28,37 +35,35 @@ #include #include "taos.h" // TAOS header file -#define nDup 3 -#define USER_LEN 24 -#define BUF_LEN 1024 -#define DB "d0" -#define DB_BUFFER 32 -#define STB "stb" -#define CTB "ctb" -#define COL "c" +typedef enum { + CHECK_ALTER_STABLE_ADD_COL = 0, + CHECK_ALTER_STABLE_ADD_TAG = 1, + CHECK_ALTER_STABLE_MODIFY_COL = 2, + CHECK_ALTER_STABLE_MODIFY_TAG = 3, + CHECK_ALTER_NTABLE_ADD_COL = 4, + CHECK_ALTER_NTABLE_MODIFY_COL = 5, +} ENUM_CHECK_ALTER_TYPE; -#if 0 -#define STB_NUM 125 -#define CTB_NUM 960 -#define COL_NUM 52 -#define COL_NCHAR_LEN 320 -#else -#define STB_NUM 5 -#define CTB_NUM 1 -#define COL_NUM 505 -#define COL_NCHAR_LEN 32 -#endif -typedef uint16_t VarDataLenT; +#define nDup 3 +#define USER_LEN 24 +#define BUF_LEN 1024 +#define DB "d0" +#define DB_BUFFER 32 +#define STB "stb" +#define NTB "ntb" +#define CTB "ctb" +#define COL "c" +#define STB_NUM 10 +#define NTB_NUM 20 +#define CTB_NUM 1 +#define COL_NUM 505 +#define TAG_NUM 127 +#define STB_NUM_MODIFY 100 // for modify columns/tags(increase the number if not easy to reproduced) +#define NTB_NUM_MODIFY 500 +#define COL_NCHAR_LEN 32 int32_t isDropDb = 0; - -#define TSDB_NCHAR_SIZE sizeof(int32_t) -#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) - -#define GET_FLOAT_VAL(x) (*(float *)(x)) -#define GET_DOUBLE_VAL(x) (*(double *)(x)) - -#define varDataLen(v) ((VarDataLenT *)(v))[0] +int32_t checkType = 0; static int32_t queryDB(TAOS *taos, char *command, bool skipError) { int i; @@ -92,16 +97,7 @@ static int32_t queryDB(TAOS *taos, char *command, bool skipError) { taos_free_result(pSql); } -static void createDatabaseAlterStbColumns(TAOS *taos, const char *host, char *qstr, int32_t addColMode) { - if (isDropDb) { - sprintf(qstr, "drop database if exists %s", DB); - queryDB(taos, qstr, false); - sprintf(qstr, "create database if not exists %s vgroups 2 buffer %d", DB, DB_BUFFER); - queryDB(taos, qstr, false); - } - sprintf(qstr, "use %s", DB); - queryDB(taos, qstr, false); - +static void createDatabaseAlterStbAddColumn(TAOS *taos, const char *host, char *qstr, int32_t type) { // create stb if (isDropDb) { for (int i = 0; i < STB_NUM; ++i) { @@ -117,27 +113,21 @@ static void createDatabaseAlterStbColumns(TAOS *taos, const char *host, char *qs } if (isDropDb) { - printf("sleep 86400s to wait other terminal\n"); + printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n"); sleep(86400); } + int32_t colNum = type == CHECK_ALTER_STABLE_ADD_COL ? COL_NUM : TAG_NUM; + const char *colName = type == CHECK_ALTER_STABLE_ADD_COL ? "column" : "tag"; + // alter stb cols - if (addColMode == 0) { - for (int i = 0; i < STB_NUM; ++i) { - for (int c = 1; c < COL_NUM; ++c) { - sprintf(qstr, "alter table %s_%d add column c_%d NCHAR(%d);", STB, i, c, COL_NCHAR_LEN); - queryDB(taos, qstr, true); - } - sprintf(qstr, "desc %s_%d;", STB, i); - queryDB(taos, qstr, false); - } - } else if (addColMode == 1) { + for (int i = 0; i < STB_NUM; ++i) { for (int c = 1; c < COL_NUM; ++c) { - for (int i = 0; i < STB_NUM; ++i) { - sprintf(qstr, "alter table %s_%d add column c_%d NCHAR(%d);", STB, i, c, COL_NCHAR_LEN); - queryDB(taos, qstr, true); - } + sprintf(qstr, "alter table %s_%d add %s c_%d NCHAR(%d);", STB, i, colName, c, COL_NCHAR_LEN); + queryDB(taos, qstr, true); } + sprintf(qstr, "desc %s_%d;", STB, i); + queryDB(taos, qstr, false); } // check @@ -147,28 +137,188 @@ static void createDatabaseAlterStbColumns(TAOS *taos, const char *host, char *qs } } +static void createDatabaseAlterStbModifyColumn(TAOS *taos, const char *host, char *qstr, int32_t type) { + // create stb + if (isDropDb) { + for (int i = 0; i < STB_NUM_MODIFY; ++i) { + sprintf( + qstr, + "CREATE table if not exists %s_%d (ts timestamp, c_0 NCHAR(160), c_1 NCHAR(160), c_2 NCHAR(160), c_3 " + "NCHAR(160),c_4 NCHAR(160),c_5 NCHAR(160),c_6 NCHAR(160),c_7 NCHAR(160),c_8 NCHAR(160),c_9 NCHAR(160),c_10 " + "NCHAR(160),c_11 NCHAR(160),c_12 NCHAR(160),c_13 NCHAR(160),c_14 NCHAR(160),c_15 NCHAR(160),c_16 " + "NCHAR(160),c_17 NCHAR(160),c_18 NCHAR(160),c_19 NCHAR(160),c_20 NCHAR(160),c_21 NCHAR(160),c_22 " + "NCHAR(160),c_23 NCHAR(160),c_24 NCHAR(160),c_25 NCHAR(160),c_26 NCHAR(160),c_27 NCHAR(160),c_28 " + "NCHAR(160),c_29 NCHAR(160),c_30 NCHAR(160),c_31 NCHAR(160),c_32 NCHAR(160),c_33 NCHAR(160),c_34 " + "NCHAR(160),c_35 NCHAR(160)) tags(t_0 NCHAR(80), t_1 NCHAR(80), t_2 NCHAR(80), t_3 NCHAR(80),t_4 " + "NCHAR(80),t_5 NCHAR(80),t_6 NCHAR(80),t_7 NCHAR(80),t_8 NCHAR(80),t_9 NCHAR(80),t_10 NCHAR(80),t_11 " + "NCHAR(80),t_12 NCHAR(80),t_13 NCHAR(80),t_14 NCHAR(80),t_15 NCHAR(80),t_16 NCHAR(80),t_17 NCHAR(80),t_18 " + "NCHAR(80),t_19 NCHAR(80),t_20 NCHAR(80),t_21 NCHAR(80),t_22 NCHAR(80),t_23 NCHAR(80),t_24 NCHAR(80),t_25 " + "NCHAR(80),t_26 NCHAR(80),t_27 NCHAR(80),t_28 NCHAR(80),t_29 NCHAR(80),t_30 NCHAR(80),t_31 NCHAR(80),t_32 " + "NCHAR(80),t_33 NCHAR(80),t_34 NCHAR(80),t_35 NCHAR(80));", + STB, i); + queryDB(taos, qstr, false); + } + } + + if (isDropDb) { + printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n"); + sleep(86400); + } + + int32_t colLen = type == CHECK_ALTER_STABLE_MODIFY_COL ? 455 : 115; + const char *colName = type == CHECK_ALTER_STABLE_MODIFY_COL ? "column c_" : "tag t_"; + + // alter stb cols + for (int i = 0; i < STB_NUM_MODIFY; ++i) { + for (int c = 0; c < 36; ++c) { + sprintf(qstr, "alter table %s_%d modify %s%d NCHAR(%d);", STB, i, colName, c, colLen); + queryDB(taos, qstr, true); + // usleep(1000); + } + sprintf(qstr, "desc %s_%d;", STB, i); + queryDB(taos, qstr, false); + } + + // check + for (int i = 0; i < STB_NUM_MODIFY; ++i) { + sprintf(qstr, "desc %s_%d;", STB, i); + queryDB(taos, qstr, false); + } +} + +static void createDatabaseAlterNtbAddColumn(TAOS *taos, const char *host, char *qstr) { + // create ntb + if (isDropDb) { + for (int i = 0; i < NTB_NUM; ++i) { + sprintf(qstr, "CREATE table if not exists %s_%d (ts timestamp, %s_%d NCHAR(32));", NTB, i, COL, 0); + queryDB(taos, qstr, false); + } + } + + if (isDropDb) { + printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n"); + sleep(86400); + } + + // alter ntb cols + for (int i = 0; i < NTB_NUM; ++i) { + for (int c = 1; c < COL_NUM; ++c) { + sprintf(qstr, "alter table %s_%d add column c_%d NCHAR(%d);", NTB, i, c, COL_NCHAR_LEN); + queryDB(taos, qstr, true); + } + sprintf(qstr, "desc %s_%d;", NTB, i); + queryDB(taos, qstr, false); + } + + // check + for (int i = 0; i < NTB_NUM; ++i) { + sprintf(qstr, "desc %s_%d;", NTB, i); + queryDB(taos, qstr, false); + } +} + +static void createDatabaseAlterNtbModifyColumn(TAOS *taos, const char *host, char *qstr) { + // create stb + if (isDropDb) { + for (int i = 0; i < NTB_NUM_MODIFY; ++i) { + sprintf( + qstr, + "CREATE table if not exists %s_%d (ts timestamp, c_0 NCHAR(160), c_1 NCHAR(160), c_2 NCHAR(160), c_3 " + "NCHAR(160),c_4 NCHAR(160),c_5 NCHAR(160),c_6 NCHAR(160),c_7 NCHAR(160),c_8 NCHAR(160),c_9 NCHAR(160),c_10 " + "NCHAR(160),c_11 NCHAR(160),c_12 NCHAR(160),c_13 NCHAR(160),c_14 NCHAR(160),c_15 NCHAR(160),c_16 " + "NCHAR(160),c_17 NCHAR(160),c_18 NCHAR(160),c_19 NCHAR(160),c_20 NCHAR(160),c_21 NCHAR(160),c_22 " + "NCHAR(160),c_23 NCHAR(160),c_24 NCHAR(160),c_25 NCHAR(160),c_26 NCHAR(160),c_27 NCHAR(160),c_28 " + "NCHAR(160),c_29 NCHAR(160),c_30 NCHAR(160),c_31 NCHAR(160),c_32 NCHAR(160),c_33 NCHAR(160),c_34 " + "NCHAR(160),c_35 NCHAR(160));", + NTB, i); + queryDB(taos, qstr, false); + } + } + + if (isDropDb) { + printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n"); + sleep(86400); + } + + // alter ntb cols + for (int i = 0; i < NTB_NUM_MODIFY; ++i) { + for (int c = 0; c < 36; ++c) { + sprintf(qstr, "alter table %s_%d modify column c_%d NCHAR(%d);", NTB, i, c, 455); + queryDB(taos, qstr, true); + // usleep(1000); + } + sprintf(qstr, "desc %s_%d;", NTB, i); + queryDB(taos, qstr, false); + } + + // check + for (int i = 0; i < NTB_NUM_MODIFY; ++i) { + sprintf(qstr, "desc %s_%d;", NTB, i); + queryDB(taos, qstr, false); + } +} + int main(int argc, char *argv[]) { char qstr[1024]; // connect to server if (argc < 2) { - printf("please input server-ip \n"); + printf("please input server-ip \n"); // e.g. localhost return 0; } if (argc < 3) { - printf("please specify if drop DB\n"); + printf("please specify if drop DB to clear env\n"); // 0 not drop, 1 drop return 0; } isDropDb = atoi(argv[2]); + if (argc < 4) { + printf("please specify check type\n"); // enum of ENUM_CHECK_ALTER_TYPE + return 0; + } + + checkType = atoi(argv[3]); + TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); if (taos == NULL) { printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/); exit(1); } - createDatabaseAlterStbColumns(taos, argv[1], qstr, 0); + + if (isDropDb) { + sprintf(qstr, "drop database if exists %s", DB); + queryDB(taos, qstr, false); + sprintf(qstr, "create database if not exists %s vgroups 2 buffer %d", DB, DB_BUFFER); + queryDB(taos, qstr, false); + } + sprintf(qstr, "use %s", DB); + queryDB(taos, qstr, false); + + switch (checkType) { + case CHECK_ALTER_STABLE_ADD_COL: // reproduced in 3.0.7.1 + createDatabaseAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_COL); + break; + case CHECK_ALTER_STABLE_ADD_TAG: // reproduced in 3.0.7.1 + createDatabaseAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_TAG); + break; + case CHECK_ALTER_STABLE_MODIFY_COL: // not reproduced in 3.0.7.1 since already checked in mnode + createDatabaseAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_COL); + break; + case CHECK_ALTER_STABLE_MODIFY_TAG: // reproduced in 3.0.7.1 + createDatabaseAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_TAG); + break; + case CHECK_ALTER_NTABLE_ADD_COL: // not reproduced in 3.0.7.1 + createDatabaseAlterNtbAddColumn(taos, argv[1], qstr); + break; + case CHECK_ALTER_NTABLE_MODIFY_COL: // not reproduced in 3.0.7.1 + createDatabaseAlterNtbModifyColumn(taos, argv[1], qstr); + break; + default: + printf("unkown check type:%d\n", checkType); + break; + } taos_close(taos); taos_cleanup(); diff --git a/tests/script/api/makefile b/tests/script/api/makefile index d848dca194..2bc127d995 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -16,6 +16,7 @@ exe: gcc $(CFLAGS) ./dbTableRoute.c -o $(ROOT)dbTableRoute $(LFLAGS) gcc $(CFLAGS) ./insertSameTs.c -o $(ROOT)insertSameTs $(LFLAGS) gcc $(CFLAGS) ./passwdTest.c -o $(ROOT)passwdTest $(LFLAGS) + gcc $(CFLAGS) ./alterTableTest.c -o $(ROOT)alterTableTest $(LFLAGS) gcc $(CFLAGS) ./whiteListTest.c -o $(ROOT)whiteListTest $(LFLAGS) gcc $(CFLAGS) ./insert_stb.c -o $(ROOT)insert_stb $(LFLAGS) gcc $(CFLAGS) ./tmqViewTest.c -o $(ROOT)tmqViewTest $(LFLAGS) @@ -27,6 +28,7 @@ clean: rm $(ROOT)dbTableRoute rm $(ROOT)insertSameTs rm $(ROOT)passwdTest + rm $(ROOT)alterTableTest rm $(ROOT)whiteListTest rm $(ROOT)insert_stb rm $(ROOT)tmqViewTest From 56c570cdcabdd22bc84edfcd850dc426f8059610 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 23 Nov 2023 19:51:48 +0800 Subject: [PATCH 3/9] test: add alter table test --- tests/script/api/alterTableTest.c | 37 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/script/api/alterTableTest.c b/tests/script/api/alterTableTest.c index 402cdb347d..e1cf3f1b9a 100644 --- a/tests/script/api/alterTableTest.c +++ b/tests/script/api/alterTableTest.c @@ -24,7 +24,7 @@ * Usage Example: check add column for stable * step 1) Open terminal 1, execute: "./alterTableTest localhost 1 0" to prepare db/stables. * step 2) Open terminal 2 and 3, execute: "./alterTableTest localhost 0 0" to add columns simultaneously. - * + * * Check Result: If reproduced, "Invalid value in client" error appears during checking "desc tables ..." */ @@ -97,11 +97,11 @@ static int32_t queryDB(TAOS *taos, char *command, bool skipError) { taos_free_result(pSql); } -static void createDatabaseAlterStbAddColumn(TAOS *taos, const char *host, char *qstr, int32_t type) { +static void checkAlterStbAddColumn(TAOS *taos, const char *host, char *qstr, int32_t type) { // create stb if (isDropDb) { for (int i = 0; i < STB_NUM; ++i) { - sprintf(qstr, "CREATE table if not exists %s_%d (ts timestamp, %s_%d NCHAR(32)) tags(t0 nchar(16));", STB, i, COL, + sprintf(qstr, "CREATE table if not exists %s_%d (ts timestamp, %s_%d NCHAR(32)) tags(t0 nchar(32));", STB, i, COL, 0); queryDB(taos, qstr, false); // create ctb @@ -113,7 +113,7 @@ static void createDatabaseAlterStbAddColumn(TAOS *taos, const char *host, char * } if (isDropDb) { - printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n"); + printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n"); sleep(86400); } @@ -122,7 +122,7 @@ static void createDatabaseAlterStbAddColumn(TAOS *taos, const char *host, char * // alter stb cols for (int i = 0; i < STB_NUM; ++i) { - for (int c = 1; c < COL_NUM; ++c) { + for (int c = 1; c < colNum; ++c) { sprintf(qstr, "alter table %s_%d add %s c_%d NCHAR(%d);", STB, i, colName, c, COL_NCHAR_LEN); queryDB(taos, qstr, true); } @@ -137,7 +137,7 @@ static void createDatabaseAlterStbAddColumn(TAOS *taos, const char *host, char * } } -static void createDatabaseAlterStbModifyColumn(TAOS *taos, const char *host, char *qstr, int32_t type) { +static void checkAlterStbModifyColumn(TAOS *taos, const char *host, char *qstr, int32_t type) { // create stb if (isDropDb) { for (int i = 0; i < STB_NUM_MODIFY; ++i) { @@ -161,7 +161,7 @@ static void createDatabaseAlterStbModifyColumn(TAOS *taos, const char *host, cha } if (isDropDb) { - printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n"); + printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n"); sleep(86400); } @@ -186,7 +186,7 @@ static void createDatabaseAlterStbModifyColumn(TAOS *taos, const char *host, cha } } -static void createDatabaseAlterNtbAddColumn(TAOS *taos, const char *host, char *qstr) { +static void checkAlterNtbAddColumn(TAOS *taos, const char *host, char *qstr) { // create ntb if (isDropDb) { for (int i = 0; i < NTB_NUM; ++i) { @@ -196,7 +196,7 @@ static void createDatabaseAlterNtbAddColumn(TAOS *taos, const char *host, char * } if (isDropDb) { - printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n"); + printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n"); sleep(86400); } @@ -217,8 +217,8 @@ static void createDatabaseAlterNtbAddColumn(TAOS *taos, const char *host, char * } } -static void createDatabaseAlterNtbModifyColumn(TAOS *taos, const char *host, char *qstr) { - // create stb +static void checkAlterNtbModifyColumn(TAOS *taos, const char *host, char *qstr) { + // create ntb if (isDropDb) { for (int i = 0; i < NTB_NUM_MODIFY; ++i) { sprintf( @@ -236,7 +236,7 @@ static void createDatabaseAlterNtbModifyColumn(TAOS *taos, const char *host, cha } if (isDropDb) { - printf("sleep 86400s to wait another terminals (at least 2 terminals) executing ... \n"); + printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n"); sleep(86400); } @@ -245,7 +245,6 @@ static void createDatabaseAlterNtbModifyColumn(TAOS *taos, const char *host, cha for (int c = 0; c < 36; ++c) { sprintf(qstr, "alter table %s_%d modify column c_%d NCHAR(%d);", NTB, i, c, 455); queryDB(taos, qstr, true); - // usleep(1000); } sprintf(qstr, "desc %s_%d;", NTB, i); queryDB(taos, qstr, false); @@ -298,22 +297,22 @@ int main(int argc, char *argv[]) { switch (checkType) { case CHECK_ALTER_STABLE_ADD_COL: // reproduced in 3.0.7.1 - createDatabaseAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_COL); + checkAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_COL); break; case CHECK_ALTER_STABLE_ADD_TAG: // reproduced in 3.0.7.1 - createDatabaseAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_TAG); + checkAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_TAG); break; case CHECK_ALTER_STABLE_MODIFY_COL: // not reproduced in 3.0.7.1 since already checked in mnode - createDatabaseAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_COL); + checkAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_COL); break; case CHECK_ALTER_STABLE_MODIFY_TAG: // reproduced in 3.0.7.1 - createDatabaseAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_TAG); + checkAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_TAG); break; case CHECK_ALTER_NTABLE_ADD_COL: // not reproduced in 3.0.7.1 - createDatabaseAlterNtbAddColumn(taos, argv[1], qstr); + checkAlterNtbAddColumn(taos, argv[1], qstr); break; case CHECK_ALTER_NTABLE_MODIFY_COL: // not reproduced in 3.0.7.1 - createDatabaseAlterNtbModifyColumn(taos, argv[1], qstr); + checkAlterNtbModifyColumn(taos, argv[1], qstr); break; default: printf("unkown check type:%d\n", checkType); From 9ce28483a001160aae2d7642385c182520549d28 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 23 Nov 2023 21:22:41 +0800 Subject: [PATCH 4/9] fix: add check in mnode/vnode when alter table col/tag --- source/dnode/vnode/src/meta/metaTable.c | 33 ++++++++++++++++++------- tests/script/api/alterTableTest.c | 8 +++--- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index f600925a0b..688adbf64d 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -1233,6 +1233,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl if (pAlterTbReq->colName == NULL) { terrno = TSDB_CODE_INVALID_MSG; + metaError("meta/table: null pAlterTbReq->colName"); return -1; } @@ -1300,20 +1301,26 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl SMetaEntry oldEntry = {.type = TSDB_NORMAL_TABLE, .uid = entry.uid}; oldEntry.ntbEntry.schemaRow.nCols = pSchema->nCols; - int32_t iCol = 0; + int32_t rowLen = -1; + if (pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN || + pAlterTbReq->action == TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES) { + rowLen = 0; + } + + int32_t iCol = 0; + SSchema *qColumn = NULL; for (;;) { - pColumn = NULL; + qColumn = NULL; if (iCol >= pSchema->nCols) break; - pColumn = &pSchema->pSchema[iCol]; + qColumn = &pSchema->pSchema[iCol]; - if (NULL == pAlterTbReq->colName) { - metaError("meta/table: null pAlterTbReq->colName"); - return -1; + if (!pColumn && (strcmp(qColumn->name, pAlterTbReq->colName) == 0)) { + pColumn = qColumn; + if (rowLen < 0) break; } - - if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break; - iCol++; + rowLen += qColumn->bytes; + ++iCol; } entry.version = version; @@ -1328,6 +1335,10 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { goto _err; } + if (rowLen + pAlterTbReq->bytes > TSDB_MAX_BYTES_PER_ROW) { + terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; + goto _err; + } pSchema->version++; pSchema->nCols++; pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols); @@ -1373,6 +1384,10 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; goto _err; } + if (rowLen + pAlterTbReq->colModBytes - pColumn->bytes > TSDB_MAX_BYTES_PER_ROW) { + terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; + goto _err; + } if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { terrno = TSDB_CODE_VND_COL_SUBSCRIBED; goto _err; diff --git a/tests/script/api/alterTableTest.c b/tests/script/api/alterTableTest.c index e1cf3f1b9a..7d7df340e8 100644 --- a/tests/script/api/alterTableTest.c +++ b/tests/script/api/alterTableTest.c @@ -246,8 +246,8 @@ static void checkAlterNtbModifyColumn(TAOS *taos, const char *host, char *qstr) sprintf(qstr, "alter table %s_%d modify column c_%d NCHAR(%d);", NTB, i, c, 455); queryDB(taos, qstr, true); } - sprintf(qstr, "desc %s_%d;", NTB, i); - queryDB(taos, qstr, false); + // sprintf(qstr, "desc %s_%d;", NTB, i); + // queryDB(taos, qstr, false); } // check @@ -308,10 +308,10 @@ int main(int argc, char *argv[]) { case CHECK_ALTER_STABLE_MODIFY_TAG: // reproduced in 3.0.7.1 checkAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_TAG); break; - case CHECK_ALTER_NTABLE_ADD_COL: // not reproduced in 3.0.7.1 + case CHECK_ALTER_NTABLE_ADD_COL: // reproduced in 3.0.7.1 checkAlterNtbAddColumn(taos, argv[1], qstr); break; - case CHECK_ALTER_NTABLE_MODIFY_COL: // not reproduced in 3.0.7.1 + case CHECK_ALTER_NTABLE_MODIFY_COL: // not reproduced in 3.0.7.1(should reproduced) checkAlterNtbModifyColumn(taos, argv[1], qstr); break; default: From dc977ca2007698dfcbb31165dd67ac542737f1d2 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 23 Nov 2023 23:09:49 +0800 Subject: [PATCH 5/9] fix: stash drop column --- source/dnode/vnode/src/meta/metaTable.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 688adbf64d..eaaa9882ac 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -1307,7 +1307,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl rowLen = 0; } - int32_t iCol = 0; + int32_t iCol = 0, jCol = 0; SSchema *qColumn = NULL; for (;;) { qColumn = NULL; @@ -1317,6 +1317,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl if (!pColumn && (strcmp(qColumn->name, pAlterTbReq->colName) == 0)) { pColumn = qColumn; + jCol = iCol; if (rowLen < 0) break; } rowLen += qColumn->bytes; @@ -1367,7 +1368,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl goto _err; } pSchema->version++; - tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema); + tlen = (pSchema->nCols - jCol - 1) * sizeof(SSchema); if (tlen) { memmove(pColumn, pColumn + 1, tlen); } From c4f1babdcf4a7d46951f45fe312812112770cf4f Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 24 Nov 2023 06:42:30 +0800 Subject: [PATCH 6/9] enh: naming optimize --- source/dnode/vnode/src/meta/metaTable.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index eaaa9882ac..6967d79d12 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -1312,16 +1312,16 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl for (;;) { qColumn = NULL; - if (iCol >= pSchema->nCols) break; - qColumn = &pSchema->pSchema[iCol]; + if (jCol >= pSchema->nCols) break; + qColumn = &pSchema->pSchema[jCol]; if (!pColumn && (strcmp(qColumn->name, pAlterTbReq->colName) == 0)) { pColumn = qColumn; - jCol = iCol; + iCol = jCol; if (rowLen < 0) break; } rowLen += qColumn->bytes; - ++iCol; + ++jCol; } entry.version = version; @@ -1368,7 +1368,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl goto _err; } pSchema->version++; - tlen = (pSchema->nCols - jCol - 1) * sizeof(SSchema); + tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema); if (tlen) { memmove(pColumn, pColumn + 1, tlen); } From ac47d3ea7df72e61d1a4976c1f3f020d119adf46 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 24 Nov 2023 06:51:57 +0800 Subject: [PATCH 7/9] enh: more check for column bytes --- source/dnode/vnode/src/meta/metaTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 6967d79d12..a4e665e225 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -1381,7 +1381,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl terrno = TSDB_CODE_VND_COL_NOT_EXISTS; goto _err; } - if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes > pAlterTbReq->colModBytes) { + if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes >= pAlterTbReq->colModBytes) { terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; goto _err; } From 0b27cb487b75912d34d1334a6e4504a1991f50eb Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 25 Nov 2023 00:31:07 +0800 Subject: [PATCH 8/9] test: adjust test case for alter table --- tests/script/api/alterTableTest.c | 324 --------------------- tests/script/api/makefile | 2 - tests/system-test/1-insert/alter_stable.py | 120 ++++++++ 3 files changed, 120 insertions(+), 326 deletions(-) delete mode 100644 tests/script/api/alterTableTest.c diff --git a/tests/script/api/alterTableTest.c b/tests/script/api/alterTableTest.c deleted file mode 100644 index 7d7df340e8..0000000000 --- a/tests/script/api/alterTableTest.c +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// TAOS standard API example. The same syntax as MySQL, but only a subset -// to compile: gcc -o demo demo.c -ltaos - -/** - * alterTableTest.c - * - for JIRA: PI-23 - * - Run the test case in clear TDengine environment with default root passwd 'taosdata' - * - * Usage Example: check add column for stable - * step 1) Open terminal 1, execute: "./alterTableTest localhost 1 0" to prepare db/stables. - * step 2) Open terminal 2 and 3, execute: "./alterTableTest localhost 0 0" to add columns simultaneously. - * - * Check Result: If reproduced, "Invalid value in client" error appears during checking "desc tables ..." - */ - -#include -#include -#include -#include -#include -#include "taos.h" // TAOS header file - -typedef enum { - CHECK_ALTER_STABLE_ADD_COL = 0, - CHECK_ALTER_STABLE_ADD_TAG = 1, - CHECK_ALTER_STABLE_MODIFY_COL = 2, - CHECK_ALTER_STABLE_MODIFY_TAG = 3, - CHECK_ALTER_NTABLE_ADD_COL = 4, - CHECK_ALTER_NTABLE_MODIFY_COL = 5, -} ENUM_CHECK_ALTER_TYPE; - -#define nDup 3 -#define USER_LEN 24 -#define BUF_LEN 1024 -#define DB "d0" -#define DB_BUFFER 32 -#define STB "stb" -#define NTB "ntb" -#define CTB "ctb" -#define COL "c" -#define STB_NUM 10 -#define NTB_NUM 20 -#define CTB_NUM 1 -#define COL_NUM 505 -#define TAG_NUM 127 -#define STB_NUM_MODIFY 100 // for modify columns/tags(increase the number if not easy to reproduced) -#define NTB_NUM_MODIFY 500 -#define COL_NCHAR_LEN 32 - -int32_t isDropDb = 0; -int32_t checkType = 0; - -static int32_t queryDB(TAOS *taos, char *command, bool skipError) { - int i; - TAOS_RES *pSql = NULL; - int32_t code = -1; - - for (i = 0; i < nDup; ++i) { - if (NULL != pSql) { - taos_free_result(pSql); - pSql = NULL; - } - - pSql = taos_query(taos, command); - code = taos_errno(pSql); - if (0 == code) { - break; - } - } - - if (code != 0) { - fprintf(stderr, "failed to run: %s, reason: %s\n", command, taos_errstr(pSql)); - if (!skipError) { - taos_free_result(pSql); - taos_close(taos); - exit(EXIT_FAILURE); - } - } else { - fprintf(stderr, "success to run: %s\n", command); - } - - taos_free_result(pSql); -} - -static void checkAlterStbAddColumn(TAOS *taos, const char *host, char *qstr, int32_t type) { - // create stb - if (isDropDb) { - for (int i = 0; i < STB_NUM; ++i) { - sprintf(qstr, "CREATE table if not exists %s_%d (ts timestamp, %s_%d NCHAR(32)) tags(t0 nchar(32));", STB, i, COL, - 0); - queryDB(taos, qstr, false); - // create ctb - for (int j = 0; j < CTB_NUM; ++j) { - sprintf(qstr, "CREATE table %s_%d_%s_%d using %s_%d tags('%d_%d');", STB, i, CTB, j, STB, i, i, j); - queryDB(taos, qstr, false); - } - } - } - - if (isDropDb) { - printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n"); - sleep(86400); - } - - int32_t colNum = type == CHECK_ALTER_STABLE_ADD_COL ? COL_NUM : TAG_NUM; - const char *colName = type == CHECK_ALTER_STABLE_ADD_COL ? "column" : "tag"; - - // alter stb cols - for (int i = 0; i < STB_NUM; ++i) { - for (int c = 1; c < colNum; ++c) { - sprintf(qstr, "alter table %s_%d add %s c_%d NCHAR(%d);", STB, i, colName, c, COL_NCHAR_LEN); - queryDB(taos, qstr, true); - } - sprintf(qstr, "desc %s_%d;", STB, i); - queryDB(taos, qstr, false); - } - - // check - for (int i = 0; i < STB_NUM; ++i) { - sprintf(qstr, "desc %s_%d;", STB, i); - queryDB(taos, qstr, false); - } -} - -static void checkAlterStbModifyColumn(TAOS *taos, const char *host, char *qstr, int32_t type) { - // create stb - if (isDropDb) { - for (int i = 0; i < STB_NUM_MODIFY; ++i) { - sprintf( - qstr, - "CREATE table if not exists %s_%d (ts timestamp, c_0 NCHAR(160), c_1 NCHAR(160), c_2 NCHAR(160), c_3 " - "NCHAR(160),c_4 NCHAR(160),c_5 NCHAR(160),c_6 NCHAR(160),c_7 NCHAR(160),c_8 NCHAR(160),c_9 NCHAR(160),c_10 " - "NCHAR(160),c_11 NCHAR(160),c_12 NCHAR(160),c_13 NCHAR(160),c_14 NCHAR(160),c_15 NCHAR(160),c_16 " - "NCHAR(160),c_17 NCHAR(160),c_18 NCHAR(160),c_19 NCHAR(160),c_20 NCHAR(160),c_21 NCHAR(160),c_22 " - "NCHAR(160),c_23 NCHAR(160),c_24 NCHAR(160),c_25 NCHAR(160),c_26 NCHAR(160),c_27 NCHAR(160),c_28 " - "NCHAR(160),c_29 NCHAR(160),c_30 NCHAR(160),c_31 NCHAR(160),c_32 NCHAR(160),c_33 NCHAR(160),c_34 " - "NCHAR(160),c_35 NCHAR(160)) tags(t_0 NCHAR(80), t_1 NCHAR(80), t_2 NCHAR(80), t_3 NCHAR(80),t_4 " - "NCHAR(80),t_5 NCHAR(80),t_6 NCHAR(80),t_7 NCHAR(80),t_8 NCHAR(80),t_9 NCHAR(80),t_10 NCHAR(80),t_11 " - "NCHAR(80),t_12 NCHAR(80),t_13 NCHAR(80),t_14 NCHAR(80),t_15 NCHAR(80),t_16 NCHAR(80),t_17 NCHAR(80),t_18 " - "NCHAR(80),t_19 NCHAR(80),t_20 NCHAR(80),t_21 NCHAR(80),t_22 NCHAR(80),t_23 NCHAR(80),t_24 NCHAR(80),t_25 " - "NCHAR(80),t_26 NCHAR(80),t_27 NCHAR(80),t_28 NCHAR(80),t_29 NCHAR(80),t_30 NCHAR(80),t_31 NCHAR(80),t_32 " - "NCHAR(80),t_33 NCHAR(80),t_34 NCHAR(80),t_35 NCHAR(80));", - STB, i); - queryDB(taos, qstr, false); - } - } - - if (isDropDb) { - printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n"); - sleep(86400); - } - - int32_t colLen = type == CHECK_ALTER_STABLE_MODIFY_COL ? 455 : 115; - const char *colName = type == CHECK_ALTER_STABLE_MODIFY_COL ? "column c_" : "tag t_"; - - // alter stb cols - for (int i = 0; i < STB_NUM_MODIFY; ++i) { - for (int c = 0; c < 36; ++c) { - sprintf(qstr, "alter table %s_%d modify %s%d NCHAR(%d);", STB, i, colName, c, colLen); - queryDB(taos, qstr, true); - // usleep(1000); - } - sprintf(qstr, "desc %s_%d;", STB, i); - queryDB(taos, qstr, false); - } - - // check - for (int i = 0; i < STB_NUM_MODIFY; ++i) { - sprintf(qstr, "desc %s_%d;", STB, i); - queryDB(taos, qstr, false); - } -} - -static void checkAlterNtbAddColumn(TAOS *taos, const char *host, char *qstr) { - // create ntb - if (isDropDb) { - for (int i = 0; i < NTB_NUM; ++i) { - sprintf(qstr, "CREATE table if not exists %s_%d (ts timestamp, %s_%d NCHAR(32));", NTB, i, COL, 0); - queryDB(taos, qstr, false); - } - } - - if (isDropDb) { - printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n"); - sleep(86400); - } - - // alter ntb cols - for (int i = 0; i < NTB_NUM; ++i) { - for (int c = 1; c < COL_NUM; ++c) { - sprintf(qstr, "alter table %s_%d add column c_%d NCHAR(%d);", NTB, i, c, COL_NCHAR_LEN); - queryDB(taos, qstr, true); - } - sprintf(qstr, "desc %s_%d;", NTB, i); - queryDB(taos, qstr, false); - } - - // check - for (int i = 0; i < NTB_NUM; ++i) { - sprintf(qstr, "desc %s_%d;", NTB, i); - queryDB(taos, qstr, false); - } -} - -static void checkAlterNtbModifyColumn(TAOS *taos, const char *host, char *qstr) { - // create ntb - if (isDropDb) { - for (int i = 0; i < NTB_NUM_MODIFY; ++i) { - sprintf( - qstr, - "CREATE table if not exists %s_%d (ts timestamp, c_0 NCHAR(160), c_1 NCHAR(160), c_2 NCHAR(160), c_3 " - "NCHAR(160),c_4 NCHAR(160),c_5 NCHAR(160),c_6 NCHAR(160),c_7 NCHAR(160),c_8 NCHAR(160),c_9 NCHAR(160),c_10 " - "NCHAR(160),c_11 NCHAR(160),c_12 NCHAR(160),c_13 NCHAR(160),c_14 NCHAR(160),c_15 NCHAR(160),c_16 " - "NCHAR(160),c_17 NCHAR(160),c_18 NCHAR(160),c_19 NCHAR(160),c_20 NCHAR(160),c_21 NCHAR(160),c_22 " - "NCHAR(160),c_23 NCHAR(160),c_24 NCHAR(160),c_25 NCHAR(160),c_26 NCHAR(160),c_27 NCHAR(160),c_28 " - "NCHAR(160),c_29 NCHAR(160),c_30 NCHAR(160),c_31 NCHAR(160),c_32 NCHAR(160),c_33 NCHAR(160),c_34 " - "NCHAR(160),c_35 NCHAR(160));", - NTB, i); - queryDB(taos, qstr, false); - } - } - - if (isDropDb) { - printf("sleep 86400s to wait another terminals (at least 2 terminals) to execute \n"); - sleep(86400); - } - - // alter ntb cols - for (int i = 0; i < NTB_NUM_MODIFY; ++i) { - for (int c = 0; c < 36; ++c) { - sprintf(qstr, "alter table %s_%d modify column c_%d NCHAR(%d);", NTB, i, c, 455); - queryDB(taos, qstr, true); - } - // sprintf(qstr, "desc %s_%d;", NTB, i); - // queryDB(taos, qstr, false); - } - - // check - for (int i = 0; i < NTB_NUM_MODIFY; ++i) { - sprintf(qstr, "desc %s_%d;", NTB, i); - queryDB(taos, qstr, false); - } -} - -int main(int argc, char *argv[]) { - char qstr[1024]; - - // connect to server - if (argc < 2) { - printf("please input server-ip \n"); // e.g. localhost - return 0; - } - - if (argc < 3) { - printf("please specify if drop DB to clear env\n"); // 0 not drop, 1 drop - return 0; - } - - isDropDb = atoi(argv[2]); - - if (argc < 4) { - printf("please specify check type\n"); // enum of ENUM_CHECK_ALTER_TYPE - return 0; - } - - checkType = atoi(argv[3]); - - TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); - if (taos == NULL) { - printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/); - exit(1); - } - - if (isDropDb) { - sprintf(qstr, "drop database if exists %s", DB); - queryDB(taos, qstr, false); - sprintf(qstr, "create database if not exists %s vgroups 2 buffer %d", DB, DB_BUFFER); - queryDB(taos, qstr, false); - } - sprintf(qstr, "use %s", DB); - queryDB(taos, qstr, false); - - switch (checkType) { - case CHECK_ALTER_STABLE_ADD_COL: // reproduced in 3.0.7.1 - checkAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_COL); - break; - case CHECK_ALTER_STABLE_ADD_TAG: // reproduced in 3.0.7.1 - checkAlterStbAddColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_ADD_TAG); - break; - case CHECK_ALTER_STABLE_MODIFY_COL: // not reproduced in 3.0.7.1 since already checked in mnode - checkAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_COL); - break; - case CHECK_ALTER_STABLE_MODIFY_TAG: // reproduced in 3.0.7.1 - checkAlterStbModifyColumn(taos, argv[1], qstr, CHECK_ALTER_STABLE_MODIFY_TAG); - break; - case CHECK_ALTER_NTABLE_ADD_COL: // reproduced in 3.0.7.1 - checkAlterNtbAddColumn(taos, argv[1], qstr); - break; - case CHECK_ALTER_NTABLE_MODIFY_COL: // not reproduced in 3.0.7.1(should reproduced) - checkAlterNtbModifyColumn(taos, argv[1], qstr); - break; - default: - printf("unkown check type:%d\n", checkType); - break; - } - - taos_close(taos); - taos_cleanup(); -} \ No newline at end of file diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 2bc127d995..d848dca194 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -16,7 +16,6 @@ exe: gcc $(CFLAGS) ./dbTableRoute.c -o $(ROOT)dbTableRoute $(LFLAGS) gcc $(CFLAGS) ./insertSameTs.c -o $(ROOT)insertSameTs $(LFLAGS) gcc $(CFLAGS) ./passwdTest.c -o $(ROOT)passwdTest $(LFLAGS) - gcc $(CFLAGS) ./alterTableTest.c -o $(ROOT)alterTableTest $(LFLAGS) gcc $(CFLAGS) ./whiteListTest.c -o $(ROOT)whiteListTest $(LFLAGS) gcc $(CFLAGS) ./insert_stb.c -o $(ROOT)insert_stb $(LFLAGS) gcc $(CFLAGS) ./tmqViewTest.c -o $(ROOT)tmqViewTest $(LFLAGS) @@ -28,7 +27,6 @@ clean: rm $(ROOT)dbTableRoute rm $(ROOT)insertSameTs rm $(ROOT)passwdTest - rm $(ROOT)alterTableTest rm $(ROOT)whiteListTest rm $(ROOT)insert_stb rm $(ROOT)tmqViewTest diff --git a/tests/system-test/1-insert/alter_stable.py b/tests/system-test/1-insert/alter_stable.py index 52f185a868..201610abf0 100644 --- a/tests/system-test/1-insert/alter_stable.py +++ b/tests/system-test/1-insert/alter_stable.py @@ -13,6 +13,7 @@ import random import string +import threading from util.log import * from util.cases import * from util.sql import * @@ -25,10 +26,24 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.setsql = TDSetSql() + self.fname = __file__ + '.tmp.sql' + self.dbname = 'db1' self.ntbname = 'ntb' self.stbname = 'stb' + self.stbnum = 10 + self.ntbnum = 10 + self.colnum = 52 + self.tagnum = 15 + self.collen = 320 + self.colnum_modify = 40 + self.tagnum_modify = 40 + self.collen_old_modify = 160 + self.collen_new_modify = 455 + self.taglen_old_modify = 80 + self.taglen_new_modify = 155 self.binary_length = 20 # the length of binary for column_dict self.nchar_length = 20 # the length of nchar for column_dict + self.threadnum = 2 self.column_dict = { 'ts' : 'timestamp', 'col1': 'tinyint', @@ -183,9 +198,114 @@ class TDTestCase: tdLog.info(res) assert(res[1][2] == 39001) + def prepareAlterEnv(self): + tdSql.execute(f'drop database if exists {self.dbname}') + tdSql.execute(f'create database if not exists {self.dbname} vgroups 2') + tdSql.execute(f'use {self.dbname}') + + def destroyAlterEnv(self): + tdSql.execute(f'drop database if exists {self.dbname}') + + def alterTableTask(self, i): + os.system(f'taos -f {self.fname}.{i};') + + def alterTableCheck(self, opt): + if opt in ["stb_add_col", "stb_add_tag"]: + for i in range(self.stbnum): + tdSql.execute(f'desc {self.stbname}_{i}') + elif opt in ["stb_modify_col", "stb_modify_tag"]: + for i in range(self.stbnum): + tdSql.execute(f'desc {self.stbname}_{i}') + elif opt in ["ntb_add_col", "ntb_modify_col"]: + for i in range(self.ntbnum): + tdSql.execute(f'desc {self.ntbname}_{i}') + + def executeAlterTableAndCheck(self, opt): + threads = [] + for i in range(self.threadnum): + thread = threading.Thread(target=self.alterTableTask, args=(i,)) + threads.append(thread) + thread.start() + for i in range(self.threadnum): + threads[i].join() + self.alterTableCheck(opt) + + def destroyAlterTableEnv(self): + for i in range(self.threadnum): + if os.path.isfile(f'{self.fname}.{i}'): + os.remove(f'{self.fname}.{i}') + + def prepareAlterTableEnv(self, opt): + self.destroyAlterTableEnv() + lines = [f'use {self.dbname};\n'] + if opt in ["stb_add_col", "stb_add_tag"]: + for i in range(self.stbnum): + tdSql.execute(f'create table if not exists {self.stbname}_{i} (ts timestamp, c_0 NCHAR({self.collen})) tags(t0 nchar({self.collen}));') + for i in range(self.stbnum): + if opt == 'stb_add_col': + for c in range(1, self.colnum): + lines.append(f'alter table {self.stbname}_{i} add column c_{c} NCHAR({self.collen});\n') + else: + for c in range(1, self.tagnum): + lines.append(f'alter table {self.stbname}_{i} add tag t_{c} NCHAR({self.collen});\n') + elif opt in ["stb_modify_col", "stb_modify_tag"]: + for i in range(self.stbnum): + createTbSql = f'CREATE table if not exists {self.stbname}_{i} (ts timestamp' + for j in range(self.colnum_modify): + createTbSql += f',c_{j} NCHAR({self.collen_old_modify})' + createTbSql += f') tags(t_0 NCHAR({self.taglen_old_modify})' + for k in range(1,self.tagnum_modify): + createTbSql += f',t_{k} NCHAR({self.taglen_old_modify})' + createTbSql += f');' + tdLog.info(createTbSql) + tdSql.execute(createTbSql) + for i in range(self.stbnum): + if opt == 'stb_modify_col': + for c in range(self.colnum_modify): + lines.append(f'alter table {self.stbname}_{i} modify column c_{c} NCHAR({self.collen_new_modify});\n') + else: + for c in range(self.tagnum_modify): + lines.append(f'alter table {self.stbname}_{i} modify tag t_{c} NCHAR({self.taglen_new_modify});\n') + elif opt in ['ntb_add_col']: + for i in range(self.ntbnum): + tdSql.execute(f'create table if not exists {self.ntbname}_{i} (ts timestamp, c_0 NCHAR({self.collen}));') + for i in range(self.ntbnum): + for c in range(1, self.colnum): + lines.append(f'alter table {self.ntbname}_{i} add column c_{c} NCHAR({self.collen});\n') + elif opt in ['ntb_modify_col']: + for i in range(self.ntbnum): + createTbSql = f'CREATE table if not exists {self.ntbname}_{i} (ts timestamp' + for j in range(self.colnum_modify): + createTbSql += f',c_{j} NCHAR({self.collen_old_modify})' + createTbSql += f');' + tdLog.info(createTbSql) + tdSql.execute(createTbSql) + for i in range(self.ntbnum): + for c in range(self.colnum_modify): + lines.append(f'alter table {self.ntbname}_{i} modify column c_{c} NCHAR({self.collen_new_modify});\n') + # generate sql file + with open(f'{self.fname}.0', "a") as f: + f.writelines(lines) + # clone sql file in case of race condition + for i in range(1, self.threadnum): + shutil.copy(f'{self.fname}.0', f'{self.fname}.{i}') + + def alter_stable_multi_client_check(self): + """Check alter stable/ntable var type column/tag(PI-23) + """ + alter_table_check_type = ["stb_add_col", "stb_add_tag", "stb_modify_col", "stb_modify_tag", "ntb_add_col", "ntb_modify_col"] + + for opt in alter_table_check_type: + self.prepareAlterEnv() + self.prepareAlterTableEnv(opt) + self.executeAlterTableAndCheck(opt) + self.destroyAlterTableEnv() + self.destroyAlterEnv() + def run(self): self.alter_stable_check() self.alter_stable_column_varchar_39001() + self.alter_stable_multi_client_check() def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 9ceddd3f60467d2be931179b67962495131e098e Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 25 Nov 2023 00:49:49 +0800 Subject: [PATCH 9/9] test: adjust test case for alter table --- tests/system-test/1-insert/alter_stable.py | 34 +++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/system-test/1-insert/alter_stable.py b/tests/system-test/1-insert/alter_stable.py index 201610abf0..40d7f04ceb 100644 --- a/tests/system-test/1-insert/alter_stable.py +++ b/tests/system-test/1-insert/alter_stable.py @@ -209,7 +209,16 @@ class TDTestCase: def alterTableTask(self, i): os.system(f'taos -f {self.fname}.{i};') - def alterTableCheck(self, opt): + def executeAlterTable(self, opt): + threads = [] + for i in range(self.threadnum): + thread = threading.Thread(target=self.alterTableTask, args=(i,)) + threads.append(thread) + thread.start() + for i in range(self.threadnum): + threads[i].join() + + def checkAlterTable(self, opt): if opt in ["stb_add_col", "stb_add_tag"]: for i in range(self.stbnum): tdSql.execute(f'desc {self.stbname}_{i}') @@ -220,23 +229,13 @@ class TDTestCase: for i in range(self.ntbnum): tdSql.execute(f'desc {self.ntbname}_{i}') - def executeAlterTableAndCheck(self, opt): - threads = [] - for i in range(self.threadnum): - thread = threading.Thread(target=self.alterTableTask, args=(i,)) - threads.append(thread) - thread.start() - for i in range(self.threadnum): - threads[i].join() - self.alterTableCheck(opt) - - def destroyAlterTableEnv(self): + def destroyAlterTable(self): for i in range(self.threadnum): if os.path.isfile(f'{self.fname}.{i}'): os.remove(f'{self.fname}.{i}') - def prepareAlterTableEnv(self, opt): - self.destroyAlterTableEnv() + def prepareAlterTable(self, opt): + self.destroyAlterTable() lines = [f'use {self.dbname};\n'] if opt in ["stb_add_col", "stb_add_tag"]: for i in range(self.stbnum): @@ -297,9 +296,10 @@ class TDTestCase: for opt in alter_table_check_type: self.prepareAlterEnv() - self.prepareAlterTableEnv(opt) - self.executeAlterTableAndCheck(opt) - self.destroyAlterTableEnv() + self.prepareAlterTable(opt) + self.executeAlterTable(opt) + self.checkAlterTable(opt) + self.destroyAlterTable() self.destroyAlterEnv() def run(self):