From 956aa1742cb640a60b4b88a7894ce3f61de08f37 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 24 Oct 2023 05:32:48 +0000 Subject: [PATCH 01/15] TD-26884 --- source/common/src/tglobal.c | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d9b1b2e0ed..f8feb949c7 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -98,7 +98,7 @@ int32_t tsMonitorMaxLogs = 100; bool tsMonitorComp = false; // audit -bool tsEnableAudit = false; +bool tsEnableAudit = true; char tsAuditFqdn[TSDB_FQDN_LEN] = {0}; uint16_t tsAuditPort = 6043; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 1f8c3b161b..159774efdc 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -314,8 +314,7 @@ _CONNECT: sprintf(obj, "%s:%d", ip, pConn->port); char detail[1000] = {0}; - sprintf(detail, "connType:%d, db:%s, pid:%d, startTime:%" PRId64 ", sVer:%s, app:%s", - connReq.connType, connReq.db, connReq.pid, connReq.startTime, connReq.sVer, connReq.app); + sprintf(detail, "app:%s", connReq.app); auditRecord(pReq, pMnode->clusterId, "login", connReq.user, obj, detail, strlen(detail)); From 162066235b29533b05806425d1b6c4870271f8b4 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 24 Oct 2023 06:17:13 +0000 Subject: [PATCH 02/15] create table --- include/common/tglobal.h | 1 + packaging/cfg/taos.cfg | 3 +++ source/common/src/tglobal.c | 3 +++ source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++-- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index c4037ed2ea..3e29703070 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -107,6 +107,7 @@ extern bool tsMonitorComp; extern bool tsEnableAudit; extern char tsAuditFqdn[]; extern uint16_t tsAuditPort; +extern bool tsEnableAuditCreateTable; // telem extern bool tsEnableTelem; diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 743edb6ca3..6d25db843d 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -101,6 +101,9 @@ # enable/disable audit log # audit 1 +# enable/disable audit create table +# auditCreateTable 1 + # The following parameter is used to limit the maximum number of lines in log files. # max number of lines per log filters # numOfLogLines 10000000 diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f8feb949c7..0a155f4ea1 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -101,6 +101,7 @@ bool tsMonitorComp = false; bool tsEnableAudit = true; char tsAuditFqdn[TSDB_FQDN_LEN] = {0}; uint16_t tsAuditPort = 6043; +bool tsEnableAuditCreateTable = true; // telem bool tsEnableTelem = true; @@ -632,6 +633,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddString(pCfg, "auditFqdn", tsAuditFqdn, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "auditPort", tsAuditPort, 1, 65056, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH) != 0) return -1; if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH) != 0) return -1; @@ -1050,6 +1052,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsEnableAudit = cfgGetItem(pCfg, "audit")->bval; tstrncpy(tsAuditFqdn, cfgGetItem(pCfg, "auditFqdn")->str, TSDB_FQDN_LEN); tsAuditPort = (uint16_t)cfgGetItem(pCfg, "auditPort")->i32; + tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval; tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index efa722d41a..e3aa459e1b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -947,13 +947,13 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, taosArrayPush(rsp.pArray, &cRsp); - if(pCreateReq->sqlLen > 0){ //skip auto create table, not set sql when auto create table + if(tsEnableAuditCreateTable){ int32_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; SName name = {0}; tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); - auditRecord(pReq, clusterId, "createTable", name.dbname, pCreateReq->name, pCreateReq->sql, pCreateReq->sqlLen); + auditRecord(pReq, clusterId, "createTable", name.dbname, "", pCreateReq->name, strlen(pCreateReq->name)); } } From b9fa620b26d23cb652122ac5b59072be1f6083d5 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Tue, 24 Oct 2023 15:05:24 +0800 Subject: [PATCH 03/15] shell query --- tools/shell/src/shellEngine.c | 227 +++++++++++++++++++++------------- 1 file changed, 143 insertions(+), 84 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index a7f79fc9db..6cde16856e 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -22,6 +22,23 @@ #include "shellAuto.h" #include "shellInt.h" +typedef struct { + const char *sql; + bool vertical; + tsem_t sem; + int64_t numOfRows; // the num of this batch + int64_t numOfAllRows; + + int32_t numFields; + TAOS_FIELD *fields; + int32_t precision; + + int32_t maxColNameLen; // for vertical print + int32_t width[TSDB_MAX_COLUMNS]; // for horizontal print + + uint64_t resShowMaxNum; +} tsDumpInfo; + static bool shellIsEmptyCommand(const char *cmd); static int32_t shellRunSingleCommand(char *command); static void shellRecordCommandToHistory(char *command); @@ -31,8 +48,8 @@ static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision); static int64_t shellDumpResultToFile(const char *fname, TAOS_RES *tres); static void shellPrintNChar(const char *str, int32_t length, int32_t width); static void shellPrintGeometry(const unsigned char *str, int32_t length, int32_t width); -static int64_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql); -static int64_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql); +static void shellVerticalPrintResult(TAOS_RES *tres, tsDumpInfo* dump_info); +static void shellHorizontalPrintResult(TAOS_RES *tres, tsDumpInfo* dump_info); static int64_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql); static void shellReadHistory(); static void shellWriteHistory(); @@ -702,64 +719,88 @@ bool shellIsShowQuery(const char *sql) { return false; } -int64_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql) { - TAOS_ROW row = taos_fetch_row(tres); - if (row == NULL) { - return 0; +void init_dump_info(tsDumpInfo *dump_info, TAOS_RES *tres, const char *sql, bool vertical) { + dump_info->sql = sql; + dump_info->vertical = vertical; + tsem_init(&dump_info->sem, 0, 0); + dump_info->numOfAllRows = 0; + + dump_info->numFields = taos_num_fields(tres); + dump_info->fields = taos_fetch_fields(tres); + dump_info->precision = taos_result_precision(tres); + + dump_info->resShowMaxNum = UINT64_MAX; + + if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsShowWhole(dump_info->sql)) { + dump_info->resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM; } - int32_t num_fields = taos_num_fields(tres); - TAOS_FIELD *fields = taos_fetch_fields(tres); - int32_t precision = taos_result_precision(tres); - - int32_t maxColNameLen = 0; - for (int32_t col = 0; col < num_fields; col++) { - int32_t len = (int32_t)strlen(fields[col].name); - if (len > maxColNameLen) { - maxColNameLen = len; + if (vertical) { + dump_info->maxColNameLen = 0; + for (int32_t col = 0; col < dump_info->numFields; col++) { + int32_t len = (int32_t)strlen(dump_info->fields[col].name); + if (len > dump_info->maxColNameLen) { + dump_info->maxColNameLen = len; + } + } + } else { + for (int32_t col = 0; col < dump_info->numFields; col++) { + dump_info->width[col] = shellCalcColWidth(dump_info->fields + col, dump_info->precision); } } +} - uint64_t resShowMaxNum = UINT64_MAX; - - if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsShowWhole(sql)) { - resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM; +void shellVerticalPrintResult(TAOS_RES *tres, tsDumpInfo *dump_info) { + if (dump_info->numOfRows <= 0) return; + if (dump_info->numOfAllRows >= dump_info->resShowMaxNum) { + return; } - int64_t numOfRows = 0; - int32_t showMore = 1; - do { - if (numOfRows < resShowMaxNum) { - printf("*************************** %"PRId64".row ***************************\r\n", numOfRows + 1); + TAOS_ROW row = taos_fetch_row(tres); + if (row == NULL) { + return; + } - int32_t *length = taos_fetch_lengths(tres); + int64_t numOfPintRows = dump_info->numOfAllRows; + int numOfPrintRowsThisOne = 0; - for (int32_t i = 0; i < num_fields; i++) { - TAOS_FIELD *field = fields + i; + while (row != NULL) { + printf("*************************** %" PRId64 ".row ***************************\r\n", numOfPintRows + 1); - int32_t padding = (int32_t)(maxColNameLen - strlen(field->name)); - printf("%*.s%s: ", padding, " ", field->name); + int32_t *length = taos_fetch_lengths(tres); - shellPrintField((const char *)row[i], field, 0, length[i], precision); - putchar('\r'); - putchar('\n'); - } - } else if (showMore) { + for (int32_t i = 0; i < dump_info->numFields; i++) { + TAOS_FIELD *field = dump_info->fields + i; + + int32_t padding = (int32_t)(dump_info->maxColNameLen - strlen(field->name)); + printf("%*.s%s: ", padding, " ", field->name); + + shellPrintField((const char *)row[i], field, 0, length[i], dump_info->precision); + putchar('\r'); + putchar('\n'); + } + + numOfPintRows++; + numOfPrintRowsThisOne++; + + if (numOfPintRows == dump_info->resShowMaxNum) { printf("\r\n"); - printf(" Notice: The result shows only the first %d rows.\r\n", SHELL_DEFAULT_RES_SHOW_NUM); + printf(" Notice: The result shows only the first %d rows.\r\n", dump_info->resShowMaxNum); printf(" You can use the `LIMIT` clause to get fewer result to show.\r\n"); printf(" Or use '>>' to redirect the whole set of the result to a specified file.\r\n"); printf("\r\n"); printf(" You can use Ctrl+C to stop the underway fetching.\r\n"); printf("\r\n"); - showMore = 0; + return; } - numOfRows++; - row = taos_fetch_row(tres); - } while (row != NULL); + if (numOfPrintRowsThisOne == dump_info->numOfRows) { + return; + } - return numOfRows; + row = taos_fetch_row(tres); + } + return; } int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) { @@ -856,47 +897,42 @@ void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields) { putchar('\n'); } -int64_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql) { +void shellHorizontalPrintResult(TAOS_RES *tres, tsDumpInfo *dump_info) { + if (dump_info->numOfRows <= 0) return; + if (dump_info->numOfAllRows >= dump_info->resShowMaxNum) { + return; + } + TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { - return 0; + return; } - int32_t num_fields = taos_num_fields(tres); - TAOS_FIELD *fields = taos_fetch_fields(tres); - int32_t precision = taos_result_precision(tres); - - int32_t width[TSDB_MAX_COLUMNS]; - for (int32_t col = 0; col < num_fields; col++) { - width[col] = shellCalcColWidth(fields + col, precision); + int64_t numOfPintRows = dump_info->numOfAllRows; + int numOfPrintRowsThisOne = 0; + if (numOfPintRows == 0) { + shellPrintHeader(dump_info->fields, dump_info->width, dump_info->numFields); } - shellPrintHeader(fields, width, num_fields); - - uint64_t resShowMaxNum = UINT64_MAX; - - if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsShowWhole(sql)) { - resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM; - } - - int64_t numOfRows = 0; - int32_t showMore = 1; - - do { + while (row != NULL) { int32_t *length = taos_fetch_lengths(tres); - if (numOfRows < resShowMaxNum) { - for (int32_t i = 0; i < num_fields; i++) { - putchar(' '); - shellPrintField((const char *)row[i], fields + i, width[i], length[i], precision); - putchar(' '); - putchar('|'); - } - putchar('\r'); - putchar('\n'); - } else if (showMore) { + for (int32_t i = 0; i < dump_info->numFields; i++) { + putchar(' '); + shellPrintField((const char *)row[i], dump_info->fields + i, dump_info->width[i], length[i], + dump_info->precision); + putchar(' '); + putchar('|'); + } + putchar('\r'); + putchar('\n'); + + numOfPintRows++; + numOfPrintRowsThisOne++; + + if (numOfPintRows == dump_info->resShowMaxNum) { printf("\r\n"); - printf(" Notice: The result shows only the first %d rows.\r\n", SHELL_DEFAULT_RES_SHOW_NUM); - if (shellIsShowQuery(sql)) { + printf(" Notice: The result shows only the first %d rows.\r\n", dump_info->resShowMaxNum); + if (shellIsShowQuery(dump_info->sql)) { printf(" You can use '>>' to redirect the whole set of the result to a specified file.\r\n"); } else { printf(" You can use the `LIMIT` clause to get fewer result to show.\r\n"); @@ -905,28 +941,51 @@ int64_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql) { printf("\r\n"); printf(" You can use Ctrl+C to stop the underway fetching.\r\n"); printf("\r\n"); - showMore = 0; + return; } - numOfRows++; - row = taos_fetch_row(tres); - } while (row != NULL); + if (numOfPrintRowsThisOne == dump_info->numOfRows) { + return; + } - return numOfRows; + row = taos_fetch_row(tres); + } + return; +} + +void shellDumpResultCallback(void *param, TAOS_RES *tres, int num_of_rows) { + tsDumpInfo *dump_info = (tsDumpInfo *)param; + if (num_of_rows > 0) { + dump_info->numOfRows = num_of_rows; + if (dump_info->vertical) { + shellVerticalPrintResult(tres, dump_info); + } else { + shellHorizontalPrintResult(tres, dump_info); + } + dump_info->numOfAllRows += num_of_rows; + taos_fetch_rows_a(tres, shellDumpResultCallback, param); + } else { + if (num_of_rows < 0) { + printf("\033[31masync retrieve failed, code: %d\033[0m\n", num_of_rows); + } + tsem_post(&dump_info->sem); + } } int64_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql) { - int64_t numOfRows = 0; + int64_t num_of_rows = 0; if (fname != NULL) { - numOfRows = shellDumpResultToFile(fname, tres); - } else if (vertical) { - numOfRows = shellVerticalPrintResult(tres, sql); + num_of_rows = shellDumpResultToFile(fname, tres); } else { - numOfRows = shellHorizontalPrintResult(tres, sql); + tsDumpInfo dump_info; + init_dump_info(&dump_info, tres, sql, vertical); + taos_fetch_rows_a(tres, shellDumpResultCallback, &dump_info); + tsem_wait(&dump_info.sem); + num_of_rows = dump_info.numOfAllRows; } *error_no = taos_errno(tres); - return numOfRows; + return num_of_rows; } void shellReadHistory() { From c14dbbdd01fe3833b6245dc1fd137530a0ebf689 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Tue, 24 Oct 2023 15:15:39 +0800 Subject: [PATCH 04/15] print exception log --- tools/shell/src/shellEngine.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 6cde16856e..f911637cc3 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -751,13 +751,9 @@ void init_dump_info(tsDumpInfo *dump_info, TAOS_RES *tres, const char *sql, bool } void shellVerticalPrintResult(TAOS_RES *tres, tsDumpInfo *dump_info) { - if (dump_info->numOfRows <= 0) return; - if (dump_info->numOfAllRows >= dump_info->resShowMaxNum) { - return; - } - TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { + printf("\033[31mtaos_fetch_row failed.\033[0m\n"); return; } @@ -898,13 +894,9 @@ void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields) { } void shellHorizontalPrintResult(TAOS_RES *tres, tsDumpInfo *dump_info) { - if (dump_info->numOfRows <= 0) return; - if (dump_info->numOfAllRows >= dump_info->resShowMaxNum) { - return; - } - TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { + printf("\033[31mtaos_fetch_row failed.\033[0m\n"); return; } @@ -957,10 +949,12 @@ void shellDumpResultCallback(void *param, TAOS_RES *tres, int num_of_rows) { tsDumpInfo *dump_info = (tsDumpInfo *)param; if (num_of_rows > 0) { dump_info->numOfRows = num_of_rows; - if (dump_info->vertical) { - shellVerticalPrintResult(tres, dump_info); - } else { - shellHorizontalPrintResult(tres, dump_info); + if (dump_info->numOfAllRows < dump_info->resShowMaxNum) { + if (dump_info->vertical) { + shellVerticalPrintResult(tres, dump_info); + } else { + shellHorizontalPrintResult(tres, dump_info); + } } dump_info->numOfAllRows += num_of_rows; taos_fetch_rows_a(tres, shellDumpResultCallback, param); From 9c5f8165ccb4523ff8f8f876b182be2166193fe6 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Tue, 24 Oct 2023 16:02:24 +0800 Subject: [PATCH 05/15] print uint64 --- tools/shell/src/shellEngine.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index f911637cc3..ec7bb26563 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -781,7 +781,7 @@ void shellVerticalPrintResult(TAOS_RES *tres, tsDumpInfo *dump_info) { if (numOfPintRows == dump_info->resShowMaxNum) { printf("\r\n"); - printf(" Notice: The result shows only the first %d rows.\r\n", dump_info->resShowMaxNum); + printf(" Notice: The result shows only the first %ld rows.\r\n", dump_info->resShowMaxNum); printf(" You can use the `LIMIT` clause to get fewer result to show.\r\n"); printf(" Or use '>>' to redirect the whole set of the result to a specified file.\r\n"); printf("\r\n"); @@ -923,7 +923,7 @@ void shellHorizontalPrintResult(TAOS_RES *tres, tsDumpInfo *dump_info) { if (numOfPintRows == dump_info->resShowMaxNum) { printf("\r\n"); - printf(" Notice: The result shows only the first %d rows.\r\n", dump_info->resShowMaxNum); + printf(" Notice: The result shows only the first %ld rows.\r\n", dump_info->resShowMaxNum); if (shellIsShowQuery(dump_info->sql)) { printf(" You can use '>>' to redirect the whole set of the result to a specified file.\r\n"); } else { From ebd5f7c20d84e0540f515d6deaeae09e3ecff6bc Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Wed, 25 Oct 2023 07:15:29 +0800 Subject: [PATCH 06/15] fix print log on mac --- tools/shell/src/shellEngine.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index ec7bb26563..3b150230e7 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -781,7 +781,7 @@ void shellVerticalPrintResult(TAOS_RES *tres, tsDumpInfo *dump_info) { if (numOfPintRows == dump_info->resShowMaxNum) { printf("\r\n"); - printf(" Notice: The result shows only the first %ld rows.\r\n", dump_info->resShowMaxNum); + printf(" Notice: The result shows only the first %d rows.\r\n", SHELL_DEFAULT_RES_SHOW_NUM); printf(" You can use the `LIMIT` clause to get fewer result to show.\r\n"); printf(" Or use '>>' to redirect the whole set of the result to a specified file.\r\n"); printf("\r\n"); @@ -923,7 +923,7 @@ void shellHorizontalPrintResult(TAOS_RES *tres, tsDumpInfo *dump_info) { if (numOfPintRows == dump_info->resShowMaxNum) { printf("\r\n"); - printf(" Notice: The result shows only the first %ld rows.\r\n", dump_info->resShowMaxNum); + printf(" Notice: The result shows only the first %d rows.\r\n", SHELL_DEFAULT_RES_SHOW_NUM); if (shellIsShowQuery(dump_info->sql)) { printf(" You can use '>>' to redirect the whole set of the result to a specified file.\r\n"); } else { From 14be9c81e268ee9665c138869c24fd2544a5feed Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 25 Oct 2023 06:49:23 +0000 Subject: [PATCH 07/15] create table user and clusterid --- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index e3aa459e1b..c46ea15111 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -948,12 +948,12 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, taosArrayPush(rsp.pArray, &cRsp); if(tsEnableAuditCreateTable){ - int32_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; + int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; SName name = {0}; tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); - auditRecord(pReq, clusterId, "createTable", name.dbname, "", pCreateReq->name, strlen(pCreateReq->name)); + auditRecord(NULL, clusterId, "createTable", name.dbname, "", pCreateReq->name, strlen(pCreateReq->name)); } } From d77ecbbb1b150611e5ccb9a6314213499ae0769c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 25 Oct 2023 17:41:55 +0800 Subject: [PATCH 08/15] cmake/xml2: use sha256 to verify distribution release --- cmake/xml2_CMakeLists.txt.in | 6 ++++-- source/dnode/vnode/CMakeLists.txt | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmake/xml2_CMakeLists.txt.in b/cmake/xml2_CMakeLists.txt.in index 25eb5a69ac..ad0704cdb9 100644 --- a/cmake/xml2_CMakeLists.txt.in +++ b/cmake/xml2_CMakeLists.txt.in @@ -1,7 +1,9 @@ # xml2 ExternalProject_Add(xml2 - URL https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz + URL https://download.gnome.org/sources/libxml2/2.11/libxml2-2.11.5.tar.xz + URL_HASH SHA256=3727b078c360ec69fa869de14bd6f75d7ee8d36987b071e6928d4720a28df3a6 + #https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz #GIT_REPOSITORY https://github.com/GNOME/libxml2 #GIT_TAG v2.11.5 DOWNLOAD_NO_PROGRESS 1 @@ -9,7 +11,7 @@ ExternalProject_Add(xml2 SOURCE_DIR "${TD_CONTRIB_DIR}/xml2" #BINARY_DIR "" BUILD_IN_SOURCE TRUE - CONFIGURE_COMMAND ./autogen.sh && ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python --without-lzma BUILD_COMMAND make -j INSTALL_COMMAND make install && ln -s $ENV{HOME}/.cos-local.2/include/libxml2/libxml $ENV{HOME}/.cos-local.2/include/libxml TEST_COMMAND "" diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index a10b38a1bf..dcc9f9a115 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -173,7 +173,8 @@ if(${BUILD_WITH_S3}) set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) find_library(S3_LIBRARY s3) - find_library(CURL_LIBRARY curl) + find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) + find_library(XML2_LIBRARY xml2) find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) target_link_libraries( @@ -184,7 +185,7 @@ if(${BUILD_WITH_S3}) PUBLIC ${CURL_LIBRARY} PUBLIC ${SSL_LIBRARY} PUBLIC ${CRYPTO_LIBRARY} - PUBLIC xml2 + PUBLIC ${XML2_LIBRARY} ) add_definitions(-DUSE_S3) From 6c9c06552d8679e7f65da4008b856ad3e9caa2ac Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 26 Oct 2023 06:10:09 +0000 Subject: [PATCH 09/15] fix case --- tests/system-test/0-others/taosdMonitor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index a07d7f411e..169c70e675 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -268,6 +268,8 @@ class TDTestCase: updatecfgDict["monitorMaxLogs"] = "10" updatecfgDict["monitorComp"] = "1" + updatecfgDict["audit"] = '0' + print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): From b3147581086a8bd0adfb540b3ea8abe71039d547 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 26 Oct 2023 16:24:06 +0800 Subject: [PATCH 10/15] tsdb/commit: make ts big enough to skip row --- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index a2de5a81ca..6dc492f420 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -137,7 +137,12 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) { } int64_t ts = TSDBROW_TS(&row->row); - if (ts > committer->ctx->maxKey || skipRow) { + + if (skipRow && ts <= committer->ctx->maxKey) { + ts = committer->ctx->maxKey + 1; + } + + if (ts > committer->ctx->maxKey) { committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts); code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); TSDB_CHECK_CODE(code, lino, _exit); From 58bab7cf68ab9a6eb6d97d51fba9426e8dd6d93e Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 26 Oct 2023 19:09:37 +0800 Subject: [PATCH 11/15] test: support replica3 in splited/transform tmqvnode --- tests/system-test/7-tmq/tmqVnodeTransform-stb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py index 2dd756b788..18c5f8880b 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb.py @@ -140,7 +140,7 @@ class TDTestCase: 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 30, + 'pollDelay': 60, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} From 3f6f928399df2a35d0d779a2b05386536af77c91 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 26 Oct 2023 20:25:01 +0800 Subject: [PATCH 12/15] test:delete zlib cache in ci --- tests/parallel_test/container_build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 94704b1c25..8e309a8182 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -68,9 +68,10 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ - -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ +# -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ if [[ -d ${WORKDIR}/debugNoSan ]] ;then echo "delete ${WORKDIR}/debugNoSan" @@ -97,10 +98,11 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ - -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ + -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " + # -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan ret=$? From 310793cbf0ccaa502a3f85838cfb299febfa3c2e Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 26 Oct 2023 20:33:51 +0800 Subject: [PATCH 13/15] test:delete zlib cache in ci --- tests/parallel_test/container_build.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 8e309a8182..8652161d17 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -68,7 +68,6 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ # -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ @@ -98,7 +97,6 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ - -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " From 2b0c144c8eeaa8389b7b442fd904c36c78ebd722 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Fri, 27 Oct 2023 09:52:59 +0800 Subject: [PATCH 14/15] Revert "test:delete zlib cache in ci" --- tests/parallel_test/container_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 8652161d17..94704b1c25 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -68,9 +68,9 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ + -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ -# -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ if [[ -d ${WORKDIR}/debugNoSan ]] ;then echo "delete ${WORKDIR}/debugNoSan" @@ -97,10 +97,10 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ + -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " - # -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan ret=$? From e6df389151d25994ff51c9db9d05760f09c384c6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 27 Oct 2023 11:46:59 +0800 Subject: [PATCH 15/15] fix:add req_id for raw block interface --- include/client/taos.h | 3 +++ source/client/src/clientRawBlockWrite.c | 13 +++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index f75a84baa8..45dc85f6d9 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -338,8 +338,11 @@ typedef struct tmq_raw_data { DLL_EXPORT int32_t tmq_get_raw(TAOS_RES *res, tmq_raw_data *raw); DLL_EXPORT int32_t tmq_write_raw(TAOS *taos, tmq_raw_data raw); DLL_EXPORT int taos_write_raw_block(TAOS *taos, int numOfRows, char *pData, const char *tbname); +DLL_EXPORT int taos_write_raw_block_with_reqid(TAOS *taos, int numOfRows, char *pData, const char *tbname, int64_t reqid); DLL_EXPORT int taos_write_raw_block_with_fields(TAOS *taos, int rows, char *pData, const char *tbname, TAOS_FIELD *fields, int numFields); +DLL_EXPORT int taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pData, const char *tbname, + TAOS_FIELD *fields, int numFields, int64_t reqid); DLL_EXPORT void tmq_free_raw(tmq_raw_data raw); // Returning null means error. Returned result need to be freed by tmq_free_json_meta diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index e7ba30d78c..a12dc8d839 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1338,6 +1338,11 @@ end: int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD* fields, int numFields) { + return taos_write_raw_block_with_fields_with_reqid(taos, rows, pData, tbname, fields, numFields, 0); +} + +int taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pData, const char *tbname, + TAOS_FIELD *fields, int numFields, int64_t reqid){ if (!taos || !pData || !tbname) { return TSDB_CODE_INVALID_PARA; } @@ -1347,7 +1352,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch SHashObj* pVgHash = NULL; uDebug("taos_write_raw_block_with_fields called"); - SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); + SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, reqid); if (!pRequest) { uError("WriteRaw:createRequest error request is null"); code = terrno; @@ -1427,6 +1432,10 @@ end: } int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) { + return taos_write_raw_block_with_reqid(taos, rows, pData, tbname, 0); +} + +int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const char* tbname, int64_t reqid) { if (!taos || !pData || !tbname) { return TSDB_CODE_INVALID_PARA; } @@ -1436,7 +1445,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) SHashObj* pVgHash = NULL; uDebug("taos_write_raw_block called"); - SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); + SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, reqid); if (!pRequest) { uError("WriteRaw:createRequest error request is null"); code = terrno;