xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of github.com:taosdata/TDengine into szhou/bi-mode-tag-scan

This commit is contained in:
slzhou 2023-10-30 09:50:38 +08:00
parent 6edf954cd3 40022b48fc
commit 4367975ade
13 changed files with 177 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmake/xml2_CMakeLists.txt.in
View File

@ -1,7 +1,9 @@
# xml2
ExternalProject_Add(xml2
URL https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz
URL https://download.gnome.org/sources/libxml2/2.11/libxml2-2.11.5.tar.xz
URL_HASH SHA256=3727b078c360ec69fa869de14bd6f75d7ee8d36987b071e6928d4720a28df3a6
#https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.tar.gz
#GIT_REPOSITORY https://github.com/GNOME/libxml2
#GIT_TAG v2.11.5
DOWNLOAD_NO_PROGRESS 1
@ -9,7 +11,7 @@ ExternalProject_Add(xml2
SOURCE_DIR "${TD_CONTRIB_DIR}/xml2"
#BINARY_DIR ""
BUILD_IN_SOURCE TRUE
CONFIGURE_COMMAND ./autogen.sh && ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python
CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --enable-shared=no --enable-static=yes --without-python --without-lzma
BUILD_COMMAND make -j
INSTALL_COMMAND make install && ln -s $ENV{HOME}/.cos-local.2/include/libxml2/libxml $ENV{HOME}/.cos-local.2/include/libxml
TEST_COMMAND ""

3
include/client/taos.h
View File

@ -338,8 +338,11 @@ typedef struct tmq_raw_data {
DLL_EXPORT int32_t tmq_get_raw(TAOS_RES *res, tmq_raw_data *raw);
DLL_EXPORT int32_t tmq_write_raw(TAOS *taos, tmq_raw_data raw);
DLL_EXPORT int taos_write_raw_block(TAOS *taos, int numOfRows, char *pData, const char *tbname);
DLL_EXPORT int taos_write_raw_block_with_reqid(TAOS *taos, int numOfRows, char *pData, const char *tbname, int64_t reqid);
DLL_EXPORT int taos_write_raw_block_with_fields(TAOS *taos, int rows, char *pData, const char *tbname,
TAOS_FIELD *fields, int numFields);
DLL_EXPORT int taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pData, const char *tbname,
TAOS_FIELD *fields, int numFields, int64_t reqid);
DLL_EXPORT void tmq_free_raw(tmq_raw_data raw);
// Returning null means error. Returned result need to be freed by tmq_free_json_meta

1
include/common/tglobal.h
View File

@ -107,6 +107,7 @@ extern bool tsMonitorComp;
extern bool tsEnableAudit;
extern char tsAuditFqdn[];
extern uint16_t tsAuditPort;
extern bool tsEnableAuditCreateTable;
// telem
extern bool tsEnableTelem;

3
packaging/cfg/taos.cfg
View File

@ -101,6 +101,9 @@
# enable/disable audit log
# audit 1
# enable/disable audit create table
# auditCreateTable 1
# The following parameter is used to limit the maximum number of lines in log files.
# max number of lines per log filters
# numOfLogLines 10000000

13
source/client/src/clientRawBlockWrite.c
View File

@ -1338,6 +1338,11 @@ end:
int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD* fields,
int numFields) {
return taos_write_raw_block_with_fields_with_reqid(taos, rows, pData, tbname, fields, numFields, 0);
}
int taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pData, const char *tbname,
TAOS_FIELD *fields, int numFields, int64_t reqid){
if (!taos || !pData || !tbname) {
return TSDB_CODE_INVALID_PARA;
}
@ -1347,7 +1352,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
SHashObj* pVgHash = NULL;
uDebug("taos_write_raw_block_with_fields called");
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0);
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, reqid);
if (!pRequest) {
uError("WriteRaw:createRequest error request is null");
code = terrno;
@ -1427,6 +1432,10 @@ end:
}
int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) {
return taos_write_raw_block_with_reqid(taos, rows, pData, tbname, 0);
}
int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const char* tbname, int64_t reqid) {
if (!taos || !pData || !tbname) {
return TSDB_CODE_INVALID_PARA;
}
@ -1436,7 +1445,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
SHashObj* pVgHash = NULL;
uDebug("taos_write_raw_block called");
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0);
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, reqid);
if (!pRequest) {
uError("WriteRaw:createRequest error request is null");
code = terrno;

5
source/common/src/tglobal.c
View File

@ -98,9 +98,10 @@ int32_t tsMonitorMaxLogs = 100;
bool tsMonitorComp = false;
// audit
bool tsEnableAudit = false;
bool tsEnableAudit = true;
char tsAuditFqdn[TSDB_FQDN_LEN] = {0};
uint16_t tsAuditPort = 6043;
bool tsEnableAuditCreateTable = true;
// telem
bool tsEnableTelem = true;
@ -632,6 +633,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddString(pCfg, "auditFqdn", tsAuditFqdn, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "auditPort", tsAuditPort, 1, 65056, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH) != 0) return -1;
if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH) != 0) return -1;
@ -1050,6 +1052,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsEnableAudit = cfgGetItem(pCfg, "audit")->bval;
tstrncpy(tsAuditFqdn, cfgGetItem(pCfg, "auditFqdn")->str, TSDB_FQDN_LEN);
tsAuditPort = (uint16_t)cfgGetItem(pCfg, "auditPort")->i32;
tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval;
tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval;
tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval;

3
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -315,8 +315,7 @@ _CONNECT:
sprintf(obj, "%s:%d", ip, pConn->port);
char detail[1000] = {0};
sprintf(detail, "connType:%d, db:%s, pid:%d, startTime:%" PRId64 ", sVer:%s, app:%s",
connReq.connType, connReq.db, connReq.pid, connReq.startTime, connReq.sVer, connReq.app);
sprintf(detail, "app:%s", connReq.app);
auditRecord(pReq, pMnode->clusterId, "login", connReq.user, obj, detail, strlen(detail));

5
source/dnode/vnode/CMakeLists.txt
View File

@ -173,7 +173,8 @@ if(${BUILD_WITH_S3})
set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2)
find_library(S3_LIBRARY s3)
find_library(CURL_LIBRARY curl)
find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
find_library(XML2_LIBRARY xml2)
find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
target_link_libraries(
@ -184,7 +185,7 @@ if(${BUILD_WITH_S3})
PUBLIC ${CURL_LIBRARY}
PUBLIC ${SSL_LIBRARY}
PUBLIC ${CRYPTO_LIBRARY}
PUBLIC xml2
PUBLIC ${XML2_LIBRARY}
)
add_definitions(-DUSE_S3)

7
source/dnode/vnode/src/tsdb/tsdbCommit2.c
View File

@ -137,7 +137,12 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
}
int64_t ts = TSDBROW_TS(&row->row);
if (ts > committer->ctx->maxKey || skipRow) {
if (skipRow && ts <= committer->ctx->maxKey) {
ts = committer->ctx->maxKey + 1;
}
if (ts > committer->ctx->maxKey) {
committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts);
code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid);
TSDB_CHECK_CODE(code, lino, _exit);

6
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -947,13 +947,13 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
taosArrayPush(rsp.pArray, &cRsp);
if(pCreateReq->sqlLen > 0){ //skip auto create table, not set sql when auto create table
int32_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId;
if(tsEnableAuditCreateTable){
int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId;
SName name = {0};
tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB);
auditRecord(pReq, clusterId, "createTable", name.dbname, pCreateReq->name, pCreateReq->sql, pCreateReq->sqlLen);
auditRecord(NULL, clusterId, "createTable", name.dbname, "", pCreateReq->name, strlen(pCreateReq->name));
}
}

2
tests/system-test/0-others/taosdMonitor.py
View File

@ -268,6 +268,8 @@ class TDTestCase:
updatecfgDict["monitorMaxLogs"] = "10"
updatecfgDict["monitorComp"] = "1"
updatecfgDict["audit"] = '0'
print ("===================: ", updatecfgDict)
def init(self, conn, logSql, replicaVar=1):

2
tests/system-test/7-tmq/tmqVnodeTransform-stb.py
View File

@ -140,7 +140,7 @@ class TDTestCase:
'rowsPerTbl': 10000,
'batchNum': 10,
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
'pollDelay': 30,
'pollDelay': 60,
'showMsg': 1,
'showRow': 1,
'snapshot': 0}

217
tools/shell/src/shellEngine.c
View File

@ -22,6 +22,23 @@
#include "shellAuto.h"
#include "shellInt.h"
typedef struct {
const char *sql;
bool vertical;
tsem_t sem;
int64_t numOfRows; // the num of this batch
int64_t numOfAllRows;
int32_t numFields;
TAOS_FIELD *fields;
int32_t precision;
int32_t maxColNameLen; // for vertical print
int32_t width[TSDB_MAX_COLUMNS]; // for horizontal print
uint64_t resShowMaxNum;
} tsDumpInfo;
static bool shellIsEmptyCommand(const char *cmd);
static int32_t shellRunSingleCommand(char *command);
static void shellRecordCommandToHistory(char *command);
@ -31,8 +48,8 @@ static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision);
static int64_t shellDumpResultToFile(const char *fname, TAOS_RES *tres);
static void shellPrintNChar(const char *str, int32_t length, int32_t width);
static void shellPrintGeometry(const unsigned char *str, int32_t length, int32_t width);
static int64_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql);
static int64_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql);
static void shellVerticalPrintResult(TAOS_RES *tres, tsDumpInfo* dump_info);
static void shellHorizontalPrintResult(TAOS_RES *tres, tsDumpInfo* dump_info);
static int64_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql);
static void shellReadHistory();
static void shellWriteHistory();
@ -702,49 +719,67 @@ bool shellIsShowQuery(const char *sql) {
return false;
}
int64_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql) {
TAOS_ROW row = taos_fetch_row(tres);
if (row == NULL) {
return 0;
void init_dump_info(tsDumpInfo *dump_info, TAOS_RES *tres, const char *sql, bool vertical) {
dump_info->sql = sql;
dump_info->vertical = vertical;
tsem_init(&dump_info->sem, 0, 0);
dump_info->numOfAllRows = 0;
dump_info->numFields = taos_num_fields(tres);
dump_info->fields = taos_fetch_fields(tres);
dump_info->precision = taos_result_precision(tres);
dump_info->resShowMaxNum = UINT64_MAX;
if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsShowWhole(dump_info->sql)) {
dump_info->resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM;
}
int32_t num_fields = taos_num_fields(tres);
TAOS_FIELD *fields = taos_fetch_fields(tres);
int32_t precision = taos_result_precision(tres);
int32_t maxColNameLen = 0;
for (int32_t col = 0; col < num_fields; col++) {
int32_t len = (int32_t)strlen(fields[col].name);
if (len > maxColNameLen) {
maxColNameLen = len;
if (vertical) {
dump_info->maxColNameLen = 0;
for (int32_t col = 0; col < dump_info->numFields; col++) {
int32_t len = (int32_t)strlen(dump_info->fields[col].name);
if (len > dump_info->maxColNameLen) {
dump_info->maxColNameLen = len;
}
}
} else {
for (int32_t col = 0; col < dump_info->numFields; col++) {
dump_info->width[col] = shellCalcColWidth(dump_info->fields + col, dump_info->precision);
}
}
}
uint64_t resShowMaxNum = UINT64_MAX;
if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsShowWhole(sql)) {
resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM;
void shellVerticalPrintResult(TAOS_RES *tres, tsDumpInfo *dump_info) {
TAOS_ROW row = taos_fetch_row(tres);
if (row == NULL) {
printf("\033[31mtaos_fetch_row failed.\033[0m\n");
return;
}
int64_t numOfRows = 0;
int32_t showMore = 1;
do {
if (numOfRows < resShowMaxNum) {
printf("*************************** %"PRId64".row ***************************\r\n", numOfRows + 1);
int64_t numOfPintRows = dump_info->numOfAllRows;
int numOfPrintRowsThisOne = 0;
int32_t *length = taos_fetch_lengths(tres);
while (row != NULL) {
printf("*************************** %" PRId64 ".row ***************************\r\n", numOfPintRows + 1);
for (int32_t i = 0; i < num_fields; i++) {
TAOS_FIELD *field = fields + i;
int32_t *length = taos_fetch_lengths(tres);
int32_t padding = (int32_t)(maxColNameLen - strlen(field->name));
printf("%*.s%s: ", padding, " ", field->name);
for (int32_t i = 0; i < dump_info->numFields; i++) {
TAOS_FIELD *field = dump_info->fields + i;
shellPrintField((const char *)row[i], field, 0, length[i], precision);
putchar('\r');
putchar('\n');
}
} else if (showMore) {
int32_t padding = (int32_t)(dump_info->maxColNameLen - strlen(field->name));
printf("%*.s%s: ", padding, " ", field->name);
shellPrintField((const char *)row[i], field, 0, length[i], dump_info->precision);
putchar('\r');
putchar('\n');
}
numOfPintRows++;
numOfPrintRowsThisOne++;
if (numOfPintRows == dump_info->resShowMaxNum) {
printf("\r\n");
printf(" Notice: The result shows only the first %d rows.\r\n", SHELL_DEFAULT_RES_SHOW_NUM);
printf(" You can use the `LIMIT` clause to get fewer result to show.\r\n");
@ -752,14 +787,16 @@ int64_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql) {
printf("\r\n");
printf(" You can use Ctrl+C to stop the underway fetching.\r\n");
printf("\r\n");
showMore = 0;
return;
}
numOfRows++;
row = taos_fetch_row(tres);
} while (row != NULL);
if (numOfPrintRowsThisOne == dump_info->numOfRows) {
return;
}
return numOfRows;
row = taos_fetch_row(tres);
}
return;
}
int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) {
@ -856,47 +893,38 @@ void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields) {
putchar('\n');
}
int64_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql) {
void shellHorizontalPrintResult(TAOS_RES *tres, tsDumpInfo *dump_info) {
TAOS_ROW row = taos_fetch_row(tres);
if (row == NULL) {
return 0;
printf("\033[31mtaos_fetch_row failed.\033[0m\n");
return;
}
int32_t num_fields = taos_num_fields(tres);
TAOS_FIELD *fields = taos_fetch_fields(tres);
int32_t precision = taos_result_precision(tres);
int32_t width[TSDB_MAX_COLUMNS];
for (int32_t col = 0; col < num_fields; col++) {
width[col] = shellCalcColWidth(fields + col, precision);
int64_t numOfPintRows = dump_info->numOfAllRows;
int numOfPrintRowsThisOne = 0;
if (numOfPintRows == 0) {
shellPrintHeader(dump_info->fields, dump_info->width, dump_info->numFields);
}
shellPrintHeader(fields, width, num_fields);
uint64_t resShowMaxNum = UINT64_MAX;
if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsShowWhole(sql)) {
resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM;
}
int64_t numOfRows = 0;
int32_t showMore = 1;
do {
while (row != NULL) {
int32_t *length = taos_fetch_lengths(tres);
if (numOfRows < resShowMaxNum) {
for (int32_t i = 0; i < num_fields; i++) {
putchar(' ');
shellPrintField((const char *)row[i], fields + i, width[i], length[i], precision);
putchar(' ');
putchar('|');
}
putchar('\r');
putchar('\n');
} else if (showMore) {
for (int32_t i = 0; i < dump_info->numFields; i++) {
putchar(' ');
shellPrintField((const char *)row[i], dump_info->fields + i, dump_info->width[i], length[i],
dump_info->precision);
putchar(' ');
putchar('|');
}
putchar('\r');
putchar('\n');
numOfPintRows++;
numOfPrintRowsThisOne++;
if (numOfPintRows == dump_info->resShowMaxNum) {
printf("\r\n");
printf(" Notice: The result shows only the first %d rows.\r\n", SHELL_DEFAULT_RES_SHOW_NUM);
if (shellIsShowQuery(sql)) {
if (shellIsShowQuery(dump_info->sql)) {
printf(" You can use '>>' to redirect the whole set of the result to a specified file.\r\n");
} else {
printf(" You can use the `LIMIT` clause to get fewer result to show.\r\n");
@ -905,28 +933,53 @@ int64_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql) {
printf("\r\n");
printf(" You can use Ctrl+C to stop the underway fetching.\r\n");
printf("\r\n");
showMore = 0;
return;
}
numOfRows++;
row = taos_fetch_row(tres);
} while (row != NULL);
if (numOfPrintRowsThisOne == dump_info->numOfRows) {
return;
}
return numOfRows;
row = taos_fetch_row(tres);
}
return;
}
void shellDumpResultCallback(void *param, TAOS_RES *tres, int num_of_rows) {
tsDumpInfo *dump_info = (tsDumpInfo *)param;
if (num_of_rows > 0) {
dump_info->numOfRows = num_of_rows;
if (dump_info->numOfAllRows < dump_info->resShowMaxNum) {
if (dump_info->vertical) {
shellVerticalPrintResult(tres, dump_info);
} else {
shellHorizontalPrintResult(tres, dump_info);
}
}
dump_info->numOfAllRows += num_of_rows;
taos_fetch_rows_a(tres, shellDumpResultCallback, param);
} else {
if (num_of_rows < 0) {
printf("\033[31masync retrieve failed, code: %d\033[0m\n", num_of_rows);
}
tsem_post(&dump_info->sem);
}
}
int64_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql) {
int64_t numOfRows = 0;
int64_t num_of_rows = 0;
if (fname != NULL) {
numOfRows = shellDumpResultToFile(fname, tres);
} else if (vertical) {
numOfRows = shellVerticalPrintResult(tres, sql);
num_of_rows = shellDumpResultToFile(fname, tres);
} else {
numOfRows = shellHorizontalPrintResult(tres, sql);
tsDumpInfo dump_info;
init_dump_info(&dump_info, tres, sql, vertical);
taos_fetch_rows_a(tres, shellDumpResultCallback, &dump_info);
tsem_wait(&dump_info.sem);
num_of_rows = dump_info.numOfAllRows;
}
*error_no = taos_errno(tres);
return numOfRows;
return num_of_rows;
}
void shellReadHistory() {