xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into merge/mainto3.0

This commit is contained in:
Shengliang Guan 2025-01-02 17:37:00 +08:00
parent 0b62edf67a d81931569d
commit 1024bfdc43
33 changed files with 1707 additions and 599 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
include/common/tmsg.h
View File

@ -482,6 +482,7 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_STREAM_ANOMALY,
QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC,
QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERP_FUNC,
QUERY_NODE_RESET_STREAM_STMT,
} ENodeType;
typedef struct {
@ -3915,6 +3916,15 @@ typedef struct {
int32_t tSerializeSMResumeStreamReq(void* buf, int32_t bufLen, const SMResumeStreamReq* pReq);
int32_t tDeserializeSMResumeStreamReq(void* buf, int32_t bufLen, SMResumeStreamReq* pReq);
typedef struct {
char name[TSDB_STREAM_FNAME_LEN];
int8_t igNotExists;
int8_t igUntreated;
} SMResetStreamReq;
int32_t tSerializeSMResetStreamReq(void* buf, int32_t bufLen, const SMResetStreamReq* pReq);
int32_t tDeserializeSMResetStreamReq(void* buf, int32_t bufLen, SMResetStreamReq* pReq);
typedef struct {
char name[TSDB_TABLE_FNAME_LEN];
char stb[TSDB_TABLE_FNAME_LEN];

1
include/common/tmsgdef.h
View File

@ -261,6 +261,7 @@
TD_DEF_MSG_TYPE(TDMT_MND_UPDATE_DNODE_INFO, "update-dnode-info", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_AUDIT, "audit", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CONFIG, "init-config", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_RESET_STREAM, "reset-stream", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG)
TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8

6
include/libs/nodes/cmdnodes.h
View File

@ -600,6 +600,12 @@ typedef struct SResumeStreamStmt {
bool ignoreUntreated;
} SResumeStreamStmt;
typedef struct SResetStreamStmt {
ENodeType type;
char streamName[TSDB_TABLE_NAME_LEN];
bool ignoreNotExists;
} SResetStreamStmt;
typedef struct SCreateFunctionStmt {
ENodeType type;
bool orReplace;

2
include/util/tconfig.h
View File

@ -160,8 +160,6 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl
SArray *taosGetLocalCfg(SConfig *pCfg);
SArray *taosGetGlobalCfg(SConfig *pCfg);
void taosSetLocalCfg(SConfig *pCfg, SArray *pArray);
void taosSetGlobalCfg(SConfig *pCfg, SArray *pArray);
#ifdef __cplusplus
}
#endif

19
include/util/tutil.h
View File

@ -187,6 +187,15 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
} \
} while (0)
#define TAOS_CHECK_RETURN_SET_CODE(CMD, CODE, ERRNO) \
do { \
int32_t __c = (CMD); \
if (__c != TSDB_CODE_SUCCESS) { \
(CODE) = (ERRNO); \
TAOS_RETURN(__c); \
} \
} while (0)
#define TAOS_CHECK_RETURN_WITH_RELEASE(CMD, PTR1, PTR2) \
do { \
int32_t __c = (CMD); \
@ -225,6 +234,16 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
} \
} while (0)
#define TAOS_CHECK_EXIT_SET_CODE(CMD, CODE, ERRNO) \
do { \
code = (CMD); \
if (code < TSDB_CODE_SUCCESS) { \
(CODE) = (ERRNO); \
lino = __LINE__; \
goto _exit; \
} \
} while (0)
#define TAOS_UNUSED(expr) (void)(expr)
bool taosIsBigChar(char c);

3
source/client/inc/clientStmt2.h
View File

@ -221,11 +221,8 @@ int stmtPrepare2(TAOS_STMT2 *stmt, const char *sql, unsigned long length
int stmtSetTbName2(TAOS_STMT2 *stmt, const char *tbName);
int stmtSetTbTags2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *tags);
int stmtBindBatch2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *bind, int32_t colIdx);
int stmtGetTagFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields);
int stmtGetColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields);
int stmtGetStbColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_ALL **fields);
int stmtGetParamNum2(TAOS_STMT2 *stmt, int *nums);
int stmtGetParamTbName(TAOS_STMT2 *stmt, int *nums);
int stmtIsInsert2(TAOS_STMT2 *stmt, int *insert);
TAOS_RES *stmtUseResult2(TAOS_STMT2 *stmt);
const char *stmtErrstr2(TAOS_STMT2 *stmt);

82
source/client/src/clientMain.c
View File

@ -24,13 +24,13 @@
#include "query.h"
#include "scheduler.h"
#include "tcompare.h"
#include "tconv.h"
#include "tdatablock.h"
#include "tglobal.h"
#include "tmsg.h"
#include "tref.h"
#include "trpc.h"
#include "version.h"
#include "tconv.h"
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
@ -56,12 +56,12 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) {
}
#ifndef WINDOWS
static void freeTz(void *p){
static void freeTz(void *p) {
timezone_t tz = *(timezone_t *)p;
tzfree(tz);
}
int32_t tzInit(){
int32_t tzInit() {
pTimezoneMap = taosHashInit(0, MurmurHash3_32, false, HASH_ENTRY_LOCK);
if (pTimezoneMap == NULL) {
return terrno;
@ -75,15 +75,15 @@ int32_t tzInit(){
return 0;
}
void tzCleanup(){
void tzCleanup() {
taosHashCleanup(pTimezoneMap);
taosHashCleanup(pTimezoneNameMap);
}
static timezone_t setConnnectionTz(const char* val){
timezone_t tz = NULL;
static timezone_t setConnnectionTz(const char *val) {
timezone_t tz = NULL;
timezone_t *tmp = taosHashGet(pTimezoneMap, val, strlen(val));
if (tmp != NULL && *tmp != NULL){
if (tmp != NULL && *tmp != NULL) {
tz = *tmp;
goto END;
}
@ -100,20 +100,20 @@ static timezone_t setConnnectionTz(const char* val){
}
}
int32_t code = taosHashPut(pTimezoneMap, val, strlen(val), &tz, sizeof(timezone_t));
if (code != 0){
if (code != 0) {
tscError("%s put timezone to tz map error:%d", __func__, code);
tzfree(tz);
tz = NULL;
goto END;
}
time_t tx1 = taosGetTimestampSec();
char output[TD_TIMEZONE_LEN] = {0};
time_t tx1 = taosGetTimestampSec();
char output[TD_TIMEZONE_LEN] = {0};
code = taosFormatTimezoneStr(tx1, val, tz, output);
if (code == 0){
if (code == 0) {
code = taosHashPut(pTimezoneNameMap, &tz, sizeof(timezone_t), output, strlen(output) + 1);
}
if (code != 0){
if (code != 0) {
tscError("failed to put timezone %s to map", val);
}
@ -122,18 +122,18 @@ END:
}
#endif
static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, const char* val){
static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, const char *val) {
if (taos == NULL) {
return terrno = TSDB_CODE_INVALID_PARA;
}
#ifdef WINDOWS
if (option == TSDB_OPTION_CONNECTION_TIMEZONE){
if (option == TSDB_OPTION_CONNECTION_TIMEZONE) {
return terrno = TSDB_CODE_NOT_SUPPORTTED_IN_WINDOWS;
}
#endif
if (option < TSDB_OPTION_CONNECTION_CLEAR || option >= TSDB_MAX_OPTIONS_CONNECTION){
if (option < TSDB_OPTION_CONNECTION_CLEAR || option >= TSDB_MAX_OPTIONS_CONNECTION) {
return terrno = TSDB_CODE_INVALID_PARA;
}
@ -149,7 +149,7 @@ static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, co
return terrno;
}
if (option == TSDB_OPTION_CONNECTION_CLEAR){
if (option == TSDB_OPTION_CONNECTION_CLEAR) {
val = NULL;
}
@ -165,19 +165,19 @@ static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, co
goto END;
}
pObj->optionInfo.charsetCxt = tmp;
}else{
} else {
pObj->optionInfo.charsetCxt = NULL;
}
}
if (option == TSDB_OPTION_CONNECTION_TIMEZONE || option == TSDB_OPTION_CONNECTION_CLEAR) {
#ifndef WINDOWS
if (val != NULL){
if (val[0] == 0){
if (val != NULL) {
if (val[0] == 0) {
val = "UTC";
}
timezone_t tz = setConnnectionTz(val);
if (tz == NULL){
if (tz == NULL) {
code = terrno;
goto END;
}
@ -199,7 +199,7 @@ static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, co
if (option == TSDB_OPTION_CONNECTION_USER_IP || option == TSDB_OPTION_CONNECTION_CLEAR) {
if (val != NULL) {
pObj->optionInfo.userIp = taosInetAddr(val);
if (pObj->optionInfo.userIp == INADDR_NONE){
if (pObj->optionInfo.userIp == INADDR_NONE) {
code = TSDB_CODE_INVALID_PARA;
goto END;
}
@ -213,7 +213,7 @@ END:
return terrno = code;
}
int taos_options_connection(TAOS *taos, TSDB_OPTION_CONNECTION option, const void *arg, ...){
int taos_options_connection(TAOS *taos, TSDB_OPTION_CONNECTION option, const void *arg, ...) {
return setConnectionOption(taos, option, (const char *)arg);
}
@ -2129,6 +2129,11 @@ int taos_stmt_close(TAOS_STMT *stmt) {
}
TAOS_STMT2 *taos_stmt2_init(TAOS *taos, TAOS_STMT2_OPTION *option) {
if (NULL == taos) {
tscError("NULL parameter for %s", __FUNCTION__);
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
STscObj *pObj = acquireTscObj(*(int64_t *)taos);
if (NULL == pObj) {
tscError("invalid parameter for %s", __FUNCTION__);
@ -2257,16 +2262,7 @@ int taos_stmt2_close(TAOS_STMT2 *stmt) {
return stmtClose2(stmt);
}
/*
int taos_stmt2_param_count(TAOS_STMT2 *stmt, int *nums) {
if (stmt == NULL || nums == NULL) {
tscError("NULL parameter for %s", __FUNCTION__);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return stmtGetParamNum2(stmt, nums);
}
*/
int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert) {
if (stmt == NULL || insert == NULL) {
tscError("NULL parameter for %s", __FUNCTION__);
@ -2277,28 +2273,6 @@ int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert) {
return stmtIsInsert2(stmt, insert);
}
// int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields) {
// if (stmt == NULL || count == NULL) {
// tscError("NULL parameter for %s", __FUNCTION__);
// terrno = TSDB_CODE_INVALID_PARA;
// return terrno;
// }
// if (field_type == TAOS_FIELD_COL) {
// return stmtGetColFields2(stmt, count, fields);
// } else if (field_type == TAOS_FIELD_TAG) {
// return stmtGetTagFields2(stmt, count, fields);
// } else if (field_type == TAOS_FIELD_QUERY) {
// return stmtGetParamNum2(stmt, count);
// } else if (field_type == TAOS_FIELD_TBNAME) {
// return stmtGetParamTbName(stmt, count);
// } else {
// tscError("invalid parameter for %s", __FUNCTION__);
// terrno = TSDB_CODE_INVALID_PARA;
// return terrno;
// }
// }
int taos_stmt2_get_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields) {
if (stmt == NULL || count == NULL) {
tscError("NULL parameter for %s", __FUNCTION__);

161
source/client/src/clientStmt2.c
View File

@ -1037,28 +1037,6 @@ int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) {
return TSDB_CODE_SUCCESS;
}
static int stmtFetchTagFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields) {
if (pStmt->errCode != TSDB_CODE_SUCCESS) {
return pStmt->errCode;
}
if (STMT_TYPE_QUERY == pStmt->sql.type) {
tscError("invalid operation to get query tag fileds");
STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
}
STableDataCxt** pDataBlock =
(STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
if (NULL == pDataBlock) {
tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName);
STMT_ERR_RET(TSDB_CODE_APP_ERROR);
}
STMT_ERR_RET(qBuildStmtTagFields(*pDataBlock, pStmt->bInfo.boundTags, fieldNum, fields));
return TSDB_CODE_SUCCESS;
}
static int stmtFetchColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields) {
if (pStmt->errCode != TSDB_CODE_SUCCESS) {
return pStmt->errCode;
@ -1820,47 +1798,6 @@ int stmtAffectedRows(TAOS_STMT* stmt) { return ((STscStmt2*)stmt)->affectedRows;
int stmtAffectedRowsOnce(TAOS_STMT* stmt) { return ((STscStmt2*)stmt)->exec.affectedRows; }
*/
int stmtGetTagFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_E** fields) {
int32_t code = 0;
STscStmt2* pStmt = (STscStmt2*)stmt;
int32_t preCode = pStmt->errCode;
STMT_DLOG_E("start to get tag fields");
if (pStmt->errCode != TSDB_CODE_SUCCESS) {
return pStmt->errCode;
}
if (STMT_TYPE_QUERY == pStmt->sql.type) {
STMT_ERRI_JRET(TSDB_CODE_TSC_STMT_API_ERROR);
}
STMT_ERRI_JRET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 &&
STMT_TYPE_MULTI_INSERT != pStmt->sql.type) {
pStmt->bInfo.needParse = false;
}
if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) {
taos_free_result(pStmt->exec.pRequest);
pStmt->exec.pRequest = NULL;
}
STMT_ERRI_JRET(stmtCreateRequest(pStmt));
if (pStmt->bInfo.needParse) {
STMT_ERRI_JRET(stmtParseSql(pStmt));
}
STMT_ERRI_JRET(stmtFetchTagFields2(stmt, nums, fields));
_return:
pStmt->errCode = preCode;
return code;
}
int stmtParseColFields2(TAOS_STMT2* stmt) {
int32_t code = 0;
@ -1903,15 +1840,6 @@ _return:
return code;
}
int stmtGetColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_E** fields) {
int32_t code = stmtParseColFields2(stmt);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
return stmtFetchColFields2(stmt, nums, fields);
}
int stmtGetStbColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_ALL** fields) {
int32_t code = stmtParseColFields2(stmt);
if (code != TSDB_CODE_SUCCESS) {
@ -1957,95 +1885,6 @@ int stmtGetParamNum2(TAOS_STMT2* stmt, int* nums) {
return TSDB_CODE_SUCCESS;
}
int stmtGetParamTbName(TAOS_STMT2* stmt, int* nums) {
STscStmt2* pStmt = (STscStmt2*)stmt;
int32_t code = 0;
int32_t preCode = pStmt->errCode;
STMT_DLOG_E("start to get param num");
if (pStmt->errCode != TSDB_CODE_SUCCESS) {
return pStmt->errCode;
}
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 &&
STMT_TYPE_MULTI_INSERT != pStmt->sql.type) {
pStmt->bInfo.needParse = false;
}
if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) {
taos_free_result(pStmt->exec.pRequest);
pStmt->exec.pRequest = NULL;
}
STMT_ERR_RET(stmtCreateRequest(pStmt));
if (pStmt->bInfo.needParse) {
STMT_ERRI_JRET(stmtParseSql(pStmt));
}
*nums = STMT_TYPE_MULTI_INSERT == pStmt->sql.type ? 1 : 0;
_return:
if (TSDB_CODE_TSC_STMT_TBNAME_ERROR == code) {
*nums = 1;
code = TSDB_CODE_SUCCESS;
}
pStmt->errCode = preCode;
return code;
}
/*
int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) {
STscStmt2* pStmt = (STscStmt2*)stmt;
STMT_DLOG_E("start to get param");
if (pStmt->errCode != TSDB_CODE_SUCCESS) {
return pStmt->errCode;
}
if (STMT_TYPE_QUERY == pStmt->sql.type) {
STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
}
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 &&
STMT_TYPE_MULTI_INSERT != pStmt->sql.type) {
pStmt->bInfo.needParse = false;
}
if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) {
taos_free_result(pStmt->exec.pRequest);
pStmt->exec.pRequest = NULL;
}
STMT_ERR_RET(stmtCreateRequest(pStmt));
if (pStmt->bInfo.needParse) {
STMT_ERR_RET(stmtParseSql(pStmt));
}
int32_t nums = 0;
TAOS_FIELD_E* pField = NULL;
STMT_ERR_RET(stmtFetchColFields(stmt, &nums, &pField));
if (idx >= nums) {
tscError("idx %d is too big", idx);
taosMemoryFree(pField);
STMT_ERR_RET(TSDB_CODE_INVALID_PARA);
}
*type = pField[idx].type;
*bytes = pField[idx].bytes;
taosMemoryFree(pField);
return TSDB_CODE_SUCCESS;
}
*/
TAOS_RES* stmtUseResult2(TAOS_STMT2* stmt) {
STscStmt2* pStmt = (STscStmt2*)stmt;

19
source/client/test/CMakeLists.txt
View File

@ -41,6 +41,12 @@ TARGET_LINK_LIBRARIES(
PUBLIC ${TAOS_LIB}
)
ADD_EXECUTABLE(stmt2Test stmt2Test.cpp)
TARGET_LINK_LIBRARIES(
stmt2Test
os util common transport parser catalog scheduler gtest ${TAOS_LIB_STATIC} qcom executor function
)
TARGET_INCLUDE_DIRECTORIES(
clientTest
PUBLIC "${TD_SOURCE_DIR}/include/client/"
@ -62,6 +68,10 @@ IF(${TD_LINUX})
NAME connectOptionsTest
COMMAND connectOptionsTest
)
add_test(
NAME stmt2Test
COMMAND stmt2Test
)
ENDIF ()
TARGET_INCLUDE_DIRECTORIES(
@ -82,6 +92,12 @@ TARGET_INCLUDE_DIRECTORIES(
# PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
#)
TARGET_INCLUDE_DIRECTORIES(
stmt2Test
PUBLIC "${TD_SOURCE_DIR}/include/client/"
PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
)
add_test(
NAME smlTest
COMMAND smlTest
@ -95,5 +111,4 @@ add_test(
add_test(
NAME userOperTest
COMMAND userOperTest
)
)

827
source/client/test/stmt2Test.cpp Normal file
View File

@ -0,0 +1,827 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <string.h>
#include "clientInt.h"
#include "osSemaphore.h"
#include "taoserror.h"
#include "tglobal.h"
#include "thash.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "../inc/clientStmt.h"
#include "../inc/clientStmt2.h"
#include "executor.h"
#include "taos.h"
namespace {
void checkRows(TAOS* pConn, const char* sql, int32_t expectedRows) {
TAOS_RES* pRes = taos_query(pConn, sql);
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
TAOS_ROW pRow = NULL;
int rows = 0;
while ((pRow = taos_fetch_row(pRes)) != NULL) {
rows++;
}
ASSERT_EQ(rows, expectedRows);
taos_free_result(pRes);
}
void stmtAsyncQueryCb(void* param, TAOS_RES* pRes, int code) {
int affected_rows = taos_affected_rows(pRes);
return;
}
void getFieldsSuccess(TAOS* taos, const char* sql, TAOS_FIELD_ALL* expectedFields, int expectedFieldNum) {
TAOS_STMT2_OPTION option = {0};
TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
int code = taos_stmt2_prepare(stmt, sql, 0);
ASSERT_EQ(code, 0);
int fieldNum = 0;
TAOS_FIELD_ALL* pFields = NULL;
code = taos_stmt2_get_fields(stmt, &fieldNum, &pFields);
ASSERT_EQ(code, 0);
ASSERT_EQ(fieldNum, expectedFieldNum);
for (int i = 0; i < fieldNum; i++) {
ASSERT_STREQ(pFields[i].name, expectedFields[i].name);
ASSERT_EQ(pFields[i].type, expectedFields[i].type);
ASSERT_EQ(pFields[i].field_type, expectedFields[i].field_type);
ASSERT_EQ(pFields[i].precision, expectedFields[i].precision);
ASSERT_EQ(pFields[i].bytes, expectedFields[i].bytes);
ASSERT_EQ(pFields[i].scale, expectedFields[i].scale);
}
taos_stmt2_free_fields(stmt, pFields);
taos_stmt2_close(stmt);
}
void getFieldsError(TAOS* taos, const char* sql, int errorCode) {
TAOS_STMT2_OPTION option = {0};
TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
int code = taos_stmt2_prepare(stmt, sql, 0);
ASSERT_EQ(code, 0);
int fieldNum = 0;
TAOS_FIELD_ALL* pFields = NULL;
code = taos_stmt2_get_fields(stmt, &fieldNum, &pFields);
ASSERT_EQ(code, errorCode);
taos_stmt2_free_fields(stmt, pFields);
taos_stmt2_close(stmt);
}
void getQueryFields(TAOS* taos, const char* sql, int expectedFieldNum) {
TAOS_STMT2_OPTION option = {0};
TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
int code = taos_stmt2_prepare(stmt, sql, 0);
ASSERT_EQ(code, 0);
int fieldNum = 0;
TAOS_FIELD_ALL* pFields = NULL;
code = taos_stmt2_get_fields(stmt, &fieldNum, NULL);
ASSERT_EQ(code, 0);
ASSERT_EQ(fieldNum, expectedFieldNum);
taos_stmt2_free_fields(stmt, NULL);
taos_stmt2_close(stmt);
}
void do_query(TAOS* taos, const char* sql) {
TAOS_RES* result = taos_query(taos, sql);
int code = taos_errno(result);
ASSERT_EQ(code, 0);
taos_free_result(result);
}
void do_stmt(TAOS* taos, const char* sql, int CTB_NUMS, int ROW_NUMS, int CYC_NUMS, bool createTable) {
do_query(taos, "drop database if exists db");
do_query(taos, "create database db");
do_query(taos, "create table db.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))");
do_query(taos, "use db");
TAOS_STMT2_OPTION option = {0, true, true, NULL, NULL};
TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
ASSERT_NE(stmt, nullptr);
int code = taos_stmt2_prepare(stmt, sql, 0);
ASSERT_EQ(code, 0);
ASSERT_EQ(terrno, 0);
// tbname
char** tbs = (char**)taosMemoryMalloc(CTB_NUMS * sizeof(char*));
for (int i = 0; i < CTB_NUMS; i++) {
tbs[i] = (char*)taosMemoryMalloc(sizeof(char) * 20);
sprintf(tbs[i], "ctb_%d", i);
if (createTable) {
char* tmp = (char*)taosMemoryMalloc(sizeof(char) * 100);
sprintf(tmp, "create table db.%s using db.stb tags(0, 'after')", tbs[i]);
do_query(taos, tmp);
}
}
for (int r = 0; r < CYC_NUMS; r++) {
// col params
int64_t** ts = (int64_t**)taosMemoryMalloc(CTB_NUMS * sizeof(int64_t*));
char** b = (char**)taosMemoryMalloc(CTB_NUMS * sizeof(char*));
int* ts_len = (int*)taosMemoryMalloc(ROW_NUMS * sizeof(int));
int* b_len = (int*)taosMemoryMalloc(ROW_NUMS * sizeof(int));
for (int i = 0; i < ROW_NUMS; i++) {
ts_len[i] = sizeof(int64_t);
b_len[i] = 1;
}
for (int i = 0; i < CTB_NUMS; i++) {
ts[i] = (int64_t*)taosMemoryMalloc(ROW_NUMS * sizeof(int64_t));
b[i] = (char*)taosMemoryMalloc(ROW_NUMS * sizeof(char));
for (int j = 0; j < ROW_NUMS; j++) {
ts[i][j] = 1591060628000 + r * 100000 + j;
b[i][j] = 'a' + j;
}
}
// tag params
int t1 = 0;
int t1len = sizeof(int);
int t2len = 3;
// TAOS_STMT2_BIND* tagv[2] = {&tags[0][0], &tags[1][0]};
// bind params
TAOS_STMT2_BIND** paramv = (TAOS_STMT2_BIND**)taosMemoryMalloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*));
TAOS_STMT2_BIND** tags = (TAOS_STMT2_BIND**)taosMemoryMalloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*));
for (int i = 0; i < CTB_NUMS; i++) {
// create tags
tags[i] = (TAOS_STMT2_BIND*)taosMemoryMalloc(2 * sizeof(TAOS_STMT2_BIND));
tags[i][0] = {TSDB_DATA_TYPE_INT, &t1, &t1len, NULL, 0};
tags[i][1] = {TSDB_DATA_TYPE_BINARY, (void*)"after", &t2len, NULL, 0};
// create col params
paramv[i] = (TAOS_STMT2_BIND*)taosMemoryMalloc(2 * sizeof(TAOS_STMT2_BIND));
paramv[i][0] = {TSDB_DATA_TYPE_TIMESTAMP, &ts[i][0], &ts_len[0], NULL, ROW_NUMS};
paramv[i][1] = {TSDB_DATA_TYPE_BINARY, &b[i][0], &b_len[0], NULL, ROW_NUMS};
}
// bind
TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, tags, paramv};
code = taos_stmt2_bind_param(stmt, &bindv, -1);
ASSERT_EQ(code, 0);
ASSERT_EQ(errno, 0);
// exec
code = taos_stmt2_exec(stmt, NULL);
ASSERT_EQ(code, 0);
ASSERT_EQ(errno, 0);
for (int i = 0; i < CTB_NUMS; i++) {
taosMemoryFree(tags[i]);
taosMemoryFree(paramv[i]);
taosMemoryFree(ts[i]);
taosMemoryFree(b[i]);
}
taosMemoryFree(ts);
taosMemoryFree(b);
taosMemoryFree(ts_len);
taosMemoryFree(b_len);
taosMemoryFree(paramv);
taosMemoryFree(tags);
}
checkRows(taos, "select * from db.stb", CYC_NUMS * ROW_NUMS * CTB_NUMS);
for (int i = 0; i < CTB_NUMS; i++) {
taosMemoryFree(tbs[i]);
}
taosMemoryFree(tbs);
taos_stmt2_close(stmt);
}
} // namespace
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
TEST(clientCase, driverInit_Test) {
// taosInitGlobalCfg();
// taos_init();
}
TEST(stmt2Case, insert_stb_get_fields_Test) {
TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(taos, nullptr);
do_query(taos, "drop database if exists db");
do_query(taos, "create database db PRECISION 'ns'");
do_query(taos,
"create table db.stb (ts timestamp, b binary(10)) tags(t1 "
"int, t2 binary(10))");
do_query(
taos,
"create table if not exists db.all_stb(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 "
"tinyint unsigned, v7 smallint unsigned, v8 int unsigned, v9 bigint unsigned, v10 float, v11 double, v12 "
"binary(20), v13 varbinary(20), v14 geometry(100), v15 nchar(20))tags(tts timestamp, tv1 bool, tv2 tinyint, tv3 "
"smallint, tv4 int, tv5 bigint, tv6 tinyint unsigned, tv7 smallint unsigned, tv8 int unsigned, tv9 bigint "
"unsigned, tv10 float, tv11 double, tv12 binary(20), tv13 varbinary(20), tv14 geometry(100), tv15 nchar(20));");
printf("support case \n");
// case 1 : test super table
{
const char* sql = "insert into db.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)";
TAOS_FIELD_ALL expectedFields[5] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG},
{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG},
{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL},
{"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL},
{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}};
printf("case 1 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 5);
}
{
// case 2 : no tag
const char* sql = "insert into db.stb(ts,b,tbname) values(?,?,?)";
TAOS_FIELD_ALL expectedFields[3] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL},
{"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL},
{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}};
printf("case 2 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 3);
}
// case 3 : random order
{
const char* sql = "insert into db.stb(tbname,ts,t2,b,t1) values(?,?,?,?,?)";
TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME},
{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL},
{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG},
{"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL},
{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}};
printf("case 3 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 5);
}
// case 4 : random order 2
{
const char* sql = "insert into db.stb(ts,tbname,b,t2,t1) values(?,?,?,?,?)";
TAOS_FIELD_ALL expectedFields[5] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL},
{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME},
{"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL},
{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG},
{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}};
printf("case 4 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 5);
}
// case 5 : 'db'.'stb'
{
const char* sql = "insert into 'db'.'stb'(t1,t2,ts,b,tbname) values(?,?,?,?,?)";
TAOS_FIELD_ALL expectedFields[5] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG},
{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG},
{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL},
{"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL},
{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}};
printf("case 5 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 5);
}
// case 6 : use db
{
do_query(taos, "use db");
const char* sql = "insert into stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)";
TAOS_FIELD_ALL expectedFields[5] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG},
{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG},
{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL},
{"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL},
{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}};
printf("case 6 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 5);
}
// case 7 : less param
{
const char* sql = "insert into db.stb(ts,tbname) values(?,?)";
TAOS_FIELD_ALL expectedFields[2] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL},
{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}};
printf("case 7 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 2);
}
// case 8 : test all types
{
const char* sql =
"insert into "
"all_stb(tbname,tts,tv1,tv2,tv3,tv4,tv5,tv6,tv7,tv8,tv9,tv10,tv11,tv12,tv13,tv14,tv15,ts,v1,v2,v3,v4,v5,v6,v7,"
"v8,v9,v10,"
"v11,v12,v13,v14,v15) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
TAOS_FIELD_ALL expectedFields[33] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME},
{"tts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_TAG},
{"tv1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_TAG},
{"tv2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_TAG},
{"tv3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_TAG},
{"tv4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG},
{"tv5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_TAG},
{"tv6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_TAG},
{"tv7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_TAG},
{"tv8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_TAG},
{"tv9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_TAG},
{"tv10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_TAG},
{"tv11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_TAG},
{"tv12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_TAG},
{"tv13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_TAG},
{"tv14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_TAG},
{"tv15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_TAG},
{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL},
{"v1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_COL},
{"v2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_COL},
{"v3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_COL},
{"v4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL},
{"v5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_COL},
{"v6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_COL},
{"v7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_COL},
{"v8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_COL},
{"v9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_COL},
{"v10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_COL},
{"v11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_COL},
{"v12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_COL},
{"v13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_COL},
{"v14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_COL},
{"v15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_COL}};
printf("case 8 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 33);
}
// not support case
printf("not support case \n");
// case 5 : add in main TD-33353
{
const char* sql = "insert into db.stb(t1,t2,ts,b,tbname) values(1,?,?,'abc',?)";
printf("case 2 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION);
}
// case 2 : no pk
{
const char* sql = "insert into db.stb(b,tbname) values(?,?)";
printf("case 2 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION);
}
// case 3 : no tbname and tag(not support bind)
{
const char* sql = "insert into db.stb(ts,b) values(?,?)";
printf("case 3 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION);
}
// case 4 : no col and tag(not support bind)
{
const char* sql = "insert into db.stb(tbname) values(?)";
printf("case 4 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION);
}
// case 5 : no field name
{
const char* sql = "insert into db.stb(?,?,?,?,?)";
printf("case 5 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR);
}
// case 6 : test super table not exist
{
const char* sql = "insert into db.nstb(?,?,?,?,?)";
printf("case 6 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR);
}
// case 7 : no col
{
const char* sql = "insert into db.stb(t1,t2,tbname) values(?,?,?)";
printf("case 7 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION);
}
// case 8 : wrong para nums
{
const char* sql = "insert into db.stb(ts,b,tbname) values(?,?,?,?,?)";
printf("case 8 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_PAR_INVALID_COLUMNS_NUM);
}
// case 9 : wrong simbol
{
const char* sql = "insert into db.stb(t1,t2,ts,b,tbname) values(*,*,*,*,*)";
printf("case 9 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION);
}
taos_close(taos);
}
TEST(stmt2Case, insert_ctb_using_get_fields_Test) {
TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(taos, nullptr);
do_query(taos, "drop database if exists db");
do_query(taos, "create database db PRECISION 'ns'");
do_query(taos,
"create table db.stb (ts timestamp, b binary(10)) tags(t1 "
"int, t2 binary(10))");
do_query(
taos,
"create table if not exists db.all_stb(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 "
"tinyint unsigned, v7 smallint unsigned, v8 int unsigned, v9 bigint unsigned, v10 float, v11 double, v12 "
"binary(20), v13 varbinary(20), v14 geometry(100), v15 nchar(20))tags(tts timestamp, tv1 bool, tv2 tinyint, tv3 "
"smallint, tv4 int, tv5 bigint, tv6 tinyint unsigned, tv7 smallint unsigned, tv8 int unsigned, tv9 bigint "
"unsigned, tv10 float, tv11 double, tv12 binary(20), tv13 varbinary(20), tv14 geometry(100), tv15 nchar(20));");
do_query(taos, "CREATE TABLE db.t0 USING db.stb (t1,t2) TAGS (7,'Cali');");
printf("support case \n");
// case 1 : test child table already exist
{
const char* sql = "INSERT INTO db.t0(ts,b)using db.stb (t1,t2) TAGS(?,?) VALUES (?,?)";
TAOS_FIELD_ALL expectedFields[4] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG},
{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG},
{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL},
{"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}};
printf("case 1 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 4);
}
// case 2 : insert clause
{
const char* sql = "INSERT INTO db.? using db.stb (t1,t2) TAGS(?,?) (ts,b)VALUES(?,?)";
TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME},
{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG},
{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG},
{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL},
{"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}};
printf("case 2 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 5);
}
// case 3 : insert child table not exist
{
const char* sql = "INSERT INTO db.d1 using db.stb (t1,t2)TAGS(?,?) (ts,b)VALUES(?,?)";
TAOS_FIELD_ALL expectedFields[4] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG},
{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG},
{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL},
{"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}};
printf("case 3 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 4);
}
// case 4 : random order
{
const char* sql = "INSERT INTO db.? using db.stb (t2,t1)TAGS(?,?) (b,ts)VALUES(?,?)";
TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME},
{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG},
{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG},
{"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL},
{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}};
printf("case 4 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 5);
}
// case 5 : less para
{
const char* sql = "insert into db.? using db.stb (t2)tags(?) (ts)values(?)";
TAOS_FIELD_ALL expectedFields[3] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME},
{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG},
{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}};
printf("case 5 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 3);
}
// case 6 : insert into db.? using db.stb tags(?, ?) values(?,?)
// no field name
{
const char* sql = "insert into db.? using db.stb tags(?, ?) values(?,?)";
TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME},
{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG},
{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG},
{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL},
{"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}};
printf("case 6 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 5);
}
// case 7 : insert into db.d0 (ts)values(?)
// less para
{
const char* sql = "insert into db.t0 (ts)values(?)";
TAOS_FIELD_ALL expectedFields[1] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}};
printf("case 7 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 1);
}
// case 8 : 'db' 'stb'
{
const char* sql = "INSERT INTO 'db'.? using 'db'.'stb' (t1,t2) TAGS(?,?) (ts,b)VALUES(?,?)";
TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME},
{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG},
{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG},
{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL},
{"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}};
printf("case 8 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 5);
}
// case 9 : use db
{
do_query(taos, "use db");
const char* sql = "INSERT INTO ? using stb (t1,t2) TAGS(?,?) (ts,b)VALUES(?,?)";
TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME},
{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG},
{"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG},
{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL},
{"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}};
printf("case 9 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 5);
}
// case 10 : test all types
{
const char* sql =
"insert into ? using all_stb tags(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
TAOS_FIELD_ALL expectedFields[33] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME},
{"tts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_TAG},
{"tv1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_TAG},
{"tv2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_TAG},
{"tv3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_TAG},
{"tv4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG},
{"tv5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_TAG},
{"tv6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_TAG},
{"tv7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_TAG},
{"tv8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_TAG},
{"tv9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_TAG},
{"tv10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_TAG},
{"tv11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_TAG},
{"tv12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_TAG},
{"tv13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_TAG},
{"tv14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_TAG},
{"tv15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_TAG},
{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL},
{"v1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_COL},
{"v2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_COL},
{"v3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_COL},
{"v4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL},
{"v5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_COL},
{"v6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_COL},
{"v7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_COL},
{"v8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_COL},
{"v9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_COL},
{"v10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_COL},
{"v11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_COL},
{"v12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_COL},
{"v13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_COL},
{"v14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_COL},
{"v15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_COL}};
printf("case 10 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 33);
}
printf("not support case \n");
// case 1 : test super table not exist
{
const char* sql = "INSERT INTO db.?(ts,b)using db.nstb (t1,t2) TAGS(?,?) VALUES (?,?)";
printf("case 1 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR);
}
// case 2 : no pk
{
const char* sql = "INSERT INTO db.?(ts,b)using db.nstb (t1,t2) TAGS(?,?) (n)VALUES (?)";
printf("case 2 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR);
}
// case 3 : less param and no filed name
{
const char* sql = "INSERT INTO db.?(ts,b)using db.stb TAGS(?)VALUES (?,?)";
printf("case 3 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR);
}
// case 4 : none para for ctbname
{
const char* sql = "INSERT INTO db.d0 using db.stb values(?,?)";
printf("case 4 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_TSC_SQL_SYNTAX_ERROR);
}
// case 5 : none para for ctbname
{
const char* sql = "insert into ! using db.stb tags(?, ?) values(?,?)";
printf("case 5 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_TSC_SQL_SYNTAX_ERROR);
}
taos_close(taos);
}
TEST(stmt2Case, insert_ntb_get_fields_Test) {
TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(taos, nullptr);
do_query(taos, "drop database if exists db");
do_query(taos, "create database db PRECISION 'ms'");
do_query(taos, "CREATE TABLE db.ntb(nts timestamp, nb binary(10),nvc varchar(16),ni int);");
do_query(
taos,
"create table if not exists db.all_ntb(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 "
"tinyint unsigned, v7 smallint unsigned, v8 int unsigned, v9 bigint unsigned, v10 float, v11 double, v12 "
"binary(20), v13 varbinary(20), v14 geometry(100), v15 nchar(20));");
printf("support case \n");
// case 1 : test normal table no field name
{
const char* sql = "INSERT INTO db.ntb VALUES(?,?,?,?)";
TAOS_FIELD_ALL expectedFields[4] = {{"nts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0, 8, TAOS_FIELD_COL},
{"nb", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL},
{"nvc", TSDB_DATA_TYPE_BINARY, 0, 0, 18, TAOS_FIELD_COL},
{"ni", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL}};
printf("case 1 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 4);
}
// case 2 : test random order
{
const char* sql = "INSERT INTO db.ntb (ni,nb,nvc,nts)VALUES(?,?,?,?)";
TAOS_FIELD_ALL expectedFields[4] = {{"ni", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL},
{"nb", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL},
{"nvc", TSDB_DATA_TYPE_BINARY, 0, 0, 18, TAOS_FIELD_COL},
{"nts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0, 8, TAOS_FIELD_COL}};
printf("case 2 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 4);
}
// case 3 : less param
{
const char* sql = "INSERT INTO db.ntb (nts)VALUES(?)";
TAOS_FIELD_ALL expectedFields[1] = {{"nts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0, 8, TAOS_FIELD_COL}};
printf("case 3 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 1);
}
// case 4 : test all types
{
const char* sql = "insert into db.all_ntb values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
TAOS_FIELD_ALL expectedFields[16] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0, 8, TAOS_FIELD_COL},
{"v1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_COL},
{"v2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_COL},
{"v3", TSDB_DATA_TYPE_SMALLINT, 0, 0, 2, TAOS_FIELD_COL},
{"v4", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL},
{"v5", TSDB_DATA_TYPE_BIGINT, 0, 0, 8, TAOS_FIELD_COL},
{"v6", TSDB_DATA_TYPE_UTINYINT, 0, 0, 1, TAOS_FIELD_COL},
{"v7", TSDB_DATA_TYPE_USMALLINT, 0, 0, 2, TAOS_FIELD_COL},
{"v8", TSDB_DATA_TYPE_UINT, 0, 0, 4, TAOS_FIELD_COL},
{"v9", TSDB_DATA_TYPE_UBIGINT, 0, 0, 8, TAOS_FIELD_COL},
{"v10", TSDB_DATA_TYPE_FLOAT, 0, 0, 4, TAOS_FIELD_COL},
{"v11", TSDB_DATA_TYPE_DOUBLE, 0, 0, 8, TAOS_FIELD_COL},
{"v12", TSDB_DATA_TYPE_VARCHAR, 0, 0, 22, TAOS_FIELD_COL},
{"v13", TSDB_DATA_TYPE_VARBINARY, 0, 0, 22, TAOS_FIELD_COL},
{"v14", TSDB_DATA_TYPE_GEOMETRY, 0, 0, 102, TAOS_FIELD_COL},
{"v15", TSDB_DATA_TYPE_NCHAR, 0, 0, 82, TAOS_FIELD_COL}};
printf("case 4 : %s\n", sql);
getFieldsSuccess(taos, sql, expectedFields, 16);
}
printf("not support case \n");
// case 1 : wrong db
{
const char* sql = "insert into ntb values(?,?,?,?)";
printf("case 1 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION);
}
// case 2 : normal table must have tbnam
{
const char* sql = "insert into db.? values(?,?)";
printf("case 2 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_PAR_TABLE_NOT_EXIST);
}
// case 3 : wrong para nums
{
const char* sql = "insert into db.ntb(nts,ni) values(?,?,?,?,?)";
printf("case 3 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_PAR_INVALID_COLUMNS_NUM);
}
}
TEST(stmt2Case, select_get_fields_Test) {
TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(taos, nullptr);
do_query(taos, "drop database if exists db");
do_query(taos, "create database db PRECISION 'ns'");
do_query(taos, "use db");
do_query(taos, "CREATE TABLE db.ntb(nts timestamp, nb binary(10),nvc varchar(16),ni int);");
{
// case 1 :
const char* sql = "select * from ntb where ts = ?";
printf("case 1 : %s\n", sql);
getQueryFields(taos, sql, 1);
}
{
// case 2 :
const char* sql = "select * from ntb where ts = ? and b = ?";
printf("case 2 : %s\n", sql);
getQueryFields(taos, sql, 2);
}
{
// case 3 :
const char* sql = "select * from ? where ts = ?";
printf("case 3 : %s\n", sql);
getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR);
}
taos_close(taos);
}
TEST(stmt2Case, get_fields_error_Test) {
// case 1 :
{
printf("case 1 : NULL param \n");
int code = taos_stmt2_get_fields(NULL, NULL, NULL);
ASSERT_EQ(code, TSDB_CODE_INVALID_PARA);
}
}
TEST(stmt2Case, stmt2_init_prepare_Test) {
TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(taos, nullptr);
{
(void)taos_stmt2_init(NULL, NULL);
ASSERT_EQ(terrno, TSDB_CODE_INVALID_PARA);
terrno = 0;
}
{
(void)taos_stmt2_prepare(NULL, NULL, 0);
ASSERT_EQ(terrno, TSDB_CODE_INVALID_PARA);
terrno = 0;
}
{
TAOS_STMT2_OPTION option = {0, true, true, NULL, NULL};
TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
ASSERT_EQ(terrno, 0);
ASSERT_NE(stmt, nullptr);
int code = taos_stmt2_prepare(stmt, "wrong sql", 0);
ASSERT_EQ(terrno, 0);
ASSERT_NE(stmt, nullptr);
ASSERT_EQ(((STscStmt2*)stmt)->db, nullptr);
code = taos_stmt2_prepare(stmt, "insert into 'db'.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)", 0);
ASSERT_EQ(terrno, 0);
ASSERT_NE(stmt, nullptr);
ASSERT_STREQ(((STscStmt2*)stmt)->db, "db"); // add in main TD-33332
taos_stmt2_close(stmt);
}
{
TAOS_STMT2_OPTION option = {0, true, false, NULL, NULL};
TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
ASSERT_EQ(terrno, 0);
ASSERT_NE(stmt, nullptr);
taos_stmt2_close(stmt);
}
{
TAOS_STMT2_OPTION option = {0, true, true, stmtAsyncQueryCb, NULL};
TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
ASSERT_EQ(terrno, 0);
ASSERT_NE(stmt, nullptr);
taos_stmt2_close(stmt);
}
taos_close(taos);
}
TEST(stmt2Case, stmt2_all) {
TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 0);
ASSERT_NE(taos, nullptr);
{ do_stmt(taos, "insert into db.stb (tbname,ts,b,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true); }
taos_close(taos);
}
TEST(stmt2Case, stmt2_status_Test) {}
#pragma GCC diagnostic pop

37
source/common/src/msg/tmsg.c
View File

@ -12020,6 +12020,43 @@ _exit:
return code;
}
int32_t tSerializeSMResetStreamReq(void *buf, int32_t bufLen, const SMResetStreamReq *pReq) {
SEncoder encoder = {0};
int32_t code = 0;
int32_t lino;
int32_t tlen;
tEncoderInit(&encoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartEncode(&encoder));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->name));
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->igNotExists));
tEndEncode(&encoder);
_exit:
if (code) {
tlen = code;
} else {
tlen = encoder.pos;
}
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSMResetStreamReq(void *buf, int32_t bufLen, SMResetStreamReq *pReq) {
SDecoder decoder = {0};
int32_t code = 0;
int32_t lino;
tDecoderInit(&decoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartDecode(&decoder));
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->name));
TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->igNotExists));
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);
return code;
}
int32_t tEncodeMqSubTopicEp(void **buf, const SMqSubTopicEp *pTopicEp) {
int32_t tlen = 0;
tlen += taosEncodeString(buf, pTopicEp->topic);

27
source/common/src/tglobal.c
View File

@ -14,12 +14,12 @@
*/
#define _DEFAULT_SOURCE
#include "tglobal.h"
#include "cJSON.h"
#include "defines.h"
#include "os.h"
#include "osString.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tgrant.h"
#include "tjson.h"
#include "tlog.h"
@ -825,7 +825,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
tsLogBufferMemoryAllowed = TRANGE(tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL);
// clang-format off
TAOS_CHECK_RETURN(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER, CFG_CATEGORY_LOCAL));
@ -858,9 +858,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "syncLogBufferMemoryAllowed", tsLogBufferMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL));
@ -911,7 +911,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlBatchDropNum", tsTtlBatchDropNum, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
@ -919,11 +919,11 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "s3MigrateEnabled", tsS3MigrateEnabled, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL));
@ -939,11 +939,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL));
SConfigItem *pItem = NULL;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "fPrecision");
tsFPrecision = pItem->fval;
TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_GLOBAL));
@ -1932,7 +1929,7 @@ int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char *
SConfig *pCfg = NULL;
TAOS_CHECK_RETURN(cfgInit(&pCfg));
TAOS_CHECK_GOTO(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE, CFG_CATEGORY_LOCAL), &lino,
TAOS_CHECK_GOTO(cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL), &lino,
_exit);
TAOS_CHECK_GOTO(
cfgAddInt32(pCfg, "debugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER, CFG_CATEGORY_LOCAL), &lino,

1
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -182,6 +182,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TB_WITH_TSMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RESET_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_ANAL_ALGO, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;

10
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -254,7 +254,15 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
pRpc->info.wrapper = pWrapper;
EQItype itype = IsReq(pRpc) ? RPC_QITEM : DEF_QITEM; // rsp msg is not restricted by tsQueueMemoryUsed
EQItype itype = RPC_QITEM; // rsp msg is not restricted by tsQueueMemoryUsed
if (IsReq(pRpc)) {
if (pRpc->msgType == TDMT_SYNC_HEARTBEAT || pRpc->msgType == TDMT_SYNC_HEARTBEAT_REPLY)
itype = DEF_QITEM;
else
itype = RPC_QITEM;
} else {
itype = DEF_QITEM;
}
code = taosAllocateQitem(sizeof(SRpcMsg), itype, pRpc->contLen, (void **)&pMsg);
if (code) goto _OVER;

33
source/dnode/mnode/impl/src/mndStream.c
View File

@ -52,6 +52,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
static void mndCancelGetNextStreamTask(SMnode *pMnode, void *pIter);
static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq);
static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq);
static int32_t mndProcessResetStreamReq(SRpcMsg *pReq);
static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int32_t nodeId, int64_t checkpointId,
int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger);
static int32_t mndProcessNodeCheck(SRpcMsg *pReq);
@ -130,6 +131,7 @@ int32_t mndInitStream(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_PAUSE_STREAM, mndProcessPauseStreamReq);
mndSetMsgHandle(pMnode, TDMT_MND_RESUME_STREAM, mndProcessResumeStreamReq);
mndSetMsgHandle(pMnode, TDMT_MND_RESET_STREAM, mndProcessResetStreamReq);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStream);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream);
@ -1887,6 +1889,37 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
return TSDB_CODE_ACTION_IN_PROGRESS;
}
static int32_t mndProcessResetStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SStreamObj *pStream = NULL;
int32_t code = 0;
if ((code = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
return code;
}
SMResetStreamReq resetReq = {0};
if (tDeserializeSMResetStreamReq(pReq->pCont, pReq->contLen, &resetReq) < 0) {
TAOS_RETURN(TSDB_CODE_INVALID_MSG);
}
mDebug("recv reset stream req, stream:%s", resetReq.name);
code = mndAcquireStream(pMnode, resetReq.name, &pStream);
if (pStream == NULL || code != 0) {
if (resetReq.igNotExists) {
mInfo("stream:%s, not exist, not pause stream", resetReq.name);
return 0;
} else {
mError("stream:%s not exist, failed to pause stream", resetReq.name);
TAOS_RETURN(TSDB_CODE_MND_STREAM_NOT_EXIST);
}
}
//todo(liao hao jun)
return TSDB_CODE_ACTION_IN_PROGRESS;
}
static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo, bool includeAllNodes) {
SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL;

28
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -193,6 +193,8 @@ const char* nodesNodeName(ENodeType type) {
return "PauseStreamStmt";
case QUERY_NODE_RESUME_STREAM_STMT:
return "ResumeStreamStmt";
case QUERY_NODE_RESET_STREAM_STMT:
return "ResetStreamStmt";
case QUERY_NODE_BALANCE_VGROUP_STMT:
return "BalanceVgroupStmt";
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
@ -7289,6 +7291,32 @@ static int32_t jsonToDropStreamStmt(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkResetStreamStmtStreamName = "StreamName";
static const char* jkResetStreamStmtIgnoreNotExists = "IgnoreNotExists";
static int32_t resetStreamStmtToJson(const void* pObj, SJson* pJson) {
const SResetStreamStmt* pNode = (const SResetStreamStmt*)pObj;
int32_t code = tjsonAddStringToObject(pJson, jkResetStreamStmtStreamName, pNode->streamName);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkResetStreamStmtIgnoreNotExists, pNode->ignoreNotExists);
}
return code;
}
static int32_t jsonToResetStreamStmt(const SJson* pJson, void* pObj) {
SResetStreamStmt* pNode = (SResetStreamStmt*)pObj;
int32_t code = tjsonGetStringValue(pJson, jkResetStreamStmtStreamName, pNode->streamName);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkResetStreamStmtIgnoreNotExists, &pNode->ignoreNotExists);
}
return code;
}
static const char* jkMergeVgroupStmtVgroupId1 = "VgroupId1";
static const char* jkMergeVgroupStmtVgroupId2 = "VgroupId2";

4
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -615,6 +615,9 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) {
case QUERY_NODE_RESUME_STREAM_STMT:
code = makeNode(type, sizeof(SResumeStreamStmt), &pNode);
break;
case QUERY_NODE_RESET_STREAM_STMT:
code = makeNode(type, sizeof(SResetStreamStmt), &pNode);
break;
case QUERY_NODE_BALANCE_VGROUP_STMT:
code = makeNode(type, sizeof(SBalanceVgroupStmt), &pNode);
break;
@ -1483,6 +1486,7 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_DROP_STREAM_STMT: // no pointer field
case QUERY_NODE_PAUSE_STREAM_STMT: // no pointer field
case QUERY_NODE_RESUME_STREAM_STMT: // no pointer field
case QUERY_NODE_RESET_STREAM_STMT: // no pointer field
case QUERY_NODE_BALANCE_VGROUP_STMT: // no pointer field
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: // no pointer field
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT: // no pointer field

1
source/libs/parser/inc/parAst.h
View File

@ -301,6 +301,7 @@ SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken
SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName);
SNode* createPauseStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName);
SNode* createResumeStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, bool ignoreUntreated, SToken* pStreamName);
SNode* createResetStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName);
SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId);
SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId);
SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt);

1
source/libs/parser/inc/sql.y
View File

@ -779,6 +779,7 @@ cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A) stream_options(B) INTO
cmd ::= DROP STREAM exists_opt(A) stream_name(B). { pCxt->pRootNode = createDropStreamStmt(pCxt, A, &B); }
cmd ::= PAUSE STREAM exists_opt(A) stream_name(B). { pCxt->pRootNode = createPauseStreamStmt(pCxt, A, &B); }
cmd ::= RESUME STREAM exists_opt(A) ignore_opt(C) stream_name(B). { pCxt->pRootNode = createResumeStreamStmt(pCxt, A, C, &B); }
cmd ::= RESET STREAM exists_opt(A) stream_name(B). { pCxt->pRootNode = createResetStreamStmt(pCxt, A, &B); }
%type col_list_opt { SNodeList* }
%destructor col_list_opt { nodesDestroyList($$); }

14
source/libs/parser/src/parAstCreater.c
View File

@ -3726,6 +3726,20 @@ _err:
return NULL;
}
SNode* createResetStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName) {
CHECK_PARSER_STATUS(pCxt);
CHECK_NAME(checkStreamName(pCxt, pStreamName));
SPauseStreamStmt* pStmt = NULL;
pCxt->errCode = nodesMakeNode(QUERY_NODE_RESET_STREAM_STMT, (SNode**)&pStmt);
CHECK_MAKE_NODE(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->streamName, pStreamName);
pStmt->ignoreNotExists = ignoreNotExists;
return (SNode*)pStmt;
_err:
return NULL;
}
SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId) {
CHECK_PARSER_STATUS(pCxt);
SKillStmt* pStmt = NULL;

13
source/libs/parser/src/parTranslater.c
View File

@ -12358,6 +12358,16 @@ static int32_t translateResumeStream(STranslateContext* pCxt, SResumeStreamStmt*
return buildCmdMsg(pCxt, TDMT_MND_RESUME_STREAM, (FSerializeFunc)tSerializeSMResumeStreamReq, &req);
}
static int32_t translateResetStream(STranslateContext* pCxt, SResetStreamStmt* pStmt) {
SMResetStreamReq req = {0};
SName name;
int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->streamName, strlen(pStmt->streamName));
if (TSDB_CODE_SUCCESS != code) return code;
(void)tNameGetFullDbName(&name, req.name);
req.igNotExists = pStmt->ignoreNotExists;
return buildCmdMsg(pCxt, TDMT_MND_RESET_STREAM, (FSerializeFunc)tSerializeSMResetStreamReq, &req);
}
static int32_t validateCreateView(STranslateContext* pCxt, SCreateViewStmt* pStmt) {
if (QUERY_NODE_SELECT_STMT != nodeType(pStmt->pQuery) && QUERY_NODE_SET_OPERATOR != nodeType(pStmt->pQuery)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VIEW_QUERY, "Invalid view query type");
@ -13392,6 +13402,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_RESUME_STREAM_STMT:
code = translateResumeStream(pCxt, (SResumeStreamStmt*)pNode);
break;
case QUERY_NODE_RESET_STREAM_STMT:
code = translateResetStream(pCxt, (SResetStreamStmt*)pNode);
break;
case QUERY_NODE_CREATE_FUNCTION_STMT:
code = translateCreateFunction(pCxt, (SCreateFunctionStmt*)pNode);
break;

2
source/libs/sync/src/syncSnapshot.c
View File

@ -601,8 +601,10 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap
// write data
sRInfo(pReceiver, "snapshot receiver write about to finish, blockLen:%d seq:%d", pMsg->dataLen, pMsg->seq);
if (pMsg->dataLen > 0) {
(void)taosThreadMutexLock(&pReceiver->writerMutex);
code = pReceiver->pSyncNode->pFsm->FpSnapshotDoWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, pMsg->data,
pMsg->dataLen);
(void)taosThreadMutexUnlock(&pReceiver->writerMutex);
if (code != 0) {
sRError(pReceiver, "failed to finish snapshot receiver write since %s", tstrerror(code));
TAOS_RETURN(code);

2
source/libs/wal/inc/walInt.h
View File

@ -156,9 +156,11 @@ static inline void walResetVer(SWalVer* pVer) {
int32_t walLoadMeta(SWal* pWal);
int32_t walSaveMeta(SWal* pWal);
int32_t walRemoveMeta(SWal* pWal);
int32_t walRollImpl(SWal* pWal);
int32_t walRollFileInfo(SWal* pWal);
int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* lastVer);
int32_t walCheckAndRepairMeta(SWal* pWal);
int64_t walChangeWrite(SWal* pWal, int64_t ver);
int32_t walCheckAndRepairIdx(SWal* pWal);

248
source/libs/wal/src/walMeta.c
View File

@ -57,18 +57,10 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t*
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int64_t fileSize = 0;
if (taosStatFile(fnameStr, &fileSize, NULL, NULL) != 0) {
wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
code = terrno;
goto _err;
}
TAOS_CHECK_GOTO(taosStatFile(fnameStr, &fileSize, NULL, NULL), &lino, _err);
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE);
if (pFile == NULL) {
wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
*lastVer = retVer;
TAOS_RETURN(terrno);
}
TSDB_CHECK_NULL(pFile, code, lino, _err, terrno);
// ensure size as non-negative
pFileInfo->fileSize = TMAX(0, pFileInfo->fileSize);
@ -102,9 +94,7 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t*
capacity = readSize + sizeof(magic);
ptr = taosMemoryRealloc(buf, capacity);
if (ptr == NULL) {
TAOS_CHECK_GOTO(terrno, &lino, _err);
}
TSDB_CHECK_NULL(ptr, code, lino, _err, terrno);
buf = ptr;
int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET);
@ -166,9 +156,7 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t*
if (capacity < readSize + extraSize + sizeof(magic)) {
capacity += extraSize;
void* ptr = taosMemoryRealloc(buf, capacity);
if (ptr == NULL) {
TAOS_CHECK_GOTO(terrno, &lino, _err);
}
TSDB_CHECK_NULL(ptr, code, lino, _err, terrno);
buf = ptr;
}
int64_t ret = taosLSeekFile(pFile, offset + readSize, SEEK_SET);
@ -187,10 +175,7 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t*
}
logContent = (SWalCkHead*)(buf + pos);
code = decryptBody(&pWal->cfg, logContent, logContent->head.bodyLen, __FUNCTION__);
if (code) {
break;
}
TAOS_CHECK_GOTO(decryptBody(&pWal->cfg, logContent, logContent->head.bodyLen, __FUNCTION__), &lino, _err);
if (walValidBodyCksum(logContent) != 0) {
code = TSDB_CODE_WAL_CHKSUM_MISMATCH;
@ -233,13 +218,16 @@ FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t*
if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pFile) < 0) {
wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr);
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _err);
TAOS_CHECK_GOTO(terrno, &lino, _err);
}
}
pFileInfo->fileSize = lastEntryEndOffset;
_err:
if (code != 0) {
wError("vgId:%d, failed to scan log file due to %s, file:%s", pWal->cfg.vgId, tstrerror(terrno), fnameStr);
}
taosCloseFile(&pFile);
taosMemoryFree(buf);
*lastVer = retVer;
@ -371,45 +359,37 @@ static int32_t walLogEntriesComplete(const SWal* pWal) {
}
static int32_t walTrimIdxFile(SWal* pWal, int32_t fileIdx) {
int32_t code = TSDB_CODE_SUCCESS;
TdFilePtr pFile = NULL;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
TdFilePtr pFile = NULL;
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
if (!pFileInfo) {
TAOS_RETURN(TSDB_CODE_FAILED);
}
TSDB_CHECK_NULL(pFileInfo, code, lino, _exit, terrno);
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
int64_t fileSize = 0;
if (taosStatFile(fnameStr, &fileSize, NULL, NULL) != 0) {
wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
code = terrno;
goto _exit;
}
TAOS_CHECK_EXIT(taosStatFile(fnameStr, &fileSize, NULL, NULL) != 0);
int64_t records = TMAX(0, pFileInfo->lastVer - pFileInfo->firstVer + 1);
int64_t lastEndOffset = records * sizeof(SWalIdxEntry);
if (fileSize <= lastEndOffset) {
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
if (fileSize <= lastEndOffset) TAOS_RETURN(TSDB_CODE_SUCCESS);
pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE);
if (pFile == NULL) {
code = terrno;
goto _exit;
}
TSDB_CHECK_NULL(pFile, code, lino, _exit, terrno);
wInfo("vgId:%d, trim idx file. file: %s, size: %" PRId64 ", offset: %" PRId64, pWal->cfg.vgId, fnameStr, fileSize,
lastEndOffset);
code = taosFtruncateFile(pFile, lastEndOffset);
if (code < 0) {
wError("vgId:%d, failed to truncate file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
goto _exit;
}
TAOS_CHECK_EXIT(taosFtruncateFile(pFile, lastEndOffset));
_exit:
if (code != TSDB_CODE_SUCCESS) {
wError("vgId:%d, failed to trim idx file %s due to %s", pWal->cfg.vgId, fnameStr, tstrerror(code));
}
(void)taosCloseFile(&pFile);
TAOS_RETURN(code);
}
@ -425,35 +405,34 @@ static void printFileSet(int32_t vgId, SArray* fileSet, const char* str) {
}
}
void walRegfree(regex_t* ptr) {
if (ptr == NULL) {
return;
}
regfree(ptr);
}
int32_t walCheckAndRepairMeta(SWal* pWal) {
// load log files, get first/snapshot/last version info
int32_t code = 0;
int32_t lino = 0;
const char* logPattern = "^[0-9]+.log$";
const char* idxPattern = "^[0-9]+.idx$";
regex_t logRegPattern;
regex_t idxRegPattern;
regex_t logRegPattern, idxRegPattern;
TdDirPtr pDir = NULL;
SArray* actualLog = NULL;
wInfo("vgId:%d, begin to repair meta, wal path:%s, firstVer:%" PRId64 ", lastVer:%" PRId64 ", snapshotVer:%" PRId64,
pWal->cfg.vgId, pWal->path, pWal->vers.firstVer, pWal->vers.lastVer, pWal->vers.snapshotVer);
if (regcomp(&logRegPattern, logPattern, REG_EXTENDED) != 0) {
wError("failed to compile log pattern, error:%s", tstrerror(terrno));
return terrno;
}
if (regcomp(&idxRegPattern, idxPattern, REG_EXTENDED) != 0) {
wError("failed to compile idx pattern");
return terrno;
}
TAOS_CHECK_EXIT_SET_CODE(regcomp(&logRegPattern, logPattern, REG_EXTENDED), code, terrno);
TdDirPtr pDir = taosOpenDir(pWal->path);
if (pDir == NULL) {
regfree(&logRegPattern);
regfree(&idxRegPattern);
wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
return terrno;
}
TAOS_CHECK_EXIT_SET_CODE(regcomp(&idxRegPattern, idxPattern, REG_EXTENDED), code, terrno);
SArray* actualLog = taosArrayInit(8, sizeof(SWalFileInfo));
pDir = taosOpenDir(pWal->path);
TSDB_CHECK_NULL(pDir, code, lino, _exit, terrno);
actualLog = taosArrayInit(8, sizeof(SWalFileInfo));
// scan log files and build new meta
TdDirEntryPtr pDirEntry;
@ -464,28 +443,10 @@ int32_t walCheckAndRepairMeta(SWal* pWal) {
SWalFileInfo fileInfo;
(void)memset(&fileInfo, -1, sizeof(SWalFileInfo));
(void)sscanf(name, "%" PRId64 ".log", &fileInfo.firstVer);
if (!taosArrayPush(actualLog, &fileInfo)) {
regfree(&logRegPattern);
regfree(&idxRegPattern);
int32_t ret = taosCloseDir(&pDir);
if (ret != 0) {
wError("failed to close dir, ret:%s", tstrerror(ret));
return terrno;
}
return terrno;
}
TSDB_CHECK_NULL(taosArrayPush(actualLog, &fileInfo), code, lino, _exit, terrno);
}
}
int32_t ret = taosCloseDir(&pDir);
if (ret != 0) {
wError("failed to close dir, ret:%s", tstrerror(ret));
return terrno;
}
regfree(&logRegPattern);
regfree(&idxRegPattern);
taosArraySort(actualLog, compareWalFileInfo);
wInfo("vgId:%d, actual log file, wal path:%s, num:%d", pWal->cfg.vgId, pWal->path,
@ -500,11 +461,7 @@ int32_t walCheckAndRepairMeta(SWal* pWal) {
bool updateMeta = (metaFileNum != actualFileNum);
// rebuild meta of file info
code = walRebuildFileInfoSet(pWal->fileInfoSet, actualLog);
taosArrayDestroy(actualLog);
if (code) {
TAOS_RETURN(code);
}
TAOS_CHECK_EXIT(walRebuildFileInfoSet(pWal->fileInfoSet, actualLog));
wInfo("vgId:%d, log file in meta, wal path:%s, num:%d", pWal->cfg.vgId, pWal->path,
(int32_t)taosArrayGetSize(pWal->fileInfoSet));
@ -521,12 +478,7 @@ int32_t walCheckAndRepairMeta(SWal* pWal) {
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int32_t code = taosStatFile(fnameStr, &fileSize, NULL, NULL);
if (code < 0) {
wError("failed to stat file since %s. file:%s", terrstr(), fnameStr);
TAOS_RETURN(terrno);
}
TAOS_CHECK_EXIT(taosStatFile(fnameStr, &fileSize, NULL, NULL));
if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) {
totSize += pFileInfo->fileSize;
@ -581,22 +533,24 @@ int32_t walCheckAndRepairMeta(SWal* pWal) {
wInfo("vgId:%d, success to repair meta, wal path:%s, firstVer:%" PRId64 ", lastVer:%" PRId64 ", snapshotVer:%" PRId64,
pWal->cfg.vgId, pWal->path, pWal->vers.firstVer, pWal->vers.lastVer, pWal->vers.snapshotVer);
_exit:
if (code != TSDB_CODE_SUCCESS) {
wError("vgId:%d, failed to repair meta due to %s, at line:%d", pWal->cfg.vgId, tstrerror(code), lino);
}
taosArrayDestroy(actualLog);
TAOS_UNUSED(taosCloseDir(&pDir));
walRegfree(&logRegPattern);
walRegfree(&idxRegPattern);
return code;
}
static int32_t walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) {
if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) {
TAOS_RETURN(terrno);
}
if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) return terrno;
if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) {
TAOS_RETURN(terrno);
}
if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) return terrno;
if (walValidHeadCksum(pCkHead) != 0) {
TAOS_RETURN(TSDB_CODE_WAL_CHKSUM_MISMATCH);
}
if (walValidHeadCksum(pCkHead) != 0) return TSDB_CODE_WAL_CHKSUM_MISMATCH;
return TSDB_CODE_SUCCESS;
}
@ -838,29 +792,17 @@ int32_t walMetaSerialize(SWal* pWal, char** serialized) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
if (!cJSON_AddItemToObject(pRoot, "meta", pMeta)) {
wInfo("vgId:%d, failed to add meta to root", pWal->cfg.vgId);
}
if (!cJSON_AddItemToObject(pRoot, "meta", pMeta)) goto _err;
snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pWal->vers.firstVer);
if (cJSON_AddStringToObject(pMeta, "firstVer", buf) == NULL) {
wInfo("vgId:%d, failed to add firstVer to meta", pWal->cfg.vgId);
}
if (cJSON_AddStringToObject(pMeta, "firstVer", buf) == NULL) goto _err;
(void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pWal->vers.snapshotVer);
if (cJSON_AddStringToObject(pMeta, "snapshotVer", buf) == NULL) {
wInfo("vgId:%d, failed to add snapshotVer to meta", pWal->cfg.vgId);
}
if (cJSON_AddStringToObject(pMeta, "snapshotVer", buf) == NULL) goto _err;
(void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pWal->vers.commitVer);
if (cJSON_AddStringToObject(pMeta, "commitVer", buf) == NULL) {
wInfo("vgId:%d, failed to add commitVer to meta", pWal->cfg.vgId);
}
if (cJSON_AddStringToObject(pMeta, "commitVer", buf) == NULL) goto _err;
(void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pWal->vers.lastVer);
if (cJSON_AddStringToObject(pMeta, "lastVer", buf) == NULL) {
wInfo("vgId:%d, failed to add lastVer to meta", pWal->cfg.vgId);
}
if (cJSON_AddStringToObject(pMeta, "lastVer", buf) == NULL) goto _err;
if (!cJSON_AddItemToObject(pRoot, "files", pFiles)) {
wInfo("vgId:%d, failed to add files to root", pWal->cfg.vgId);
}
if (!cJSON_AddItemToObject(pRoot, "files", pFiles)) goto _err;
SWalFileInfo* pData = pWal->fileInfoSet->pData;
for (int i = 0; i < sz; i++) {
SWalFileInfo* pInfo = &pData[i];
@ -869,31 +811,20 @@ int32_t walMetaSerialize(SWal* pWal, char** serialized) {
}
if (pField == NULL) {
cJSON_Delete(pRoot);
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
// cjson only support int32_t or double
// string are used to prohibit the loss of precision
(void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pInfo->firstVer);
if (cJSON_AddStringToObject(pField, "firstVer", buf) == NULL) {
wInfo("vgId:%d, failed to add firstVer to field", pWal->cfg.vgId);
}
if (cJSON_AddStringToObject(pField, "firstVer", buf) == NULL) goto _err;
(void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pInfo->lastVer);
if (cJSON_AddStringToObject(pField, "lastVer", buf) == NULL) {
wInfo("vgId:%d, failed to add lastVer to field", pWal->cfg.vgId);
}
if (cJSON_AddStringToObject(pField, "lastVer", buf) == NULL) goto _err;
(void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pInfo->createTs);
if (cJSON_AddStringToObject(pField, "createTs", buf) == NULL) {
wInfo("vgId:%d, failed to add createTs to field", pWal->cfg.vgId);
}
if (cJSON_AddStringToObject(pField, "createTs", buf) == NULL) goto _err;
(void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pInfo->closeTs);
if (cJSON_AddStringToObject(pField, "closeTs", buf) == NULL) {
wInfo("vgId:%d, failed to add closeTs to field", pWal->cfg.vgId);
}
if (cJSON_AddStringToObject(pField, "closeTs", buf) == NULL) goto _err;
(void)snprintf(buf, WAL_JSON_BUF_SIZE, "%" PRId64, pInfo->fileSize);
if (cJSON_AddStringToObject(pField, "fileSize", buf) == NULL) {
wInfo("vgId:%d, failed to add fileSize to field", pWal->cfg.vgId);
}
if (cJSON_AddStringToObject(pField, "fileSize", buf) == NULL) goto _err;
}
char* pSerialized = cJSON_Print(pRoot);
cJSON_Delete(pRoot);
@ -901,6 +832,9 @@ int32_t walMetaSerialize(SWal* pWal, char** serialized) {
*serialized = pSerialized;
TAOS_RETURN(TSDB_CODE_SUCCESS);
_err:
cJSON_Delete(pRoot);
return TSDB_CODE_FAILED;
}
int32_t walMetaDeserialize(SWal* pWal, const char* bytes) {
@ -1109,8 +1043,12 @@ _err:
}
int32_t walLoadMeta(SWal* pWal) {
int32_t code = 0;
int n = 0;
int32_t code = 0;
int n = 0;
int32_t lino = 0;
char* buf = NULL;
TdFilePtr pFile = NULL;
// find existing meta file
int metaVer = walFindCurMetaVer(pWal);
if (metaVer == -1) {
@ -1125,11 +1063,7 @@ int32_t walLoadMeta(SWal* pWal) {
}
// read metafile
int64_t fileSize = 0;
if (taosStatFile(fnameStr, &fileSize, NULL, NULL) != 0) {
wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_EXIT(taosStatFile(fnameStr, &fileSize, NULL, NULL));
if (fileSize == 0) {
code = taosRemoveFile(fnameStr);
if (code) {
@ -1141,37 +1075,37 @@ int32_t walLoadMeta(SWal* pWal) {
TAOS_RETURN(TSDB_CODE_FAILED);
}
int size = (int)fileSize;
char* buf = taosMemoryMalloc(size + 5);
if (buf == NULL) {
TAOS_RETURN(terrno);
}
int size = (int)fileSize;
buf = taosMemoryMalloc(size + 5);
TSDB_CHECK_NULL(buf, code, lino, _exit, TSDB_CODE_OUT_OF_MEMORY);
(void)memset(buf, 0, size + 5);
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pFile == NULL) {
taosMemoryFree(buf);
pFile = taosOpenFile(fnameStr, TD_FILE_READ);
TSDB_CHECK_NULL(pFile, code, lino, _exit, terrno);
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
}
if (taosReadFile(pFile, buf, size) != size) {
(void)taosCloseFile(&pFile);
taosMemoryFree(buf);
TAOS_RETURN(terrno);
code = terrno;
goto _exit;
}
// load into fileInfoSet
code = walMetaDeserialize(pWal, buf);
if (code < 0) {
wError("failed to deserialize wal meta. file:%s", fnameStr);
code = TSDB_CODE_WAL_FILE_CORRUPTED;
}
(void)taosCloseFile(&pFile);
taosMemoryFree(buf);
wInfo("vgId:%d, meta file loaded: %s, firstVer:%" PRId64 ", lastVer:%" PRId64 ", fileInfoSet size:%d", pWal->cfg.vgId,
fnameStr, pWal->vers.firstVer, pWal->vers.lastVer, (int32_t)taosArrayGetSize(pWal->fileInfoSet));
printFileSet(pWal->cfg.vgId, pWal->fileInfoSet, "file in meta");
_exit:
if (code != TSDB_CODE_SUCCESS) {
wError("vgId:%d, failed to load meta file due to %s, at line:%d", pWal->cfg.vgId, tstrerror(code), lino);
}
taosMemoryFree(buf);
(void)taosCloseFile(&pFile);
TAOS_RETURN(code);
}

118
source/libs/wal/src/walWrite.c
View File

@ -92,18 +92,15 @@ void walApplyVer(SWal *pWal, int64_t ver) {
}
int32_t walCommit(SWal *pWal, int64_t ver) {
if (ver < pWal->vers.commitVer) {
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
if (ver > pWal->vers.lastVer || pWal->vers.commitVer < pWal->vers.snapshotVer) {
TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER);
}
if (ver < pWal->vers.commitVer) TAOS_RETURN(TSDB_CODE_SUCCESS);
if (ver > pWal->vers.lastVer || pWal->vers.commitVer < pWal->vers.snapshotVer) TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER);
pWal->vers.commitVer = ver;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static int64_t walChangeWrite(SWal *pWal, int64_t ver) {
int64_t walChangeWrite(SWal *pWal, int64_t ver) {
int code;
TdFilePtr pIdxTFile, pLogTFile;
char fnameStr[WAL_FILE_LEN];
@ -161,6 +158,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex));
wInfo("vgId:%d, wal rollback for version %" PRId64, pWal->cfg.vgId, ver);
int32_t code = 0;
int32_t lino = 0;
int64_t ret;
char fnameStr[WAL_FILE_LEN];
TdFilePtr pIdxFile = NULL, pLogFile = NULL;
@ -172,11 +170,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
// find correct file
if (ver < walGetLastFileFirstVer(pWal)) {
// change current files
ret = walChangeWrite(pWal, ver);
if (ret < 0) {
code = terrno;
goto _exit;
}
TAOS_CHECK_EXIT_SET_CODE(walChangeWrite(pWal, ver), code, terrno);
// delete files in descending order
int fileSetSize = taosArrayGetSize(pWal->fileInfoSet);
@ -198,10 +192,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
pIdxFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND);
if (pIdxFile == NULL) {
code = terrno;
goto _exit;
}
TSDB_CHECK_NULL(pIdxFile, code, lino, _exit, terrno);
int64_t idxOff = walGetVerIdxOffset(pWal, ver);
ret = taosLSeekFile(pIdxFile, idxOff, SEEK_SET);
if (ret < 0) {
@ -218,11 +209,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
pLogFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND);
wDebug("vgId:%d, wal truncate file %s", pWal->cfg.vgId, fnameStr);
if (pLogFile == NULL) {
// TODO
code = terrno;
goto _exit;
}
TSDB_CHECK_NULL(pLogFile, code, lino, _exit, terrno);
ret = taosLSeekFile(pLogFile, entry.offset, SEEK_SET);
if (ret < 0) {
// TODO
@ -238,35 +226,26 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
}
code = walValidHeadCksum(&head);
if (code != 0) {
code = TSDB_CODE_WAL_FILE_CORRUPTED;
goto _exit;
}
if (head.head.version != ver) {
if (code != 0 || head.head.version != ver) {
code = TSDB_CODE_WAL_FILE_CORRUPTED;
goto _exit;
}
// truncate old files
code = taosFtruncateFile(pLogFile, entry.offset);
if (code < 0) {
goto _exit;
}
code = taosFtruncateFile(pIdxFile, idxOff);
if (code < 0) {
goto _exit;
}
if ((code = taosFtruncateFile(pLogFile, entry.offset)) < 0) goto _exit;
if ((code = taosFtruncateFile(pIdxFile, idxOff)) < 0) goto _exit;
pWal->vers.lastVer = ver - 1;
((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset;
code = walSaveMeta(pWal);
if (code < 0) {
wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr());
goto _exit;
}
TAOS_CHECK_EXIT(walSaveMeta(pWal));
_exit:
if (code != 0) {
wError("vgId:%d, %s failed at line %d since %s", pWal->cfg.vgId, __func__, lino, tstrerror(code));
}
TAOS_UNUSED(taosCloseFile(&pIdxFile));
TAOS_UNUSED(taosCloseFile(&pLogFile));
TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex));
@ -274,7 +253,7 @@ _exit:
TAOS_RETURN(code);
}
static int32_t walRollImpl(SWal *pWal) {
int32_t walRollImpl(SWal *pWal) {
int32_t code = 0, lino = 0;
if (pWal->cfg.level == TAOS_WAL_SKIP && pWal->pIdxFile != NULL && pWal->pLogFile != NULL) {
@ -306,15 +285,12 @@ static int32_t walRollImpl(SWal *pWal) {
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, newFileFirstVer, fnameStr);
pIdxFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxFile == NULL) {
TAOS_CHECK_GOTO(terrno, &lino, _exit);
}
TSDB_CHECK_NULL(pIdxFile, code, lino, _exit, terrno);
walBuildLogName(pWal, newFileFirstVer, fnameStr);
pLogFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
wDebug("vgId:%d, wal create new file for write:%s", pWal->cfg.vgId, fnameStr);
if (pLogFile == NULL) {
TAOS_CHECK_GOTO(terrno, &lino, _exit);
}
TSDB_CHECK_NULL(pLogFile, code, lino, _exit, terrno);
TAOS_CHECK_GOTO(walRollFileInfo(pWal), &lino, _exit);
@ -358,6 +334,7 @@ static FORCE_INLINE int32_t walCheckAndRoll(SWal *pWal) {
int32_t walBeginSnapshot(SWal *pWal, int64_t ver, int64_t logRetention) {
int32_t code = 0;
int32_t lino = 0;
if (pWal->cfg.level == TAOS_WAL_SKIP) {
TAOS_RETURN(TSDB_CODE_SUCCESS);
@ -375,16 +352,13 @@ int32_t walBeginSnapshot(SWal *pWal, int64_t ver, int64_t logRetention) {
", last ver %" PRId64,
pWal->cfg.vgId, ver, pWal->vers.logRetention, pWal->vers.firstVer, pWal->vers.lastVer);
// check file rolling
if (walGetLastFileSize(pWal) != 0) {
if ((code = walRollImpl(pWal)) < 0) {
wError("vgId:%d, failed to roll wal files since %s", pWal->cfg.vgId, terrstr());
goto _exit;
}
}
if (walGetLastFileSize(pWal) != 0 && (code = walRollImpl(pWal)) < 0) goto _exit;
_exit:
if (code) {
wError("vgId:%d, %s failed since %s at line %d", pWal->cfg.vgId, __func__, tstrerror(code), lino);
}
TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex));
TAOS_RETURN(code);
}
@ -515,6 +489,13 @@ _exit:
return code;
}
static void walStopDnode(SWal *pWal) {
if (pWal->stopDnode != NULL) {
wWarn("vgId:%d, set stop dnode flag", pWal->cfg.vgId);
pWal->stopDnode();
}
}
static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
int32_t code = 0;
@ -528,12 +509,7 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
int64_t size = taosWriteFile(pWal->pIdxFile, &entry, sizeof(SWalIdxEntry));
if (size != sizeof(SWalIdxEntry)) {
wError("vgId:%d, failed to write idx entry due to %s. ver:%" PRId64, pWal->cfg.vgId, strerror(errno), ver);
if (pWal->stopDnode != NULL) {
wWarn("vgId:%d, set stop dnode flag", pWal->cfg.vgId);
pWal->stopDnode();
}
walStopDnode(pWal);
TAOS_RETURN(terrno);
}
@ -579,12 +555,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
code = terrno;
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
if (pWal->stopDnode != NULL) {
wWarn("vgId:%d, set stop dnode flag", pWal->cfg.vgId);
pWal->stopDnode();
}
walStopDnode(pWal);
TAOS_CHECK_GOTO(code, &lino, _exit);
}
@ -597,12 +568,8 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
cyptedBodyLen = ENCRYPTED_LEN(cyptedBodyLen);
newBody = taosMemoryMalloc(cyptedBodyLen);
if (newBody == NULL) {
wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
TSDB_CHECK_NULL(newBody, code, lino, _exit, terrno);
TAOS_CHECK_GOTO(terrno, &lino, _exit);
}
(void)memset(newBody, 0, cyptedBodyLen);
(void)memcpy(newBody, body, plainBodyLen);
@ -641,10 +608,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
taosMemoryFreeClear(newBodyEncrypted);
}
if (pWal->stopDnode != NULL) {
wWarn("vgId:%d, set stop dnode flag", pWal->cfg.vgId);
pWal->stopDnode();
}
walStopDnode(pWal);
TAOS_CHECK_GOTO(code, &lino, _exit);
}
@ -652,8 +616,6 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
if (pWal->cfg.encryptAlgorithm == DND_CA_SM4) {
taosMemoryFreeClear(newBody);
taosMemoryFreeClear(newBodyEncrypted);
// wInfo("vgId:%d, free newBody newBodyEncrypted %s",
// pWal->cfg.vgId, __FUNCTION__);
}
// set status
@ -668,6 +630,10 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
return 0;
_exit:
if (code) {
wError("vgId:%d, %s failed at line %d since %s", pWal->cfg.vgId, __func__, lino, tstrerror(code));
}
// recover in a reverse order
if (taosFtruncateFile(pWal->pLogFile, offset) < 0) {
wFatal("vgId:%d, failed to recover WAL logfile from write error since %s, offset:%" PRId64, pWal->cfg.vgId,

62
source/libs/wal/test/walMetaTest.cpp
View File

@ -627,6 +627,68 @@ TEST_F(WalKeepEnv, walRollback) {
ASSERT_EQ(code, 0);
}
TEST_F(WalKeepEnv, walChangeWrite) {
walResetEnv();
int code;
int i;
for (i = 0; i < 100; i++) {
char newStr[100];
sprintf(newStr, "%s-%d", ranStr, i);
int len = strlen(newStr);
code = walAppendLog(pWal, i, 0, syncMeta, newStr, len);
ASSERT_EQ(code, 0);
}
code = walChangeWrite(pWal, 50);
ASSERT_EQ(code, 0);
}
TEST_F(WalCleanEnv, walRepairLogFileTs2) {
int code;
int i;
for (i = 0; i < 100; i++) {
char newStr[100];
sprintf(newStr, "%s-%d", ranStr, i);
int len = strlen(newStr);
code = walAppendLog(pWal, i, 0, syncMeta, newStr, len);
ASSERT_EQ(code, 0);
}
code = walRollImpl(pWal);
ASSERT_EQ(code, 0);
for (i = 100; i < 200; i++) {
char newStr[100];
sprintf(newStr, "%s-%d", ranStr, i);
int len = strlen(newStr);
code = walAppendLog(pWal, i, 0, syncMeta, newStr, len);
ASSERT_EQ(code, 0);
}
code = walRollImpl(pWal);
ASSERT_EQ(code, 0);
for (i = 200; i < 300; i++) {
char newStr[100];
sprintf(newStr, "%s-%d", ranStr, i);
int len = strlen(newStr);
code = walAppendLog(pWal, i, 0, syncMeta, newStr, len);
ASSERT_EQ(code, 0);
}
code = walRollImpl(pWal);
ASSERT_EQ(code, 0);
// Try to step in ts repair logic.
SWalFileInfo* pFileInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 2);
pFileInfo->closeTs = -1;
code = walCheckAndRepairMeta(pWal);
ASSERT_EQ(code, 0);
}
TEST_F(WalRetentionEnv, repairMeta1) {
walResetEnv();
int code;

218
source/util/src/tconfig.c
View File

@ -46,24 +46,29 @@ int32_t cfgSetItemVal(SConfigItem *pItem, const char *name, const char *value, E
extern char **environ;
int32_t cfgInit(SConfig **ppCfg) {
SConfig *pCfg = taosMemoryCalloc(1, sizeof(SConfig));
if (pCfg == NULL) {
TAOS_RETURN(terrno);
}
int32_t code = 0;
int32_t lino = 0;
SConfig *pCfg = NULL;
pCfg = taosMemoryCalloc(1, sizeof(SConfig));
if (pCfg == NULL) return terrno;
pCfg->localArray = NULL, pCfg->globalArray = NULL;
pCfg->localArray = taosArrayInit(64, sizeof(SConfigItem));
if (pCfg->localArray == NULL) {
taosMemoryFree(pCfg);
TAOS_RETURN(terrno);
}
TSDB_CHECK_NULL(pCfg->localArray, code, lino, _exit, terrno);
pCfg->globalArray = taosArrayInit(64, sizeof(SConfigItem));
if (pCfg->globalArray == NULL) {
taosMemoryFree(pCfg);
TAOS_RETURN(terrno);
}
TSDB_CHECK_NULL(pCfg->globalArray, code, lino, _exit, terrno);
TAOS_CHECK_RETURN(taosThreadMutexInit(&pCfg->lock, NULL));
*ppCfg = pCfg;
_exit:
if (code != 0) {
uError("failed to init config, since %s ,at line %d", tstrerror(code), lino);
cfgCleanup(pCfg);
}
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
@ -187,14 +192,11 @@ int32_t cfgGetGlobalSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->globalAr
static int32_t cfgCheckAndSetConf(SConfigItem *pItem, const char *conf) {
cfgItemFreeVal(pItem);
if (!(pItem->str == NULL)) {
return TSDB_CODE_INVALID_PARA;
}
if (!(pItem->str == NULL)) return TSDB_CODE_INVALID_PARA;
pItem->str = taosStrdup(conf);
if (pItem->str == NULL) {
TAOS_RETURN(terrno);
}
if (pItem->str == NULL) return terrno;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
@ -209,9 +211,8 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) {
taosMemoryFreeClear(pItem->str);
pItem->str = taosStrdup(fullDir);
if (pItem->str == NULL) {
TAOS_RETURN(terrno);
}
if (pItem->str == NULL) return terrno;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
@ -219,9 +220,8 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) {
static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
int32_t code = 0;
bool tmp = false;
if (strcasecmp(value, "true") == 0) {
tmp = true;
}
if (strcasecmp(value, "true") == 0) tmp = true;
int32_t val = 0;
if ((code = taosStr2int32(value, &val)) == 0 && val > 0) {
tmp = true;
@ -441,9 +441,7 @@ static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool rese
// logflag names that should 'not' be set by 'debugFlag'
if (pDebugFlagItem->array == NULL) {
pDebugFlagItem->array = taosArrayInit(16, sizeof(SLogVar));
if (pDebugFlagItem->array == NULL) {
TAOS_RETURN(terrno);
}
if (pDebugFlagItem->array == NULL) return terrno;
}
taosArrayClear(pDebugFlagItem->array);
TAOS_RETURN(TSDB_CODE_SUCCESS);
@ -454,9 +452,7 @@ static int32_t cfgUpdateDebugFlagItem(SConfig *pCfg, const char *name, bool rese
if (pDebugFlagItem->array != NULL) {
SLogVar logVar = {0};
tstrncpy(logVar.name, name, TSDB_LOG_VAR_LEN);
if (NULL == taosArrayPush(pDebugFlagItem->array, &logVar)) {
TAOS_RETURN(terrno);
}
if (NULL == taosArrayPush(pDebugFlagItem->array, &logVar)) return terrno;
}
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
@ -518,9 +514,8 @@ _exit:
int32_t cfgSetItemVal(SConfigItem *pItem, const char *name, const char *value, ECfgSrcType stype) {
int32_t code = TSDB_CODE_SUCCESS;
if (pItem == NULL) {
TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND);
}
if (pItem == NULL) return TSDB_CODE_CFG_NOT_FOUND;
switch (pItem->dtype) {
case CFG_DTYPE_BOOL: {
code = cfgSetBool(pItem, value, stype);
@ -589,10 +584,7 @@ SConfigItem *cfgGetItem(SConfig *pCfg, const char *pName) {
}
void cfgLock(SConfig *pCfg) {
if (pCfg == NULL) {
return;
}
if (pCfg == NULL) return;
(void)taosThreadMutexLock(&pCfg->lock);
}
@ -600,7 +592,7 @@ void cfgUnLock(SConfig *pCfg) { (void)taosThreadMutexUnlock(&pCfg->lock); }
int32_t checkItemDyn(SConfigItem *pItem, bool isServer) {
if (pItem->dynScope == CFG_DYN_NONE) {
return TSDB_CODE_SUCCESS;
return TSDB_CODE_INVALID_CFG;
}
if (isServer) {
if (pItem->dynScope == CFG_DYN_CLIENT || pItem->dynScope == CFG_DYN_CLIENT_LAZY) {
@ -617,39 +609,33 @@ int32_t checkItemDyn(SConfigItem *pItem, bool isServer) {
int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer,
CfgAlterType alterType) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
cfgLock(pCfg);
SConfigItem *pItem = cfgGetItem(pCfg, name);
if (pItem == NULL) {
cfgUnLock(pCfg);
TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND);
}
int32_t code = checkItemDyn(pItem, isServer);
if (code != TSDB_CODE_SUCCESS) {
cfgUnLock(pCfg);
TAOS_RETURN(code);
}
TSDB_CHECK_NULL(pItem, code, lino, _exit, TSDB_CODE_CFG_NOT_FOUND);
TAOS_CHECK_EXIT(checkItemDyn(pItem, isServer));
if ((pItem->category == CFG_CATEGORY_GLOBAL) && alterType == CFG_ALTER_DNODE) {
uError("failed to config:%s, not support update global config on only one dnode", name);
cfgUnLock(pCfg);
TAOS_RETURN(TSDB_CODE_INVALID_CFG);
code = TSDB_CODE_INVALID_CFG;
goto _exit;
}
switch (pItem->dtype) {
case CFG_DTYPE_STRING: {
if (strcasecmp(name, "slowLogScope") == 0) {
char *tmp = taosStrdup(pVal);
if (!tmp) {
cfgUnLock(pCfg);
uError("failed to config:%s since %s", name, terrstr());
TAOS_RETURN(terrno);
code = terrno;
goto _exit;
}
int32_t scope = 0;
int32_t code = taosSetSlowLogScope(tmp, &scope);
code = taosSetSlowLogScope(tmp, &scope);
if (TSDB_CODE_SUCCESS != code) {
cfgUnLock(pCfg);
taosMemoryFree(tmp);
TAOS_RETURN(code);
goto _exit;
}
taosMemoryFree(tmp);
}
@ -659,13 +645,13 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p
code = taosStr2int32(pVal, &ival);
if (code != 0 || (ival != 0 && ival != 1)) {
uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name, cfgDtypeStr(pItem->dtype), ival);
cfgUnLock(pCfg);
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
code = TSDB_CODE_OUT_OF_RANGE;
goto _exit;
}
} break;
case CFG_DTYPE_INT32: {
int32_t ival;
int32_t code = (int32_t)taosStrHumanToInt32(pVal, &ival);
code = (int32_t)taosStrHumanToInt32(pVal, &ival);
if (code != TSDB_CODE_SUCCESS) {
cfgUnLock(pCfg);
return code;
@ -673,13 +659,13 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p
if (ival < pItem->imin || ival > pItem->imax) {
uError("cfg:%s, type:%s value:%d out of range[%" PRId64 ", %" PRId64 "]", pItem->name,
cfgDtypeStr(pItem->dtype), ival, pItem->imin, pItem->imax);
cfgUnLock(pCfg);
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
code = TSDB_CODE_OUT_OF_RANGE;
goto _exit;
}
} break;
case CFG_DTYPE_INT64: {
int64_t ival;
int32_t code = taosStrHumanToInt64(pVal, &ival);
code = taosStrHumanToInt64(pVal, &ival);
if (code != TSDB_CODE_SUCCESS) {
cfgUnLock(pCfg);
TAOS_RETURN(code);
@ -687,31 +673,32 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p
if (ival < pItem->imin || ival > pItem->imax) {
uError("cfg:%s, type:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 "]", pItem->name,
cfgDtypeStr(pItem->dtype), ival, pItem->imin, pItem->imax);
cfgUnLock(pCfg);
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
code = TSDB_CODE_OUT_OF_RANGE;
goto _exit;
}
} break;
case CFG_DTYPE_FLOAT:
case CFG_DTYPE_DOUBLE: {
float dval = 0;
int32_t code = parseCfgReal(pVal, &dval);
if (code != TSDB_CODE_SUCCESS) {
cfgUnLock(pCfg);
TAOS_RETURN(code);
}
float dval = 0;
TAOS_CHECK_EXIT(parseCfgReal(pVal, &dval));
if (dval < pItem->fmin || dval > pItem->fmax) {
uError("cfg:%s, type:%s value:%g out of range[%g, %g]", pItem->name, cfgDtypeStr(pItem->dtype), dval,
pItem->fmin, pItem->fmax);
cfgUnLock(pCfg);
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
code = TSDB_CODE_OUT_OF_RANGE;
goto _exit;
}
} break;
default:
break;
}
_exit:
if (code != TSDB_CODE_SUCCESS) {
uError("failed to check range for cfg:%s, value:%s, since %s at line:%d", name, pVal, tstrerror(code), __LINE__);
}
cfgUnLock(pCfg);
TAOS_RETURN(TSDB_CODE_SUCCESS);
TAOS_RETURN(code);
}
static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) {
@ -720,9 +707,7 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) {
pItem->stype = CFG_STYPE_DEFAULT;
pItem->name = taosStrdup(name);
if (pItem->name == NULL) {
TAOS_RETURN(terrno);
}
if (pItem->name == NULL) return terrno;
int32_t size = taosArrayGetSize(array);
for (int32_t i = 0; i < size; ++i) {
@ -819,9 +804,8 @@ int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, in
int8_t category) {
SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope, .dynScope = dynScope, .category = category};
item.str = taosStrdup(defaultVal);
if (item.str == NULL) {
TAOS_RETURN(terrno);
}
if (item.str == NULL) return terrno;
return cfgAddItem(pCfg, &item, name);
}
@ -943,13 +927,9 @@ int32_t cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t
break;
}
if (len < 0) {
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
if (len < 0) return terrno;
if (len > bufSize) {
len = bufSize;
}
if (len > bufSize) len = bufSize;
*pLen = len;
TAOS_RETURN(TSDB_CODE_SUCCESS);
@ -1310,9 +1290,7 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) {
}
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) {
TAOS_RETURN(terrno);
}
if (pFile == NULL) return terrno;
while (!taosEOFFile(pFile)) {
name = value = value2 = value3 = value4 = NULL;
@ -1466,8 +1444,10 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
}
int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
char *cfgLineBuf = NULL, *name, *value, *value2, *value3, *value4;
SJson *pJson = NULL;
char *cfgLineBuf = NULL, *buf = NULL, *name, *value, *value2, *value3, *value4;
SJson *pJson = NULL;
TdFilePtr pFile = NULL;
int32_t olen, vlen, vlen2, vlen3, vlen4;
int32_t code = 0, lino = 0;
if (url == NULL || strlen(url) == 0) {
@ -1490,36 +1470,28 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
}
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ);
if (pFile == NULL) {
TAOS_CHECK_EXIT(terrno);
}
TSDB_CHECK_NULL(pFile, code, lino, _exit, terrno);
size_t fileSize = taosLSeekFile(pFile, 0, SEEK_END);
if (fileSize <= 0) {
(void)taosCloseFile(&pFile);
(void)printf("load json file error: %s\n", filepath);
TAOS_CHECK_EXIT(terrno);
}
char *buf = taosMemoryMalloc(fileSize + 1);
if (!buf) {
(void)taosCloseFile(&pFile);
(void)printf("load json file error: %s, failed to alloc memory\n", filepath);
TAOS_RETURN(terrno);
code = terrno;
goto _exit;
}
buf = taosMemoryMalloc(fileSize + 1);
TSDB_CHECK_NULL(buf, code, lino, _exit, terrno);
buf[fileSize] = 0;
if (taosLSeekFile(pFile, 0, SEEK_SET) < 0) {
(void)taosCloseFile(&pFile);
(void)printf("load json file error: %s\n", filepath);
taosMemoryFreeClear(buf);
TAOS_RETURN(terrno);
code = terrno;
goto _exit;
}
if (taosReadFile(pFile, buf, fileSize) <= 0) {
(void)taosCloseFile(&pFile);
(void)printf("load json file error: %s\n", filepath);
taosMemoryFreeClear(buf);
TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT);
code = TSDB_CODE_INVALID_DATA_FMT;
goto _exit;
}
(void)taosCloseFile(&pFile);
pJson = tjsonParse(buf);
if (NULL == pJson) {
const char *jsonParseError = tjsonGetError();
@ -1529,7 +1501,6 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
taosMemoryFreeClear(buf);
TAOS_CHECK_EXIT(TSDB_CODE_INVALID_DATA_FMT);
}
taosMemoryFreeClear(buf);
int32_t jsonArraySize = tjsonGetArraySize(pJson);
for (int32_t i = 0; i < jsonArraySize; i++) {
@ -1596,17 +1567,20 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
taosMemoryFree(cfgLineBuf);
uInfo("load from apoll url not implemented yet");
TAOS_RETURN(TSDB_CODE_SUCCESS);
_exit:
taosMemoryFree(cfgLineBuf);
taosMemoryFree(buf);
(void)taosCloseFile(&pFile);
tjsonDelete(pJson);
if (code != 0) {
(void)printf("failed to load from apollo url:%s at line %d since %s\n", url, lino, tstrerror(code));
if (code == TSDB_CODE_CFG_NOT_FOUND) {
uTrace("load from apoll url success");
TAOS_RETURN(TSDB_CODE_SUCCESS);
} else {
(void)printf("failed to load from apoll url:%s at line %d since %s\n", url, lino, tstrerror(code));
TAOS_RETURN(code);
}
TAOS_RETURN(code);
}
int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl) {
@ -1701,9 +1675,7 @@ struct SConfigIter {
int32_t cfgCreateIter(SConfig *pConf, SConfigIter **ppIter) {
SConfigIter *pIter = taosMemoryCalloc(1, sizeof(SConfigIter));
if (pIter == NULL) {
TAOS_RETURN(terrno);
}
if (pIter == NULL) return terrno;
pIter->pConf = pConf;
@ -1721,14 +1693,10 @@ SConfigItem *cfgNextIter(SConfigIter *pIter) {
}
void cfgDestroyIter(SConfigIter *pIter) {
if (pIter == NULL) {
return;
}
if (pIter == NULL) return;
taosMemoryFree(pIter);
}
SArray *taosGetLocalCfg(SConfig *pCfg) { return pCfg->localArray; }
SArray *taosGetGlobalCfg(SConfig *pCfg) { return pCfg->globalArray; }
void taosSetLocalCfg(SConfig *pCfg, SArray *pArray) { pCfg->localArray = pArray; };
void taosSetGlobalCfg(SConfig *pCfg, SArray *pArray) { pCfg->globalArray = pArray; };
SArray *taosGetGlobalCfg(SConfig *pCfg) { return pCfg->globalArray; }

12
source/util/test/cfgTest.cpp
View File

@ -337,6 +337,12 @@ TEST_F(CfgTest, cfgLoadFromEnvFile) {
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
ASSERT_NE(pConfig, nullptr);
EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 0, 0), 0);
EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0, 0, 0), 0);
EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0, 0, 0), 0);
EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0, 0, 0), 0);
EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0, 0, 0), 0);
TdFilePtr envFile = NULL;
const char *envFilePath = TD_TMP_DIR_PATH "envFile";
envFile = taosOpenFile(envFilePath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
@ -355,6 +361,12 @@ TEST_F(CfgTest, cfgLoadFromApollUrl) {
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
ASSERT_NE(pConfig, nullptr);
EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 0, 0), 0);
EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0, 0, 0), 0);
EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0, 0, 0), 0);
EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0, 0, 0), 0);
EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0, 0, 0), 0);
TdFilePtr jsonFile = NULL;
const char *jsonFilePath = TD_TMP_DIR_PATH "envJson.json";
jsonFile = taosOpenFile(jsonFilePath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);

264
tests/army/cluster/tsdbSnapshot.py Normal file
View File

@ -0,0 +1,264 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from datetime import timedelta
import sys
import time
import random
import taos
import frame
import frame.etool
from frame.log import *
from frame.cases import *
from frame.sql import *
from frame.caseBase import *
from frame import *
from frame.srvCtl import *
from frame.clusterCommonCheck import clusterComCheck
class TDTestCase(TBase):
def init(self, conn, logSql, replicaVar=3):
# super(TDTestCase, self).init(conn, logSql, replicaVar=3, db="db")
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), logSql)
self.vgroupNum = 3
self.dbName = 'test'
self.dnode1Path = tdCom.getTaosdPath()
self.dnode1Cfg = f'{self.dnode1Path}/cfg'
self.dnode1Log = f'{self.dnode1Path}/log'
def _write_bulk_data(self):
tdLog.info("============== write bulk data ===============")
json_content = f"""
{{
"filetype": "insert",
"cfgdir": "{self.dnode1Cfg}",
"host": "localhost",
"port": 6030,
"user": "root",
"password": "taosdata",
"connection_pool_size": 8,
"thread_count": 16,
"create_table_thread_count": 10,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 5,
"num_of_records_per_req": 1540,
"prepared_rand": 10000,
"chinese": "no",
"databases": [
{{
"dbinfo": {{
"name": "{self.dbName}",
"drop": "yes",
"vgroups": 5,
"duration": "10d",
"wal_retention_period": 0,
"replica": 3,
"stt_trigger": 2
}},
"super_tables": [
{{
"name": "stb",
"child_table_exists": "no",
"childtable_count": 100,
"childtable_prefix": "ctb",
"escape_character": "yes",
"auto_create_table": "no",
"batch_create_tbl_num": 500,
"data_source": "rand",
"insert_mode": "taosc",
"non_stop_mode": "no",
"line_protocol": "line",
"insert_rows": 10000,
"interlace_rows": 0,
"insert_interval": 0,
"partial_col_num": 0,
"disorder_ratio": 0,
"disorder_range": 0,
"timestamp_step": 1000,
"start_timestamp": "{(datetime.now() - timedelta(days=1)).replace(hour=10, minute=0, second=0, microsecond=0).strftime('%Y-%m-%d %H:%M:%S')}",
"use_sample_ts": "no",
"tags_file": "",
"columns": [
{{
"type": "bigint",
"count": 10
}}
],
"tags": [
{{
"type": "TINYINT",
"name": "groupid",
"max": 10,
"min": 1
}},
{{
"name": "location",
"type": "BINARY",
"len": 16,
"values": [
"beijing",
"shanghai"
]
}}
]
}}
]
}}
]
}}
"""
json_file = '/tmp/test.json'
with open(json_file, 'w') as f:
f.write(json_content)
# Use subprocess.run() to wait for the command to finish
subprocess.run(f'taosBenchmark -f {json_file}', shell=True, check=True)
def _write_bulk_data2(self):
tdLog.info("============== write bulk data ===============")
json_content = f"""
{{
"filetype": "insert",
"cfgdir": "{self.dnode1Cfg}",
"host": "localhost",
"port": 6030,
"user": "root",
"password": "taosdata",
"connection_pool_size": 8,
"thread_count": 16,
"create_table_thread_count": 10,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 5,
"num_of_records_per_req": 1540,
"prepared_rand": 10000,
"chinese": "no",
"databases": [
{{
"dbinfo": {{
"name": "{self.dbName}",
"drop": "no",
"vgroups": 5,
"duration": "10d",
"wal_retention_period": 0,
"replica": 3,
"stt_trigger": 2
}},
"super_tables": [
{{
"name": "stb",
"child_table_exists": "yes",
"childtable_count": 100,
"childtable_prefix": "ctb",
"escape_character": "yes",
"auto_create_table": "no",
"batch_create_tbl_num": 500,
"data_source": "rand",
"insert_mode": "taosc",
"non_stop_mode": "no",
"line_protocol": "line",
"insert_rows": 10000,
"interlace_rows": 0,
"insert_interval": 0,
"partial_col_num": 0,
"disorder_ratio": 0,
"disorder_range": 0,
"timestamp_step": 1000,
"start_timestamp": "{(datetime.now() - timedelta(days=1)).replace(hour=14, minute=0, second=0, microsecond=0).strftime('%Y-%m-%d %H:%M:%S')}",
"use_sample_ts": "no",
"tags_file": "",
"columns": [
{{
"type": "bigint",
"count": 10
}}
],
"tags": [
{{
"type": "TINYINT",
"name": "groupid",
"max": 10,
"min": 1
}},
{{
"name": "location",
"type": "BINARY",
"len": 16,
"values": [
"beijing",
"shanghai"
]
}}
]
}}
]
}}
]
}}
"""
json_file = '/tmp/test.json'
with open(json_file, 'w') as f:
f.write(json_content)
# Use subprocess.run() to wait for the command to finish
subprocess.run(f'taosBenchmark -f {json_file}', shell=True, check=True)
def run(self):
tdLog.info("============== write bulk data ===============")
self._write_bulk_data()
tdSql.execute(f'flush database {self.dbName}')
tdLog.sleep(10)
tdLog.info("============== stop dnode 3 ===============")
cluster.dnodes[2].stoptaosd()
tdLog.sleep(10)
# tdLog.info("============== write more data ===============")
self._write_bulk_data2()
tdSql.execute(f'flush database {self.dbName}')
tdLog.sleep(10)
cluster.dnodes[0].stoptaosd()
cluster.dnodes[1].stoptaosd()
dnode1_wal = f'{self.dnode1Path}/data/vnode/vnode2/wal'
dnode2_wal = f'{self.dnode1Path}/../dnode2/data/vnode/vnode2/wal'
tdLog.info("============== remove wal files ===============")
tdLog.info(f"{dnode1_wal}")
tdLog.info(f"{dnode2_wal}")
os.system(f'rm -rf {dnode1_wal}/*')
os.system(f'rm -rf {dnode2_wal}/*')
tdLog.info("============== restart cluster ===============")
cluster.dnodes[0].starttaosd()
cluster.dnodes[1].starttaosd()
cluster.dnodes[2].starttaosd()
tdLog.sleep(60)
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

10
tests/army/frame/common.py
View File

@ -214,6 +214,16 @@ class TDCom:
telnet_url = "http://127.0.0.1:6041/opentsdb/v1/put/telnet"
return header, sql_url, sqlt_url, sqlutc_url, influx_url, telnet_url
def getTaosdPath(self, dnodeID="dnode1"):
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info("taosd found in %s" % buildPath)
taosdPath = buildPath + "/../sim/" + dnodeID
tdLog.info("taosdPath: %s" % taosdPath)
return taosdPath
def genTcpParam(self):
MaxBytes = 1024*1024
host ='127.0.0.1'

5
tests/parallel_test/cases.task
View File

@ -44,6 +44,7 @@
,,y,army,./pytest.sh python3 ./test.py -f cluster/test_drop_table_by_uid.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f cluster/incSnapshot.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f cluster/clusterBasic.py -N 5
,,y,army,./pytest.sh python3 ./test.py -f cluster/tsdbSnapshot.py -N 3 -M 3
,,y,army,./pytest.sh python3 ./test.py -f query/query_basic.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f query/accuracy/test_query_accuracy.py
,,y,army,./pytest.sh python3 ./test.py -f query/accuracy/test_ts5400.py
@ -419,8 +420,8 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/subscribe_stream_privilege.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/empty_identifier.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_transaction_detail.py -N 3
# ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/kill_balance_leader.py -N 3
# ,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/kill_restore_dnode.py -N 5
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/kill_balance_leader.py -N 3
,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/kill_restore_dnode.py -N 5
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/persisit_config.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/qmemCtrl.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact_vgroups.py

2
tests/pytest/util/tserror.py
View File

@ -11,3 +11,5 @@ TSDB_CODE_UDF_FUNC_EXEC_FAILURE = (TAOS_DEF_ERROR_CODE | 0x290A)
TSDB_CODE_TSC_INTERNAL_ERROR = (TAOS_DEF_ERROR_CODE | 0x02FF)
TSDB_CODE_TSC_INVALID_OPERATION = (TAOS_DEF_ERROR_CODE | 0x0200)

64
tests/system-test/2-query/join.py
View File

@ -6,6 +6,7 @@ from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.tserror import *
PRIMARY_COL = "ts"
@ -352,7 +353,66 @@ class TDTestCase:
tdSql.execute( f"insert into {dbname}.nt1 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )" )
tdSql.execute( f"insert into {dbname}.nt1 values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )" )
def join_semantic_test(self, dbname=DBNAME):
tdSql.query("select ct1.c_int from db.ct1 as ct1 join db1.ct1 as cy1 on ct1.ts=cy1.ts")
tdSql.checkRows(self.rows)
tdSql.error("select ct1.c_int from db.ct1 as ct1 semi join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION)
tdSql.error("select ct1.c_int from db.ct1 as ct1 anti join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION)
tdSql.error("select ct1.c_int from db.ct1 as ct1 outer join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION)
tdSql.error("select ct1.c_int from db.ct1 as ct1 asof join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION)
tdSql.error("select ct1.c_int from db.ct1 as ct1 window join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION)
tdSql.query("select ct1.c_int from db.ct1 as ct1 join db1.ct1 as cy1 on ct1.ts=cy1.ts")
tdSql.checkRows(self.rows)
tdSql.query("select ct1.c_int from db.ct1 as ct1 left join db1.ct1 as cy1 on ct1.ts=cy1.ts")
tdSql.checkRows(self.rows)
tdSql.query("select ct1.c_int from db.ct1 as ct1 left semi join db1.ct1 as cy1 on ct1.ts=cy1.ts")
tdSql.checkRows(self.rows)
tdSql.query("select ct1.c_int from db.ct1 as ct1 left anti join db1.ct1 as cy1 on ct1.ts=cy1.ts")
tdSql.checkRows(0)
tdSql.query("select ct1.c_int from db.ct1 as ct1 left outer join db1.ct1 as cy1 on ct1.ts=cy1.ts")
tdSql.checkRows(self.rows)
tdSql.query("select ct1.c_int from db.ct1 as ct1 left asof join db1.ct1 as cy1 on ct1.ts=cy1.ts")
tdSql.checkRows(self.rows)
tdSql.error("select ct1.c_int from db.ct1 as ct1 left window join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION)
tdSql.query("select ct1.c_int from db.ct1 as ct1 right join db1.ct1 as cy1 on ct1.ts=cy1.ts")
tdSql.checkRows(self.rows)
tdSql.query("select ct1.c_int from db.ct1 as ct1 right semi join db1.ct1 as cy1 on ct1.ts=cy1.ts")
tdSql.checkRows(self.rows)
tdSql.query("select ct1.c_int from db.ct1 as ct1 right anti join db1.ct1 as cy1 on ct1.ts=cy1.ts")
tdSql.checkRows(0)
tdSql.query("select ct1.c_int from db.ct1 as ct1 right outer join db1.ct1 as cy1 on ct1.ts=cy1.ts")
tdSql.checkRows(self.rows)
tdSql.query("select ct1.c_int from db.ct1 as ct1 right asof join db1.ct1 as cy1 on ct1.ts=cy1.ts")
tdSql.checkRows(self.rows)
tdSql.error("select ct1.c_int from db.ct1 as ct1 right window join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION)
tdSql.query("select ct1.c_int from db.ct1 as ct1 full join db1.ct1 as cy1 on ct1.ts=cy1.ts")
tdSql.checkRows(self.rows)
tdSql.error("select ct1.c_int from db.ct1 as ct1 full semi join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION)
tdSql.error("select ct1.c_int from db.ct1 as ct1 full anti join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION)
tdSql.query("select ct1.c_int from db.ct1 as ct1 full outer join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION)
tdSql.checkRows(self.rows)
tdSql.error("select ct1.c_int from db.ct1 as ct1 full asof join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION)
tdSql.error("select ct1.c_int from db.ct1 as ct1 full window join db1.ct1 as cy1 on ct1.ts=cy1.ts", TSDB_CODE_TSC_INVALID_OPERATION)
tdSql.query("select ct1.c_int from db.ct1 as ct1 full join db1.ct1 as cy1 on ct1.ts=cy1.ts join db1.ct1 as cy2 on ct1.ts=cy2.ts")
tdSql.checkRows(self.rows)
tdSql.query("select * from db.ct1 join db.ct2 join db.ct3 on ct2.ts=ct3.ts on ct1.ts=ct2.ts")
tdSql.checkRows(0)
tdSql.execute(f'create table db.ct1_2 using db.stb1 tags ( 102 )')
tdSql.execute(f'create table db.ct1_3 using db.stb1 tags ( 103 )')
tdSql.execute(f'insert into db.ct1_2 (select * from db.ct1)')
tdSql.execute(f'insert into db.ct1_3 (select * from db.ct1)')
tdSql.query("select * from db.ct1 join db.ct1_2 join db.ct1_3 on ct1_2.ts=ct1_3.ts on ct1.ts=ct1_2.ts")
tdSql.checkRows(self.rows)
tdSql.execute("drop table db.ct1_2")
tdSql.execute("drop table db.ct1_3")
def run(self):
tdSql.prepare()
@ -374,6 +434,8 @@ class TDTestCase:
tdSql.execute(f"use {dbname1}")
self.__create_tb(dbname=dbname1)
self.__insert_data(dbname=dbname1)
self.join_semantic_test({dbname1})
tdSql.query("select ct1.c_int from db.ct1 as ct1 join db1.ct1 as cy1 on ct1.ts=cy1.ts")
tdSql.checkRows(self.rows)