xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feature/vnode

This commit is contained in:
Hongze Cheng 2021-12-19 11:25:18 +08:00
parent dceb46795d a46e3a9e42
commit 2e64ea8a6d
90 changed files with 2354 additions and 1275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -26,6 +26,7 @@ if(${BUILD_TEST})
endif(${BUILD_TEST})
add_subdirectory(source)
add_subdirectory(tools)
add_subdirectory(tests)
# docs
add_subdirectory(docs)

7
cmake/cmake.options
View File

@ -37,6 +37,7 @@ option(
off
)
option(
BUILD_WITH_NURAFT
"If build with NuRaft"
@ -54,3 +55,9 @@ option(
"If use doxygen build documents"
OFF
)
option(
BUILD_WITH_INVERTEDINDEX
"If use invertedIndex"
ON
)

21
include/common/taosmsg.h
View File

@ -750,31 +750,36 @@ typedef struct {
} SReplica;
typedef struct {
char db[TSDB_FULL_DB_NAME_LEN];
int32_t vgId;
int32_t dnodeId;
char db[TSDB_FULL_DB_NAME_LEN];
uint64_t dbUid;
int32_t cacheBlockSize;
int32_t totalBlocks;
int32_t daysPerFile;
int32_t daysToKeep0;
int32_t daysToKeep1;
int32_t daysToKeep2;
int32_t minRowsPerFileBlock;
int32_t maxRowsPerFileBlock;
int32_t minRows;
int32_t maxRows;
int32_t commitTime;
int32_t fsyncPeriod;
int8_t reserved[16];
int8_t walLevel;
int8_t precision;
int8_t compression;
int8_t cacheLastRow;
int8_t update;
int8_t walLevel;
int8_t quorum;
int8_t update;
int8_t cacheLastRow;
int8_t replica;
int8_t selfIndex;
SReplica replicas[TSDB_MAX_REPLICA];
} SCreateVnodeMsg, SAlterVnodeMsg;
typedef struct {
int32_t vgId;
int32_t vgId;
int32_t dnodeId;
char db[TSDB_FULL_DB_NAME_LEN];
uint64_t dbUid;
} SDropVnodeMsg, SSyncVnodeMsg, SCompactVnodeMsg;
typedef struct {

16
include/common/tname.h
View File

@ -16,7 +16,7 @@
#ifndef TDENGINE_TNAME_H
#define TDENGINE_TNAME_H
#include "taosmsg.h"
//#include "taosmsg.h"
#define TSDB_DB_NAME_T 1
#define TSDB_TABLE_NAME_T 2
@ -27,7 +27,7 @@
typedef struct SName {
uint8_t type; //db_name_t, table_name_t
char acctId[TSDB_ACCT_ID_LEN];
int32_t acctId;
char dbname[TSDB_DB_NAME_LEN];
char tname[TSDB_TABLE_NAME_LEN];
} SName;
@ -38,7 +38,7 @@ int32_t tNameLen(const SName* name);
SName* tNameDup(const SName* name);
bool tIsValidName(const SName* name);
bool tNameIsValid(const SName* name);
const char* tNameGetTableName(const SName* name);
@ -50,14 +50,10 @@ bool tNameIsEmpty(const SName* name);
void tNameAssign(SName* dst, const SName* src);
int32_t tNameSetDbName(SName* dst, int32_t acctId, const char* dbName, size_t nameLen);
int32_t tNameFromString(SName* dst, const char* str, uint32_t type);
int32_t tNameSetAcctId(SName* dst, const char* acct);
SSchema* tGetTbnameColumnSchema();
#if 0
int32_t tNameSetDbName(SName* dst, const char* acct, SToken* dbToken);
#endif
int32_t tNameSetAcctId(SName* dst, int32_t acctId);
#endif // TDENGINE_TNAME_H

22
include/libs/index/index.h
View File

@ -24,6 +24,7 @@ extern "C" {
#endif
typedef struct SIndex SIndex;
typedef struct SIndexTerm SIndexTerm;
typedef struct SIndexOpts SIndexOpts;
typedef struct SIndexMultiTermQuery SIndexMultiTermQuery;
typedef struct SArray SIndexMultiTerm;
@ -35,7 +36,7 @@ typedef enum {
ADD_INDEX, // add index on specify column
DROP_INDEX, // drop existed index
DROP_SATBLE // drop stable
} SIndexColumnType;
} SIndexOperOnColumn;
typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType;
typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX = 3} EIndexQueryType;
@ -45,12 +46,12 @@ typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX =
*/
SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType oper);
void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery);
int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, const char *field, int32_t nFields, const char *value, int32_t nValue, EIndexQueryType type);
int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType type);
/*
* @param:
* @param:
*/
SIndex* indexOpen(SIndexOpts *opt, const char *path);
int indexOpen(SIndexOpts *opt, const char *path, SIndex **index);
void indexClose(SIndex *index);
int indexPut(SIndex *index, SIndexMultiTerm *terms, int uid);
int indexDelete(SIndex *index, SIndexMultiTermQuery *query);
@ -61,8 +62,8 @@ int indexRebuild(SIndex *index, SIndexOpts *opt);
* @param
*/
SIndexMultiTerm *indexMultiTermCreate();
int indexMultiTermAdd(SIndexMultiTerm *terms, const char *field, int32_t nFields, const char *value, int32_t nValue);
void indexMultiTermDestroy(SIndexMultiTerm *terms);
int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term);
void indexMultiTermDestroy(SIndexMultiTerm *terms);
/*
* @param:
* @param:
@ -70,6 +71,17 @@ void indexMultiTermDestroy(SIndexMultiTerm *terms);
SIndexOpts *indexOptsCreate();
void indexOptsDestroy(SIndexOpts *opts);
/*
* @param:
* @param:
*/
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType,
const char *colName, int32_t nColName, const char *colVal, int32_t nColVal);
void indexTermDestroy(SIndexTerm *p);
#ifdef __cplusplus
}
#endif

14
include/libs/planner/planner.h
View File

@ -108,7 +108,7 @@ typedef struct SProjectPhyNode {
typedef struct SExchangePhyNode {
SPhyNode node;
uint64_t srcTemplateId; // template id of datasource suplans
SArray *pSourceEpSet; // SEpSet, scheduler fill by calling qSetSuplanExecutionNode
SArray *pSrcEndPoints; // SEpAddrMsg, scheduler fill by calling qSetSuplanExecutionNode
} SExchangePhyNode;
typedef struct SSubplanId {
@ -128,6 +128,8 @@ typedef struct SSubplan {
} SSubplan;
typedef struct SQueryDag {
uint64_t queryId;
int32_t numOfSubplans;
SArray *pSubplans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0.
} SQueryDag;
@ -136,7 +138,11 @@ typedef struct SQueryDag {
*/
int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag);
int32_t qSetSuplanExecutionNode(SSubplan* subplan, SArray* nodes);
// Set datasource of this subplan, multiple calls may be made to a subplan.
// @subplan subplan to be schedule
// @templateId templateId of a group of datasource subplans of this @subplan
// @eps Execution location of this group of datasource subplans, is an array of SEpAddr structures
int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps);
int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str);
@ -147,12 +153,14 @@ int32_t qSubPlanToString(const SSubplan* subplan, char** str);
int32_t qStringToSubplan(const char* str, SSubplan** subplan);
void qDestroySubplan(SSubplan* pSubplan);
/**
* Destroy the physical plan.
* @param pQueryPhyNode
* @return
*/
void qDestroyQueryDag(struct SQueryDag* pDag);
void qDestroyQueryDag(SQueryDag* pDag);
#ifdef __cplusplus
}

4
include/libs/query/query.h → include/libs/qcom/query.h
View File

@ -81,12 +81,14 @@ typedef struct STableMetaOutput {
STableMeta *tbMeta;
} STableMetaOutput;
bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen);
extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize);
SSchema* tGetTbnameColumnSchema();
extern void msgInit();
extern int32_t qDebugFlag;
#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0)

23
include/util/taoserror.h
View File

@ -120,17 +120,18 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) //"Invalid tsc input")
// mnode-common
#define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0300)
#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0301)
#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0302)
#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0303)
#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0304)
#define TSDB_CODE_MND_INVALID_OPTIONS TAOS_DEF_ERROR_CODE(0, 0x0305)
#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0306)
#define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0307)
#define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0308)
#define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0309)
#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x030A)
#define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300)
#define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0301)
#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0302)
#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0303)
#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0304)
#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0305)
#define TSDB_CODE_MND_INVALID_OPTIONS TAOS_DEF_ERROR_CODE(0, 0x0306)
#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0307)
#define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0308)
#define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0309)
#define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x030A)
#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x030B)
// mnode-show
#define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x0310)

3
include/util/tdef.h
View File

@ -231,10 +231,11 @@ do { \
#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value
#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth
#define TSDB_CQ_SQL_SIZE 1024
#define TSDB_MIN_VNODES 64
#define TSDB_MIN_VNODES 16
#define TSDB_MAX_VNODES 512
#define TSDB_MIN_VNODES_PER_DB 1
#define TSDB_MAX_VNODES_PER_DB 4096
#define TSDB_DEFAULT_VN_PER_DB 2
#define TSDB_DNODE_ROLE_ANY 0
#define TSDB_DNODE_ROLE_MGMT 1

7
source/client/CMakeLists.txt
View File

@ -2,14 +2,13 @@ aux_source_directory(src CLIENT_SRC)
add_library(taos ${CLIENT_SRC})
target_include_directories(
taos
PUBLIC "${CMAKE_SOURCE_DIR}/include/client"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
PUBLIC "${CMAKE_SOURCE_DIR}/include/client"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
taos
PRIVATE common
INTERFACE api
PRIVATE os util common transport parser catalog function query
PRIVATE os util common transport parser catalog function qcom
)
ADD_SUBDIRECTORY(test)

3
source/client/inc/clientInt.h
View File

@ -76,11 +76,10 @@ typedef struct SAppInfo {
typedef struct STscObj {
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char acctId[TSDB_ACCT_ID_LEN];
char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
int32_t acctId;
uint32_t connId;
uint64_t id; // ref ID returned by taosAddRef
// struct SSqlObj *sqlList;
void *pTransporter;
pthread_mutex_t mutex; // used to protect the operation on db
int32_t numOfReqs; // number of sqlObj from this tscObj

10
source/client/src/clientImpl.c
View File

@ -153,7 +153,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
void* output = NULL;
int32_t outputLen = 0;
code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE);
if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW) {
if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_CREATE_DB) {
pRequest->type = type;
pRequest->body.param = output;
pRequest->body.paramLen = outputLen;
@ -430,8 +430,14 @@ void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
int32_t offset = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
pResultInfo->length[i] = pResultInfo->fields[i].bytes;
pResultInfo->row[i] = pResultInfo->pData + offset * pResultInfo->numOfRows;
pResultInfo->row[i] = (char*) (pResultInfo->pData + offset * pResultInfo->numOfRows);
pResultInfo->pCol[i] = pResultInfo->row[i];
offset += pResultInfo->fields[i].bytes;
}
}
const char *taos_get_client_info() { return version; }
int taos_affected_rows(TAOS_RES *res) { return 1; }
int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; }

50
source/client/src/clientMsgHandler.c
View File

@ -14,6 +14,7 @@
*/
#include <catalog.h>
#include <tname.h>
#include "clientInt.h"
#include "clientLog.h"
#include "os.h"
@ -2885,7 +2886,7 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg
}
int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate, bool onlyLocal) {
assert(tIsValidName(&pTableMetaInfo->name));
assert(tNameIsValid(&pTableMetaInfo->name));
char name[TSDB_TABLE_FNAME_LEN] = {0};
tNameExtractFullName(&pTableMetaInfo->name, name);
@ -3138,7 +3139,7 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
// TODO refactor
pthread_mutex_lock(&pTscObj->mutex);
char temp[TSDB_TABLE_FNAME_LEN * 2] = {0};
int32_t len = sprintf(temp, "%s%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pTscObj->db);
int32_t len = sprintf(temp, "%d%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pTscObj->db);
assert(len <= sizeof(pTscObj->db));
tstrncpy(pTscObj->db, temp, sizeof(pTscObj->db));
@ -3153,6 +3154,7 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
}
pTscObj->connId = pConnect->connId;
pTscObj->acctId = pConnect->acctId;
// update the appInstInfo
pTscObj->pAppInfo->clusterId = pConnect->clusterId;
@ -3165,19 +3167,33 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
return 0;
}
int32_t buildCreateUserMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER;
pMsgBody->msgLen = sizeof(SCreateUserMsg);
int32_t doBuildMsgSupp(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
pMsgBody->requestObjRefId = pRequest->self;
pMsgBody->pData = pRequest->body.param;
return 0;
}
int32_t buildShowMsg(SRequestObj* pRequest, SRequestMsgBody* pMsgBody) {
pMsgBody->msgType = TSDB_MSG_TYPE_SHOW;
pMsgBody->msgLen = pRequest->body.paramLen;
pMsgBody->requestObjRefId = pRequest->self;
pMsgBody->pData = pRequest->body.param;
switch(pRequest->type) {
case TSDB_SQL_CREATE_USER:
pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER;
break;
case TSDB_SQL_CREATE_DB: {
pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_DB;
SCreateDbMsg* pCreateMsg = pRequest->body.param;
SName name = {0};
int32_t ret = tNameSetDbName(&name, pRequest->pTscObj->acctId, pCreateMsg->db, strnlen(pCreateMsg->db, tListLen(pCreateMsg->db)));
if (ret != TSDB_CODE_SUCCESS) {
return -1;
}
tNameGetFullDbName(&name, pCreateMsg->db);
break;
}
case TSDB_SQL_SHOW:
pMsgBody->msgType = TSDB_MSG_TYPE_SHOW;
break;
}
}
STableMeta* createTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) {
@ -3283,6 +3299,9 @@ int32_t processRetrieveMnodeRsp(SRequestObj *pRequest, const char* pMsg, int32_t
return 0;
}
int32_t processCreateDbRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
// todo rsp with the vnode id list
}
void initMsgHandleFp() {
#if 0
@ -3363,11 +3382,14 @@ void initMsgHandleFp() {
buildRequestMsgFp[TSDB_SQL_CONNECT] = buildConnectMsg;
handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp;
buildRequestMsgFp[TSDB_SQL_CREATE_USER] = buildCreateUserMsg;
buildRequestMsgFp[TSDB_SQL_CREATE_USER] = doBuildMsgSupp;
buildRequestMsgFp[TSDB_SQL_SHOW] = buildShowMsg;
buildRequestMsgFp[TSDB_SQL_SHOW] = doBuildMsgSupp;
handleRequestRspFp[TSDB_SQL_SHOW] = processShowRsp;
buildRequestMsgFp[TSDB_SQL_RETRIEVE_MNODE] = buildRetrieveMnodeMsg;
handleRequestRspFp[TSDB_SQL_RETRIEVE_MNODE]= processRetrieveMnodeRsp;
buildRequestMsgFp[TSDB_SQL_CREATE_DB] = doBuildMsgSupp;
handleRequestRspFp[TSDB_SQL_CREATE_DB] = processCreateDbRsp;
}

50
source/client/test/clientTests.cpp
View File

@ -57,25 +57,38 @@ TEST(testCase, create_user_Test) {
taos_close(pConn);
}
//TEST(testCase, show_user_Test) {
//TEST(testCase, drop_user_Test) {
// TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "show users");
// TAOS_ROW pRow = NULL;
//
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes);
//
// char str[512] = {0};
// while((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
// TAOS_RES* pRes = taos_query(pConn, "drop user abc");
// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
// printf("failed to create user, reason:%s\n", taos_errstr(pRes));
// }
//
// taos_free_result(pRes);
// taos_close(pConn);
//}
TEST(testCase, show_user_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "show users");
TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
char str[512] = {0};
while((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
printf("%s\n", str);
}
taos_close(pConn);
}
TEST(testCase, show_db_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
@ -94,3 +107,18 @@ TEST(testCase, show_db_Test) {
taos_close(pConn);
}
TEST(testCase, create_db_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "create database abc");
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
ASSERT_TRUE(pFields == NULL);
int32_t numOfFields = taos_num_fields(pRes);
ASSERT_EQ(numOfFields, 0);
taos_close(pConn);
}

31
source/common/inc/commonInt.h
View File

@ -1,31 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_COMMON_INT_H_
#define _TD_COMMON_INT_H_
#ifdef __cplusplus
extern "C" {
#endif
extern bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
#ifdef __cplusplus
}
#endif
#endif /*_TD_COMMON_INT_H_*/

128
source/common/src/tname.c
View File

@ -4,9 +4,6 @@
#include "tname.h"
#include "taosmsg.h"
#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS)
#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS)
#define VALID_NAME_TYPE(x) ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T)
char* extractDBName(const char* tableId, char* name) {
@ -120,84 +117,11 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in
#endif
static struct SSchema _s = {
.colId = TSDB_TBNAME_COLUMN_INDEX,
.type = TSDB_DATA_TYPE_BINARY,
.bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE,
.name = "tbname",
};
SSchema* tGetTbnameColumnSchema() {
return &_s;
}
static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) {
int32_t rowLen = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
// 1. valid types
if (!isValidDataType(pSchema[i].type)) {
return false;
}
// 2. valid length for each type
if (pSchema[i].type == TSDB_DATA_TYPE_BINARY) {
if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) {
return false;
}
} else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
if (pSchema[i].bytes > TSDB_MAX_NCHAR_LEN) {
return false;
}
} else {
if (pSchema[i].bytes != tDataTypes[pSchema[i].type].bytes) {
return false;
}
}
// 3. valid column names
for (int32_t j = i + 1; j < numOfCols; ++j) {
if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) {
return false;
}
}
rowLen += pSchema[i].bytes;
}
return rowLen <= maxLen;
}
bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags) {
if (!VALIDNUMOFCOLS(numOfCols)) {
return false;
}
if (!VALIDNUMOFTAGS(numOfTags)) {
return false;
}
/* first column must be the timestamp, which is a primary key */
if (pSchema[0].type != TSDB_DATA_TYPE_TIMESTAMP) {
return false;
}
if (!doValidateSchema(pSchema, numOfCols, TSDB_MAX_BYTES_PER_ROW)) {
return false;
}
if (!doValidateSchema(&pSchema[numOfCols], numOfTags, TSDB_MAX_TAGS_LEN)) {
return false;
}
return true;
}
int32_t tNameExtractFullName(const SName* name, char* dst) {
assert(name != NULL && dst != NULL);
// invalid full name format, abort
if (!tIsValidName(name)) {
if (!tNameIsValid(name)) {
return -1;
}
@ -230,7 +154,7 @@ int32_t tNameLen(const SName* name) {
}
}
bool tIsValidName(const SName* name) {
bool tNameIsValid(const SName* name) {
assert(name != NULL);
if (!VALID_NAME_TYPE(name->type)) {
@ -265,13 +189,13 @@ int32_t tNameGetDbName(const SName* name, char* dst) {
int32_t tNameGetFullDbName(const SName* name, char* dst) {
assert(name != NULL && dst != NULL);
snprintf(dst, TSDB_ACCT_ID_LEN + TS_PATH_DELIMITER_LEN + TSDB_DB_NAME_LEN, // there is a over write risk
"%s.%s", name->acctId, name->dbname);
"%d.%s", name->acctId, name->dbname);
return 0;
}
bool tNameIsEmpty(const SName* name) {
assert(name != NULL);
return name->type == 0 || strlen(name->acctId) <= 0;
return name->type == 0 || name->acctId == 0;
}
const char* tNameGetTableName(const SName* name) {
@ -283,32 +207,23 @@ void tNameAssign(SName* dst, const SName* src) {
memcpy(dst, src, sizeof(SName));
}
//int32_t tNameSetDbName(SName* dst, const char* acct, SStrToken* dbToken) {
// assert(dst != NULL && dbToken != NULL && acct != NULL);
//
// // too long account id or too long db name
// if (strlen(acct) >= tListLen(dst->acctId) || dbToken->n >= tListLen(dst->dbname)) {
// return -1;
// }
//
// dst->type = TSDB_DB_NAME_T;
// tstrncpy(dst->acctId, acct, tListLen(dst->acctId));
// tstrncpy(dst->dbname, dbToken->z, dbToken->n + 1);
// return 0;
//}
int32_t tNameSetAcctId(SName* dst, const char* acct) {
assert(dst != NULL && acct != NULL);
int32_t tNameSetDbName(SName* dst, int32_t acct, const char* dbName, size_t nameLen) {
assert(dst != NULL && dbName != NULL && nameLen > 0);
// too long account id or too long db name
if (strlen(acct) >= tListLen(dst->acctId)) {
if (nameLen >= tListLen(dst->dbname)) {
return -1;
}
tstrncpy(dst->acctId, acct, tListLen(dst->acctId));
assert(strlen(dst->acctId) > 0);
dst->type = TSDB_DB_NAME_T;
dst->acctId = acct;
tstrncpy(dst->dbname, dbName, nameLen + 1);
return 0;
}
int32_t tNameSetAcctId(SName* dst, int32_t acctId) {
assert(dst != NULL && acct != NULL);
dst->acctId = acctId;
return 0;
}
@ -325,14 +240,11 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) {
int32_t len = (int32_t)(p - str);
// too long account id or too long db name
if ((len >= tListLen(dst->acctId)) || (len <= 0)) {
return -1;
}
memcpy (dst->acctId, str, len);
dst->acctId[len] = 0;
assert(strlen(dst->acctId) > 0);
// if ((len >= tListLen(dst->acctId)) || (len <= 0)) {
// return -1;
// }
// memcpy (dst->acctId, str, len);
dst->acctId = strtoll(str, NULL, 10);
}
if ((type & T_NAME_DB) == T_NAME_DB) {

1
source/dnode/mgmt/daemon/src/daemon.c
View File

@ -92,6 +92,7 @@ void dmnPrintVersion() {
}
int dmnReadConfig(const char *path) {
tstrncpy(configDir, global.configDir, PATH_MAX);
taosInitGlobalCfg();
taosReadGlobalLogCfg();

88
source/dnode/mgmt/impl/src/dndVnodes.c
View File

@ -95,7 +95,7 @@ static int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg);
static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) {
SVnodesMgmt *pMgmt = &pDnode->vmgmt;
SVnodeObj * pVnode = NULL;
SVnodeObj *pVnode = NULL;
int32_t refCount = 0;
taosRLockLatch(&pMgmt->latch);
@ -107,23 +107,23 @@ static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) {
}
taosRUnLockLatch(&pMgmt->latch);
dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount);
if (pVnode != NULL) {
dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount);
}
return pVnode;
}
static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) {
if (pVnode == NULL) return;
SVnodesMgmt *pMgmt = &pDnode->vmgmt;
int32_t refCount = 0;
taosRLockLatch(&pMgmt->latch);
if (pVnode != NULL) {
refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
}
int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
taosRUnLockLatch(&pMgmt->latch);
if (pVnode != NULL) {
dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount);
}
dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount);
}
static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, SVnode *pImpl) {
@ -457,7 +457,7 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
pMgmt->totalVnodes = numOfVnodes;
int32_t threadNum = tsNumOfCores;
int32_t threadNum = pDnode->opt.numOfCores;
int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread));
@ -525,33 +525,49 @@ static void dndCloseVnodes(SDnode *pDnode) {
static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg *pCfg) {
SCreateVnodeMsg *pCreate = rpcMsg->pCont;
*vgId = htonl(pCreate->vgId);
pCreate->vgId = htonl(pCreate->vgId);
pCreate->dnodeId = htonl(pCreate->dnodeId);
pCreate->dbUid = htobe64(pCreate->dbUid);
pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize);
pCreate->totalBlocks = htonl(pCreate->totalBlocks);
pCreate->daysPerFile = htonl(pCreate->daysPerFile);
pCreate->daysToKeep0 = htonl(pCreate->daysToKeep0);
pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1);
pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2);
pCreate->minRows = htonl(pCreate->minRows);
pCreate->maxRows = htonl(pCreate->maxRows);
pCreate->commitTime = htonl(pCreate->commitTime);
pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod);
for (int r = 0; r < pCreate->replica; ++r) {
SReplica *pReplica = &pCreate->replicas[r];
pReplica->id = htonl(pReplica->id);
pReplica->port = htons(pReplica->port);
}
*vgId = pCreate->vgId;
#if 0
tstrncpy(pCfg->db, pCreate->db, TSDB_FULL_DB_NAME_LEN);
pCfg->cacheBlockSize = htonl(pCreate->cacheBlockSize);
pCfg->totalBlocks = htonl(pCreate->totalBlocks);
pCfg->daysPerFile = htonl(pCreate->daysPerFile);
pCfg->daysToKeep0 = htonl(pCreate->daysToKeep0);
pCfg->daysToKeep1 = htonl(pCreate->daysToKeep1);
pCfg->daysToKeep2 = htonl(pCreate->daysToKeep2);
pCfg->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock);
pCfg->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock);
pCfg->precision = pCreate->precision;
pCfg->compression = pCreate->compression;
pCfg->cacheLastRow = pCreate->cacheLastRow;
pCfg->update = pCreate->update;
pCfg->quorum = pCreate->quorum;
pCfg->replica = pCreate->replica;
pCfg->walLevel = pCreate->walLevel;
pCfg->fsyncPeriod = htonl(pCreate->fsyncPeriod);
for (int32_t i = 0; i < pCfg->replica; ++i) {
pCfg->replicas[i].port = htons(pCreate->replicas[i].port);
tstrncpy(pCfg->replicas[i].fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN);
}
pCfg->wsize = pCreate->cacheBlockSize;
pCfg->ssize = pCreate->cacheBlockSize;
pCfg->wsize = pCreate->cacheBlockSize;
pCfg->lsize = pCreate->cacheBlockSize;
pCfg->isHeapAllocator = true;
pCfg->ttl = 4;
pCfg->keep = pCreate->daysToKeep0;
pCfg->isWeak = true;
pCfg->tsdbCfg.keep0 = pCreate->daysToKeep0;
pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2;
pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0;
pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize;
pCfg->metaCfg.lruSize = pCreate->cacheBlockSize;
pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod;
pCfg->walCfg.level = pCreate->walLevel;
pCfg->walCfg.retentionPeriod = 10;
pCfg->walCfg.retentionSize = 128;
pCfg->walCfg.rollPeriod = 128;
pCfg->walCfg.segSize = 128;
pCfg->walCfg.vgId = pCreate->vgId;
#endif
return 0;
}
@ -1016,7 +1032,7 @@ static int32_t dndInitVnodeWriteWorker(SDnode *pDnode) {
SVnodesMgmt * pMgmt = &pDnode->vmgmt;
SMWorkerPool *pPool = &pMgmt->writePool;
pPool->name = "vnode-write";
pPool->max = tsNumOfCores;
pPool->max = pDnode->opt.numOfCores;
if (tMWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@ -1050,7 +1066,7 @@ static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) {
}
static int32_t dndInitVnodeSyncWorker(SDnode *pDnode) {
int32_t maxThreads = tsNumOfCores / 2;
int32_t maxThreads = pDnode->opt.numOfCores / 2;
if (maxThreads < 1) maxThreads = 1;
SVnodesMgmt *pMgmt = &pDnode->vmgmt;

2
source/dnode/mgmt/impl/test/CMakeLists.txt
View File

@ -15,6 +15,6 @@ add_subdirectory(stb)
# add_subdirectory(telem)
# add_subdirectory(trans)
add_subdirectory(user)
# add_subdirectory(vgroup)
add_subdirectory(vgroup)
# add_subdirectory(common)

1
source/dnode/mgmt/impl/test/db/db.cpp
View File

@ -232,6 +232,7 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) {
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
// taosMsleep(1000000);
}
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL);

27
source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt Normal file
View File

@ -0,0 +1,27 @@
add_executable(dnode_test_vgroup "")
target_sources(dnode_test_vgroup
PRIVATE
"vgroup.cpp"
"../sut/deploy.cpp"
)
target_link_libraries(
dnode_test_vgroup
PUBLIC dnode
PUBLIC util
PUBLIC os
PUBLIC gtest_main
)
target_include_directories(dnode_test_vgroup
PUBLIC
"${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt"
"${CMAKE_CURRENT_SOURCE_DIR}/../../inc"
"${CMAKE_CURRENT_SOURCE_DIR}/../sut"
)
add_test(
NAME dnode_test_vgroup
COMMAND dnode_test_vgroup
)

224
source/dnode/mgmt/impl/test/vgroup/vgroup.cpp Normal file
View File

@ -0,0 +1,224 @@
/**
* @file db.cpp
* @author slguan (slguan@taosdata.com)
* @brief DNODE module vgroup-msg tests
* @version 0.1
* @date 2021-12-20
*
* @copyright Copyright (c) 2021
*
*/
#include "deploy.h"
class DndTestVgroup : public ::testing::Test {
protected:
static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) {
SServer* pServer = createServer(path, fqdn, port, firstEp);
ASSERT(pServer);
return pServer;
}
static void SetUpTestSuite() {
initLog("/tmp/tdlog");
const char* fqdn = "localhost";
const char* firstEp = "localhost:9150";
pServer = CreateServer("/tmp/dnode_test_vgroup", fqdn, 9150, firstEp);
pClient = createClient("root", "taosdata", fqdn, 9150);
taosMsleep(1100);
}
static void TearDownTestSuite() {
stopServer(pServer);
dropClient(pClient);
pServer = NULL;
pClient = NULL;
}
static SServer* pServer;
static SClient* pClient;
static int32_t connId;
public:
void SetUp() override {}
void TearDown() override {}
void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) {
SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg));
pShow->type = showType;
if (db != NULL) {
strcpy(pShow->db, db);
}
SRpcMsg showRpcMsg = {0};
showRpcMsg.pCont = pShow;
showRpcMsg.contLen = sizeof(SShowMsg);
showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW;
sendMsg(pClient, &showRpcMsg);
ASSERT_NE(pClient->pRsp, nullptr);
ASSERT_EQ(pClient->pRsp->code, 0);
ASSERT_NE(pClient->pRsp->pCont, nullptr);
SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont;
ASSERT_NE(pShowRsp, nullptr);
pShowRsp->showId = htonl(pShowRsp->showId);
pMeta = &pShowRsp->tableMeta;
pMeta->numOfTags = htonl(pMeta->numOfTags);
pMeta->numOfColumns = htonl(pMeta->numOfColumns);
pMeta->sversion = htonl(pMeta->sversion);
pMeta->tversion = htonl(pMeta->tversion);
pMeta->tuid = htobe64(pMeta->tuid);
pMeta->suid = htobe64(pMeta->suid);
showId = pShowRsp->showId;
EXPECT_NE(pShowRsp->showId, 0);
EXPECT_STREQ(pMeta->tbFname, showName);
EXPECT_EQ(pMeta->numOfTags, 0);
EXPECT_EQ(pMeta->numOfColumns, columns);
EXPECT_EQ(pMeta->precision, 0);
EXPECT_EQ(pMeta->tableType, 0);
EXPECT_EQ(pMeta->update, 0);
EXPECT_EQ(pMeta->sversion, 0);
EXPECT_EQ(pMeta->tversion, 0);
EXPECT_EQ(pMeta->tuid, 0);
EXPECT_EQ(pMeta->suid, 0);
}
void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) {
SSchema* pSchema = &pMeta->pSchema[index];
pSchema->bytes = htonl(pSchema->bytes);
EXPECT_EQ(pSchema->colId, 0);
EXPECT_EQ(pSchema->type, type);
EXPECT_EQ(pSchema->bytes, bytes);
EXPECT_STREQ(pSchema->name, name);
}
void SendThenCheckShowRetrieveMsg(int32_t rows) {
SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg));
pRetrieve->showId = htonl(showId);
pRetrieve->free = 0;
SRpcMsg retrieveRpcMsg = {0};
retrieveRpcMsg.pCont = pRetrieve;
retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg);
retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE;
sendMsg(pClient, &retrieveRpcMsg);
ASSERT_NE(pClient->pRsp, nullptr);
ASSERT_EQ(pClient->pRsp->code, 0);
ASSERT_NE(pClient->pRsp->pCont, nullptr);
pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont;
ASSERT_NE(pRetrieveRsp, nullptr);
pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows);
pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds);
pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen);
EXPECT_EQ(pRetrieveRsp->numOfRows, rows);
EXPECT_EQ(pRetrieveRsp->useconds, 0);
// EXPECT_EQ(pRetrieveRsp->completed, completed);
EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(pRetrieveRsp->compressed, 0);
EXPECT_EQ(pRetrieveRsp->compLen, 0);
pData = pRetrieveRsp->data;
pos = 0;
}
void CheckInt8(int8_t val) {
int8_t data = *((int8_t*)(pData + pos));
pos += sizeof(int8_t);
EXPECT_EQ(data, val);
}
void CheckInt16(int16_t val) {
int16_t data = *((int16_t*)(pData + pos));
pos += sizeof(int16_t);
EXPECT_EQ(data, val);
}
void CheckInt32(int32_t val) {
int32_t data = *((int32_t*)(pData + pos));
pos += sizeof(int32_t);
EXPECT_EQ(data, val);
}
void CheckInt64(int64_t val) {
int64_t data = *((int64_t*)(pData + pos));
pos += sizeof(int64_t);
EXPECT_EQ(data, val);
}
void CheckTimestamp() {
int64_t data = *((int64_t*)(pData + pos));
pos += sizeof(int64_t);
EXPECT_GT(data, 0);
}
void CheckBinary(const char* val, int32_t len) {
pos += sizeof(VarDataLenT);
char* data = (char*)(pData + pos);
pos += len;
EXPECT_STREQ(data, val);
}
int32_t showId;
STableMetaMsg* pMeta;
SRetrieveTableRsp* pRetrieveRsp;
char* pData;
int32_t pos;
};
SServer* DndTestVgroup::pServer;
SClient* DndTestVgroup::pClient;
int32_t DndTestVgroup::connId;
TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) {
{
SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(sizeof(SCreateVnodeMsg));
pReq->vgId = htonl(2);
pReq->dnodeId = htonl(1);
strcpy(pReq->db, "1.d1");
pReq->dbUid = htobe64(9527);
pReq->cacheBlockSize = htonl(16);
pReq->totalBlocks = htonl(10);
pReq->daysPerFile = htonl(10);
pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650);
pReq->minRows = htonl(100);
pReq->minRows = htonl(4096);
pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1;
pReq->precision = 0;
pReq->compression = 2;
pReq->replica = 1;
pReq->quorum = 1;
pReq->update = 0;
pReq->cacheLastRow = 0;
pReq->selfIndex = 0;
for (int r = 0; r < pReq->replica; ++r) {
SReplica* pReplica = &pReq->replicas[r];
pReplica->id = htonl(1);
pReplica->port = htons(9150);
}
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SCreateVnodeMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
taosMsleep(1000000);
}
}

2
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -182,6 +182,7 @@ typedef struct {
} SUserObj;
typedef struct {
int32_t numOfVgroups;
int32_t cacheBlockSize;
int32_t totalBlocks;
int32_t daysPerFile;
@ -209,7 +210,6 @@ typedef struct {
int64_t uid;
int32_t cfgVersion;
int32_t vgVersion;
int32_t numOfVgroups;
int8_t hashMethod; // default is 1
SDbCfg cfg;
} SDbObj;

4
source/dnode/mnode/impl/inc/mndTrans.h
View File

@ -30,8 +30,8 @@ void mndTransDrop(STrans *pTrans);
int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw);
int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw);
int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw);
int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *, void *pMsg);
int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *, void *pMsg);
int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont);
int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont);
int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);
void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code);
char *mndTransStageStr(ETrnStage stage);

4
source/dnode/mnode/impl/inc/mndVgroup.h
View File

@ -28,7 +28,9 @@ SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId);
void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup);
int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups);
SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup);
SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw);
SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup);
SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup);
#ifdef __cplusplus
}

73
source/dnode/mnode/impl/src/mndDb.c
View File

@ -77,8 +77,8 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SDB_SET_INT64(pRaw, dataPos, pDb->uid)
SDB_SET_INT32(pRaw, dataPos, pDb->cfgVersion)
SDB_SET_INT32(pRaw, dataPos, pDb->vgVersion)
SDB_SET_INT32(pRaw, dataPos, pDb->numOfVgroups)
SDB_SET_INT8(pRaw, dataPos, pDb->hashMethod)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfVgroups)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile)
@ -124,8 +124,8 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->uid)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfgVersion)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->vgVersion)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->numOfVgroups)
SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->hashMethod)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.numOfVgroups)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.cacheBlockSize)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.totalBlocks)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysPerFile)
@ -163,7 +163,6 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) {
pOldDb->updateTime = pNewDb->createdTime;
pOldDb->cfgVersion = pNewDb->cfgVersion;
pOldDb->vgVersion = pNewDb->vgVersion;
pOldDb->numOfVgroups = pNewDb->numOfVgroups;
memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg));
return 0;
}
@ -195,6 +194,7 @@ static int32_t mndCheckDbName(char *dbName, SUserObj *pUser) {
}
static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1;
if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) return -1;
if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) return -1;
if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1;
@ -222,6 +222,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
}
static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->numOfVgroups < 0) pCfg->numOfVgroups = TSDB_DEFAULT_VN_PER_DB;
if (pCfg->cacheBlockSize < 0) pCfg->cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
if (pCfg->totalBlocks < 0) pCfg->totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS;
if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
@ -246,7 +247,7 @@ static int32_t mndSetRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgOb
if (pDbRaw == NULL || mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1;
sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING);
for (int v = 0; v < pDb->numOfVgroups; ++v) {
for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v);
if (pVgRaw == NULL || mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1;
sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING);
@ -260,7 +261,7 @@ static int32_t mndSetUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgOb
if (pDbRaw == NULL || mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1;
sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED);
for (int v = 0; v < pDb->numOfVgroups; ++v) {
for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v);
if (pVgRaw == NULL || mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1;
sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED);
@ -274,7 +275,7 @@ static int32_t mndSetCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVg
if (pDbRaw == NULL || mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1;
sdbSetRawStatus(pDbRaw, SDB_STATUS_READY);
for (int v = 0; v < pDb->numOfVgroups; ++v) {
for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v);
if (pVgRaw == NULL || mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1;
sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
@ -284,10 +285,60 @@ static int32_t mndSetCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVg
}
static int32_t mndSetRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SVgObj *pVgroup = pVgroups + v;
for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pDnode == NULL) {
return -1;
}
SEpSet epset = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode);
SCreateVnodeMsg *pMsg = mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup);
if (pMsg == NULL) {
return -1;
}
if (mndTransAppendRedoAction(pTrans, &epset, TSDB_MSG_TYPE_ALTER_VNODE_IN, sizeof(SCreateVnodeMsg), pMsg) != 0) {
free(pMsg);
return -1;
}
}
}
return 0;
}
static int32_t mndSetUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SVgObj *pVgroup = pVgroups + v;
for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pDnode == NULL) {
return -1;
}
SEpSet epset = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode);
SDropVnodeMsg *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup);
if (pMsg == NULL) {
return -1;
}
if (mndTransAppendUndoAction(pTrans, &epset, TSDB_MSG_TYPE_DROP_VNODE_IN, sizeof(SDropVnodeMsg), pMsg) != 0) {
free(pMsg);
return -1;
}
}
}
return 0;
}
@ -298,11 +349,11 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat
dbObj.createdTime = taosGetTimestampMs();
dbObj.updateTime = dbObj.createdTime;
dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN);
dbObj.numOfVgroups = pCreate->numOfVgroups;
dbObj.hashMethod = 1;
dbObj.cfgVersion = 1;
dbObj.vgVersion = 1;
dbObj.cfg = (SDbCfg){.cacheBlockSize = pCreate->cacheBlockSize,
dbObj.cfg = (SDbCfg){.numOfVgroups = pCreate->numOfVgroups,
.cacheBlockSize = pCreate->cacheBlockSize,
.totalBlocks = pCreate->totalBlocks,
.daysPerFile = pCreate->daysPerFile,
.daysToKeep0 = pCreate->daysToKeep0,
@ -643,7 +694,7 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) {
return -1;
}
int32_t contLen = sizeof(SUseDbRsp) + pDb->numOfVgroups * sizeof(SVgroupInfo);
int32_t contLen = sizeof(SUseDbRsp) + pDb->cfg.numOfVgroups * sizeof(SVgroupInfo);
SUseDbRsp *pRsp = rpcMallocCont(contLen);
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -654,7 +705,7 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) {
if (pUse->vgVersion < pDb->vgVersion) {
void *pIter = NULL;
while (vindex < pDb->numOfVgroups) {
while (vindex < pDb->cfg.numOfVgroups) {
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
@ -888,7 +939,7 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *)pWrite = pDb->numOfVgroups;
*(int16_t *)pWrite = pDb->cfg.numOfVgroups;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

8
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -180,8 +180,12 @@ static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj
}
SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId) {
SSdb *pSdb = pMnode->pSdb;
return sdbAcquire(pSdb, SDB_DNODE, &dnodeId);
SSdb *pSdb = pMnode->pSdb;
SDnodeObj *pDnode = sdbAcquire(pSdb, SDB_DNODE, &dnodeId);
if (pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
}
return pDnode;
}
void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode) {

208
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -21,6 +21,13 @@
#define TSDB_TRN_ARRAY_SIZE 8
#define TSDB_TRN_RESERVE_SIZE 64
typedef struct {
SEpSet epSet;
int8_t msgType;
int32_t contLen;
void *pCont;
} STransAction;
static SSdbRaw *mndTransActionEncode(STrans *pTrans);
static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw);
static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans);
@ -29,9 +36,12 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans);
static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle);
static void mndTransSendRpcRsp(STrans *pTrans, int32_t code);
static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw);
static void mndTransDropArray(SArray *pArray);
static int32_t mndTransExecuteArray(SMnode *pMnode, SArray *pArray);
static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw);
static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont);
static void mndTransDropLogs(SArray *pArray);
static void mndTransDropActions(SArray *pArray);
static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray);
static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray);
static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans);
static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans);
static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans);
@ -58,7 +68,7 @@ int32_t mndInitTrans(SMnode *pMnode) {
void mndCleanupTrans(SMnode *pMnode) {}
static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
int32_t rawDataLen = 16 * sizeof(int32_t) + TSDB_TRN_RESERVE_SIZE;
int32_t rawDataLen = sizeof(STrans) + TSDB_TRN_RESERVE_SIZE;
int32_t redoLogNum = taosArrayGetSize(pTrans->redoLogs);
int32_t undoLogNum = taosArrayGetSize(pTrans->undoLogs);
int32_t commitLogNum = taosArrayGetSize(pTrans->commitLogs);
@ -80,6 +90,16 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
rawDataLen += sdbGetRawTotalSize(pTmp);
}
for (int32_t i = 0; i < redoActionNum; ++i) {
STransAction *pAction = taosArrayGet(pTrans->redoActions, i);
rawDataLen += (sizeof(STransAction) + pAction->contLen);
}
for (int32_t i = 0; i < undoActionNum; ++i) {
STransAction *pAction = taosArrayGet(pTrans->undoActions, i);
rawDataLen += (sizeof(STransAction) + pAction->contLen);
}
SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER, rawDataLen);
if (pRaw == NULL) {
mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr());
@ -116,6 +136,22 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len)
}
for (int32_t i = 0; i < redoActionNum; ++i) {
STransAction *pAction = taosArrayGet(pTrans->redoActions, i);
SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet));
SDB_SET_INT8(pRaw, dataPos, pAction->msgType)
SDB_SET_INT32(pRaw, dataPos, pAction->contLen)
SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen);
}
for (int32_t i = 0; i < undoActionNum; ++i) {
STransAction *pAction = taosArrayGet(pTrans->undoActions, i);
SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet));
SDB_SET_INT8(pRaw, dataPos, pAction->msgType)
SDB_SET_INT32(pRaw, dataPos, pAction->contLen)
SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen);
}
SDB_SET_RESERVE(pRaw, dataPos, TSDB_TRN_RESERVE_SIZE)
SDB_SET_DATALEN(pRaw, dataPos);
mTrace("trans:%d, encode to raw:%p, len:%d", pTrans->id, pRaw, dataPos);
@ -147,8 +183,8 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
pTrans->redoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->undoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->commitLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction));
pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction));
if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL ||
pTrans->redoActions == NULL || pTrans->undoActions == NULL) {
@ -175,42 +211,77 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
for (int32_t i = 0; i < redoLogNum; ++i) {
int32_t dataLen = 0;
SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen)
char *pData = malloc(dataLen);
SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen);
void *ret = taosArrayPush(pTrans->redoLogs, &pData);
if (ret == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto TRANS_DECODE_OVER;
break;
}
}
for (int32_t i = 0; i < undoLogNum; ++i) {
int32_t dataLen = 0;
SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen)
char *pData = malloc(dataLen);
SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen);
void *ret = taosArrayPush(pTrans->undoLogs, &pData);
if (ret == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto TRANS_DECODE_OVER;
break;
}
}
for (int32_t i = 0; i < commitLogNum; ++i) {
int32_t dataLen = 0;
SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen)
char *pData = malloc(dataLen);
SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen);
void *ret = taosArrayPush(pTrans->commitLogs, &pData);
if (ret == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto TRANS_DECODE_OVER;
break;
}
}
for (int32_t i = 0; i < redoActionNum; ++i) {
STransAction action = {0};
SDB_GET_BINARY(pRaw, pRow, dataPos, (void *)&action.epSet, sizeof(SEpSet));
SDB_GET_INT8(pRaw, pRow, dataPos, &action.msgType)
SDB_GET_INT32(pRaw, pRow, dataPos, &action.contLen)
action.pCont = malloc(action.contLen);
if (action.pCont == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto TRANS_DECODE_OVER;
}
SDB_GET_BINARY(pRaw, pRow, dataPos, action.pCont, action.contLen);
void *ret = taosArrayPush(pTrans->redoActions, &action);
if (ret == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto TRANS_DECODE_OVER;
}
}
for (int32_t i = 0; i < undoActionNum; ++i) {
STransAction action = {0};
SDB_GET_BINARY(pRaw, pRow, dataPos, (void *)&action.epSet, sizeof(SEpSet));
SDB_GET_INT8(pRaw, pRow, dataPos, &action.msgType)
SDB_GET_INT32(pRaw, pRow, dataPos, &action.contLen)
action.pCont = malloc(action.contLen);
if (action.pCont == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto TRANS_DECODE_OVER;
}
SDB_GET_BINARY(pRaw, pRow, dataPos, action.pCont, action.contLen);
void *ret = taosArrayPush(pTrans->undoActions, &action);
if (ret == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto TRANS_DECODE_OVER;
}
}
@ -237,11 +308,11 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) {
static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) {
mTrace("trans:%d, perform delete action, stage:%s", pTrans->id, mndTransStageStr(pTrans->stage));
mndTransDropArray(pTrans->redoLogs);
mndTransDropArray(pTrans->undoLogs);
mndTransDropArray(pTrans->commitLogs);
mndTransDropArray(pTrans->redoActions);
mndTransDropArray(pTrans->undoActions);
mndTransDropLogs(pTrans->redoLogs);
mndTransDropLogs(pTrans->undoLogs);
mndTransDropLogs(pTrans->commitLogs);
mndTransDropActions(pTrans->redoActions);
mndTransDropActions(pTrans->undoActions);
return 0;
}
@ -274,6 +345,8 @@ char *mndTransStageStr(ETrnStage stage) {
return "rollback";
case TRN_STAGE_RETRY:
return "retry";
case TRN_STAGE_OVER:
return "stop";
default:
return "undefined";
}
@ -305,8 +378,8 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) {
pTrans->redoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->undoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->commitLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction));
pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction));
if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL ||
pTrans->redoActions == NULL || pTrans->undoActions == NULL) {
@ -319,7 +392,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) {
return pTrans;
}
static void mndTransDropArray(SArray *pArray) {
static void mndTransDropLogs(SArray *pArray) {
for (int32_t i = 0; i < pArray->size; ++i) {
SSdbRaw *pRaw = taosArrayGetP(pArray, i);
tfree(pRaw);
@ -328,12 +401,21 @@ static void mndTransDropArray(SArray *pArray) {
taosArrayDestroy(pArray);
}
static void mndTransDropActions(SArray *pArray) {
for (int32_t i = 0; i < pArray->size; ++i) {
STransAction *pAction = taosArrayGet(pArray, i);
free(pAction->pCont);
}
taosArrayDestroy(pArray);
}
void mndTransDrop(STrans *pTrans) {
mndTransDropArray(pTrans->redoLogs);
mndTransDropArray(pTrans->undoLogs);
mndTransDropArray(pTrans->commitLogs);
mndTransDropArray(pTrans->redoActions);
mndTransDropArray(pTrans->undoActions);
mndTransDropLogs(pTrans->redoLogs);
mndTransDropLogs(pTrans->undoLogs);
mndTransDropLogs(pTrans->commitLogs);
mndTransDropActions(pTrans->redoActions);
mndTransDropActions(pTrans->undoActions);
mDebug("trans:%d, data:%p is dropped", pTrans->id, pTrans);
tfree(pTrans);
@ -344,7 +426,7 @@ static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle) {
mTrace("trans:%d, set rpc handle:%p", pTrans->id, rpcHandle);
}
static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw) {
static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw) {
if (pArray == NULL || pRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@ -360,32 +442,44 @@ static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw) {
}
int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw) {
int32_t code = mndTransAppendArray(pTrans->redoLogs, pRaw);
int32_t code = mndTransAppendLog(pTrans->redoLogs, pRaw);
mTrace("trans:%d, raw:%p append to redo logs, code:0x%x", pTrans->id, pRaw, code);
return code;
}
int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw) {
int32_t code = mndTransAppendArray(pTrans->undoLogs, pRaw);
int32_t code = mndTransAppendLog(pTrans->undoLogs, pRaw);
mTrace("trans:%d, raw:%p append to undo logs, code:0x%x", pTrans->id, pRaw, code);
return code;
}
int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) {
int32_t code = mndTransAppendArray(pTrans->commitLogs, pRaw);
int32_t code = mndTransAppendLog(pTrans->commitLogs, pRaw);
mTrace("trans:%d, raw:%p append to commit logs, code:0x%x", pTrans->id, pRaw, code);
return code;
}
int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) {
int32_t code = mndTransAppendArray(pTrans->redoActions, pMsg);
mTrace("trans:%d, msg:%p append to redo actions", pTrans->id, pMsg);
static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) {
STransAction action = {.epSet = *pEpSet, .msgType = msgType, .contLen = contLen, .pCont = pCont};
void *ptr = taosArrayPush(pArray, &action);
if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) {
int32_t code = mndTransAppendAction(pTrans->redoActions, pEpSet, msgType, contLen, pCont);
mTrace("trans:%d, msg:%s len:%d append to redo actions", pTrans->id, taosMsg[msgType], contLen);
return code;
}
int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) {
int32_t code = mndTransAppendArray(pTrans->undoActions, pMsg);
mTrace("trans:%d, msg:%p append to undo actions", pTrans->id, pMsg);
int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) {
int32_t code = mndTransAppendAction(pTrans->undoActions, pEpSet, msgType, contLen, pCont);
mTrace("trans:%d, msg:%s len:%d append to undo actions", pTrans->id, taosMsg[msgType], contLen);
return code;
}
@ -502,7 +596,7 @@ void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code)
// todo
}
static int32_t mndTransExecuteArray(SMnode *pMnode, SArray *pArray) {
static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) {
SSdb *pSdb = pMnode->pSdb;
int32_t arraySize = taosArrayGetSize(pArray);
@ -520,7 +614,7 @@ static int32_t mndTransExecuteArray(SMnode *pMnode, SArray *pArray) {
static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) {
int32_t code = 0;
if (taosArrayGetSize(pTrans->redoLogs) != 0) {
code = mndTransExecuteArray(pMnode, pTrans->redoLogs);
code = mndTransExecuteLogs(pMnode, pTrans->redoLogs);
if (code != 0) {
mError("trans:%d, failed to execute redo logs since %s", pTrans->id, terrstr())
} else {
@ -534,7 +628,7 @@ static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) {
static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) {
int32_t code = 0;
if (taosArrayGetSize(pTrans->undoLogs) != 0) {
code = mndTransExecuteArray(pMnode, pTrans->undoLogs);
code = mndTransExecuteLogs(pMnode, pTrans->undoLogs);
if (code != 0) {
mError("trans:%d, failed to execute undo logs since %s", pTrans->id, terrstr())
} else {
@ -548,7 +642,7 @@ static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) {
static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) {
int32_t code = 0;
if (taosArrayGetSize(pTrans->commitLogs) != 0) {
code = mndTransExecuteArray(pMnode, pTrans->commitLogs);
code = mndTransExecuteLogs(pMnode, pTrans->commitLogs);
if (code != 0) {
mError("trans:%d, failed to execute commit logs since %s", pTrans->id, terrstr())
} else {
@ -559,18 +653,40 @@ static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) {
return code;
}
static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) {
if (taosArrayGetSize(pTrans->redoActions) != 0) {
mTrace("trans:%d, execute redo actions finished", pTrans->id);
static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray) {
#if 0
int32_t arraySize = taosArrayGetSize(pArray);
for (int32_t i = 0; i < arraySize; ++i) {
STransAction *pAction = taosArrayGet(pArray, i);
SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen};
rpcMsg.pCont = rpcMallocCont(pAction->contLen);
if (rpcMsg.pCont == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen);
mndSendMsgToDnode(pMnode, &pAction->epSet, &rpcMsg);
}
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
#else
return 0;
#endif
}
static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) {
if (taosArrayGetSize(pTrans->redoActions) <= 0) return 0;
mTrace("trans:%d, start to execute redo actions", pTrans->id);
return mndTransExecuteActions(pMnode, pTrans->redoActions);
}
static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) {
if (taosArrayGetSize(pTrans->undoActions) != 0) {
mTrace("trans:%d, execute undo actions finished", pTrans->id);
}
return 0;
if (taosArrayGetSize(pTrans->undoActions) <= 0) return 0;
mTrace("trans:%d, start to execute undo actions", pTrans->id);
return mndTransExecuteActions(pMnode, pTrans->undoActions);
}
static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) {

104
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -24,9 +24,10 @@
#define TSDB_VGROUP_VER_NUM 1
#define TSDB_VGROUP_RESERVE_SIZE 64
static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup);
static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw);
static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup);
static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg);
static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg);
@ -156,9 +157,78 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) {
sdbRelease(pSdb, pVgroup);
}
static int32_t mndGetDefaultVgroupSize(SMnode *pMnode) {
// todo
return 2;
SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) {
SCreateVnodeMsg *pCreate = malloc(sizeof(SCreateVnodeMsg));
if (pCreate == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pCreate->dnodeId = htonl(pDnode->id);
pCreate->vgId = htonl(pVgroup->vgId);
memcpy(pCreate->db, pDb->name, TSDB_FULL_DB_NAME_LEN);
pCreate->dbUid = htobe64(pDb->uid);
pCreate->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize);
pCreate->totalBlocks = htonl(pDb->cfg.totalBlocks);
pCreate->daysPerFile = htonl(pDb->cfg.daysPerFile);
pCreate->daysToKeep0 = htonl(pDb->cfg.daysToKeep0);
pCreate->daysToKeep1 = htonl(pDb->cfg.daysToKeep1);
pCreate->daysToKeep2 = htonl(pDb->cfg.daysToKeep2);
pCreate->minRows = htonl(pDb->cfg.minRows);
pCreate->maxRows = htonl(pDb->cfg.maxRows);
pCreate->commitTime = htonl(pDb->cfg.commitTime);
pCreate->fsyncPeriod = htonl(pDb->cfg.fsyncPeriod);
pCreate->walLevel = pDb->cfg.walLevel;
pCreate->precision = pDb->cfg.precision;
pCreate->compression = pDb->cfg.compression;
pCreate->quorum = pDb->cfg.quorum;
pCreate->update = pDb->cfg.update;
pCreate->cacheLastRow = pDb->cfg.cacheLastRow;
pCreate->replica = pVgroup->replica;
pCreate->selfIndex = -1;
for (int32_t v = 0; v < pVgroup->replica; ++v) {
SReplica *pReplica = &pCreate->replicas[v];
SVnodeGid *pVgid = &pVgroup->vnodeGid[v];
SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pVgidDnode == NULL) {
free(pCreate);
terrno = TSDB_CODE_MND_APP_ERROR;
return NULL;
}
pReplica->id = htonl(pVgidDnode->id);
pReplica->port = htons(pVgidDnode->port);
memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN);
mndReleaseDnode(pMnode, pVgidDnode);
if (pDnode->id == pVgid->dnodeId) {
pCreate->selfIndex = v;
}
}
if (pCreate->selfIndex == -1) {
free(pCreate);
terrno = TSDB_CODE_MND_APP_ERROR;
return NULL;
}
return pCreate;
}
SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) {
SDropVnodeMsg *pDrop = malloc(sizeof(SDropVnodeMsg));
if (pDrop == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pDrop->dnodeId = htonl(pDnode->id);
pDrop->vgId = htonl(pVgroup->vgId);
memcpy(pDrop->db, pDb->name, TSDB_FULL_DB_NAME_LEN);
pDrop->dbUid = htobe64(pDb->uid);
return pDrop;
}
static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup) {
@ -193,21 +263,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup) {
}
int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
if (pDb->numOfVgroups != -1 &&
(pDb->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pDb->numOfVgroups > TSDB_MAX_VNODES_PER_DB)) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
return -1;
}
if (pDb->numOfVgroups == -1) {
pDb->numOfVgroups = mndGetDefaultVgroupSize(pMnode);
if (pDb->numOfVgroups < 0) {
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
return -1;
}
}
SVgObj *pVgroups = calloc(pDb->numOfVgroups, sizeof(SVgObj));
SVgObj *pVgroups = calloc(pDb->cfg.numOfVgroups, sizeof(SVgObj));
if (pVgroups == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@ -217,9 +273,9 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP);
uint32_t hashMin = 0;
uint32_t hashMax = UINT32_MAX;
uint32_t hashInterval = (hashMax - hashMin) / pDb->numOfVgroups;
uint32_t hashInterval = (hashMax - hashMin) / pDb->cfg.numOfVgroups;
for (uint32_t v = 0; v < pDb->numOfVgroups; v++) {
for (uint32_t v = 0; v < pDb->cfg.numOfVgroups; v++) {
SVgObj *pVgroup = &pVgroups[v];
pVgroup->vgId = maxVgId++;
pVgroup->createdTime = taosGetTimestampMs();
@ -227,7 +283,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
pVgroup->version = 1;
pVgroup->dbUid = pDb->uid;
pVgroup->hashBegin = hashMin + hashInterval * v;
if (v == pDb->numOfVgroups - 1) {
if (v == pDb->cfg.numOfVgroups - 1) {
pVgroup->hashEnd = hashMax;
} else {
pVgroup->hashEnd = hashMin + hashInterval * (v + 1) - 1;

2
source/libs/CMakeLists.txt
View File

@ -10,4 +10,4 @@ add_subdirectory(catalog)
add_subdirectory(executor)
add_subdirectory(planner)
add_subdirectory(function)
add_subdirectory(query)
add_subdirectory(qcom)

2
source/libs/catalog/CMakeLists.txt
View File

@ -8,7 +8,7 @@ target_include_directories(
target_link_libraries(
catalog
PRIVATE os util common transport query
PRIVATE os util transport qcom
)
ADD_SUBDIRECTORY(test)

2
source/libs/catalog/test/CMakeLists.txt
View File

@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(catalogTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
catalogTest
PUBLIC os util common catalog transport gtest query taos
PUBLIC os util common catalog transport gtest qcom taos
)
TARGET_INCLUDE_DIRECTORIES(

7
source/libs/index/CMakeLists.txt
View File

@ -9,6 +9,7 @@ target_link_libraries(
index
PUBLIC os
PUBLIC util
PUBLIC common
)
if (${BUILD_WITH_LUCENE})
@ -21,9 +22,13 @@ if (${BUILD_WITH_LUCENE})
index
PUBLIC lucene++
)
endif(${BUILD_WITH_LUCENE})
if (${BUILD_WITH_INVERTEDINDEX})
add_definitions(-DUSE_INVERTED_INDEX)
endif(${BUILD_WITH_INVERTEDINDEX})
if (${BUILD_TEST})
add_subdirectory(test)
endif(${BUILD_TEST})

26
source/libs/index/inc/indexInt.h
View File

@ -37,9 +37,11 @@ struct SIndex {
#endif
void *cache;
void *tindex;
SHashObj *fieldObj; // <field name, field id>
uint64_t suid;
int fieldId;
SHashObj *colObj;// < field name, field id>
int64_t suid; // current super table id, -1 is normal table
int colId; // field id allocated to cache
int32_t cVersion; // current version allocated to cache
pthread_mutex_t mtx;
};
@ -58,21 +60,21 @@ struct SIndexMultiTermQuery {
// field and key;
typedef struct SIndexTerm {
char *key;
int32_t nKey;
char *val;
int32_t nVal;
int64_t suid;
SIndexOperOnColumn operType; // oper type, add/del/update
uint8_t colType; // term data type, str/interger/json
char *colName;
int32_t nColName;
char *colVal;
int32_t nColVal;
} SIndexTerm;
typedef struct SIndexTermQuery {
SIndexTerm* field_value;
EIndexQueryType type;
SIndexTerm* term;
EIndexQueryType qType;
} SIndexTermQuery;
SIndexTerm *indexTermCreate(const char *key, int32_t nKey, const char *val, int32_t nVal);
void indexTermDestroy(SIndexTerm *p);
#define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); }} while(0)
#define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); }} while(0)

16
source/libs/index/inc/index_cache.h
View File

@ -17,11 +17,12 @@
#include "index.h"
#include "tlockfree.h"
#include "tskiplist.h"
// ----------------- row structure in skiplist ---------------------
/* A data row, the format is like below:
* |<--totalLen-->|<-- fieldId-->|<-- value len--->|<-- value-->|<--version--->|<-- itermType -->|
*
* content: |<--totalLen-->|<-- fieldid-->|<--field type -->|<-- value len--->|<-- value -->|<-- uid -->|<--version--->|<-- itermType -->|
* len : |<--int32_t -->|<-- int16_t-->|<-- int16_t --->|<--- int32_t --->|<--valuelen->|<--uint64_t->|<-- int32_t-->|<-- int8_t --->|
*/
#ifdef __cplusplus
@ -30,19 +31,20 @@ extern "C" {
typedef struct IndexCache {
T_REF_DECLARE()
int cVersion; //
SSkipList *skiplist;
} IndexCache;
//
IndexCache *indexCacheCreate();
void indexCacheDestroy(IndexCache *cache);
void indexCacheDestroy(void *cache);
int indexCachePut(IndexCache *cache, int32_t fieldId, const char *fieldVale, int32_t fvlen, uint64_t uid, int8_t operaType);
int indexCachePut(void *cache, int16_t fieldId, int16_t fieldType, const char *fieldValue, int32_t fvLen,
uint32_t version, uint64_t uid, int8_t operType);
int indexCacheGet(IndexCache *cache, uint64_t *rst);
int indexCacheSearch(IndexCache *cache, SIndexMultiTermQuery *query, SArray *result);
int indexCacheGet(void *cache, uint64_t *rst);
int indexCacheSearch(void *cache, SIndexMultiTermQuery *query, SArray *result);
#ifdef __cplusplus
}

1
source/libs/index/inc/index_fst.h
View File

@ -315,7 +315,6 @@ typedef struct StreamWithStateResult {
FstSlice data;
FstOutput out;
void *state;
} StreamWithStateResult;
StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state);

131
source/libs/index/src/index.c
View File

@ -22,10 +22,10 @@
#endif
typedef struct SIdxFieldInfo {
int id; // generated by index internal
int type; // field type
} SIdxFieldInfo;
typedef struct SIdxColInfo {
int colId; // generated by index internal
int cVersion;
} SIdxColInfo;
static pthread_once_t isInit = PTHREAD_ONCE_INIT;
static void indexInit();
@ -37,9 +37,10 @@ static int indexMergeCacheIntoTindex(struct SIndex *sIdx) {
indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid);
return 0;
}
SIndex *indexOpen(SIndexOpts *opts, const char *path) {
int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) {
pthread_once(&isInit, indexInit);
SIndex *sIdx = malloc(sizeof(SIndex));
SIndex *sIdx = calloc(1, sizeof(SIndex));
if (sIdx == NULL) { return -1; }
#ifdef USE_LUCENE
index_t *index = index_open(path);
@ -48,9 +49,13 @@ SIndex *indexOpen(SIndexOpts *opts, const char *path) {
sIdx->cache = (void*)indexCacheCreate();
sIdx->tindex = NULL;
sIdx->fieldObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
sIdx->colId = 1;
sIdx->cVersion = 1;
pthread_mutex_init(&sIdx->mtx, NULL);
return sIdx;
*index = sIdx;
return 0;
}
void indexClose(SIndex *sIdx) {
@ -58,14 +63,17 @@ void indexClose(SIndex *sIdx) {
index_close(sIdex->index);
sIdx->index = NULL;
#endif
#ifdef USE_INVERTED_INDEX
indexCacheDestroy(sIdx->cache);
taosHashCleanup(sIdx->fieldObj);
taosHashCleanup(sIdx->colObj);
pthread_mutex_destroy(&sIdx->mtx);
#endif
free(sIdx);
return;
}
int indexPut(SIndex *index, SArray* field_vals, int uid) {
int indexPut(SIndex *index, SIndexMultiTerm * fVals, int uid) {
#ifdef USE_LUCENE
index_document_t *doc = index_document_create();
@ -73,8 +81,8 @@ int indexPut(SIndex *index, SArray* field_vals, int uid) {
char buf[16] = {0};
sprintf(buf, "%d", uid);
for (int i = 0; i < taosArrayGetSize(field_vals); i++) {
SIndexTerm *p = taosArrayGetP(field_vals, i);
for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm *p = taosArrayGetP(fVals, i);
index_document_add(doc, (const char *)(p->key), p->nKey, (const char *)(p->val), p->nVal, 1);
}
index_document_add(doc, NULL, 0, buf, strlen(buf), 0);
@ -82,10 +90,39 @@ int indexPut(SIndex *index, SArray* field_vals, int uid) {
index_put(index->index, doc);
index_document_destroy(doc);
#endif
pthread_mutex_lock(&index->mtx);
pthread_mutex_unlock(&index->mtx);
return 1;
#ifdef USE_INVERTED_INDEX
//TODO(yihao): reduce the lock range
pthread_mutex_lock(&index->mtx);
for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm *p = taosArrayGetP(fVals, i);
SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName);
if (fi == NULL) {
SIdxColInfo tfi = {.colId = index->colId};
index->cVersion++;
index->colId++;
taosHashPut(index->colObj, p->colName, p->nColName, &tfi, sizeof(tfi));
} else {
//TODO, del
}
}
pthread_mutex_unlock(&index->mtx);
for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm *p = taosArrayGetP(fVals, i);
SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName);
assert(fi != NULL);
int32_t colId = fi->colId;
int32_t version = index->cVersion;
int ret = indexCachePut(index->cache, colId, p->colType, p->colVal, p->nColVal, version, uid, p->operType);
if (ret != 0) {
return ret;
}
}
#endif
return 0;
}
int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result) {
#ifdef USE_LUCENE
@ -122,16 +159,26 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
free(fields);
free(keys);
free(types);
#endif
#ifdef USE_INVERTED_INDEX
#endif
return 1;
}
int indexDelete(SIndex *index, SIndexMultiTermQuery *query) {
#ifdef USE_INVERTED_INDEX
#endif
return 1;
}
int indexRebuild(SIndex *index, SIndexOpts *opts);
int indexRebuild(SIndex *index, SIndexOpts *opts) {
#ifdef USE_INVERTED_INDEX
#endif
}
SIndexOpts *indexOptsCreate() {
@ -152,59 +199,61 @@ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) {
SIndexMultiTermQuery *p = (SIndexMultiTermQuery *)malloc(sizeof(SIndexMultiTermQuery));
if (p == NULL) { return NULL; }
p->opera = opera;
p->query = taosArrayInit(1, sizeof(SIndexTermQuery));
p->query = taosArrayInit(4, sizeof(SIndexTermQuery));
return p;
}
void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) {
for (int i = 0; i < taosArrayGetSize(pQuery->query); i++) {
SIndexTermQuery *p = (SIndexTermQuery *)taosArrayGet(pQuery->query, i);
indexTermDestroy(p->field_value);
indexTermDestroy(p->term);
}
taosArrayDestroy(pQuery->query);
free(pQuery);
};
int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, const char *field, int32_t nFields, const char *value, int32_t nValue, EIndexQueryType type){
SIndexTerm *t = indexTermCreate(field, nFields, value, nValue);
if (t == NULL) {return -1;}
SIndexTermQuery q = {.type = type, .field_value = t};
int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType qType){
SIndexTermQuery q = {.qType = qType, .term = term};
taosArrayPush(pQuery->query, &q);
return 0;
}
SIndexTerm *indexTermCreate(const char *key, int32_t nKey, const char *val, int32_t nVal) {
SIndexTerm *t = (SIndexTerm *)malloc(sizeof(SIndexTerm));
t->key = (char *)calloc(nKey + 1, 1);
memcpy(t->key, key, nKey);
t->nKey = nKey;
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, int32_t nColName, const char *colVal, int32_t nColVal) {
SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm)));
if (t == NULL) { return NULL; }
t->val = (char *)calloc(nVal + 1, 1);
memcpy(t->val, val, nVal);
t->nVal = nVal;
t->suid = suid;
t->operType= oper;
t->colType = colType;
t->colName = (char *)calloc(1, nColName + 1);
memcpy(t->colName, colName, nColName);
t->nColName = nColName;
t->colVal = (char *)calloc(1, nColVal + 1);
memcpy(t->colVal, colVal, nColVal);
t->nColVal = nColVal;
return t;
}
void indexTermDestroy(SIndexTerm *p) {
free(p->key);
free(p->val);
free(p->colName);
free(p->colVal);
free(p);
}
SArray *indexMultiTermCreate() {
SIndexMultiTerm *indexMultiTermCreate() {
return taosArrayInit(4, sizeof(SIndexTerm *));
}
int indexMultiTermAdd(SArray *array, const char *field, int32_t nField, const char *val, int32_t nVal) {
SIndexTerm *term = indexTermCreate(field, nField, val, nVal);
if (term == NULL) { return -1; }
taosArrayPush(array, &term);
int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term) {
taosArrayPush(terms, &term);
return 0;
}
void indexMultiTermDestroy(SArray *array) {
for (int32_t i = 0; i < taosArrayGetSize(array); i++) {
SIndexTerm *p = taosArrayGetP(array, i);
void indexMultiTermDestroy(SIndexMultiTerm *terms) {
for (int32_t i = 0; i < taosArrayGetSize(terms); i++) {
SIndexTerm *p = taosArrayGetP(terms, i);
indexTermDestroy(p);
}
taosArrayDestroy(array);
taosArrayDestroy(terms);
}
void indexInit() {

95
source/libs/index/src/index_cache.c
View File

@ -14,12 +14,18 @@
*/
#include "index_cache.h"
#include "tcompare.h"
#define MAX_INDEX_KEY_LEN 256// test only, change later
static char* getIndexKey(const void *pData) {
return NULL;
}
static int32_t compareKey(const void *l, const void *r) {
char *lp = (char *)l;
char *rp = (char *)r;
// skip total len
// skip total len, not compare
int32_t ll, rl; // len
memcpy(&ll, lp, sizeof(int32_t));
memcpy(&rl, rp, sizeof(int32_t));
@ -27,7 +33,7 @@ static int32_t compareKey(const void *l, const void *r) {
rp += sizeof(int32_t);
// compare field id
int32_t lf, rf; // field id
int16_t lf, rf; // field id
memcpy(&lf, lp, sizeof(lf));
memcpy(&rf, rp, sizeof(rf));
if (lf != rf) {
@ -36,14 +42,22 @@ static int32_t compareKey(const void *l, const void *r) {
lp += sizeof(lf);
rp += sizeof(rf);
// compare field value
// compare field type
int16_t lft, rft;
memcpy(&lft, lp, sizeof(lft));
memcpy(&rft, rp, sizeof(rft));
lp += sizeof(lft);
rp += sizeof(rft);
assert(rft == rft);
// skip value len
int32_t lfl, rfl;
memcpy(&lfl, lp, sizeof(lfl));
memcpy(&rfl, rp, sizeof(rfl));
lp += sizeof(lfl);
rp += sizeof(rfl);
//refator later
// compare value
int32_t i, j;
for (i = 0, j = 0; i < lfl && j < rfl; i++, j++) {
if (lp[i] == rp[j]) { continue; }
@ -54,7 +68,14 @@ static int32_t compareKey(const void *l, const void *r) {
lp += lfl;
rp += rfl;
// compare version
// skip uid
uint64_t lu, ru;
memcpy(&lu, lp, sizeof(lu));
memcpy(&ru, rp, sizeof(ru));
lp += sizeof(lu);
rp += sizeof(ru);
// compare version, desc order
int32_t lv, rv;
memcpy(&lv, lp, sizeof(lv));
memcpy(&rv, rp, sizeof(rv));
@ -63,57 +84,69 @@ static int32_t compareKey(const void *l, const void *r) {
}
lp += sizeof(lv);
rp += sizeof(rv);
// not care item type
return 0;
}
IndexCache *indexCacheCreate() {
IndexCache *cache = calloc(1, sizeof(IndexCache));
cache->skiplist = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey);
return cache;
}
void indexCacheDestroy(IndexCache *cache) {
free(cache);
void indexCacheDestroy(void *cache) {
IndexCache *pCache = cache;
if (pCache == NULL) { return; }
tSkipListDestroy(pCache->skiplist);
free(pCache);
}
int indexCachePut(IndexCache *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) {
int indexCachePut(void *cache, int16_t fieldId, int16_t fieldType, const char *fieldValue, int32_t fvLen,
uint32_t version, uint64_t uid, int8_t operType) {
if (cache == NULL) { return -1;}
int32_t version = T_REF_INC(cache);
int32_t total = sizeof(int32_t) + sizeof(fieldId) + 4 + fvlen + sizeof(version) + sizeof(uid) + sizeof(operType);
IndexCache *pCache = cache;
// encode data
int32_t total = sizeof(int32_t) + sizeof(fieldId) + sizeof(fieldType) + sizeof(fvLen) + fvLen + sizeof(version) + sizeof(uid) + sizeof(operType);
char *buf = calloc(1, total);
char *p = buf;
memcpy(buf, &total, sizeof(total));
total += total;
memcpy(p, &total, sizeof(total));
p += sizeof(total);
memcpy(buf, &fieldId, sizeof(fieldId));
buf += sizeof(fieldId);
memcpy(p, &fieldId, sizeof(fieldId));
p += sizeof(fieldId);
memcpy(buf, &fvlen, sizeof(fvlen));
buf += sizeof(fvlen);
memcpy(buf, fieldValue, fvlen);
buf += fvlen;
memcpy(p, &fieldType, sizeof(fieldType));
p += sizeof(fieldType);
memcpy(buf, &version, sizeof(version));
buf += sizeof(version);
memcpy(p, &fvLen, sizeof(fvLen));
p += sizeof(fvLen);
memcpy(p, fieldValue, fvLen);
p += fvLen;
memcpy(buf, &uid, sizeof(uid));
buf += sizeof(uid);
memcpy(p, &version, sizeof(version));
p += sizeof(version);
memcpy(buf, &operType, sizeof(operType));
buf += sizeof(operType);
memcpy(p, &uid, sizeof(uid));
p += sizeof(uid);
memcpy(p, &operType, sizeof(operType));
p += sizeof(operType);
tSkipListPut(pCache->skiplist, (void *)buf);
// encode end
}
int indexCacheSearch(IndexCache *cache, SIndexMultiTermQuery *query, SArray *result) {
int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) {
IndexCache *pCache = cache;
return 0;
}
int indexCacheSearch(void *cache, SIndexMultiTermQuery *query, SArray *result) {
return 0;
}

2
source/libs/index/test/CMakeLists.txt
View File

@ -1,7 +1,7 @@
add_executable(indexTest "")
target_sources(indexTest
PRIVATE
"indexTests.cpp"
"indexTests.cc"
)
target_include_directories ( indexTest
PUBLIC

86
source/libs/index/test/indexTests.cpp → source/libs/index/test/indexTests.cc
View File

@ -1,3 +1,17 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <string>
#include <iostream>
@ -61,7 +75,7 @@ class FstReadMemory {
// add later
bool Search(AutomationCtx *ctx, std::vector<uint64_t> &result) {
FstStreamBuilder *sb = fstSearch(_fst, ctx);
StreamWithState *st = streamBuilderIntoStream(sb);
StreamWithState *st = streamBuilderIntoStream(sb);
StreamWithStateResult *rt = NULL;
while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
@ -279,15 +293,71 @@ void validateFst() {
delete m;
}
class IndexEnv : public ::testing::Test {
protected:
virtual void SetUp() {
taosRemoveDir(path);
opts = indexOptsCreate();
int ret = indexOpen(opts, path, &index);
assert(ret == 0);
}
virtual void TearDown() {
indexClose(index);
indexOptsDestroy(opts);
}
int main(int argc, char** argv) {
checkFstPerf();
//checkFstPrefixSearch();
return 1;
const char *path = "/tmp/tindex";
SIndexOpts *opts;
SIndex *index;
};
TEST_F(IndexEnv, testPut) {
// single index column
{
std::string colName("tag1"), colVal("Hello world");
SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
SIndexMultiTerm *terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 100; i++) {
int tableId = i;
int ret = indexPut(index, terms, tableId);
assert(ret == 0);
}
indexMultiTermDestroy(terms);
}
// multi index column
{
SIndexMultiTerm *terms = indexMultiTermCreate();
{
std::string colName("tag1"), colVal("Hello world");
SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
indexMultiTermAdd(terms, term);
}
{
std::string colName("tag2"), colVal("Hello world");
SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
indexMultiTermAdd(terms, term);
}
for (int i = 0; i < 100; i++) {
int tableId = i;
int ret = indexPut(index, terms, tableId);
assert(ret == 0);
}
indexMultiTermDestroy(terms);
}
//
}
//TEST(IndexFstBuilder, IndexFstInput) {
//
//}
TEST_F(IndexEnv, testDel) {
}

2
source/libs/parser/CMakeLists.txt
View File

@ -8,7 +8,7 @@ target_include_directories(
target_link_libraries(
parser
PRIVATE os util common catalog function transport query
PRIVATE os util catalog function transport qcom
)
ADD_SUBDIRECTORY(test)

4
source/libs/parser/inc/astGenerator.h
View File

@ -171,8 +171,8 @@ typedef struct SCreateDbInfo {
int8_t update;
int8_t cachelast;
SArray *keep;
int8_t dbType;
int16_t partitions;
// int8_t dbType;
// int16_t partitions;
} SCreateDbInfo;
typedef struct SCreateFuncInfo {

2
source/libs/parser/inc/astToMsg.h
View File

@ -5,5 +5,7 @@
#include "taosmsg.h"
SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen);
SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen);
SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen);
#endif // TDENGINE_ASTTOMSG_H

1
source/libs/parser/inc/parserUtil.h
View File

@ -46,6 +46,7 @@ SInternalField* getInternalField(SFieldInfo* pFieldInfo, int32_t index);
int32_t parserValidateIdToken(SToken* pToken);
int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf);
int32_t parserValidateNameToken(SToken* pToken);
int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg);
int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr);

19
source/libs/parser/src/astGenerator.c
View File

@ -948,27 +948,24 @@ void setCompactVnodeSql(SSqlInfo *pInfo, int32_t type, SArray *pParam) {
void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) {
pDBInfo->compressionLevel = -1;
pDBInfo->walLevel = -1;
pDBInfo->walLevel = -1;
pDBInfo->fsyncPeriod = -1;
pDBInfo->commitTime = -1;
pDBInfo->commitTime = -1;
pDBInfo->maxTablesPerVnode = -1;
pDBInfo->cacheBlockSize = -1;
pDBInfo->numOfBlocks = -1;
pDBInfo->cacheBlockSize = -1;
pDBInfo->numOfBlocks = -1;
pDBInfo->maxRowsPerBlock = -1;
pDBInfo->minRowsPerBlock = -1;
pDBInfo->daysPerFile = -1;
pDBInfo->daysPerFile = -1;
pDBInfo->replica = -1;
pDBInfo->quorum = -1;
pDBInfo->keep = NULL;
pDBInfo->quorum = -1;
pDBInfo->keep = NULL;
pDBInfo->update = -1;
pDBInfo->update = -1;
pDBInfo->cachelast = -1;
pDBInfo->dbType = -1;
pDBInfo->partitions = -1;
memset(&pDBInfo->precision, 0, sizeof(SToken));
}

145
source/libs/parser/src/astToMsg.c
View File

@ -1,23 +1,160 @@
#include "parserInt.h"
#include "parserUtil.h"
SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen) {
SCreateUserMsg *pMsg = (SCreateUserMsg *)calloc(1, sizeof(SCreateUserMsg));
SCreateUserMsg* pMsg = (SCreateUserMsg*)calloc(1, sizeof(SCreateUserMsg));
if (pMsg == NULL) {
// tscError("0x%" PRIx64 " failed to malloc for query msg", id);
// tscError("0x%" PRIx64 " failed to malloc for query msg", id);
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
return NULL;
}
SUserInfo *pUser = &pInfo->pMiscInfo->user;
SUserInfo* pUser = &pInfo->pMiscInfo->user;
strncpy(pMsg->user, pUser->user.z, pUser->user.n);
pMsg->type = pUser->type;
pMsg->superUser = (int8_t)pUser->type;
if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) {
// pMsg->privilege = (char)pCmd->count;
// pMsg->privilege = (char)pCmd->count;
} else {
strncpy(pMsg->pass, pUser->passwd.z, pUser->passwd.n);
}
return pMsg;
}
SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen) {
SShowMsg* pShowMsg = calloc(1, sizeof(SShowMsg));
pShowMsg->type = pShowInfo->showType;
if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) {
SToken* pPattern = &pShowInfo->pattern;
if (pPattern->type > 0) { // only show tables support wildcard query
strncpy(pShowMsg->payload, pPattern->z, pPattern->n);
pShowMsg->payloadLen = htons(pPattern->n);
}
} else {
SToken* pEpAddr = &pShowInfo->prefix;
assert(pEpAddr->n > 0 && pEpAddr->type > 0);
strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n);
pShowMsg->payloadLen = htons(pEpAddr->n);
}
return pShowMsg;
}
static int32_t setKeepOption(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb, SMsgBuf* pMsgBuf) {
const char* msg1 = "invalid number of keep options";
const char* msg2 = "invalid keep value";
const char* msg3 = "invalid keep value, should be keep0 <= keep1 <= keep2";
pMsg->daysToKeep0 = htonl(-1);
pMsg->daysToKeep1 = htonl(-1);
pMsg->daysToKeep2 = htonl(-1);
SArray* pKeep = pCreateDb->keep;
if (pKeep != NULL) {
size_t s = taosArrayGetSize(pKeep);
#ifdef _STORAGE
if (s >= 4 ||s <= 0) {
#else
if (s != 1) {
#endif
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
// tListI* p0 = taosArrayGet(pKeep, 0);
// tVariantListItem* p1 = (s > 1) ? taosArrayGet(pKeep, 1) : p0;
// tVariantListItem* p2 = (s > 2) ? taosArrayGet(pKeep, 2) : p1;
//
// if ((int32_t)p0->pVar.i64 <= 0 || (int32_t)p1->pVar.i64 <= 0 || (int32_t)p2->pVar.i64 <= 0) {
// return buildInvalidOperationMsg(pMsgBuf, msg2);
// }
// if (!(((int32_t)p0->pVar.i64 <= (int32_t)p1->pVar.i64) && ((int32_t)p1->pVar.i64 <= (int32_t)p2->pVar.i64))) {
// return buildInvalidOperationMsg(pMsgBuf, msg3);
// }
//
// pMsg->daysToKeep0 = htonl((int32_t)p0->pVar.i64);
// pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64);
// pMsg->daysToKeep2 = htonl((int32_t)p2->pVar.i64);
}
return TSDB_CODE_SUCCESS;
}
static int32_t setTimePrecision(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDbInfo, SMsgBuf* pMsgBuf) {
const char* msg = "invalid time precision";
pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default
SToken* pToken = &pCreateDbInfo->precision;
if (pToken->n > 0) {
pToken->n = strdequote(pToken->z);
if (strncmp(pToken->z, TSDB_TIME_PRECISION_MILLI_STR, pToken->n) == 0 &&
strlen(TSDB_TIME_PRECISION_MILLI_STR) == pToken->n) {
// time precision for this db: million second
pMsg->precision = TSDB_TIME_PRECISION_MILLI;
} else if (strncmp(pToken->z, TSDB_TIME_PRECISION_MICRO_STR, pToken->n) == 0 &&
strlen(TSDB_TIME_PRECISION_MICRO_STR) == pToken->n) {
pMsg->precision = TSDB_TIME_PRECISION_MICRO;
} else if (strncmp(pToken->z, TSDB_TIME_PRECISION_NANO_STR, pToken->n) == 0 &&
strlen(TSDB_TIME_PRECISION_NANO_STR) == pToken->n) {
pMsg->precision = TSDB_TIME_PRECISION_NANO;
} else {
return buildInvalidOperationMsg(pMsgBuf, msg);
}
}
return TSDB_CODE_SUCCESS;
}
static void doSetDbOptions(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb) {
pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize);
pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks);
pMsg->daysPerFile = htonl(pCreateDb->daysPerFile);
pMsg->commitTime = htonl((int32_t)pCreateDb->commitTime);
pMsg->minRowsPerFileBlock = htonl(pCreateDb->minRowsPerBlock);
pMsg->maxRowsPerFileBlock = htonl(pCreateDb->maxRowsPerBlock);
pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod);
pMsg->compression = pCreateDb->compressionLevel;
pMsg->walLevel = (char)pCreateDb->walLevel;
pMsg->replications = pCreateDb->replica;
pMsg->quorum = pCreateDb->quorum;
pMsg->ignoreExist = pCreateDb->ignoreExists;
pMsg->update = pCreateDb->update;
pMsg->cacheLastRow = pCreateDb->cachelast;
}
int32_t setDbOptions(SCreateDbMsg* pCreateDbMsg, const SCreateDbInfo* pCreateDbSql, SMsgBuf* pMsgBuf) {
doSetDbOptions(pCreateDbMsg, pCreateDbSql);
if (setKeepOption(pCreateDbMsg, pCreateDbSql, pMsgBuf) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
if (setTimePrecision(pCreateDbMsg, pCreateDbSql, pMsgBuf) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
// todo configurable
pCreateDbMsg->numOfVgroups = htonl(2);
return TSDB_CODE_SUCCESS;
}
SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen) {
SCreateDbMsg* pCreateMsg = calloc(1, sizeof(SCreateDbMsg));
SMsgBuf msg = {.buf = msgBuf, .len = msgLen};
if (setDbOptions(pCreateMsg, pCreateDbInfo, &msg) != TSDB_CODE_SUCCESS) {
tfree(pCreateMsg);
terrno = TSDB_CODE_TSC_INVALID_OPERATION;
return NULL;
}
return pCreateMsg;
}

160
source/libs/parser/src/astValidate.c
View File

@ -759,11 +759,6 @@ int32_t validateIntervalNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMs
// It is a time window query
pQueryInfo->info.timewindow = true;
return TSDB_CODE_SUCCESS;
// disable it temporarily
// bool interpQuery = tscIsPointInterpQuery(pQueryInfo);
// if ((pSqlNode->interval.token == TK_EVERY && (!interpQuery)) || (pSqlNode->interval.token == TK_INTERVAL && interpQuery)) {
// return buildInvalidOperationMsg(pMsgBuf, msg4);
// }
}
int32_t validateSessionNode(SQueryStmtInfo *pQueryInfo, SSessionWindowVal* pSession, int32_t precision, SMsgBuf* pMsgBuf) {
@ -3707,14 +3702,6 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
return TSDB_CODE_SUCCESS;
}
case TSDB_SQL_SHOW: {
if (setShowInfo(pSql, pInfo) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
break;
}
case TSDB_SQL_CREATE_FUNCTION:
case TSDB_SQL_DROP_FUNCTION: {
code = handleUserDefinedFunc(pSql, pInfo);
@ -3725,35 +3712,6 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
break;
}
case TSDB_SQL_ALTER_DB:
case TSDB_SQL_CREATE_DB: {
const char* msg1 = "invalid db name";
const char* msg2 = "name too long";
SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt);
if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) {
return buildInvalidOperationMsg(pMsgBuf, msg2);
}
char buf[TSDB_DB_NAME_LEN] = {0};
SToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf));
if (tscValidateName(&token) != TSDB_CODE_SUCCESS) {
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), &token);
if (ret != TSDB_CODE_SUCCESS) {
return buildInvalidOperationMsg(pMsgBuf, msg2);
}
if (parseCreateDBOptions(pCmd, pCreateDB) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
break;
}
case TSDB_SQL_CREATE_DNODE: {
const char* msg = "invalid host name (ip address)";
@ -4133,25 +4091,83 @@ static int32_t setShowInfo(struct SSqlInfo* pInfo, void** output, int32_t* msgLe
}
}
SShowMsg* pShowMsg = calloc(1, sizeof(SShowMsg));
pShowMsg->type = pShowInfo->showType;
*output = buildShowMsg(pShowInfo, 0, pMsgBuf->buf, pMsgBuf->len);
*msgLen = sizeof(SShowMsg)/* + htons(pShowMsg->payloadLen)*/;
return TSDB_CODE_SUCCESS;
}
if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) {
SToken* pPattern = &pShowInfo->pattern;
if (pPattern->type > 0) { // only show tables support wildcard query
strncpy(pShowMsg->payload, pPattern->z, pPattern->n);
pShowMsg->payloadLen = htons(pPattern->n);
}
} else {
SToken* pEpAddr = &pShowInfo->prefix;
assert(pEpAddr->n > 0 && pEpAddr->type > 0);
// can only perform the parameters based on the macro definitation
static int32_t doCheckDbOptions(SCreateDbMsg* pCreate, SMsgBuf* pMsgBuf) {
char msg[512] = {0};
strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n);
pShowMsg->payloadLen = htons(pEpAddr->n);
if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) {
snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 1-2 allowed", pCreate->walLevel);
return buildInvalidOperationMsg(pMsgBuf, msg);
}
if (pCreate->replications != -1 &&
(pCreate->replications < TSDB_MIN_DB_REPLICA_OPTION || pCreate->replications > TSDB_MAX_DB_REPLICA_OPTION)) {
snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications,
TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION);
return buildInvalidOperationMsg(pMsgBuf, msg);
}
int32_t blocks = ntohl(pCreate->totalBlocks);
if (blocks != -1 && (blocks < TSDB_MIN_TOTAL_BLOCKS || blocks > TSDB_MAX_TOTAL_BLOCKS)) {
snprintf(msg, tListLen(msg), "invalid db option totalBlocks: %d valid range: [%d, %d]", blocks,
TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS);
return buildInvalidOperationMsg(pMsgBuf, msg);
}
if (pCreate->quorum != -1 &&
(pCreate->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCreate->quorum > TSDB_MAX_DB_QUORUM_OPTION)) {
snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum,
TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION);
return buildInvalidOperationMsg(pMsgBuf, msg);
}
int32_t val = htonl(pCreate->daysPerFile);
if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) {
snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val,
TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE);
return buildInvalidOperationMsg(pMsgBuf, msg);
}
val = htonl(pCreate->cacheBlockSize);
if (val != -1 && (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE)) {
snprintf(msg, tListLen(msg), "invalid db option cacheBlockSize: %d valid range: [%d, %d]", val,
TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE);
return buildInvalidOperationMsg(pMsgBuf, msg);
}
if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO &&
pCreate->precision != TSDB_TIME_PRECISION_NANO) {
snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d, %d]", pCreate->precision,
TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO);
return buildInvalidOperationMsg(pMsgBuf, msg);
}
val = htonl(pCreate->commitTime);
if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) {
snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val,
TSDB_MIN_COMMIT_TIME, TSDB_MAX_COMMIT_TIME);
return buildInvalidOperationMsg(pMsgBuf, msg);
}
val = htonl(pCreate->fsyncPeriod);
if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) {
snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val,
TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD);
return buildInvalidOperationMsg(pMsgBuf, msg);
}
if (pCreate->compression != -1 &&
(pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) {
snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression,
TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL);
return buildInvalidOperationMsg(pMsgBuf, msg);
}
*output = pShowMsg;
*msgLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen);
return TSDB_CODE_SUCCESS;
}
@ -4216,6 +4232,36 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, in
code = setShowInfo(pInfo, output, outputLen, pMsgBuf);
break;
}
case TSDB_SQL_ALTER_DB:
case TSDB_SQL_CREATE_DB: {
const char* msg1 = "invalid db name";
const char* msg2 = "name too long";
SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt);
if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) {
return buildInvalidOperationMsg(pMsgBuf, msg2);
}
char buf[TSDB_DB_NAME_LEN] = {0};
SToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf));
if (parserValidateNameToken(&token) != TSDB_CODE_SUCCESS) {
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
SCreateDbMsg* pCreateMsg = buildCreateDbMsg(pCreateDB, pMsgBuf->buf, pMsgBuf->len);
if (doCheckDbOptions(pCreateMsg, pMsgBuf) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
strncpy(pCreateMsg->db, token.z, token.n);
*output = pCreateMsg;
*outputLen = sizeof(SCreateDbMsg);
break;
}
default:
break;
}

2
source/libs/parser/src/parser.c
View File

@ -77,7 +77,7 @@ static int32_t tnameComparFn(const void* p1, const void* p2) {
SName* pn1 = (SName*)p1;
SName* pn2 = (SName*)p2;
int32_t ret = strncmp(pn1->acctId, pn2->acctId, tListLen(pn1->acctId));
int32_t ret = pn1->acctId - pn2->acctId;
if (ret != 0) {
return ret > 0? 1:-1;
} else {

19
source/libs/parser/src/parserUtil.c
View File

@ -122,6 +122,25 @@ int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf) {
return TSDB_CODE_SUCCESS;
}
int32_t parserValidateNameToken(SToken* pToken) {
if (pToken == NULL || pToken->z == NULL || pToken->type != TK_ID) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
// it is a token quoted with escape char '`'
if (pToken->z[0] == TS_ESCAPE_CHAR && pToken->z[pToken->n - 1] == TS_ESCAPE_CHAR) {
return TSDB_CODE_SUCCESS;
}
char* sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true);
if (sep != NULL) { // It is a complex type, not allow
return TSDB_CODE_TSC_INVALID_OPERATION;
}
strntolower(pToken->z, pToken->z, pToken->n);
return TSDB_CODE_SUCCESS;
}
int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) {
strncpy(pBuf->buf, msg, pBuf->len);
return TSDB_CODE_TSC_INVALID_OPERATION;

8
source/libs/parser/src/sql.c
View File

@ -2519,7 +2519,7 @@ static void yy_reduce(
{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; }
break;
case 105: /* db_optr ::= */
{setDefaultCreateDbOption(&yymsp[1].minor.yy256); yymsp[1].minor.yy256.dbType = TSDB_DB_TYPE_DEFAULT;}
{setDefaultCreateDbOption(&yymsp[1].minor.yy256);}
break;
case 106: /* db_optr ::= db_optr cache */
{ yylhsminor.yy256 = yymsp[-1].minor.yy256; yylhsminor.yy256.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
@ -2590,16 +2590,16 @@ static void yy_reduce(
break;
case 121: /* topic_optr ::= db_optr */
case 131: /* alter_topic_optr ::= alter_db_optr */ yytestcase(yyruleno==131);
{ yylhsminor.yy256 = yymsp[0].minor.yy256; yylhsminor.yy256.dbType = TSDB_DB_TYPE_TOPIC; }
{ yylhsminor.yy256 = yymsp[0].minor.yy256;}
yymsp[0].minor.yy256 = yylhsminor.yy256;
break;
case 122: /* topic_optr ::= topic_optr partitions */
case 132: /* alter_topic_optr ::= alter_topic_optr partitions */ yytestcase(yyruleno==132);
{ yylhsminor.yy256 = yymsp[-1].minor.yy256; yylhsminor.yy256.partitions = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
{ yylhsminor.yy256 = yymsp[-1].minor.yy256; }
yymsp[-1].minor.yy256 = yylhsminor.yy256;
break;
case 123: /* alter_db_optr ::= */
{ setDefaultCreateDbOption(&yymsp[1].minor.yy256); yymsp[1].minor.yy256.dbType = TSDB_DB_TYPE_DEFAULT;}
{ setDefaultCreateDbOption(&yymsp[1].minor.yy256); }
break;
case 133: /* typename ::= ids */
{

2
source/libs/parser/test/CMakeLists.txt
View File

@ -15,7 +15,7 @@ TARGET_INCLUDE_DIRECTORIES(
TARGET_LINK_LIBRARIES(
parserTest
PUBLIC os util common parser catalog transport gtest function planner query
PUBLIC os util common parser catalog transport gtest function planner qcom
)
TARGET_LINK_OPTIONS(parserTest PRIVATE -Wl,-wrap,malloc)

2
source/libs/planner/CMakeLists.txt
View File

@ -8,7 +8,7 @@ target_include_directories(
target_link_libraries(
planner
PRIVATE os util common cjson catalog parser transport function query
PRIVATE os util catalog cjson parser transport function qcom
)
ADD_SUBDIRECTORY(test)

5
source/libs/planner/inc/plannerInt.h
View File

@ -100,8 +100,9 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str);
int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql);
int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag);
int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps);
int32_t subPlanToString(const SSubplan *pPhyNode, char** str);
int32_t stringToSubplan(const char* str, SSubplan** subplan);
/**
* Destroy the query plan object.
@ -116,6 +117,8 @@ void destroyQueryPlan(struct SQueryPlanNode* pQueryNode);
*/
void* destroyQueryPhyPlan(struct SPhyNode* pQueryPhyNode);
int32_t opNameToOpType(const char* name);
#ifdef __cplusplus
}
#endif

27
source/libs/planner/src/physicalPlan.c
View File

@ -19,6 +19,13 @@
#define STORE_CURRENT_SUBPLAN(cxt) SSubplan* _ = cxt->pCurrentSubplan
#define RECOVERY_CURRENT_SUBPLAN(cxt) cxt->pCurrentSubplan = _
typedef struct SPlanContext {
struct SCatalog* pCatalog;
struct SQueryDag* pDag;
SSubplan* pCurrentSubplan;
SSubplanId nextId;
} SPlanContext;
static const char* gOpName[] = {
"Unknown",
#define INCLUDE_AS_NAME
@ -26,12 +33,14 @@ static const char* gOpName[] = {
#undef INCLUDE_AS_NAME
};
typedef struct SPlanContext {
struct SCatalog* pCatalog;
struct SQueryDag* pDag;
SSubplan* pCurrentSubplan;
SSubplanId nextId;
} SPlanContext;
int32_t opNameToOpType(const char* name) {
for (int32_t i = 1; i < sizeof(gOpName) / sizeof(gOpName[0]); ++i) {
if (strcmp(name, gOpName[i])) {
return i;
}
}
return OP_Unknown;
}
static void toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) {
SWAP(dataBlockSchema->pSchema, pPlanNode->pSchema, SSchema*);
@ -179,7 +188,7 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
assert(false);
}
if (pPlanNode->pChildren != NULL && taosArrayGetSize(pPlanNode->pChildren) > 0) {
node->pChildren = taosArrayInit(4, POINTER_BYTES);
node->pChildren = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
size_t size = taosArrayGetSize(pPlanNode->pChildren);
for(int32_t i = 0; i < size; ++i) {
SPhyNode* child = createPhyNode(pCxt, taosArrayGet(pPlanNode->pChildren, i));
@ -215,3 +224,7 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD
*pDag = context.pDag;
return TSDB_CODE_SUCCESS;
}
int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps) {
//todo
}

799
source/libs/planner/src/physicalPlanJson.c
View File

@ -17,29 +17,52 @@
#include "parser.h"
#include "cJSON.h"
typedef cJSON* (*FToObj)(const void* obj);
typedef bool (*FToJson)(const void* obj, cJSON* json);
typedef bool (*FFromJson)(const cJSON* json, void* obj);
static bool addObject(cJSON* json, const char* name, FToObj func, const void* obj) {
static bool addObject(cJSON* json, const char* name, FToJson func, const void* obj) {
if (NULL == obj) {
return true;
}
cJSON* jObj = func(obj);
if (NULL == jObj) {
cJSON* jObj = cJSON_CreateObject();
if (NULL == jObj || !func(obj, jObj)) {
cJSON_Delete(jObj);
return false;
}
return cJSON_AddItemToObject(json, name, jObj);
}
static bool addItem(cJSON* json, FToObj func, const void* item) {
cJSON* jItem = func(item);
if (NULL == jItem) {
static bool addItem(cJSON* json, FToJson func, const void* obj) {
cJSON* jObj = cJSON_CreateObject();
if (NULL == jObj || !func(obj, jObj)) {
cJSON_Delete(jObj);
return false;
}
return cJSON_AddItemToArray(json, jItem);
return cJSON_AddItemToArray(json, jObj);
}
static bool addArray(cJSON* json, const char* name, FToObj func, const SArray* array) {
static bool fromObject(const cJSON* json, const char* name, FFromJson func, void* obj, bool required) {
cJSON* jObj = cJSON_GetObjectItem(json, name);
if (NULL == jObj) {
return !required;
}
return func(jObj, obj);
}
static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson func, void** obj, int32_t size, bool required) {
cJSON* jObj = cJSON_GetObjectItem(json, name);
if (NULL == jObj) {
return !required;
}
*obj = calloc(1, size);
if (NULL == *obj) {
return false;
}
return func(jObj, *obj);
}
static bool addArray(cJSON* json, const char* name, FToJson func, const SArray* array) {
size_t size = (NULL == array) ? 0 : taosArrayGetSize(array);
if (size > 0) {
cJSON* jArray = cJSON_AddArrayToObject(json, name);
@ -55,7 +78,26 @@ static bool addArray(cJSON* json, const char* name, FToObj func, const SArray* a
return true;
}
static bool addRawArray(cJSON* json, const char* name, FToObj func, const void* array, int32_t itemSize, int32_t size) {
static bool fromArray(const cJSON* json, const char* name, FFromJson func, SArray** array, int32_t itemSize) {
const cJSON* jArray = cJSON_GetObjectItem(json, name);
int32_t size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray));
if (size > 0) {
*array = taosArrayInit(size, POINTER_BYTES);
if (NULL == *array) {
return false;
}
}
for (int32_t i = 0; i < size; ++i) {
void* item = calloc(1, itemSize);
if (NULL == item || !func(cJSON_GetArrayItem(jArray, i), item)) {
return false;
}
taosArrayPush(*array, &item);
}
return true;
}
static bool addRawArray(cJSON* json, const char* name, FToJson func, const void* array, int32_t itemSize, int32_t size) {
if (size > 0) {
cJSON* jArray = cJSON_AddArrayToObject(json, name);
if (NULL == jArray) {
@ -70,310 +112,616 @@ static bool addRawArray(cJSON* json, const char* name, FToObj func, const void*
return true;
}
static cJSON* schemaToJson(const void* obj) {
static const cJSON* getArray(const cJSON* json, const char* name, int32_t* size) {
const cJSON* jArray = cJSON_GetObjectItem(json, name);
*size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray));
return jArray;
}
static bool fromItem(const cJSON* jArray, FFromJson func, void* array, int32_t itemSize, int32_t size) {
for (int32_t i = 0; i < size; ++i) {
if (!func(cJSON_GetArrayItem(jArray, i), (char*)array + itemSize)) {
return false;
}
}
return true;
}
static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, int32_t* size) {
const cJSON* jArray = getArray(json, name, size);
if (*size > 0) {
*array = calloc(1, itemSize * (*size));
if (NULL == *array) {
return false;
}
}
return fromItem(jArray, func, *array, itemSize, *size);
}
static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, int32_t* size) {
const cJSON* jArray = getArray(json, name, size);
return fromItem(jArray, func, array, itemSize, *size);
}
static char* getString(const cJSON* json, const char* name) {
char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name));
char* res = calloc(1, strlen(p) + 1);
strcpy(res, p);
return res;
}
static void copyString(const cJSON* json, const char* name, char* dst) {
strcpy(dst, cJSON_GetStringValue(cJSON_GetObjectItem(json, name)));
}
static int64_t getNumber(const cJSON* json, const char* name) {
return cJSON_GetNumberValue(cJSON_GetObjectItem(json, name));
}
static const char* jkSchemaType = "Type";
static const char* jkSchemaColId = "ColId";
static const char* jkSchemaBytes = "Bytes";
// The 'name' field do not need to be serialized.
static bool schemaToJson(const void* obj, cJSON* jSchema) {
const SSlotSchema* schema = (const SSlotSchema*)obj;
cJSON* jSchema = cJSON_CreateObject();
if (NULL == jSchema) {
return NULL;
}
// The 'name' field do not need to be serialized.
bool res = cJSON_AddNumberToObject(jSchema, "Type", schema->type);
bool res = cJSON_AddNumberToObject(jSchema, jkSchemaType, schema->type);
if (res) {
res = cJSON_AddNumberToObject(jSchema, "ColId", schema->colId);
res = cJSON_AddNumberToObject(jSchema, jkSchemaColId, schema->colId);
}
if (res) {
res = cJSON_AddNumberToObject(jSchema, "Bytes", schema->bytes);
res = cJSON_AddNumberToObject(jSchema, jkSchemaBytes, schema->bytes);
}
if (!res) {
cJSON_Delete(jSchema);
return NULL;
}
return jSchema;
return res;
}
static cJSON* columnFilterInfoToJson(const void* obj) {
static bool schemaFromJson(const cJSON* json, void* obj) {
SSlotSchema* schema = (SSlotSchema*)obj;
schema->type = getNumber(json, jkSchemaType);
schema->colId = getNumber(json, jkSchemaColId);
schema->bytes = getNumber(json, jkSchemaBytes);
return true;
}
static const char* jkColumnFilterInfoLowerRelOptr = "LowerRelOptr";
static const char* jkColumnFilterInfoUpperRelOptr = "UpperRelOptr";
static const char* jkColumnFilterInfoFilterstr = "Filterstr";
static const char* jkColumnFilterInfoLowerBnd = "LowerBnd";
static const char* jkColumnFilterInfoUpperBnd = "UpperBnd";
static bool columnFilterInfoToJson(const void* obj, cJSON* jFilter) {
const SColumnFilterInfo* filter = (const SColumnFilterInfo*)obj;
cJSON* jFilter = cJSON_CreateObject();
if (NULL == jFilter) {
return NULL;
}
bool res = cJSON_AddNumberToObject(jFilter, "LowerRelOptr", filter->lowerRelOptr);
bool res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoLowerRelOptr, filter->lowerRelOptr);
if (res) {
res = cJSON_AddNumberToObject(jFilter, "UpperRelOptr", filter->upperRelOptr);
res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoUpperRelOptr, filter->upperRelOptr);
}
if (res) {
res = cJSON_AddNumberToObject(jFilter, "Filterstr", filter->filterstr);
res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoFilterstr, filter->filterstr);
}
if (res) {
res = cJSON_AddNumberToObject(jFilter, "LowerBnd", filter->lowerBndd);
res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoLowerBnd, filter->lowerBndd);
}
if (res) {
res = cJSON_AddNumberToObject(jFilter, "UpperBnd", filter->upperBndd);
res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoUpperBnd, filter->upperBndd);
}
if (!res) {
cJSON_Delete(jFilter);
return NULL;
}
return jFilter;
return res;
}
static cJSON* columnInfoToJson(const void* obj) {
static bool columnFilterInfoFromJson(const cJSON* json, void* obj) {
SColumnFilterInfo* filter = (SColumnFilterInfo*)obj;
filter->lowerRelOptr = getNumber(json, jkColumnFilterInfoLowerRelOptr);
filter->upperRelOptr = getNumber(json, jkColumnFilterInfoUpperRelOptr);
filter->filterstr = getNumber(json, jkColumnFilterInfoFilterstr);
filter->lowerBndd = getNumber(json, jkColumnFilterInfoLowerBnd);
filter->upperBndd = getNumber(json, jkColumnFilterInfoUpperBnd);
return true;
}
static const char* jkColumnInfoColId = "ColId";
static const char* jkColumnInfoType = "Type";
static const char* jkColumnInfoBytes = "Bytes";
static const char* jkColumnInfoFilterList = "FilterList";
static bool columnInfoToJson(const void* obj, cJSON* jCol) {
const SColumnInfo* col = (const SColumnInfo*)obj;
cJSON* jCol = cJSON_CreateObject();
if (NULL == jCol) {
return NULL;
}
bool res = cJSON_AddNumberToObject(jCol, "ColId", col->colId);
bool res = cJSON_AddNumberToObject(jCol, jkColumnInfoColId, col->colId);
if (res) {
res = cJSON_AddNumberToObject(jCol, "Type", col->type);
res = cJSON_AddNumberToObject(jCol, jkColumnInfoType, col->type);
}
if (res) {
res = cJSON_AddNumberToObject(jCol, "Bytes", col->bytes);
res = cJSON_AddNumberToObject(jCol, jkColumnInfoBytes, col->bytes);
}
if (res) {
res = addRawArray(jCol, "FilterList", columnFilterInfoToJson, col->flist.filterInfo, sizeof(SColumnFilterInfo), col->flist.numOfFilters);
res = addRawArray(jCol, jkColumnInfoFilterList, columnFilterInfoToJson, col->flist.filterInfo, sizeof(SColumnFilterInfo), col->flist.numOfFilters);
}
if (!res) {
cJSON_Delete(jCol);
return NULL;
}
return jCol;
return res;
}
static cJSON* columnToJson(const void* obj) {
static bool columnInfoFromJson(const cJSON* json, void* obj) {
SColumnInfo* col = (SColumnInfo*)obj;
col->colId = getNumber(json, jkColumnInfoColId);
col->type = getNumber(json, jkColumnInfoType);
col->bytes = getNumber(json, jkColumnInfoBytes);
int32_t size = 0;
bool res = fromRawArrayWithAlloc(json, jkColumnInfoFilterList, columnFilterInfoFromJson, (void**)&col->flist.filterInfo, sizeof(SColumnFilterInfo), &size);
col->flist.numOfFilters = size;
return res;
}
static const char* jkColumnTableId = "TableId";
static const char* jkColumnFlag = "Flag";
static const char* jkColumnInfo = "Info";
static bool columnToJson(const void* obj, cJSON* jCol) {
const SColumn* col = (const SColumn*)obj;
cJSON* jCol = cJSON_CreateObject();
if (NULL == jCol) {
return NULL;
}
bool res = cJSON_AddNumberToObject(jCol, "TableId", col->uid);
bool res = cJSON_AddNumberToObject(jCol, jkColumnTableId, col->uid);
if (res) {
res = cJSON_AddNumberToObject(jCol, "Flag", col->flag);
res = cJSON_AddNumberToObject(jCol, jkColumnFlag, col->flag);
}
if (res) {
res = addObject(jCol, "Info", columnInfoToJson, &col->info);
res = addObject(jCol, jkColumnInfo, columnInfoToJson, &col->info);
}
if (!res) {
cJSON_Delete(jCol);
return NULL;
}
return jCol;
return res;
}
static cJSON* exprNodeToJson(const void* obj);
static bool columnFromJson(const cJSON* json, void* obj) {
SColumn* col = (SColumn*)obj;
col->uid = getNumber(json, jkColumnTableId);
col->flag = getNumber(json, jkColumnFlag);
return fromObject(json, jkColumnInfo, columnInfoFromJson, &col->info, true);
}
static cJSON* operatorToJson(const void* obj) {
static bool exprNodeToJson(const void* obj, cJSON* jExprInfo);
static bool exprNodeFromJson(const cJSON* json, void* obj);
static const char* jkExprNodeOper = "Oper";
static const char* jkExprNodeLeft = "Left";
static const char* jkExprNodeRight = "Right";
static bool operatorToJson(const void* obj, cJSON* jOper) {
const tExprNode* exprInfo = (const tExprNode*)obj;
cJSON* jOper = cJSON_CreateObject();
if (NULL == jOper) {
return NULL;
}
bool res = cJSON_AddNumberToObject(jOper, "Oper", exprInfo->_node.optr);
bool res = cJSON_AddNumberToObject(jOper, jkExprNodeOper, exprInfo->_node.optr);
if (res) {
res = addObject(jOper, "Left", exprNodeToJson, exprInfo->_node.pLeft);
res = addObject(jOper, jkExprNodeLeft, exprNodeToJson, exprInfo->_node.pLeft);
}
if (res) {
res = addObject(jOper, "Right", exprNodeToJson, exprInfo->_node.pRight);
res = addObject(jOper, jkExprNodeRight, exprNodeToJson, exprInfo->_node.pRight);
}
if (!res) {
cJSON_Delete(jOper);
return NULL;
}
return jOper;
return res;
}
static cJSON* functionToJson(const void* obj) {
static bool operatorFromJson(const cJSON* json, void* obj) {
tExprNode* exprInfo = (tExprNode*)obj;
exprInfo->_node.optr = getNumber(json, jkExprNodeOper);
bool res = fromObject(json, jkExprNodeLeft, exprNodeFromJson, exprInfo->_node.pLeft, false);
if (res) {
res = fromObject(json, jkExprNodeRight, exprNodeFromJson, exprInfo->_node.pRight, false);
}
return res;
}
static const char* jkFunctionName = "Name";
static const char* jkFunctionChild = "Child";
static bool functionToJson(const void* obj, cJSON* jFunc) {
const tExprNode* exprInfo = (const tExprNode*)obj;
cJSON* jFunc = cJSON_CreateObject();
if (NULL == jFunc) {
return NULL;
}
bool res = cJSON_AddStringToObject(jFunc, "Name", exprInfo->_function.functionName);
bool res = cJSON_AddStringToObject(jFunc, jkFunctionName, exprInfo->_function.functionName);
if (res) {
res = addRawArray(jFunc, "Child", exprNodeToJson, exprInfo->_function.pChild, sizeof(tExprNode*), exprInfo->_function.num);
res = addRawArray(jFunc, jkFunctionChild, exprNodeToJson, exprInfo->_function.pChild, sizeof(tExprNode*), exprInfo->_function.num);
}
if (!res) {
cJSON_Delete(jFunc);
return NULL;
}
return jFunc;
return res;
}
static cJSON* variantToJson(const void* obj) {
static bool functionFromJson(const cJSON* json, void* obj) {
tExprNode* exprInfo = (tExprNode*)obj;
copyString(json, jkFunctionName, exprInfo->_function.functionName);
return fromRawArrayWithAlloc(json, jkFunctionChild, exprNodeFromJson, (void**)exprInfo->_function.pChild, sizeof(tExprNode*), &exprInfo->_function.num);
}
static const char* jkVariantType = "Type";
static const char* jkVariantLen = "Len";
static const char* jkVariantvalues = "values";
static const char* jkVariantValue = "Value";
static bool variantToJson(const void* obj, cJSON* jVar) {
const SVariant* var = (const SVariant*)obj;
cJSON* jVar = cJSON_CreateObject();
if (NULL == jVar) {
return NULL;
}
bool res = cJSON_AddNumberToObject(jVar, "Type", var->nType);
bool res = cJSON_AddNumberToObject(jVar, jkVariantType, var->nType);
if (res) {
res = cJSON_AddNumberToObject(jVar, "Len", var->nLen);
res = cJSON_AddNumberToObject(jVar, jkVariantLen, var->nLen);
}
if (res) {
if (0/* in */) {
res = addArray(jVar, "values", variantToJson, var->arr);
res = addArray(jVar, jkVariantvalues, variantToJson, var->arr);
} else if (IS_NUMERIC_TYPE(var->nType)) {
res = cJSON_AddNumberToObject(jVar, "Value", var->d);
res = cJSON_AddNumberToObject(jVar, jkVariantValue, var->d);
} else {
res = cJSON_AddStringToObject(jVar, "Value", var->pz);
res = cJSON_AddStringToObject(jVar, jkVariantValue, var->pz);
}
}
if (!res) {
cJSON_Delete(jVar);
return NULL;
}
return jVar;
return res;
}
static cJSON* exprNodeToJson(const void* obj) {
const tExprNode* exprInfo = (const tExprNode*)obj;
cJSON* jExprInfo = cJSON_CreateObject();
if (NULL == jExprInfo) {
return NULL;
static bool variantFromJson(const cJSON* json, void* obj) {
SVariant* var = (SVariant*)obj;
var->nType = getNumber(json, jkVariantType);
var->nLen = getNumber(json, jkVariantLen);
if (0/* in */) {
return fromArray(json, jkVariantvalues, variantFromJson, &var->arr, sizeof(SVariant));
} else if (IS_NUMERIC_TYPE(var->nType)) {
var->d = getNumber(json, jkVariantValue);
} else {
var->pz = getString(json, jkVariantValue);
}
return true;
}
bool res = cJSON_AddNumberToObject(jExprInfo, "Type", exprInfo->nodeType);
static const char* jkExprNodeType = "Type";
static const char* jkExprNodeOperator = "Operator";
static const char* jkExprNodeFunction = "Function";
static const char* jkExprNodeColumn = "Column";
static const char* jkExprNodeValue = "Value";
static bool exprNodeToJson(const void* obj, cJSON* jExprInfo) {
const tExprNode* exprInfo = (const tExprNode*)obj;
bool res = cJSON_AddNumberToObject(jExprInfo, jkExprNodeType, exprInfo->nodeType);
if (res) {
switch (exprInfo->nodeType) {
case TEXPR_BINARYEXPR_NODE:
case TEXPR_UNARYEXPR_NODE:
res = addObject(jExprInfo, "Operator", operatorToJson, exprInfo);
res = addObject(jExprInfo, jkExprNodeOperator, operatorToJson, exprInfo);
break;
case TEXPR_FUNCTION_NODE:
res = addObject(jExprInfo, "Function", functionToJson, exprInfo);
res = addObject(jExprInfo, jkExprNodeFunction, functionToJson, exprInfo);
break;
case TEXPR_COL_NODE:
res = addObject(jExprInfo, "Column", schemaToJson, exprInfo->pSchema);
res = addObject(jExprInfo, jkExprNodeColumn, schemaToJson, exprInfo->pSchema);
break;
case TEXPR_VALUE_NODE:
res = addObject(jExprInfo, "Value", variantToJson, exprInfo->pVal);
res = addObject(jExprInfo, jkExprNodeValue, variantToJson, exprInfo->pVal);
break;
default:
res = false;
break;
}
}
if (!res) {
cJSON_Delete(jExprInfo);
return NULL;
}
return jExprInfo;
return res;
}
static cJSON* sqlExprToJson(const void* obj) {
static bool exprNodeFromJson(const cJSON* json, void* obj) {
tExprNode* exprInfo = (tExprNode*)obj;
exprInfo->nodeType = getNumber(json, jkExprNodeType);
switch (exprInfo->nodeType) {
case TEXPR_BINARYEXPR_NODE:
case TEXPR_UNARYEXPR_NODE:
return fromObject(json, jkExprNodeOperator, operatorFromJson, exprInfo, false);
case TEXPR_FUNCTION_NODE:
return fromObject(json, jkExprNodeFunction, functionFromJson, exprInfo, false);
case TEXPR_COL_NODE:
return fromObject(json, jkExprNodeColumn, schemaFromJson, exprInfo->pSchema, false);
case TEXPR_VALUE_NODE:
return fromObject(json, jkExprNodeValue, variantFromJson, exprInfo->pVal, false);
default:
break;
}
return false;
}
static const char* jkSqlExprSchema = "Schema";
static const char* jkSqlExprColumns = "Columns";
static const char* jkSqlExprInterBytes = "InterBytes";
static const char* jkSqlExprParams = "Params";
// token does not need to be serialized.
static bool sqlExprToJson(const void* obj, cJSON* jExpr) {
const SSqlExpr* expr = (const SSqlExpr*)obj;
cJSON* jExpr = cJSON_CreateObject();
if (NULL == jExpr) {
return NULL;
}
// token does not need to be serialized.
bool res = addObject(jExpr, "Schema", schemaToJson, &expr->resSchema);
bool res = addObject(jExpr, jkSqlExprSchema, schemaToJson, &expr->resSchema);
if (res) {
res = addRawArray(jExpr, "Columns", columnToJson, expr->pColumns, sizeof(SColumn), expr->numOfCols);
res = addRawArray(jExpr, jkSqlExprColumns, columnToJson, expr->pColumns, sizeof(SColumn), expr->numOfCols);
}
if (res) {
res = cJSON_AddNumberToObject(jExpr, "InterBytes", expr->interBytes);
res = cJSON_AddNumberToObject(jExpr, jkSqlExprInterBytes, expr->interBytes);
}
if (res) {
res = addRawArray(jExpr, "Params", variantToJson, expr->param, sizeof(SVariant), expr->numOfParams);
res = addRawArray(jExpr, jkSqlExprParams, variantToJson, expr->param, sizeof(SVariant), expr->numOfParams);
}
if (!res) {
cJSON_Delete(jExpr);
return NULL;
}
return jExpr;
return res;
}
static cJSON* exprInfoToJson(const void* obj) {
static bool sqlExprFromJson(const cJSON* json, void* obj) {
SSqlExpr* expr = (SSqlExpr*)obj;
bool res = fromObject(json, jkSqlExprSchema, schemaFromJson, &expr->resSchema, false);
if (res) {
res = fromRawArrayWithAlloc(json, jkSqlExprColumns, columnFromJson, (void**)&expr->pColumns, sizeof(SColumn), &expr->numOfCols);
}
if (res) {
expr->interBytes = getNumber(json, jkSqlExprInterBytes);
}
if (res) {
int32_t size = 0;
res = fromRawArray(json, jkSqlExprParams, variantFromJson, expr->param, sizeof(SVariant), &size);
expr->numOfParams = size;
}
return res;
}
static const char* jkExprInfoBase = "Base";
static const char* jkExprInfoExpr = "Expr";
static bool exprInfoToJson(const void* obj, cJSON* jExprInfo) {
const SExprInfo* exprInfo = (const SExprInfo*)obj;
cJSON* jExprInfo = cJSON_CreateObject();
if (NULL == jExprInfo) {
return NULL;
}
bool res = addObject(jExprInfo, "Base", sqlExprToJson, &exprInfo->base);
bool res = addObject(jExprInfo, jkExprInfoBase, sqlExprToJson, &exprInfo->base);
if (res) {
res = addObject(jExprInfo, "Expr", exprNodeToJson, exprInfo->pExpr);
res = addObject(jExprInfo, jkExprInfoExpr, exprNodeToJson, exprInfo->pExpr);
}
if (!res) {
cJSON_Delete(jExprInfo);
return NULL;
}
return jExprInfo;
return res;
}
static cJSON* phyNodeToJson(const void* obj) {
static bool exprInfoFromJson(const cJSON* json, void* obj) {
SExprInfo* exprInfo = (SExprInfo*)obj;
bool res = fromObject(json, jkExprInfoBase, sqlExprFromJson, &exprInfo->base, true);
if (res) {
res = fromObjectWithAlloc(json, jkExprInfoExpr, exprNodeFromJson, (void**)&exprInfo->pExpr, sizeof(tExprNode), true);
}
return res;
}
static const char* jkTimeWindowStartKey = "StartKey";
static const char* jkTimeWindowEndKey = "EndKey";
static bool timeWindowToJson(const void* obj, cJSON* json) {
const STimeWindow* win = (const STimeWindow*)obj;
bool res = cJSON_AddNumberToObject(json, jkTimeWindowStartKey, win->skey);
if (res) {
res = cJSON_AddNumberToObject(json, jkTimeWindowEndKey, win->ekey);
}
return res;
}
static bool timeWindowFromJson(const cJSON* json, void* obj) {
STimeWindow* win = (STimeWindow*)obj;
win->skey = getNumber(json, jkTimeWindowStartKey);
win->ekey = getNumber(json, jkTimeWindowEndKey);
return true;
}
static const char* jkScanNodeTableId = "TableId";
static const char* jkScanNodeTableType = "TableType";
static bool scanNodeToJson(const void* obj, cJSON* json) {
const SScanPhyNode* scan = (const SScanPhyNode*)obj;
bool res = cJSON_AddNumberToObject(json, jkScanNodeTableId, scan->uid);
if (res) {
res = cJSON_AddNumberToObject(json, jkScanNodeTableType, scan->tableType);
}
return res;
}
static bool scanNodeFromJson(const cJSON* json, void* obj) {
SScanPhyNode* scan = (SScanPhyNode*)obj;
scan->uid = getNumber(json, jkScanNodeTableId);
scan->tableType = getNumber(json, jkScanNodeTableType);
return true;
}
static const char* jkTableScanNodeFlag = "Flag";
static const char* jkTableScanNodeWindow = "Window";
static const char* jkTableScanNodeTagsConditions = "TagsConditions";
static bool tableScanNodeToJson(const void* obj, cJSON* json) {
const STableScanPhyNode* scan = (const STableScanPhyNode*)obj;
bool res = scanNodeToJson(obj, json);
if (res) {
res = cJSON_AddNumberToObject(json, jkTableScanNodeFlag, scan->scanFlag);
}
if (res) {
res = addObject(json, jkTableScanNodeWindow, timeWindowToJson, &scan->window);
}
if (res) {
res = addArray(json, jkTableScanNodeTagsConditions, exprInfoToJson, scan->pTagsConditions);
}
return res;
}
static bool tableScanNodeFromJson(const cJSON* json, void* obj) {
STableScanPhyNode* scan = (STableScanPhyNode*)obj;
bool res = scanNodeFromJson(json, obj);
if (res) {
scan->scanFlag = getNumber(json, jkTableScanNodeFlag);
}
if (res) {
res = fromObject(json, jkTableScanNodeWindow, timeWindowFromJson, &scan->window, true);
}
if (res) {
res = fromArray(json, jkTableScanNodeTagsConditions, exprInfoFromJson, &scan->pTagsConditions, sizeof(SExprInfo));
}
return res;
}
static const char* jkEpAddrFqdn = "Fqdn";
static const char* jkEpAddrPort = "Port";
static bool epAddrToJson(const void* obj, cJSON* json) {
const SEpAddrMsg* ep = (const SEpAddrMsg*)obj;
bool res = cJSON_AddStringToObject(json, jkEpAddrFqdn, ep->fqdn);
if (res) {
res = cJSON_AddNumberToObject(json, jkEpAddrPort, ep->port);
}
return res;
}
static bool epAddrFromJson(const cJSON* json, void* obj) {
SEpAddrMsg* ep = (SEpAddrMsg*)obj;
copyString(json, jkEpAddrFqdn, ep->fqdn);
ep->port = getNumber(json, jkEpAddrPort);
return true;
}
static const char* jkExchangeNodeSrcTemplateId = "SrcTemplateId";
static const char* jkExchangeNodeSrcEndPoints = "SrcEndPoints";
static bool exchangeNodeToJson(const void* obj, cJSON* json) {
const SExchangePhyNode* exchange = (const SExchangePhyNode*)obj;
bool res = cJSON_AddNumberToObject(json, jkExchangeNodeSrcTemplateId, exchange->srcTemplateId);
if (res) {
res = addArray(json, jkExchangeNodeSrcEndPoints, epAddrToJson, exchange->pSrcEndPoints);
}
return res;
}
static bool exchangeNodeFromJson(const cJSON* json, void* obj) {
SExchangePhyNode* exchange = (SExchangePhyNode*)obj;
exchange->srcTemplateId = getNumber(json, jkExchangeNodeSrcTemplateId);
return fromArray(json, jkExchangeNodeSrcEndPoints, epAddrFromJson, &exchange->pSrcEndPoints, sizeof(SEpAddrMsg));
}
static bool specificPhyNodeToJson(const void* obj, cJSON* json) {
const SPhyNode* phyNode = (const SPhyNode*)obj;
cJSON* jNode = cJSON_CreateObject();
if (NULL == jNode) {
return NULL;
switch (phyNode->info.type) {
case OP_TableScan:
case OP_DataBlocksOptScan:
case OP_TableSeqScan:
return tableScanNodeToJson(obj, json);
case OP_TagScan:
case OP_SystemTableScan:
return scanNodeToJson(obj, json);
case OP_Aggregate:
break; // todo
case OP_Project:
return true;
case OP_Groupby:
case OP_Limit:
case OP_SLimit:
case OP_TimeWindow:
case OP_SessionWindow:
case OP_StateWindow:
case OP_Fill:
case OP_MultiTableAggregate:
case OP_MultiTableTimeInterval:
case OP_Filter:
case OP_Distinct:
case OP_Join:
case OP_AllTimeWindow:
case OP_AllMultiTableTimeInterval:
case OP_Order:
break; // todo
case OP_Exchange:
return exchangeNodeToJson(obj, json);
default:
break;
}
// The 'pParent' field do not need to be serialized.
bool res = cJSON_AddStringToObject(jNode, "Name", phyNode->info.name);
if (res) {
res = addArray(jNode, "Targets", exprInfoToJson, phyNode->pTargets);
}
if (res) {
res = addArray(jNode, "Conditions", exprInfoToJson, phyNode->pConditions);
}
if (res) {
res = addRawArray(jNode, "Schema", schemaToJson, phyNode->targetSchema.pSchema, sizeof(SSlotSchema), phyNode->targetSchema.numOfCols);
}
if (res) {
res = addArray(jNode, "Children", phyNodeToJson, phyNode->pChildren);
}
if (!res) {
cJSON_Delete(jNode);
return NULL;
}
return jNode;
return false;
}
static cJSON* subplanIdToJson(const void* obj) {
static bool specificPhyNodeFromJson(const cJSON* json, void* obj) {
SPhyNode* phyNode = (SPhyNode*)obj;
switch (phyNode->info.type) {
case OP_TableScan:
case OP_DataBlocksOptScan:
case OP_TableSeqScan:
return tableScanNodeFromJson(json, obj);
case OP_TagScan:
case OP_SystemTableScan:
return scanNodeFromJson(json, obj);
case OP_Aggregate:
break; // todo
case OP_Project:
return true;
case OP_Groupby:
case OP_Limit:
case OP_SLimit:
case OP_TimeWindow:
case OP_SessionWindow:
case OP_StateWindow:
case OP_Fill:
case OP_MultiTableAggregate:
case OP_MultiTableTimeInterval:
case OP_Filter:
case OP_Distinct:
case OP_Join:
case OP_AllTimeWindow:
case OP_AllMultiTableTimeInterval:
case OP_Order:
break; // todo
case OP_Exchange:
return exchangeNodeFromJson(json, obj);
default:
break;
}
return false;
}
static const char* jkPnodeName = "Name";
static const char* jkPnodeTargets = "Targets";
static const char* jkPnodeConditions = "Conditions";
static const char* jkPnodeSchema = "Schema";
static const char* jkPnodeChildren = "Children";
// The 'pParent' field do not need to be serialized.
static bool phyNodeToJson(const void* obj, cJSON* jNode) {
const SPhyNode* phyNode = (const SPhyNode*)obj;
bool res = cJSON_AddStringToObject(jNode, jkPnodeName, phyNode->info.name);
if (res) {
res = addArray(jNode, jkPnodeTargets, exprInfoToJson, phyNode->pTargets);
}
if (res) {
res = addArray(jNode, jkPnodeConditions, exprInfoToJson, phyNode->pConditions);
}
if (res) {
res = addRawArray(jNode, jkPnodeSchema, schemaToJson, phyNode->targetSchema.pSchema, sizeof(SSlotSchema), phyNode->targetSchema.numOfCols);
}
if (res) {
res = addArray(jNode, jkPnodeChildren, phyNodeToJson, phyNode->pChildren);
}
if (res) {
res = addObject(jNode, phyNode->info.name, specificPhyNodeToJson, phyNode);
}
return res;
}
static bool phyNodeFromJson(const cJSON* json, void* obj) {
SPhyNode* node = (SPhyNode*)obj;
node->info.name = getString(json, jkPnodeName);
node->info.type = opNameToOpType(node->info.name);
bool res = fromArray(json, jkPnodeTargets, exprInfoFromJson, &node->pTargets, sizeof(SExprInfo));
if (res) {
res = fromArray(json, jkPnodeConditions, exprInfoFromJson, &node->pConditions, sizeof(SExprInfo));
}
if (res) {
res = fromRawArray(json, jkPnodeSchema, schemaFromJson, node->targetSchema.pSchema, sizeof(SSlotSchema), &node->targetSchema.numOfCols);
}
if (res) {
res = fromArray(json, jkPnodeChildren, phyNodeFromJson, &node->pChildren, sizeof(SSlotSchema));
}
if (res) {
res = fromObject(json, node->info.name, specificPhyNodeFromJson, node, true);
}
return res;
}
static const char* jkIdQueryId = "QueryId";
static const char* jkIdTemplateId = "TemplateId";
static const char* jkIdSubplanId = "SubplanId";
static bool subplanIdToJson(const void* obj, cJSON* jId) {
const SSubplanId* id = (const SSubplanId*)obj;
cJSON* jId = cJSON_CreateObject();
if (NULL == jId) {
return NULL;
}
bool res = cJSON_AddNumberToObject(jId, "QueryId", id->queryId);
bool res = cJSON_AddNumberToObject(jId, jkIdQueryId, id->queryId);
if (res) {
res = cJSON_AddNumberToObject(jId, "TemplateId", id->templateId);
res = cJSON_AddNumberToObject(jId, jkIdTemplateId, id->templateId);
}
if (res) {
res = cJSON_AddNumberToObject(jId, "SubplanId", id->subplanId);
res = cJSON_AddNumberToObject(jId, jkIdSubplanId, id->subplanId);
}
if (!res) {
cJSON_Delete(jId);
return NULL;
}
return jId;
return res;
}
static bool subplanIdFromJson(const cJSON* json, void* obj) {
SSubplanId* id = (SSubplanId*)obj;
id->queryId = getNumber(json, jkIdQueryId);
id->templateId = getNumber(json, jkIdTemplateId);
id->subplanId = getNumber(json, jkIdSubplanId);
return true;
}
static const char* jkSubplanId = "Id";
static const char* jkSubplanNode = "Node";
static cJSON* subplanToJson(const SSubplan* subplan) {
cJSON* jSubplan = cJSON_CreateObject();
if (NULL == jSubplan) {
@ -382,9 +730,9 @@ static cJSON* subplanToJson(const SSubplan* subplan) {
// The 'type', 'level', 'execEpSet', 'pChildern' and 'pParents' fields do not need to be serialized.
bool res = addObject(jSubplan, "Id", subplanIdToJson, &subplan->id);
bool res = addObject(jSubplan, jkSubplanId, subplanIdToJson, &subplan->id);
if (res) {
res = addObject(jSubplan, "Node", phyNodeToJson, subplan->pNode);
res = addObject(jSubplan, jkSubplanNode, phyNodeToJson, subplan->pNode);
}
if (!res) {
@ -394,6 +742,23 @@ static cJSON* subplanToJson(const SSubplan* subplan) {
return jSubplan;
}
static SSubplan* subplanFromJson(const cJSON* json) {
SSubplan* subplan = calloc(1, sizeof(SSubplan));
if (NULL == subplan) {
return NULL;
}
bool res = fromObject(json, jkSubplanId, subplanIdFromJson, &subplan->id, true);
if (res) {
res = fromObjectWithAlloc(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode, sizeof(SPhyNode), false);
}
if (!res) {
qDestroySubplan(subplan);
return NULL;
}
return subplan;
}
int32_t subPlanToString(const SSubplan* subplan, char** str) {
cJSON* json = subplanToJson(subplan);
if (NULL == json) {
@ -405,6 +770,10 @@ int32_t subPlanToString(const SSubplan* subplan, char** str) {
}
int32_t stringToSubplan(const char* str, SSubplan** subplan) {
// todo
return TSDB_CODE_SUCCESS;
cJSON* json = cJSON_Parse(str);
if (NULL == json) {
return TSDB_CODE_FAILED;
}
*subplan = subplanFromJson(json);
return (NULL == *subplan ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS);
}

8
source/libs/planner/src/planner.c
View File

@ -16,6 +16,10 @@
#include "parser.h"
#include "plannerInt.h"
void qDestroySubplan(SSubplan* pSubplan) {
// todo
}
void qDestroyQueryDag(struct SQueryDag* pDag) {
// todo
}
@ -42,6 +46,10 @@ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet*
return TSDB_CODE_SUCCESS;
}
int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps) {
return setSubplanExecutionNode(subplan, templateId, eps);
}
int32_t qSubPlanToString(const SSubplan *subplan, char** str) {
return subPlanToString(subplan, str);
}

2
source/libs/planner/test/CMakeLists.txt
View File

@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(plannerTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
plannerTest
PUBLIC os util common planner parser catalog transport gtest function query
PUBLIC os util common planner parser catalog transport gtest function qcom
)
TARGET_INCLUDE_DIRECTORIES(

12
source/libs/qcom/CMakeLists.txt Normal file
View File

@ -0,0 +1,12 @@
aux_source_directory(src QUERY_SRC)
add_library(qcom ${QUERY_SRC})
target_include_directories(
qcom
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qcom"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
qcom
PRIVATE os util transport
)

0
source/libs/query/inc/queryInt.h → source/libs/qcom/inc/queryInt.h
View File

78
source/libs/qcom/src/queryUtil.c Normal file
View File

@ -0,0 +1,78 @@
#include "os.h"
#include "taosmsg.h"
#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS)
#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS)
static struct SSchema _s = {
.colId = TSDB_TBNAME_COLUMN_INDEX,
.type = TSDB_DATA_TYPE_BINARY,
.bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE,
.name = "tbname",
};
SSchema* tGetTbnameColumnSchema() {
return &_s;
}
static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) {
int32_t rowLen = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
// 1. valid types
if (!isValidDataType(pSchema[i].type)) {
return false;
}
// 2. valid length for each type
if (pSchema[i].type == TSDB_DATA_TYPE_BINARY) {
if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) {
return false;
}
} else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
if (pSchema[i].bytes > TSDB_MAX_NCHAR_LEN) {
return false;
}
} else {
if (pSchema[i].bytes != tDataTypes[pSchema[i].type].bytes) {
return false;
}
}
// 3. valid column names
for (int32_t j = i + 1; j < numOfCols; ++j) {
if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) {
return false;
}
}
rowLen += pSchema[i].bytes;
}
return rowLen <= maxLen;
}
bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags) {
if (!VALIDNUMOFCOLS(numOfCols)) {
return false;
}
if (!VALIDNUMOFTAGS(numOfTags)) {
return false;
}
/* first column must be the timestamp, which is a primary key */
if (pSchema[0].type != TSDB_DATA_TYPE_TIMESTAMP) {
return false;
}
if (!doValidateSchema(pSchema, numOfCols, TSDB_MAX_BYTES_PER_ROW)) {
return false;
}
if (!doValidateSchema(&pSchema[numOfCols], numOfTags, TSDB_MAX_TAGS_LEN)) {
return false;
}
return true;
}

0
source/libs/query/src/querymsg.c → source/libs/qcom/src/querymsg.c
View File

12
source/libs/query/CMakeLists.txt
View File

@ -1,12 +0,0 @@
aux_source_directory(src QUERY_SRC)
add_library(query ${QUERY_SRC})
target_include_directories(
query
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/query"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
query
PRIVATE os util common transport
)

2
source/libs/scheduler/CMakeLists.txt
View File

@ -9,5 +9,5 @@ target_include_directories(
target_link_libraries(
scheduler
PRIVATE os util planner common query
PRIVATE os util planner qcom common
)

1
source/util/src/terror.c
View File

@ -130,6 +130,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_VALUE_OUT_OF_RANGE, "Value out of range")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_INPUT, "Invalid tsc input")
// mnode-common
TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Mnode internal error")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, "Cluster not ready")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS, "Message is progressing")

17
tests/CMakeLists.txt
View File

@ -1,15 +1,4 @@
# generate debug version:
# mkdir debug; cd debug; cmake -DCMAKE_BUILD_TYPE=Debug ..
# generate release version:
# mkdir release; cd release; cmake -DCMAKE_BUILD_TYPE=Release ..
CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
PROJECT(TDengine)
SET(CMAKE_C_STANDARD 11)
SET(CMAKE_VERBOSE_MAKEFILE ON)
ADD_SUBDIRECTORY(examples/c)
#ADD_SUBDIRECTORY(examples/c)
ADD_SUBDIRECTORY(tsim)
ADD_SUBDIRECTORY(test/c)
ADD_SUBDIRECTORY(comparisonTest/tdengine)
#ADD_SUBDIRECTORY(test/c)
#ADD_SUBDIRECTORY(comparisonTest/tdengine)

10
tests/script/general/user/basic1.sim
View File

@ -5,7 +5,7 @@ sql connect
print =============== show users
sql show users
if $rows != 3 then
if $rows != 1 then
return -1
endi
@ -21,7 +21,7 @@ sql_error drop account root
print =============== create user1
sql create user user1 PASS 'user1'
sql show users
if $rows != 4 then
if $rows != 2 then
return -1
endi
@ -33,7 +33,7 @@ print $data30 $data31 $data32
print =============== create user2
sql create user user2 PASS 'user2'
sql show users
if $rows != 5 then
if $rows != 3 then
return -1
endi
@ -46,7 +46,7 @@ print $data40 $data41 $data42
print =============== drop user1
sql drop user user1
sql show users
if $rows != 4 then
if $rows != 2 then
return -1
endi
@ -62,7 +62,7 @@ system sh/exec.sh -n dnode1 -s start
print =============== show users
sql show users
if $rows != 4 then
if $rows != 2 then
return -1
endi

9
tests/script/sh/deploy.sh
View File

@ -1,5 +1,8 @@
#!/bin/bash
set +e
#set -x
echo "Executing deploy.sh"
if [ $# != 4 ]; then
@ -50,12 +53,12 @@ else
fi
if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then
BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3`
BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2,3`
else
BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2`
BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2`
fi
BUILD_DIR=$TAOS_DIR/$BIN_DIR/build
BUILD_DIR=$TAOS_DIR/$BIN_DIR
SIM_DIR=$TAOS_DIR/sim

11
tests/script/sh/exec.sh
View File

@ -8,6 +8,9 @@
# exit 1
# fi
set +e
#set -x
UNAME_BIN=`which uname`
OS_TYPE=`$UNAME_BIN`
@ -62,16 +65,16 @@ else
fi
if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then
BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3`
BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2,3`
else
BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2`
BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2`
fi
BUILD_DIR=$TAOS_DIR/$BIN_DIR/build
BUILD_DIR=$TAOS_DIR/$BIN_DIR
SIM_DIR=$TAOS_DIR/sim
NODE_DIR=$SIM_DIR/$NODE_NAME
EXE_DIR=$BUILD_DIR/bin
EXE_DIR=$BUILD_DIR/source/dnode/mgmt/daemon
CFG_DIR=$NODE_DIR/cfg
LOG_DIR=$NODE_DIR/log
DATA_DIR=$NODE_DIR/data

26
tests/script/test.sh
View File

@ -22,9 +22,6 @@ do
f)
FILE_NAME=$OPTARG
;;
a)
ASYNC=1
;;
v)
VALGRIND=1
;;
@ -60,32 +57,22 @@ else
fi
if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then
BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3`
BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2,3`
else
BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2`
BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2`
fi
BUILD_DIR=$TOP_DIR/$BIN_DIR/build
BUILD_DIR=$TOP_DIR/$BIN_DIR
SIM_DIR=$TOP_DIR/sim
if [ $ASYNC -eq 0 ]; then
PROGRAM=$BUILD_DIR/bin/tsim
else
PROGRAM="$BUILD_DIR/bin/tsim -a"
fi
PROGRAM=$BUILD_DIR/tests/tsim/tsim
PRG_DIR=$SIM_DIR/tsim
CFG_DIR=$PRG_DIR/cfg
LOG_DIR=$PRG_DIR/log
DATA_DIR=$PRG_DIR/data
ARBITRATOR_PRG_DIR=$SIM_DIR/arbitrator
ARBITRATOR_LOG_DIR=$ARBITRATOR_PRG_DIR/log
chmod -R 777 $PRG_DIR
echo "------------------------------------------------------------------------"
echo "Start TDengine Testing Case ..."
@ -96,12 +83,10 @@ echo "CFG_DIR : $CFG_DIR"
rm -rf $LOG_DIR
rm -rf $CFG_DIR
rm -rf $ARBITRATOR_LOG_DIR
mkdir -p $PRG_DIR
mkdir -p $LOG_DIR
mkdir -p $CFG_DIR
mkdir -p $ARBITRATOR_LOG_DIR
TAOS_CFG=$PRG_DIR/cfg/taos.cfg
touch -f $TAOS_CFG
@ -115,7 +100,7 @@ echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG
echo "serverPort 7100" >> $TAOS_CFG
echo "dataDir $DATA_DIR" >> $TAOS_CFG
echo "logDir $LOG_DIR" >> $TAOS_CFG
echo "scriptDir ${CODE_DIR}/../script" >> $TAOS_CFG
echo "scriptDir ${CODE_DIR}" >> $TAOS_CFG
echo "numOfLogLines 100000000" >> $TAOS_CFG
echo "rpcDebugFlag 143" >> $TAOS_CFG
echo "tmrDebugFlag 131" >> $TAOS_CFG
@ -141,7 +126,6 @@ if [ -n "$FILE_NAME" ]; then
else
echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME
$PROGRAM -c $CFG_DIR -f $FILE_NAME
# valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${CODE_DIR}/../script/valgrind.log $PROGRAM -c $CFG_DIR -f $FILE_NAME
fi
else
echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f basicSuite.sim

23
tests/tsim/CMakeLists.txt
View File

@ -1,9 +1,14 @@
PROJECT(TDengine)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_EXECUTABLE(tsim ${SRC})
TARGET_LINK_LIBRARIES(tsim taos_static trpc tutil pthread cJson)
aux_source_directory(src TSIM_SRC)
add_executable(tsim ${TSIM_SRC})
target_link_libraries(
tsim
PUBLIC taos
PUBLIC util
PUBLIC common
PUBLIC os
PUBLIC cjson
)
target_include_directories(
tsim
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)

61
tests/tsim/inc/sim.h
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __SIM_H__
#define __SIM_H__
#ifndef _TD_SIM_H_
#define _TD_SIM_H_
#include <semaphore.h>
#include <stdbool.h>
@ -102,18 +102,18 @@ typedef struct _cmd_t {
int16_t cmdno;
int16_t nlen;
char name[MAX_SIM_CMD_NAME_LEN];
bool (*parseCmd)(char *, struct _cmd_t *, int32_t);
bool (*executeCmd)(struct _script_t *script, char *option);
bool (*parseCmd)(char *, struct _cmd_t *, int32_t);
bool (*executeCmd)(struct _script_t *script, char *option);
struct _cmd_t *next;
} SCommand;
typedef struct {
int16_t cmdno;
int16_t jump; // jump position
int16_t errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag
// will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */
int16_t lineNum; // correspodning line number in original file
int32_t optionOffset;// relative option offset
int16_t jump; // jump position
int16_t errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag
// will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */
int16_t lineNum; // correspodning line number in original file
int32_t optionOffset; // relative option offset
} SCmdLine;
typedef struct _var_t {
@ -123,24 +123,24 @@ typedef struct _var_t {
} SVariable;
typedef struct _script_t {
int32_t type;
bool killed;
void * taos;
char rows[12]; // number of rows data retrieved
char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN]; // query results
char system_exit_code[12];
char system_ret_content[MAX_SYSTEM_RESULT_LEN];
int32_t varLen;
int32_t linePos; // current cmd position
int32_t numOfLines; // number of lines in the script
int32_t bgScriptLen;
char fileName[MAX_FILE_NAME_LEN]; // script file name
char error[MAX_ERROR_LEN];
char * optionBuffer;
SCmdLine *lines; // command list
SVariable variables[MAX_VAR_LEN];
pthread_t bgPid;
char auth[128];
int32_t type;
bool killed;
void *taos;
char rows[12]; // number of rows data retrieved
char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN]; // query results
char system_exit_code[12];
char system_ret_content[MAX_SYSTEM_RESULT_LEN];
int32_t varLen;
int32_t linePos; // current cmd position
int32_t numOfLines; // number of lines in the script
int32_t bgScriptLen;
char fileName[MAX_FILE_NAME_LEN]; // script file name
char error[MAX_ERROR_LEN];
char *optionBuffer;
SCmdLine *lines; // command list
SVariable variables[MAX_VAR_LEN];
pthread_t bgPid;
char auth[128];
struct _script_t *bgScripts[MAX_BACKGROUND_SCRIPT_NUM];
} SScript;
@ -150,16 +150,15 @@ extern int32_t simScriptPos;
extern int32_t simScriptSucced;
extern int32_t simDebugFlag;
extern char tsScriptDir[];
extern bool simAsyncQuery;
extern bool abortExecution;
SScript *simParseScript(char *fileName);
SScript *simProcessCallOver(SScript *script);
void * simExecuteScript(void *script);
void *simExecuteScript(void *script);
void simInitsimCmdList();
bool simSystemInit();
void simSystemCleanUp();
char * simGetVariable(SScript *script, char *varName, int32_t varLen);
char *simGetVariable(SScript *script, char *varName, int32_t varLen);
bool simExecuteExpCmd(SScript *script, char *option);
bool simExecuteTestCmd(SScript *script, char *option);
bool simExecuteGotoCmd(SScript *script, char *option);
@ -178,4 +177,4 @@ bool simExecuteLineInsertCmd(SScript *script, char *option);
bool simExecuteLineInsertErrorCmd(SScript *script, char *option);
void simVisuallizeOption(SScript *script, char *src, char *dst);
#endif
#endif /*_TD_SIM_H_*/

14
tests/tsim/inc/simParse.h
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __SIM_PARSE_H__
#define __SIM_PARSE_H__
#ifndef _TD_SIM_PARSE_H_
#define _TD_SIM_PARSE_H_
#define MAX_NUM_CMD 64
#define MAX_NUM_LABLES 100
@ -40,10 +40,10 @@ typedef struct {
/* block definition */
typedef struct {
char top; /* the number of blocks stacked */
char type[MAX_NUM_BLOCK]; /* the block type */
int16_t *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */
int16_t back[MAX_NUM_BLOCK]; /* go back, endw and continue */
char top; /* the number of blocks stacked */
char type[MAX_NUM_BLOCK]; /* the block type */
int16_t *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */
int16_t back[MAX_NUM_BLOCK]; /* go back, endw and continue */
char numJump[MAX_NUM_BLOCK];
int16_t *jump[MAX_NUM_BLOCK][MAX_NUM_JUMP]; /* break or elif */
char sexp[MAX_NUM_BLOCK][40]; /*switch expression */
@ -52,4 +52,4 @@ typedef struct {
bool simParseExpression(char *token, int32_t lineNum);
#endif
#endif /*_TD_SIM_PARSE_H_*/

291
tests/tsim/src/simExe.c
View File

@ -14,18 +14,18 @@
*/
#define _DEFAULT_SOURCE
#include "../../../include/client/taos.h"
#include "cJSON.h"
#include "os.h"
#include "sim.h"
#include "taos.h"
#include "taoserror.h"
#include "tglobal.h"
#include "ttypes.h"
#include "tutil.h"
#undef TAOS_MEM_CHECK
void simLogSql(char *sql, bool useSharp) {
static FILE *fp = NULL;
char filename[256];
char filename[256];
sprintf(filename, "%s/sim.sql", tsScriptDir);
if (fp == NULL) {
fp = fopen(filename, "w");
@ -74,7 +74,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) {
return "null";
}
char * keyName;
char *keyName;
int32_t keyLen;
paGetToken(varName + 6, &keyName, &keyLen);
@ -91,7 +91,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) {
return "null";
}
char * keyName;
char *keyName;
int32_t keyLen;
paGetToken(varName + 7, &keyName, &keyLen);
@ -144,7 +144,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) {
}
int32_t simExecuteExpression(SScript *script, char *exp) {
char * op1, *op2, *var1, *var2, *var3, *rest;
char *op1, *op2, *var1, *var2, *var3, *rest;
int32_t op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1;
char t0[1024], t1[1024], t2[1024], t3[2048];
int32_t result;
@ -302,10 +302,10 @@ bool simExecuteRunBackCmd(SScript *script, char *option) {
}
void simReplaceShToBat(char *dst) {
char* sh = strstr(dst, ".sh");
char *sh = strstr(dst, ".sh");
if (sh != NULL) {
int32_t dstLen = (int32_t)strlen(dst);
char *end = dst + dstLen;
char *end = dst + dstLen;
*(end + 1) = 0;
for (char *p = end; p >= sh; p--) {
@ -436,7 +436,7 @@ bool simExecuteReturnCmd(SScript *script, char *option) {
}
void simVisuallizeOption(SScript *script, char *src, char *dst) {
char * var, *token, *value;
char *var, *token, *value;
int32_t dstLen, srcLen, tokenLen;
dst[0] = 0, dstLen = 0;
@ -466,10 +466,6 @@ void simVisuallizeOption(SScript *script, char *src, char *dst) {
strcpy(dst + dstLen, src);
}
void simCloseRestFulConnect(SScript *script) {
memset(script->auth, 0, sizeof(script->auth));
}
void simCloseNativeConnect(SScript *script) {
if (script->taos == NULL) return;
@ -479,168 +475,7 @@ void simCloseNativeConnect(SScript *script) {
script->taos = NULL;
}
void simCloseTaosdConnect(SScript *script) {
if (simAsyncQuery) {
simCloseRestFulConnect(script);
} else {
simCloseNativeConnect(script);
}
}
// {"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"}
// {"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}
// {"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10}
int32_t simParseHttpCommandResult(SScript *script, char *command) {
cJSON* root = cJSON_Parse(command);
if (root == NULL) {
simError("script:%s, failed to parse json, response:%s", script->fileName, command);
return -1;
}
cJSON *status = cJSON_GetObjectItem(root, "status");
if (status == NULL) {
simError("script:%s, failed to parse json, status is null, response:%s", script->fileName, command);
cJSON_Delete(root);
return -1;
}
if (status->valuestring == NULL || strlen(status->valuestring) == 0) {
simError("script:%s, failed to parse json, status value is null, response:%s", script->fileName, command);
cJSON_Delete(root);
return -1;
}
if (strcmp(status->valuestring, "succ") != 0) {
cJSON *code = cJSON_GetObjectItem(root, "code");
if (code == NULL) {
simError("script:%s, failed to parse json, code is null, response:%s", script->fileName, command);
cJSON_Delete(root);
return -1;
}
int32_t retcode = (int32_t)code->valueint;
if (retcode != 1017) {
simError("script:%s, json:status:%s not equal to succ, response:%s", script->fileName, status->valuestring,
command);
cJSON_Delete(root);
return retcode;
} else {
simDebug("script:%s, json:status:%s not equal to succ, but code is %d, response:%s", script->fileName,
status->valuestring, retcode, command);
cJSON_Delete(root);
return 0;
}
}
cJSON *desc = cJSON_GetObjectItem(root, "desc");
if (desc != NULL) {
if (desc->valuestring == NULL || strlen(desc->valuestring) == 0) {
simError("script:%s, failed to parse json, desc value is null, response:%s", script->fileName, command);
cJSON_Delete(root);
return -1;
}
strcpy(script->auth, desc->valuestring);
cJSON_Delete(root);
return 0;
}
cJSON *data = cJSON_GetObjectItem(root, "data");
if (data == NULL) {
simError("script:%s, failed to parse json, data is null, response:%s", script->fileName, command);
cJSON_Delete(root);
return -1;
}
int32_t rowsize = cJSON_GetArraySize(data);
if (rowsize < 0) {
simError("script:%s, failed to parse json:data, data size %d, response:%s", script->fileName, rowsize, command);
cJSON_Delete(root);
return -1;
}
int32_t rowIndex = 0;
sprintf(script->rows, "%d", rowsize);
for (int32_t r = 0; r < rowsize; ++r) {
cJSON *row = cJSON_GetArrayItem(data, r);
if (row == NULL) continue;
if (rowIndex++ >= 10) break;
int32_t colsize = cJSON_GetArraySize(row);
if (colsize < 0) {
break;
}
colsize = MIN(10, colsize);
for (int32_t c = 0; c < colsize; ++c) {
cJSON *col = cJSON_GetArrayItem(row, c);
if (col->valuestring != NULL) {
strcpy(script->data[r][c], col->valuestring);
} else {
if (col->numberstring[0] == 0) {
strcpy(script->data[r][c], "null");
} else {
strcpy(script->data[r][c], col->numberstring);
}
}
}
}
return 0;
}
int32_t simExecuteRestFulCommand(SScript *script, char *command) {
char buf[5000] = {0};
sprintf(buf, "%s 2>/dev/null", command);
FILE *fp = popen(buf, "r");
if (fp == NULL) {
simError("failed to execute %s", buf);
return -1;
}
int32_t mallocSize = 2000;
int32_t alreadyReadSize = 0;
char * content = malloc(mallocSize);
while (!feof(fp)) {
int32_t availSize = mallocSize - alreadyReadSize;
int32_t len = (int32_t)fread(content + alreadyReadSize, 1, availSize, fp);
if (len >= availSize) {
alreadyReadSize += len;
mallocSize *= 2;
content = realloc(content, mallocSize);
}
}
pclose(fp);
return simParseHttpCommandResult(script, content);
}
bool simCreateRestFulConnect(SScript *script, char *user, char *pass) {
char command[4096];
sprintf(command, "curl 127.0.0.1:6041/rest/login/%s/%s", user, pass);
bool success = false;
for (int32_t attempt = 0; attempt < 10; ++attempt) {
success = simExecuteRestFulCommand(script, command) == 0;
if (!success) {
simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL),
attempt);
taosMsleep(1000);
} else {
simDebug("script:%s, user:%s connect taosd successed, attempt:%d", script->fileName, user, attempt);
break;
}
}
if (!success) {
sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum,
taos_errstr(NULL));
return false;
}
simDebug("script:%s, connect taosd successed, auth:%p", script->fileName, script->auth);
return true;
}
void simCloseTaosdConnect(SScript *script) { simCloseNativeConnect(script); }
bool simCreateNativeConnect(SScript *script, char *user, char *pass) {
simCloseTaosdConnect(script);
@ -651,7 +486,7 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) {
return false;
}
taos = taos_connect(NULL, user, pass, NULL, tsDnodeShellPort);
taos = taos_connect(NULL, user, pass, NULL, 0);
if (taos == NULL) {
simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL),
attempt);
@ -675,8 +510,8 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) {
}
bool simCreateTaosdConnect(SScript *script, char *rest) {
char * user = TSDB_DEFAULT_USER;
char * token;
char *user = TSDB_DEFAULT_USER;
char *token;
int32_t tokenLen;
rest = paGetToken(rest, &token, &tokenLen);
rest = paGetToken(rest, &token, &tokenLen);
@ -684,18 +519,14 @@ bool simCreateTaosdConnect(SScript *script, char *rest) {
user = token;
}
if (simAsyncQuery) {
return simCreateRestFulConnect(script, user, TSDB_DEFAULT_PASS);
} else {
return simCreateNativeConnect(script, user, TSDB_DEFAULT_PASS);
}
return simCreateNativeConnect(script, user, TSDB_DEFAULT_PASS);
}
bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
char timeStr[30] = {0};
time_t tt;
struct tm *tp;
SCmdLine * line = &script->lines[script->linePos];
SCmdLine *line = &script->lines[script->linePos];
int32_t ret = -1;
TAOS_RES *pSql = NULL;
@ -710,7 +541,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
pSql = taos_query(script->taos, rest);
ret = taos_errno(pSql);
if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) {
if (ret == TSDB_CODE_MND_STB_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) {
simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF,
tstrerror(ret));
ret = 0;
@ -756,7 +587,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
while ((row = taos_fetch_row(pSql))) {
if (numOfRows < MAX_QUERY_ROW_NUM) {
TAOS_FIELD *fields = taos_fetch_fields(pSql);
int32_t * length = taos_fetch_lengths(pSql);
int32_t *length = taos_fetch_lengths(pSql);
for (int32_t i = 0; i < num_fields; i++) {
char *value = NULL;
@ -780,7 +611,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
sprintf(value, "%d", *((int8_t *)row[i]));
break;
case TSDB_DATA_TYPE_UTINYINT:
sprintf(value, "%u", *((uint8_t*)row[i]));
sprintf(value, "%u", *((uint8_t *)row[i]));
break;
case TSDB_DATA_TYPE_SMALLINT:
sprintf(value, "%d", *((int16_t *)row[i]));
@ -877,43 +708,8 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
return true;
}
bool simExecuteRestFulSqlCommand(SScript *script, char *rest) {
SCmdLine *line = &script->lines[script->linePos];
char command[4096];
sprintf(command, "curl -H 'Authorization: Taosd %s' -d \"%s\" 127.0.0.1:6041/rest/sql", script->auth, rest);
int32_t ret = -1;
for (int32_t attempt = 0; attempt < 10; ++attempt) {
ret = simExecuteRestFulCommand(script, command);
if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) {
simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF,
tstrerror(ret));
ret = 0;
break;
} else if (ret != 0) {
simDebug("script:%s, taos:%p, %s failed, ret:%d", script->fileName, script->taos, rest, ret);
if (line->errorJump == SQL_JUMP_TRUE) {
script->linePos = line->jump;
return true;
}
taosMsleep(1000);
} else {
break;
}
}
if (ret) {
sprintf(script->error, "lineNum:%d. sql:%s failed, ret:%d", line->lineNum, rest, ret);
return false;
}
script->linePos++;
return true;
}
bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) {
char buf[3000];
char buf[3000];
SCmdLine *line = &script->lines[script->linePos];
simVisuallizeOption(script, rest, buf);
@ -935,7 +731,7 @@ bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) {
return true;
}
if ((!simAsyncQuery && script->taos == NULL) || (simAsyncQuery && script->auth[0] == 0)) {
if (script->taos == NULL) {
if (!simCreateTaosdConnect(script, "connect root")) {
if (line->errorJump == SQL_JUMP_TRUE) {
script->linePos = line->jump;
@ -951,11 +747,7 @@ bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) {
return true;
}
if (simAsyncQuery) {
return simExecuteRestFulSqlCommand(script, rest);
} else {
return simExecuteNativeSqlCommand(script, rest, isSlow);
}
return simExecuteNativeSqlCommand(script, rest, isSlow);
}
bool simExecuteSqlCmd(SScript *script, char *rest) {
@ -1010,7 +802,7 @@ bool simExecuteRestfulCmd(SScript *script, char *rest) {
}
bool simExecuteSqlErrorCmd(SScript *script, char *rest) {
char buf[3000];
char buf[3000];
SCmdLine *line = &script->lines[script->linePos];
simVisuallizeOption(script, rest, buf);
@ -1032,7 +824,7 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) {
return true;
}
if ((!simAsyncQuery && script->taos == NULL) || (simAsyncQuery && script->auth[0] == 0)) {
if (script->taos == NULL) {
if (!simCreateTaosdConnect(script, "connect root")) {
if (line->errorJump == SQL_JUMP_TRUE) {
script->linePos = line->jump;
@ -1048,17 +840,9 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) {
return true;
}
int32_t ret;
TAOS_RES *pSql = NULL;
if (simAsyncQuery) {
char command[4096];
sprintf(command, "curl -H 'Authorization: Taosd %s' -d '%s' 127.0.0.1:6041/rest/sql", script->auth, rest);
ret = simExecuteRestFulCommand(script, command);
} else {
pSql = taos_query(script->taos, rest);
ret = taos_errno(pSql);
taos_free_result(pSql);
}
TAOS_RES *pSql = pSql = taos_query(script->taos, rest);
int32_t ret = taos_errno(pSql);
taos_free_result(pSql);
if (ret != TSDB_CODE_SUCCESS) {
simDebug("script:%s, taos:%p, %s execute, expect failed, so success, ret:%d:%s", script->fileName, script->taos,
@ -1083,15 +867,19 @@ bool simExecuteLineInsertCmd(SScript *script, char *rest) {
simInfo("script:%s, %s", script->fileName, rest);
simLogSql(buf, true);
char * lines[] = {rest};
char *lines[] = {rest};
#if 0
int32_t ret = taos_insert_lines(script->taos, lines, 1);
#else
int32_t ret = 0;
#endif
if (ret == TSDB_CODE_SUCCESS) {
simDebug("script:%s, taos:%p, %s executed. success.", script->fileName, script->taos, rest);
script->linePos++;
return true;
} else {
sprintf(script->error, "lineNum: %d. line: %s failed, ret:%d:%s", line->lineNum, rest,
ret & 0XFFFF, tstrerror(ret));
sprintf(script->error, "lineNum: %d. line: %s failed, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF,
tstrerror(ret));
return false;
}
}
@ -1106,15 +894,20 @@ bool simExecuteLineInsertErrorCmd(SScript *script, char *rest) {
simInfo("script:%s, %s", script->fileName, rest);
simLogSql(buf, true);
char * lines[] = {rest};
char *lines[] = {rest};
#if 0
int32_t ret = taos_insert_lines(script->taos, lines, 1);
#else
int32_t ret = 0;
#endif
if (ret == TSDB_CODE_SUCCESS) {
sprintf(script->error, "script:%s, taos:%p, %s executed. expect failed, but success.", script->fileName, script->taos, rest);
sprintf(script->error, "script:%s, taos:%p, %s executed. expect failed, but success.", script->fileName,
script->taos, rest);
script->linePos++;
return false;
} else {
simDebug("lineNum: %d. line: %s failed, ret:%d:%s. Expect failed, so success", line->lineNum, rest,
ret & 0XFFFF, tstrerror(ret));
simDebug("lineNum: %d. line: %s failed, ret:%d:%s. Expect failed, so success", line->lineNum, rest, ret & 0XFFFF,
tstrerror(ret));
return true;
}
}

10
tests/tsim/src/simMain.c
View File

@ -15,19 +15,17 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "tglobal.h"
#include "sim.h"
#undef TAOS_MEM_CHECK
#include "tglobal.h"
bool simAsyncQuery = false;
bool simExecSuccess = false;
bool abortExecution = false;
void simHandleSignal(int32_t signo, void *sigInfo, void *context) {
simSystemCleanUp();
abortExecution = true;
// runningScript->killed = true;
// exit(1);
// runningScript->killed = true;
// exit(1);
}
int32_t main(int32_t argc, char *argv[]) {
@ -38,8 +36,6 @@ int32_t main(int32_t argc, char *argv[]) {
tstrncpy(configDir, argv[++i], 128);
} else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) {
strcpy(scriptFile, argv[++i]);
} else if (strcmp(argv[i], "-a") == 0) {
simAsyncQuery = true;
} else {
printf("usage: %s [options] \n", argv[0]);
printf(" [-c config]: config directory, default is: %s\n", configDir);

30
tests/tsim/src/simParse.c
View File

@ -60,9 +60,9 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "sim.h"
#include "simParse.h"
#include "tutil.h"
#undef TAOS_MEM_CHECK
#include "simParse.h"
static SCommand *cmdHashList[MAX_NUM_CMD];
static SCmdLine cmdLine[MAX_CMD_LINES];
@ -177,11 +177,11 @@ SScript *simBuildScriptObj(char *fileName) {
}
SScript *simParseScript(char *fileName) {
FILE * fd;
FILE *fd;
int32_t tokenLen, lineNum = 0;
char buffer[MAX_LINE_LEN], name[128], *token, *rest;
SCommand *pCmd;
SScript * script;
SScript *script;
if ((fileName[0] == '.') || (fileName[0] == '/')) {
strcpy(name, fileName);
@ -252,7 +252,7 @@ SScript *simParseScript(char *fileName) {
}
int32_t simCheckExpression(char *exp) {
char * op1, *op2, *op, *rest;
char *op1, *op2, *op, *rest;
int32_t op1Len, op2Len, opLen;
rest = paGetToken(exp, &op1, &op1Len);
@ -336,7 +336,7 @@ bool simParseExpression(char *token, int32_t lineNum) {
}
bool simParseIfCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
char * ret;
char *ret;
int32_t expLen;
expLen = simCheckExpression(rest);
@ -502,7 +502,7 @@ bool simParseEndwCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
}
bool simParseSwitchCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
char * token;
char *token;
int32_t tokenLen;
rest = paGetToken(rest, &token, &tokenLen);
@ -525,7 +525,7 @@ bool simParseSwitchCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
}
bool simParseCaseCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
char * token;
char *token;
int32_t tokenLen;
rest = paGetToken(rest, &token, &tokenLen);
@ -666,7 +666,7 @@ bool simParsePrintCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
void simCheckSqlOption(char *rest) {
int32_t valueLen;
char * value, *xpos;
char *value, *xpos;
xpos = strstr(rest, " -x"); // need a blank
if (xpos) {
@ -750,7 +750,7 @@ bool simParseSystemContentCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
}
bool simParseSleepCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
char * token;
char *token;
int32_t tokenLen;
cmdLine[numOfLines].cmdno = SIM_CMD_SLEEP;
@ -769,7 +769,7 @@ bool simParseSleepCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
}
bool simParseReturnCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
char * token;
char *token;
int32_t tokenLen;
cmdLine[numOfLines].cmdno = SIM_CMD_RETURN;
@ -788,7 +788,7 @@ bool simParseReturnCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
}
bool simParseGotoCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
char * token;
char *token;
int32_t tokenLen;
rest = paGetToken(rest, &token, &tokenLen);
@ -811,7 +811,7 @@ bool simParseGotoCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
}
bool simParseRunCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
char * token;
char *token;
int32_t tokenLen;
rest = paGetToken(rest, &token, &tokenLen);
@ -838,7 +838,7 @@ bool simParseRunBackCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
return true;
}
bool simParseLineInsertCmd(char* rest, SCommand* pCmd, int32_t lineNum) {
bool simParseLineInsertCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
int32_t expLen;
rest++;
@ -854,7 +854,7 @@ bool simParseLineInsertCmd(char* rest, SCommand* pCmd, int32_t lineNum) {
return true;
}
bool simParseLineInsertErrorCmd(char* rest, SCommand* pCmd, int32_t lineNum) {
bool simParseLineInsertErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
int32_t expLen;
rest++;

15
tests/tsim/src/simSystem.c
View File

@ -14,15 +14,15 @@
*/
#define _DEFAULT_SOURCE
#include "../../../include/client/taos.h"
#include "os.h"
#include "sim.h"
#include "taos.h"
#include "taoserror.h"
#include "tglobal.h"
#include "tsocket.h"
#include "ttimer.h"
#include "tutil.h"
#undef TAOS_MEM_CHECK
#include "tglobal.h"
#include "tconfig.h"
SScript *simScriptList[MAX_MAIN_SCRIPT_NUM];
SCommand simCmdList[SIM_CMD_END];
@ -81,10 +81,11 @@ char *simParseHostName(char *varName) {
}
bool simSystemInit() {
if (taos_init()) {
return false;
}
taosGetFqdn(simHostName);
taosInitGlobalCfg();
taosReadCfgFromFile();
simInitsimCmdList();
memset(simScriptList, 0, sizeof(SScript *) * MAX_MAIN_SCRIPT_NUM);
return true;
@ -171,7 +172,7 @@ void *simExecuteScript(void *inputScript) {
}
} else {
SCmdLine *line = &script->lines[script->linePos];
char * option = script->optionBuffer + line->optionOffset;
char *option = script->optionBuffer + line->optionOffset;
simDebug("script:%s, line:%d with option \"%s\"", script->fileName, line->lineNum, option);
SCommand *cmd = &simCmdList[line->cmdno];

2
tools/CMakeLists.txt
View File

@ -1 +1 @@
#add_subdirectory(shell)
add_subdirectory(shell)

9
tools/shell/CMakeLists.txt
View File

@ -1,13 +1,18 @@
aux_source_directory(src SHELL_SRC)
list(REMOVE_ITEM SHELL_SRC ./src/shellWindows.c)
list(REMOVE_ITEM SHELL_SRC ./src/shellDarwin.c)
list(REMOVE_ITEM SHELL_SRC src/shellWindows.c)
list(REMOVE_ITEM SHELL_SRC src/shellDarwin.c)
add_executable(shell ${SHELL_SRC})
target_link_libraries(
shell
PUBLIC taos
PUBLIC util
PUBLIC common
PUBLIC os
)
target_include_directories(
shell
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos)

10
tools/shell/inc/shell.h
View File

@ -13,13 +13,13 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __SHELL__
#define __SHELL__
#ifndef _TD_SHELL_H_
#define _TD_SHELL_H_
#include "../../../../include/client/taos.h"
#include "stdbool.h"
#include "os.h"
#include "taos.h"
#include "taosdef.h"
#include "tsclient.h"
#define MAX_USERNAME_SIZE 64
#define MAX_DBNAME_SIZE 64

4
tools/shell/inc/shellCommand.h
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __COMMAND_STRUCT__
#define __COMMAND_STRUCT__
#ifndef _TD_SHELL_COMMAND_H_
#define _TD_SHELL_COMMAND_H_
#include "shell.h"

0
tools/shell/src/shellCheck.c → tools/shell/src/backup/shellCheck.c
View File

0
tools/shell/src/shellDarwin.c → tools/shell/src/backup/shellDarwin.c
View File

0
tools/shell/src/shellImport.c → tools/shell/src/backup/shellImport.c
View File

0
tools/shell/src/shellWindows.c → tools/shell/src/backup/shellWindows.c
View File

0
tools/shell/src/tnettest.c → tools/shell/src/backup/tnettest.c
View File

0
tools/shell/inc/tnettest.h → tools/shell/src/backup/tnettest.h
View File

2
tools/shell/src/shellCommand.c
View File

@ -19,6 +19,8 @@
#include "shell.h"
#include "shellCommand.h"
#include <regex.h>
extern int wcwidth(wchar_t c);
extern int wcswidth(const wchar_t *s, size_t n);
typedef struct {

184
tools/shell/src/shellEngine.c
View File

@ -21,13 +21,14 @@
#include "os.h"
#include "shell.h"
#include "shellCommand.h"
#include "tutil.h"
#include "taosdef.h"
#include "taoserror.h"
#include "tglobal.h"
#include "tsclient.h"
#include "ttypes.h"
#include "tutil.h"
#include <regex.h>
#include <wordexp.h>
/**************** Global variables ****************/
#ifdef _TD_POWER_
@ -58,7 +59,7 @@ SShellHistory history;
#define DEFAULT_MAX_BINARY_DISPLAY_WIDTH 30
extern int32_t tsMaxBinaryDisplayWidth;
extern TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
extern TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
/*
* FUNCTION: Initialize the shell.
@ -89,12 +90,6 @@ TAOS *shellInit(SShellArguments *_args) {
_args->user = TSDB_DEFAULT_USER;
}
if (taos_init()) {
printf("failed to init taos\n");
fflush(stdout);
return NULL;
}
// Connect to the database.
TAOS *con = NULL;
if (_args->auth == NULL) {
@ -127,6 +122,7 @@ TAOS *shellInit(SShellArguments *_args) {
exit(EXIT_SUCCESS);
}
#if 0
#ifndef WINDOWS
if (_args->dir[0] != 0) {
source_dir(con, _args);
@ -139,12 +135,13 @@ TAOS *shellInit(SShellArguments *_args) {
taos_close(con);
exit(EXIT_SUCCESS);
}
#endif
#endif
return con;
}
static bool isEmptyCommand(const char* cmd) {
static bool isEmptyCommand(const char *cmd) {
for (char c = *cmd++; c != 0; c = *cmd++) {
if (c != ' ' && c != '\t' && c != ';') {
return false;
@ -153,7 +150,6 @@ static bool isEmptyCommand(const char* cmd) {
return true;
}
static int32_t shellRunSingleCommand(TAOS *con, char *command) {
/* If command is empty just return */
if (isEmptyCommand(command)) {
@ -176,10 +172,11 @@ static int32_t shellRunSingleCommand(TAOS *con, char *command) {
return 0;
}
if (regex_match(command, "^[\t ]*set[ \t]+max_binary_display_width[ \t]+(default|[1-9][0-9]*)[ \t;]*$", REG_EXTENDED | REG_ICASE)) {
if (regex_match(command, "^[\t ]*set[ \t]+max_binary_display_width[ \t]+(default|[1-9][0-9]*)[ \t;]*$",
REG_EXTENDED | REG_ICASE)) {
strtok(command, " \t");
strtok(NULL, " \t");
char* p = strtok(NULL, " \t");
char *p = strtok(NULL, " \t");
if (strcasecmp(p, "default") == 0) {
tsMaxBinaryDisplayWidth = DEFAULT_MAX_BINARY_DISPLAY_WIDTH;
} else {
@ -202,8 +199,7 @@ static int32_t shellRunSingleCommand(TAOS *con, char *command) {
return 0;
}
int32_t shellRunCommand(TAOS* con, char* command) {
int32_t shellRunCommand(TAOS *con, char *command) {
/* If command is empty just return */
if (isEmptyCommand(command)) {
return 0;
@ -255,7 +251,7 @@ int32_t shellRunCommand(TAOS* con, char* command) {
if (c == '\\') {
if (quote != 0 && (*command == '_' || *command == '\\')) {
//DO nothing
// DO nothing
} else {
esc = true;
continue;
@ -284,21 +280,22 @@ int32_t shellRunCommand(TAOS* con, char* command) {
return shellRunSingleCommand(con, cmd);
}
void freeResultWithRid(int64_t rid) {
#if 0
SSqlObj* pSql = taosAcquireRef(tscObjRef, rid);
if(pSql){
taos_free_result(pSql);
taosReleaseRef(tscObjRef, rid);
}
#endif
}
void shellRunCommandOnServer(TAOS *con, char command[]) {
int64_t st, et;
wordexp_t full_path;
char * sptr = NULL;
char * cptr = NULL;
char * fname = NULL;
char *sptr = NULL;
char *cptr = NULL;
char *fname = NULL;
bool printMode = false;
if ((sptr = strstr(command, ">>")) != NULL) {
@ -327,7 +324,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
st = taosGetTimestampUs();
TAOS_RES* pSql = taos_query_h(con, command, &result);
TAOS_RES *pSql = taos_query(con, command);
if (taos_errno(pSql)) {
taos_error(pSql, st);
return;
@ -344,7 +341,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
return;
}
if (!tscIsUpdateQuery(pSql)) { // select and show kinds of commands
TAOS_FIELD* pFields = taos_fetch_fields(pSql);
if (pFields != NULL) { // select and show kinds of commands
int error_no = 0;
int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode);
@ -405,14 +403,13 @@ int regex_match(const char *s, const char *reg, int cflags) {
return 0;
}
static char* formatTimestamp(char* buf, int64_t val, int precision) {
static char *formatTimestamp(char *buf, int64_t val, int precision) {
if (args.is_raw_time) {
sprintf(buf, "%" PRId64, val);
return buf;
}
time_t tt;
time_t tt;
int32_t ms = 0;
if (precision == TSDB_TIME_PRECISION_NANO) {
tt = (time_t)(val / 1000000000);
@ -425,13 +422,13 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
ms = val % 1000;
}
/* comment out as it make testcases like select_with_tags.sim fail.
but in windows, this may cause the call to localtime crash if tt < 0,
need to find a better solution.
if (tt < 0) {
tt = 0;
}
*/
/* comment out as it make testcases like select_with_tags.sim fail.
but in windows, this may cause the call to localtime crash if tt < 0,
need to find a better solution.
if (tt < 0) {
tt = 0;
}
*/
#ifdef WINDOWS
if (tt < 0) tt = 0;
@ -447,8 +444,8 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
}
}
struct tm* ptm = localtime(&tt);
size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm);
struct tm *ptm = localtime(&tt);
size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm);
if (precision == TSDB_TIME_PRECISION_NANO) {
sprintf(buf + pos, ".%09d", ms);
@ -461,8 +458,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
return buf;
}
static void dumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_t length, int precision) {
static void dumpFieldToFile(FILE *fp, const char *val, TAOS_FIELD *field, int32_t length, int precision) {
if (val == NULL) {
fprintf(fp, "%s", TSDB_DATA_NULL_STR);
return;
@ -498,7 +494,7 @@ static void dumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_
fprintf(fp, "\'%s\'", buf);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
formatTimestamp(buf, *(int64_t*)val, precision);
formatTimestamp(buf, *(int64_t *)val, precision);
fprintf(fp, "'%s'", buf);
break;
default:
@ -506,7 +502,7 @@ static void dumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_
}
}
static int dumpResultToFile(const char* fname, TAOS_RES* tres) {
static int dumpResultToFile(const char *fname, TAOS_RES *tres) {
TAOS_ROW row = taos_fetch_row(tres);
if (row == NULL) {
return 0;
@ -519,7 +515,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) {
return -1;
}
FILE* fp = fopen(full_path.we_wordv[0], "w");
FILE *fp = fopen(full_path.we_wordv[0], "w");
if (fp == NULL) {
fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]);
wordfree(&full_path);
@ -528,9 +524,9 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) {
wordfree(&full_path);
int num_fields = taos_num_fields(tres);
int num_fields = taos_num_fields(tres);
TAOS_FIELD *fields = taos_fetch_fields(tres);
int precision = taos_result_precision(tres);
int precision = taos_result_precision(tres);
for (int col = 0; col < num_fields; col++) {
if (col > 0) {
@ -542,18 +538,18 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) {
int numOfRows = 0;
do {
int32_t* length = taos_fetch_lengths(tres);
int32_t *length = taos_fetch_lengths(tres);
for (int i = 0; i < num_fields; i++) {
if (i > 0) {
fputc(',', fp);
}
dumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
dumpFieldToFile(fp, (const char *)row[i], fields + i, length[i], precision);
}
fputc('\n', fp);
numOfRows++;
row = taos_fetch_row(tres);
} while( row != NULL);
} while (row != NULL);
result = 0;
fclose(fp);
@ -561,14 +557,13 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) {
return numOfRows;
}
static void shellPrintNChar(const char *str, int length, int width) {
wchar_t tail[3];
int pos = 0, cols = 0, totalCols = 0, tailLen = 0;
int pos = 0, cols = 0, totalCols = 0, tailLen = 0;
while (pos < length) {
wchar_t wc;
int bytes = mbtowc(&wc, str + pos, MB_CUR_MAX);
int bytes = mbtowc(&wc, str + pos, MB_CUR_MAX);
if (bytes == 0) {
break;
}
@ -625,8 +620,7 @@ static void shellPrintNChar(const char *str, int length, int width) {
}
}
static void printField(const char* val, TAOS_FIELD* field, int width, int32_t length, int precision) {
static void printField(const char *val, TAOS_FIELD *field, int width, int32_t length, int precision) {
if (val == NULL) {
int w = width;
if (field->type < TSDB_DATA_TYPE_TINYINT || field->type > TSDB_DATA_TYPE_DOUBLE) {
@ -679,7 +673,7 @@ static void printField(const char* val, TAOS_FIELD* field, int width, int32_t le
shellPrintNChar(val, length, width);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
formatTimestamp(buf, *(int64_t*)val, precision);
formatTimestamp(buf, *(int64_t *)val, precision);
printf("%s", buf);
break;
default:
@ -687,27 +681,26 @@ static void printField(const char* val, TAOS_FIELD* field, int width, int32_t le
}
}
bool isSelectQuery(TAOS_RES* tres) {
bool isSelectQuery(TAOS_RES *tres) {
#if 0
char *sql = tscGetSqlStr(tres);
if (regex_match(sql, "^[\t ]*select[ \t]*", REG_EXTENDED | REG_ICASE)) {
return true;
}
#endif
return false;
}
static int verticalPrintResult(TAOS_RES* tres) {
static int verticalPrintResult(TAOS_RES *tres) {
TAOS_ROW row = taos_fetch_row(tres);
if (row == NULL) {
return 0;
}
int num_fields = taos_num_fields(tres);
int num_fields = taos_num_fields(tres);
TAOS_FIELD *fields = taos_fetch_fields(tres);
int precision = taos_result_precision(tres);
int precision = taos_result_precision(tres);
int maxColNameLen = 0;
for (int col = 0; col < num_fields; col++) {
@ -719,7 +712,7 @@ static int verticalPrintResult(TAOS_RES* tres) {
uint64_t resShowMaxNum = UINT64_MAX;
if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) && !tscIsQueryWithLimit(tres)) {
if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) /*&& !tscIsQueryWithLimit(tres)*/) {
resShowMaxNum = DEFAULT_RES_SHOW_NUM;
}
@ -729,52 +722,52 @@ static int verticalPrintResult(TAOS_RES* tres) {
if (numOfRows < resShowMaxNum) {
printf("*************************** %d.row ***************************\n", numOfRows + 1);
int32_t* length = taos_fetch_lengths(tres);
int32_t *length = taos_fetch_lengths(tres);
for (int i = 0; i < num_fields; i++) {
TAOS_FIELD* field = fields + i;
TAOS_FIELD *field = fields + i;
int padding = (int)(maxColNameLen - strlen(field->name));
printf("%*.s%s: ", padding, " ", field->name);
printField((const char*)row[i], field, 0, length[i], precision);
printField((const char *)row[i], field, 0, length[i], precision);
putchar('\n');
}
} else if (showMore) {
printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n");
printf("[You can add limit statement to get more or redirect results to specific file to get all.]\n");
showMore = 0;
printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n");
printf("[You can add limit statement to get more or redirect results to specific file to get all.]\n");
showMore = 0;
}
numOfRows++;
row = taos_fetch_row(tres);
} while(row != NULL);
} while (row != NULL);
return numOfRows;
}
static int calcColWidth(TAOS_FIELD* field, int precision) {
static int calcColWidth(TAOS_FIELD *field, int precision) {
int width = (int)strlen(field->name);
switch (field->type) {
case TSDB_DATA_TYPE_BOOL:
return MAX(5, width); // 'false'
return MAX(5, width); // 'false'
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_UTINYINT:
return MAX(4, width); // '-127'
return MAX(4, width); // '-127'
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_USMALLINT:
return MAX(6, width); // '-32767'
return MAX(6, width); // '-32767'
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_UINT:
return MAX(11, width); // '-2147483648'
return MAX(11, width); // '-2147483648'
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_UBIGINT:
return MAX(21, width); // '-9223372036854775807'
return MAX(21, width); // '-9223372036854775807'
case TSDB_DATA_TYPE_FLOAT:
return MAX(20, width);
@ -801,12 +794,13 @@ static int calcColWidth(TAOS_FIELD* field, int precision) {
case TSDB_DATA_TYPE_TIMESTAMP:
if (args.is_raw_time) {
return MAX(14, width);
} if (precision == TSDB_TIME_PRECISION_NANO) {
}
if (precision == TSDB_TIME_PRECISION_NANO) {
return MAX(29, width);
} else if (precision == TSDB_TIME_PRECISION_MICRO) {
return MAX(26, width); // '2020-01-01 00:00:00.000000'
return MAX(26, width); // '2020-01-01 00:00:00.000000'
} else {
return MAX(23, width); // '2020-01-01 00:00:00.000'
return MAX(23, width); // '2020-01-01 00:00:00.000'
}
default:
@ -816,13 +810,12 @@ static int calcColWidth(TAOS_FIELD* field, int precision) {
return 0;
}
static void printHeader(TAOS_FIELD* fields, int* width, int num_fields) {
static void printHeader(TAOS_FIELD *fields, int *width, int num_fields) {
int rowWidth = 0;
for (int col = 0; col < num_fields; col++) {
TAOS_FIELD* field = fields + col;
int padding = (int)(width[col] - strlen(field->name));
int left = padding / 2;
TAOS_FIELD *field = fields + col;
int padding = (int)(width[col] - strlen(field->name));
int left = padding / 2;
printf(" %*.s%s%*.s |", left, " ", field->name, padding - left, " ");
rowWidth += width[col] + 3;
}
@ -834,16 +827,15 @@ static void printHeader(TAOS_FIELD* fields, int* width, int num_fields) {
putchar('\n');
}
static int horizontalPrintResult(TAOS_RES* tres) {
static int horizontalPrintResult(TAOS_RES *tres) {
TAOS_ROW row = taos_fetch_row(tres);
if (row == NULL) {
return 0;
}
int num_fields = taos_num_fields(tres);
int num_fields = taos_num_fields(tres);
TAOS_FIELD *fields = taos_fetch_fields(tres);
int precision = taos_result_precision(tres);
int precision = taos_result_precision(tres);
int width[TSDB_MAX_COLUMNS];
for (int col = 0; col < num_fields; col++) {
@ -854,7 +846,7 @@ static int horizontalPrintResult(TAOS_RES* tres) {
uint64_t resShowMaxNum = UINT64_MAX;
if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) && !tscIsQueryWithLimit(tres)) {
if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) /* && !tscIsQueryWithLimit(tres)*/) {
resShowMaxNum = DEFAULT_RES_SHOW_NUM;
}
@ -862,34 +854,33 @@ static int horizontalPrintResult(TAOS_RES* tres) {
int showMore = 1;
do {
int32_t* length = taos_fetch_lengths(tres);
int32_t *length = taos_fetch_lengths(tres);
if (numOfRows < resShowMaxNum) {
for (int i = 0; i < num_fields; i++) {
putchar(' ');
printField((const char*)row[i], fields + i, width[i], length[i], precision);
printField((const char *)row[i], fields + i, width[i], length[i], precision);
putchar(' ');
putchar('|');
}
putchar('\n');
} else if (showMore) {
printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n");
printf("[You can add limit statement to show more or redirect results to specific file to get all.]\n");
showMore = 0;
printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n");
printf("[You can add limit statement to show more or redirect results to specific file to get all.]\n");
showMore = 0;
}
numOfRows++;
row = taos_fetch_row(tres);
} while(row != NULL);
} while (row != NULL);
return numOfRows;
}
int shellDumpResult(TAOS_RES *tres, char *fname, int *error_no, bool vertical) {
int numOfRows = 0;
if (fname != NULL) {
numOfRows = dumpResultToFile(fname, tres);
} else if(vertical) {
} else if (vertical) {
numOfRows = verticalPrintResult(tres);
} else {
numOfRows = horizontalPrintResult(tres);
@ -899,13 +890,12 @@ int shellDumpResult(TAOS_RES *tres, char *fname, int *error_no, bool vertical) {
return numOfRows;
}
void read_history() {
// Initialize history
memset(history.hist, 0, sizeof(char *) * MAX_HISTORY_SIZE);
history.hstart = 0;
history.hend = 0;
char * line = NULL;
char *line = NULL;
size_t line_size = 0;
int read_size = 0;
@ -975,9 +965,9 @@ int isCommentLine(char *line) {
void source_file(TAOS *con, char *fptr) {
wordexp_t full_path;
int read_len = 0;
char * cmd = calloc(1, tsMaxSQLStringLen+1);
char *cmd = calloc(1, tsMaxSQLStringLen + 1);
size_t cmd_len = 0;
char * line = NULL;
char *line = NULL;
size_t line_len = 0;
if (wordexp(fptr, &full_path, 0) != 0) {
@ -1087,5 +1077,5 @@ void shellGetGrantInfo(void *con) {
}
fprintf(stdout, "\n");
#endif
#endif
}

9
tools/shell/src/shellLinux.c
View File

@ -19,7 +19,11 @@
#include "shell.h"
#include "shellCommand.h"
#include "tkey.h"
#include "tulog.h"
#include "ulog.h"
#include <wordexp.h>
#include <argp.h>
#include <termio.h>
#define OPT_ABORT 1 /* <20>Cabort */
@ -68,7 +72,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
break;
case 'P':
if (arg) {
tsDnodeShellPort = atoi(arg);
arguments->port = atoi(arg);
} else {
fprintf(stderr, "Invalid port\n");
@ -216,7 +219,9 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
argp_parse(&argp, argc, argv, 0, 0, arguments);
if (arguments->abort) {
#ifndef _ALPINE
#if 0
error(10, 0, "ABORTED");
#endif
#else
abort();
#endif

48
tools/shell/src/shellMain.c
View File

@ -16,7 +16,7 @@
#include "os.h"
#include "shell.h"
#include "tconfig.h"
#include "tnettest.h"
#include "tglobal.h"
pthread_t pid;
static tsem_t cancelSem;
@ -28,21 +28,25 @@ void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) {
void *cancelHandler(void *arg) {
setThreadName("cancelHandler");
while(1) {
while (1) {
if (tsem_wait(&cancelSem) != 0) {
taosMsleep(10);
continue;
}
#ifdef LINUX
#if 0
int64_t rid = atomic_val_compare_exchange_64(&result, result, 0);
SSqlObj* pSql = taosAcquireRef(tscObjRef, rid);
taos_stop_query(pSql);
taosReleaseRef(tscObjRef, rid);
#endif
#else
printf("\nReceive ctrl+c or other signal, quit shell.\n");
exit(0);
#endif
printf("\nReceive ctrl+c or other signal, quit shell.\n");
exit(0);
}
return NULL;
@ -69,31 +73,29 @@ int checkVersion() {
}
// Global configurations
SShellArguments args = {
.host = NULL,
SShellArguments args = {.host = NULL,
#ifndef TD_WINDOWS
.password = NULL,
.password = NULL,
#endif
.user = NULL,
.database = NULL,
.timezone = NULL,
.is_raw_time = false,
.is_use_passwd = false,
.dump_config = false,
.file = "\0",
.dir = "\0",
.threadNum = 5,
.commands = NULL,
.pktLen = 1000,
.pktNum = 100,
.pktType = "TCP",
.netTestRole = NULL
};
.user = NULL,
.database = NULL,
.timezone = NULL,
.is_raw_time = false,
.is_use_passwd = false,
.dump_config = false,
.file = "\0",
.dir = "\0",
.threadNum = 5,
.commands = NULL,
.pktLen = 1000,
.pktNum = 100,
.pktType = "TCP",
.netTestRole = NULL};
/*
* Main function.
*/
int main(int argc, char* argv[]) {
int main(int argc, char *argv[]) {
/*setlocale(LC_ALL, "en_US.UTF-8"); */
if (!checkVersion()) {
@ -102,6 +104,7 @@ int main(int argc, char* argv[]) {
shellParseArgument(argc, argv, &args);
#if 0
if (args.dump_config) {
taosInitGlobalCfg();
taosReadGlobalLogCfg();
@ -123,9 +126,10 @@ int main(int argc, char* argv[]) {
taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktNum, args.pktType);
exit(0);
}
#endif
/* Initialize the shell */
TAOS* con = shellInit(&args);
TAOS *con = shellInit(&args);
if (con == NULL) {
exit(EXIT_FAILURE);
}