xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0_query_integrate' into feature/scheduler

This commit is contained in:
dapan 2022-03-05 13:44:57 +08:00
parent 9b4bda4f18 b0a4cc81bf
commit 21af31f496
109 changed files with 5544 additions and 2760 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/client/taos.h
View File

@ -51,6 +51,7 @@ typedef void **TAOS_ROW;
#define TSDB_DATA_TYPE_JSON 17 // json
#define TSDB_DATA_TYPE_DECIMAL 18 // decimal
#define TSDB_DATA_TYPE_BLOB 19 // binary
#define TSDB_DATA_TYPE_MEDIUMBLOB 20
typedef enum {
TSDB_OPTION_LOCALE,

6
include/common/tmsg.h
View File

@ -419,7 +419,11 @@ typedef struct {
* But for data in vnode side, we need all the following information.
*/
typedef struct {
int16_t colId;
union {
int16_t colId;
int16_t slotId;
};
int16_t type;
int16_t bytes;
SColumnFilterList flist;

133
include/common/ttokendef.h
View File

@ -57,63 +57,86 @@
#define TK_VGROUPS 39
#define TK_SINGLESTABLE 40
#define TK_STREAMMODE 41
#define TK_NK_FLOAT 42
#define TK_NK_BOOL 43
#define TK_TIMESTAMP 44
#define TK_NK_VARIABLE 45
#define TK_NK_COMMA 46
#define TK_NK_ID 47
#define TK_NK_LP 48
#define TK_NK_RP 49
#define TK_NK_DOT 50
#define TK_BETWEEN 51
#define TK_IS 52
#define TK_NULL 53
#define TK_NK_LT 54
#define TK_NK_GT 55
#define TK_NK_LE 56
#define TK_NK_GE 57
#define TK_NK_NE 58
#define TK_NK_EQ 59
#define TK_LIKE 60
#define TK_MATCH 61
#define TK_NMATCH 62
#define TK_IN 63
#define TK_FROM 64
#define TK_AS 65
#define TK_JOIN 66
#define TK_ON 67
#define TK_INNER 68
#define TK_SELECT 69
#define TK_DISTINCT 70
#define TK_WHERE 71
#define TK_PARTITION 72
#define TK_BY 73
#define TK_SESSION 74
#define TK_STATE_WINDOW 75
#define TK_INTERVAL 76
#define TK_SLIDING 77
#define TK_FILL 78
#define TK_VALUE 79
#define TK_NONE 80
#define TK_PREV 81
#define TK_LINEAR 82
#define TK_NEXT 83
#define TK_GROUP 84
#define TK_HAVING 85
#define TK_ORDER 86
#define TK_SLIMIT 87
#define TK_SOFFSET 88
#define TK_LIMIT 89
#define TK_OFFSET 90
#define TK_ASC 91
#define TK_DESC 92
#define TK_NULLS 93
#define TK_FIRST 94
#define TK_LAST 95
#define TK_USE 42
#define TK_TABLE 43
#define TK_NK_LP 44
#define TK_NK_RP 45
#define TK_NK_ID 46
#define TK_NK_DOT 47
#define TK_NK_COMMA 48
#define TK_COMMENT 49
#define TK_BOOL 50
#define TK_TINYINT 51
#define TK_SMALLINT 52
#define TK_INT 53
#define TK_INTEGER 54
#define TK_BIGINT 55
#define TK_FLOAT 56
#define TK_DOUBLE 57
#define TK_BINARY 58
#define TK_TIMESTAMP 59
#define TK_NCHAR 60
#define TK_UNSIGNED 61
#define TK_JSON 62
#define TK_VARCHAR 63
#define TK_MEDIUMBLOB 64
#define TK_BLOB 65
#define TK_VARBINARY 66
#define TK_DECIMAL 67
#define TK_SHOW 68
#define TK_DATABASES 69
#define TK_TABLES 70
#define TK_NK_FLOAT 71
#define TK_NK_BOOL 72
#define TK_NK_VARIABLE 73
#define TK_BETWEEN 74
#define TK_IS 75
#define TK_NULL 76
#define TK_NK_LT 77
#define TK_NK_GT 78
#define TK_NK_LE 79
#define TK_NK_GE 80
#define TK_NK_NE 81
#define TK_NK_EQ 82
#define TK_LIKE 83
#define TK_MATCH 84
#define TK_NMATCH 85
#define TK_IN 86
#define TK_FROM 87
#define TK_AS 88
#define TK_JOIN 89
#define TK_ON 90
#define TK_INNER 91
#define TK_SELECT 92
#define TK_DISTINCT 93
#define TK_WHERE 94
#define TK_PARTITION 95
#define TK_BY 96
#define TK_SESSION 97
#define TK_STATE_WINDOW 98
#define TK_INTERVAL 99
#define TK_SLIDING 100
#define TK_FILL 101
#define TK_VALUE 102
#define TK_NONE 103
#define TK_PREV 104
#define TK_LINEAR 105
#define TK_NEXT 106
#define TK_GROUP 107
#define TK_HAVING 108
#define TK_ORDER 109
#define TK_SLIMIT 110
#define TK_SOFFSET 111
#define TK_LIMIT 112
#define TK_OFFSET 113
#define TK_ASC 114
#define TK_DESC 115
#define TK_NULLS 116
#define TK_FIRST 117
#define TK_LAST 118
#define TK_SPACE 300
#define TK_COMMENT 301
#define TK_NK_COMMENT 301
#define TK_ILLEGAL 302
#define TK_HEX 303 // hex number 0x123
#define TK_OCT 304 // oct number

114
include/libs/index/index.h
View File

@ -29,6 +29,12 @@ typedef struct SIndexOpts SIndexOpts;
typedef struct SIndexMultiTermQuery SIndexMultiTermQuery;
typedef struct SArray SIndexMultiTerm;
typedef struct SIndex SIndexJson;
typedef struct SIndexTerm SIndexJsonTerm;
typedef struct SIndexOpts SIndexJsonOpts;
typedef struct SIndexMultiTermQuery SIndexJsonMultiTermQuery;
typedef struct SArray SIndexJsonMultiTerm;
typedef enum {
ADD_VALUE, // add index colume value
DEL_VALUE, // delete index column value
@ -39,24 +45,108 @@ typedef enum {
} SIndexOperOnColumn;
typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType;
typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3 } EIndexQueryType;
typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3, QUERY_RANGE = 4 } EIndexQueryType;
/*
* @param: oper
*
* create multi query
* @param oper (input, relation between querys)
*/
SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType oper);
void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery);
int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType type);
/*
* @param:
* @param:
* destroy multi query
* @param pQuery (input, multi-query-object to be destory)
*/
void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery);
/*
* add query to multi query
* @param pQuery (input, multi-query-object)
* @param term (input, single query term)
* @param type (input, single query type)
* @return error code
*/
int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType type);
/*
* open index
* @param opt (input, index opt)
* @param path (input, index path)
* @param index (output, index object)
* @return error code
*/
int indexOpen(SIndexOpts* opt, const char* path, SIndex** index);
/*
* close index
* @param index (input, index to be closed)
* @return error code
*/
int indexOpen(SIndexOpts* opt, const char* path, SIndex** index);
void indexClose(SIndex* index);
int indexPut(SIndex* index, SIndexMultiTerm* terms, uint64_t uid);
int indexDelete(SIndex* index, SIndexMultiTermQuery* query);
int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result);
int indexRebuild(SIndex* index, SIndexOpts* opt);
/*
* insert terms into index
* @param index (input, index object)
* @param term (input, terms inserted into index)
* @param uid (input, uid of terms)
* @return error code
*/
int indexPut(SIndex* index, SIndexMultiTerm* terms, uint64_t uid);
/*
* delete terms that meet query condition
* @param index (input, index object)
* @param query (input, condition query to deleted)
* @return error code
*/
int indexDelete(SIndex* index, SIndexMultiTermQuery* query);
/*
* search index
* @param index (input, index object)
* @param query (input, multi query condition)
* @param result(output, query result)
* @return error code
*/
int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result);
/*
* rebuild index
* @param index (input, index object)
* @parma opt (input, rebuild index opts)
* @return error code
*/
int indexRebuild(SIndex* index, SIndexOpts* opt);
/*
* open index
* @param opt (input,index json opt)
* @param path (input, index json path)
* @param index (output, index json object)
* @return error code
*/
int tIndexJsonOpen(SIndexJsonOpts* opts, const char* path, SIndexJson** index);
/*
* close index
* @param index (input, index to be closed)
* @return void
*/
void tIndexJsonClose(SIndexJson* index);
/*
* insert terms into index
* @param index (input, index object)
* @param term (input, terms inserted into index)
* @param uid (input, uid of terms)
* @return error code
*/
int tIndexJsonPut(SIndexJson* index, SIndexJsonMultiTerm* terms, uint64_t uid);
/*
* search index
* @param index (input, index object)
* @param query (input, multi query condition)
* @param result(output, query result)
* @return error code
*/
int tIndexJsonSearch(SIndexJson* index, SIndexJsonMultiTermQuery* query, SArray* result);
/*
* @param
* @param

46
include/libs/monitor/monitor.h
View File

@ -23,6 +23,11 @@
extern "C" {
#endif
#define MON_STATUS_LEN 8
#define MON_ROLE_LEN 9
#define MON_VER_LEN 12
#define MON_LOG_LEN 1024
typedef struct {
int32_t dnode_id;
char dnode_ep[TSDB_EP_LEN];
@ -31,19 +36,19 @@ typedef struct {
typedef struct {
int32_t dnode_id;
char dnode_ep[TSDB_EP_LEN];
char status[8];
char status[MON_STATUS_LEN];
} SMonDnodeDesc;
typedef struct {
int32_t mnode_id;
char mnode_ep[TSDB_EP_LEN];
char role[8];
char role[MON_ROLE_LEN];
} SMonMnodeDesc;
typedef struct {
char first_ep[TSDB_EP_LEN];
int32_t first_ep_dnode_id;
char version[12];
char version[MON_VER_LEN];
float master_uptime; // day
int32_t monitor_interval; // sec
int32_t vgroups_total;
@ -57,19 +62,18 @@ typedef struct {
typedef struct {
int32_t dnode_id;
int8_t vnode_online;
char vnode_role[8];
char vnode_role[MON_ROLE_LEN];
} SMonVnodeDesc;
typedef struct {
int32_t vgroup_id;
char database_name[TSDB_DB_NAME_LEN];
int32_t tables_num;
char status[MON_STATUS_LEN];
SMonVnodeDesc vnodes[TSDB_MAX_REPLICA];
} SMonVgroupDesc;
typedef struct {
char database_name[TSDB_DB_NAME_LEN];
int32_t tables_num;
int8_t status;
SArray *vgroups; // array of SMonVgroupDesc
} SMonVgroupInfo;
@ -84,18 +88,18 @@ typedef struct {
float cpu_engine;
float cpu_system;
float cpu_cores;
float mem_engine; // MB
float mem_system; // MB
float mem_total; // MB
int64_t mem_engine; // KB
int64_t mem_system; // KB
int64_t mem_total; // KB
float disk_engine; // GB
float disk_used; // GB
float disk_total; // GB
float net_in; // Kb/s
float net_out; // Kb/s
float io_read; // Mb/s
float io_write; // Mb/s
float io_read_disk; // Mb/s
float io_write_disk; // Mb/s
int64_t net_in;
int64_t net_out;
float io_read;
float io_write;
float io_read_disk;
float io_write_disk;
int32_t req_select;
float req_select_rate;
int32_t req_insert;
@ -107,7 +111,7 @@ typedef struct {
int32_t errors;
int32_t vnodes_num;
int32_t masters;
int32_t has_mnode;
int8_t has_mnode;
} SMonDnodeInfo;
typedef struct {
@ -117,13 +121,15 @@ typedef struct {
} SMonDiskDesc;
typedef struct {
SArray *disks; // array of SMonDiskDesc
SArray *datadirs; // array of SMonDiskDesc
SMonDiskDesc logdir;
SMonDiskDesc tempdir;
} SMonDiskInfo;
typedef struct {
int64_t ts;
int8_t level;
char content[1024];
char content[MON_LOG_LEN];
} SMonLogItem;
typedef struct SMonInfo SMonInfo;

11
include/libs/nodes/cmdnodes.h
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_PLANN_NODES_H_
#define _TD_PLANN_NODES_H_
#ifndef _TD_CMD_NODES_H_
#define _TD_CMD_NODES_H_
#ifdef __cplusplus
extern "C" {
@ -49,6 +49,11 @@ typedef struct SCreateDatabaseStmt {
SDatabaseOptions options;
} SCreateDatabaseStmt;
typedef struct SUseDatabaseStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
} SUseDatabaseStmt;
typedef struct STableOptions {
int32_t keep;
int32_t ttl;
@ -77,4 +82,4 @@ typedef struct SCreateTableStmt {
}
#endif
#endif /*_TD_PLANN_NODES_H_*/
#endif /*_TD_CMD_NODES_H_*/

8
include/libs/nodes/nodes.h
View File

@ -70,16 +70,19 @@ typedef enum ENodeType {
// Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR,
QUERY_NODE_SELECT_STMT,
QUERY_NODE_SHOW_STMT,
QUERY_NODE_VNODE_MODIF_STMT,
QUERY_NODE_CREATE_DATABASE_STMT,
QUERY_NODE_CREATE_TABLE_STMT,
QUERY_NODE_USE_DATABASE_STMT,
QUERY_NODE_SHOW_DATABASES_STMT, // temp
QUERY_NODE_SHOW_TABLES_STMT, // temp
// logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN,
QUERY_NODE_LOGIC_PLAN_JOIN,
QUERY_NODE_LOGIC_PLAN_AGG,
QUERY_NODE_LOGIC_PLAN_PROJECT,
QUERY_NODE_LOGIC_PLAN_VNODE_MODIF,
QUERY_NODE_LOGIC_SUBPLAN,
QUERY_NODE_LOGIC_PLAN,
@ -94,6 +97,7 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_EXCHANGE,
QUERY_NODE_PHYSICAL_PLAN_SORT,
QUERY_NODE_PHYSICAL_PLAN_DISPATCH,
QUERY_NODE_PHYSICAL_PLAN_INSERT,
QUERY_NODE_PHYSICAL_SUBPLAN,
QUERY_NODE_PHYSICAL_PLAN
} ENodeType;
@ -125,6 +129,7 @@ void nodesDestroyNode(SNodeptr pNode);
SNodeList* nodesMakeList();
int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode);
int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode);
int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
SNodeptr nodesListGetNode(SNodeList* pList, int32_t index);
@ -153,6 +158,7 @@ bool nodesEqualNode(const SNodeptr a, const SNodeptr b);
SNodeptr nodesCloneNode(const SNodeptr pNode);
SNodeList* nodesCloneList(const SNodeList* pList);
const char* nodesNodeName(ENodeType type);
int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_t* pLen);
int32_t nodesStringToNode(const char* pStr, SNode** pNode);

28
include/libs/nodes/plannodes.h
View File

@ -43,6 +43,7 @@ typedef struct SScanLogicNode {
SLogicNode node;
SNodeList* pScanCols;
struct STableMeta* pMeta;
SVgroupsInfo* pVgroupList;
EScanType scanType;
uint8_t scanFlag; // denotes reversed scan of data or not
STimeWindow scanRange;
@ -65,11 +66,27 @@ typedef struct SProjectLogicNode {
SNodeList* pProjections;
} SProjectLogicNode;
typedef struct SVnodeModifLogicNode {
ENodeType type;;
int32_t msgType;
SArray* pDataBlocks;
SVgDataBlocks* pVgDataBlocks;
} SVnodeModifLogicNode;
typedef enum ESubplanType {
SUBPLAN_TYPE_MERGE = 1,
SUBPLAN_TYPE_PARTIAL,
SUBPLAN_TYPE_SCAN,
SUBPLAN_TYPE_MODIFY
} ESubplanType;
typedef struct SSubLogicPlan {
ENodeType type;
SNodeList* pChildren;
SNodeList* pParents;
SLogicNode* pNode;
ESubplanType subplanType;
int32_t level;
} SSubLogicPlan;
typedef struct SQueryLogicPlan {
@ -96,7 +113,7 @@ typedef struct SDataBlockDescNode {
typedef struct SPhysiNode {
ENodeType type;
SDataBlockDescNode outputDataBlockDesc;
SDataBlockDescNode* pOutputDataBlockDesc;
SNode* pConditions;
SNodeList* pChildren;
struct SPhysiNode* pParent;
@ -158,7 +175,7 @@ typedef struct SExchangePhysiNode {
typedef struct SDataSinkNode {
ENodeType type;;
SDataBlockDescNode inputDataBlockDesc;
SDataBlockDescNode* pInputDataBlockDesc;
} SDataSinkNode;
typedef struct SDataDispatcherNode {
@ -178,13 +195,6 @@ typedef struct SSubplanId {
int32_t subplanId;
} SSubplanId;
typedef enum ESubplanType {
SUBPLAN_TYPE_MERGE = 1,
SUBPLAN_TYPE_PARTIAL,
SUBPLAN_TYPE_SCAN,
SUBPLAN_TYPE_MODIFY
} ESubplanType;
typedef struct SSubplan {
ENodeType type;
SSubplanId id; // unique id of the subplan

2
include/libs/nodes/querynodes.h
View File

@ -123,6 +123,7 @@ struct STableMeta;
typedef struct SRealTableNode {
STableNode table; // QUERY_NODE_REAL_TABLE
struct STableMeta* pMeta;
SVgroupsInfo* pVgroupList;
} SRealTableNode;
typedef struct STempTableNode {
@ -264,6 +265,7 @@ typedef struct SVgDataBlocks {
typedef struct SVnodeModifOpStmt {
ENodeType nodeType;
ENodeType sqlNodeType;
SArray* pDataBlocks; // data block for each vgroup, SArray<SVgDataBlocks*>.
int8_t schemaAttache; // denote if submit block is built with table schema or not
uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert

8
include/libs/parser/parser.h
View File

@ -38,16 +38,20 @@ typedef struct SParseContext {
typedef struct SCmdMsgInfo {
int16_t msgType;
SEpSet epSet;
char* pMsg;
void* pMsg;
int32_t msgLen;
void* pExtension; // todo remove it soon
} SCmdMsgInfo;
typedef struct SQuery {
bool isCmd;
bool directRpc;
bool haveResultSet;
ENodeType sqlNodeType;
SNode* pRoot;
int32_t numOfResCols;
SSchema* pResSchema;
SCmdMsgInfo* pCmdMsg;
int32_t msgType;
} SQuery;
int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);

2
include/libs/planner/planner.h
View File

@ -28,7 +28,7 @@ typedef struct SPlanContext {
} SPlanContext;
// Create the physical plan for the query, according to the AST.
int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan);
int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNodeList);
// Set datasource of this subplan, multiple calls may be made to a subplan.
// @subplan subplan to be schedule

4
include/libs/sync/sync.h
View File

@ -34,9 +34,7 @@ typedef enum {
TAOS_SYNC_STATE_FOLLOWER = 0,
TAOS_SYNC_STATE_CANDIDATE = 1,
TAOS_SYNC_STATE_LEADER = 2,
} ESyncRole;
typedef ESyncRole ESyncState;
} ESyncState;
typedef struct SSyncBuffer {
void* data;

9
include/libs/tfs/tfs.h
View File

@ -17,6 +17,7 @@
#define _TD_TFS_H_
#include "tdef.h"
#include "monitor.h"
#ifdef __cplusplus
extern "C" {
@ -237,6 +238,14 @@ const STfsFile *tfsReaddir(STfsDir *pDir);
*/
void tfsClosedir(STfsDir *pDir);
/**
* @brief Get disk info of tfs.
*
* @param pTfs The fs object.
* @param pInfo The info object.
*/
int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo);
#ifdef __cplusplus
}
#endif

2
include/os/osDir.h
View File

@ -21,7 +21,7 @@ extern "C" {
#endif
void taosRemoveDir(const char *dirname);
int32_t taosDirExist(char *dirname);
bool taosDirExist(char *dirname);
int32_t taosMkDir(const char *dirname);
void taosRemoveOldFiles(const char *dirname, int32_t keepDays);
int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);

6
include/os/osEnv.h
View File

@ -28,11 +28,11 @@ extern char tsCharset[];
extern char tsLocale[];
extern int8_t tsDaylight;
extern bool tsEnableCoreFile;
extern int64_t tsPageSize;
extern int64_t tsPageSizeKB;
extern int64_t tsOpenMax;
extern int64_t tsStreamMax;
extern int32_t tsNumOfCores;
extern int32_t tsTotalMemoryMB;
extern float tsNumOfCores;
extern int64_t tsTotalMemoryKB;
extern char configDir[];
extern char tsDataDir[];

24
include/os/osFile.h
View File

@ -25,8 +25,12 @@ extern "C" {
#ifndef ALLOW_FORBID_FUNC
#define open OPEN_FUNC_TAOS_FORBID
#define fopen FOPEN_FUNC_TAOS_FORBID
// #define close CLOSE_FUNC_TAOS_FORBID
// #define fclose FCLOSE_FUNC_TAOS_FORBID
#define access ACCESS_FUNC_TAOS_FORBID
#define stat STAT_FUNC_TAOS_FORBID
#define lstat LSTAT_FUNC_TAOS_FORBID
#define fstat FSTAT_FUNC_TAOS_FORBID
#define close CLOSE_FUNC_TAOS_FORBID
#define fclose FCLOSE_FUNC_TAOS_FORBID
#endif
#ifndef PATH_MAX
@ -44,6 +48,12 @@ typedef struct TdFile *TdFilePtr;
#define TD_FILE_AUTO_DEL 0x0040
#define TD_FILE_EXCL 0x0080
#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosGetLineFile, taosEOFFile
TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions);
#define TD_FILE_ACCESS_EXIST_OK 0x1
#define TD_FILE_ACCESS_READ_OK 0x2
#define TD_FILE_ACCESS_WRITE_OK 0x4
bool taosCheckAccessFile(const char *pathname, int mode);
int32_t taosLockFile(TdFilePtr pFile);
int32_t taosUnLockFile(TdFilePtr pFile);
@ -51,9 +61,9 @@ int32_t taosUnLockFile(TdFilePtr pFile);
int32_t taosUmaskFile(int32_t maskVal);
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime);
int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno);
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions);
bool taosCheckExistFile(const char *pathname);
int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence);
int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length);
@ -62,7 +72,7 @@ int32_t taosFsyncFile(TdFilePtr pFile);
int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count);
int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset);
int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
void taosFprintfFile(TdFilePtr pFile, const char *format, ...);
void taosFprintfFile(TdFilePtr pFile, const char *format, ...);
int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf);
int32_t taosEOFFile(TdFilePtr pFile);
@ -71,7 +81,7 @@ int64_t taosCloseFile(TdFilePtr *ppFile);
int32_t taosRenameFile(const char *oldName, const char *newName);
int64_t taosCopyFile(const char *from, const char *to);
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size);
int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
@ -79,7 +89,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in
void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
bool taosValidFile(TdFilePtr pFile);
int taosGetErrorFile(TdFilePtr pFile);
int32_t taosGetErrorFile(TdFilePtr pFile);
#ifdef __cplusplus
}

24
include/os/osSysinfo.h
View File

@ -34,19 +34,19 @@ typedef struct {
} SDiskSpace;
void taosGetSystemInfo();
bool taosGetEmail(char *email, int32_t maxLen);
bool taosGetOsReleaseName(char *releaseName, int32_t maxLen);
bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores);
int32_t taosGetCpuCores();
bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage);
bool taosGetTotalSysMemoryKB(uint64_t *kb);
bool taosGetProcMemory(float *memoryUsedMB); //
bool taosGetSysMemory(float *memoryUsedMB); //
int32_t taosGetEmail(char *email, int32_t maxLen);
int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen);
int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores);
int32_t taosGetCpuCores(float *numOfCores);
int32_t taosGetCpuUsage(float *cpu_system, float *cpu_engine);
int32_t taosGetTotalMemory(int64_t *totalKB);
int32_t taosGetProcMemory(int64_t *usedKB);
int32_t taosGetSysMemory(int64_t *usedKB);
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
bool taosReadProcIO(int64_t *rchars, int64_t *wchars);
bool taosGetProcIO(float *readKB, float *writeKB);
bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes);
bool taosGetBandSpeed(float *bandSpeedKb);
int32_t taosReadProcIO(int64_t *rchars, int64_t *wchars);
int32_t taosGetProcIO(float *readKB, float *writeKB);
int32_t taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes);
int32_t taosGetBandSpeed(float *bandSpeedKb);
int32_t taosSystem(const char *cmd);
void taosKillSystem();

27
source/client/src/clientImpl.c
View File

@ -161,7 +161,7 @@ int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) {
}
code = qParseQuerySql(&cxt, pQuery);
if (TSDB_CODE_SUCCESS == code && !((*pQuery)->isCmd)) {
if (TSDB_CODE_SUCCESS == code && ((*pQuery)->haveResultSet)) {
setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols);
}
@ -176,6 +176,14 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
STscObj* pTscObj = pRequest->pTscObj;
SMsgSendInfo* pSendMsg = buildMsgInfoImpl(pRequest);
if (pMsgInfo->msgType == TDMT_VND_SHOW_TABLES) {
SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo;
if (pShowReqInfo->pArray == NULL) {
pShowReqInfo->currentIndex = 0; // set the first vnode/ then iterate the next vnode
pShowReqInfo->pArray = pMsgInfo->pExtension;
}
}
int64_t transporterId = 0;
asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pMsgInfo->epSet, &transporterId, pSendMsg);
@ -183,20 +191,13 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
return TSDB_CODE_SUCCESS;
}
int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pDag, SArray* pNodeList) {
// pRequest->type = pQuery->type;
int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList) {
pRequest->type = pQuery->msgType;
SPlanContext cxt = { .queryId = pRequest->requestId, .pAstRoot = pQuery->pRoot };
int32_t code = qCreateQueryPlan(&cxt, pDag);
int32_t code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
if (code != 0) {
return code;
}
// if (pQuery->type == TSDB_SQL_SELECT) {
// setResSchemaInfo(&pRequest->body.resInfo, pSchema, numOfCols);
// pRequest->type = TDMT_VND_QUERY;
// }
return code;
}
@ -226,7 +227,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
return pRequest->code;
}
if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) {
if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_CREATE_TABLE == pRequest->type) {
pRequest->body.resInfo.numOfRows = res.numOfRows;
if (pRequest->body.queryJob != 0) {
@ -254,7 +255,7 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) {
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return);
if (pQuery->isCmd) {
if (pQuery->directRpc) {
CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return);
} else {
CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList), _return);

10
source/common/src/tglobal.c
View File

@ -279,11 +279,11 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1;
if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1;
if (cfgAddBool(pCfg, "enableCoreFile", 1, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSize, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 0, 100000, 1) != 0) return -1;
if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "totalMemory(MB)", tsTotalMemoryMB, 0, INT32_MAX, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddInt64(pCfg, "totalMemory(KB)", tsTotalMemoryKB, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1;
if (cfgAddString(pCfg, "os nodename", info.nodename, 1) != 0) return -1;
if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1;
@ -404,10 +404,6 @@ static void taosSetSystemCfg(SConfig *pCfg) {
const char *charset = cfgGetItem(pCfg, "charset")->str;
taosSetSystemLocale(locale, charset);
if (tsNumOfCores <= 1) {
tsNumOfCores = 2;
}
bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval;
taosSetConsoleEcho(enableCore);

4
source/common/test/commonTests.cpp
View File

@ -1,6 +1,4 @@
#include "tcommon.h"
#include <gtest/gtest.h>
#include <tep.h>
#include <iostream>
#pragma GCC diagnostic push
@ -10,6 +8,8 @@
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "tep.h"
#include "tcommon.h"
#include "taos.h"
#include "tvariant.h"
#include "tdef.h"

2
source/common/test/tmsgTest.cpp
View File

@ -1,6 +1,6 @@
#include <iostream>
#include "gtest/gtest.h"
#include <gtest/gtest.h>
#include "tmsg.h"

4
source/dnode/mgmt/impl/inc/dndEnv.h
View File

@ -31,7 +31,7 @@ typedef struct {
SDnode *pDnode;
STaosQueue *queue;
union {
SQWorkerPool pool;
SQWorkerPool pool;
SWWorkerPool mpool;
};
} SDnodeWorker;
@ -137,6 +137,8 @@ typedef struct SDnode {
SStartupReq startup;
} SDnode;
int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo);
#ifdef __cplusplus
}
#endif

1
source/dnode/mgmt/impl/inc/dndMnode.h
View File

@ -34,6 +34,7 @@ int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo);
int8_t dndIsMnode(SDnode *pDnode);
#ifdef __cplusplus
}

9
source/dnode/mgmt/impl/src/dndEnv.c
View File

@ -323,3 +323,12 @@ void dndCleanup() {
taosStopCacheRefreshWorker();
dInfo("dnode env is cleaned up");
}
int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) {
tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name));
pInfo->logdir.size = tsLogSpace.size;
tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name));
pInfo->tempdir.size = tsTempSpace.size;
return tfsGetMonitorInfo(pDnode->pTfs, pInfo);
}

52
source/dnode/mgmt/impl/src/dndMgmt.c
View File

@ -474,20 +474,48 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
rpcSendResponse(&rpcRsp);
}
void dndGetBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
pInfo->dnode_id = dndGetDnodeId(pDnode);
tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
}
static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
pInfo->uptime = (taosGetTimestampMs() - pDnode->dmgmt.rebootTime) / (86400000.0f);
taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system);
pInfo->cpu_cores = tsNumOfCores;
taosGetProcMemory(&pInfo->mem_engine);
taosGetSysMemory(&pInfo->mem_system);
pInfo->mem_total = tsTotalMemoryKB;
pInfo->disk_engine = 0;
pInfo->disk_used = tsDataSpace.size.used / (1024 * 1024 * 1024.0);
pInfo->disk_total = tsDataSpace.size.avail / (1024 * 1024 * 1024.0);
taosGetCardInfo(NULL, &pInfo->net_in, &pInfo->net_out);
taosGetProcIO(&pInfo->io_read, &pInfo->io_write);
pInfo->io_read_disk = 0;
pInfo->io_write_disk = 0;
pInfo->req_select = 0;
pInfo->req_select_rate = 0;
pInfo->req_insert = 0;
pInfo->req_insert_success = 0;
pInfo->req_insert_rate = 0;
pInfo->req_insert_batch = 0;
pInfo->req_insert_batch_success = 0;
pInfo->req_insert_batch_rate = 0;
pInfo->errors = 0;
pInfo->vnodes_num = 0;
pInfo->masters = 0;
pInfo->has_mnode = dndIsMnode(pDnode);
}
static void dndSendMonitorReport(SDnode *pDnode) {
if (!tsEnableMonitor || tsMonitorFqdn[0] == 0) return;
if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
dTrace("pDnode:%p, send monitor report to %s:%u", pDnode, tsMonitorFqdn, tsMonitorPort);
SMonInfo *pMonitor = monCreateMonitorInfo();
if (pMonitor == NULL) return;
dTrace("pDnode:%p, send monitor report to %s:%u", pDnode, tsMonitorFqdn, tsMonitorPort);
SMonBasicInfo basicInfo = {0};
dndGetBasicInfo(pDnode, &basicInfo);
dndGetMonitorBasicInfo(pDnode, &basicInfo);
monSetBasicInfo(pMonitor, &basicInfo);
SMonClusterInfo clusterInfo = {0};
@ -499,6 +527,20 @@ static void dndSendMonitorReport(SDnode *pDnode) {
monSetGrantInfo(pMonitor, &grantInfo);
}
SMonDnodeInfo dnodeInfo = {0};
dndGetMonitorDnodeInfo(pDnode, &dnodeInfo);
monSetDnodeInfo(pMonitor, &dnodeInfo);
SMonDiskInfo diskInfo = {0};
if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) {
monSetDiskInfo(pMonitor, &diskInfo);
}
taosArrayDestroy(clusterInfo.dnodes);
taosArrayDestroy(clusterInfo.mnodes);
taosArrayDestroy(vgroupInfo.vgroups);
taosArrayDestroy(diskInfo.datadirs);
monSendReport(pMonitor);
monCleanupMonitorInfo(pMonitor);
}

7
source/dnode/mgmt/impl/src/dndMnode.c
View File

@ -639,4 +639,11 @@ int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SM
int32_t code = mndGetMonitorInfo(pMnode, pClusterInfo, pVgroupInfo, pGrantInfo);
dndReleaseMnode(pDnode, pMnode);
return code;
}
int8_t dndIsMnode(SDnode *pDnode) {
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode == NULL) return 0;
dndReleaseMnode(pDnode, pMnode);
return 1;
}

15
source/dnode/mnode/impl/src/mndTelem.c
View File

@ -52,23 +52,20 @@ static char* mndBuildTelemetryReport(SMnode* pMnode) {
tjsonAddStringToObject(pJson, "instanceId", clusterName);
tjsonAddDoubleToObject(pJson, "reportVersion", 1);
if (taosGetOsReleaseName(tmp, sizeof(tmp))) {
if (taosGetOsReleaseName(tmp, sizeof(tmp)) == 0) {
tjsonAddStringToObject(pJson, "os", tmp);
}
int32_t numOfCores = 0;
if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores)) {
float numOfCores = 0;
if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) {
tjsonAddStringToObject(pJson, "cpuModel", tmp);
tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores);
} else {
tjsonAddDoubleToObject(pJson, "numOfCpu", taosGetCpuCores());
tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores);
}
uint64_t memoryKB = 0;
if (taosGetTotalSysMemoryKB(&memoryKB)) {
snprintf(tmp, sizeof(tmp), "%" PRIu64 " kB", memoryKB);
tjsonAddStringToObject(pJson, "memory", tmp);
}
snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB);
tjsonAddStringToObject(pJson, "memory", tmp);
tjsonAddStringToObject(pJson, "version", version);
tjsonAddStringToObject(pJson, "buildInfo", buildinfo);

2
source/dnode/vnode/src/tq/tqMetaStore.c
View File

@ -90,7 +90,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
char name[pathLen + 10];
strcpy(name, path);
if (taosDirExist(name) != 0 && taosMkDir(name) != 0) {
if (!taosDirExist(name) && taosMkDir(name) != 0) {
terrno = TSDB_CODE_TQ_FAILED_TO_CREATE_DIR;
tqError("failed to create dir:%s since %s ", name, terrstr());
}

4
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -580,7 +580,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
SDFile *pRSmadF = TSDB_READ_SMAD_FILE(&(pCommith->readh));
SDFile *pWSmadF = TSDB_COMMIT_SMAD_FILE(pCommith);
if (access(TSDB_FILE_FULL_NAME(pRSmadF), F_OK) != 0) {
if (!taosCheckExistFile(TSDB_FILE_FULL_NAME(pRSmadF))) {
tsdbDebug("vgId:%d create data file %s as not exist", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pRSmadF));
tsdbInitDFile(pRepo, pWSmadF, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_SMAD);
@ -614,7 +614,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
SDFile *pRSmalF = TSDB_READ_SMAL_FILE(&(pCommith->readh));
SDFile *pWSmalF = TSDB_COMMIT_SMAL_FILE(pCommith);
if ((pCommith->isLFileSame) && access(TSDB_FILE_FULL_NAME(pRSmalF), F_OK) == 0) {
if ((pCommith->isLFileSame) && taosCheckExistFile(TSDB_FILE_FULL_NAME(pRSmalF))) {
tsdbInitDFileEx(pWSmalF, pRSmalF);
if (tsdbOpenDFile(pWSmalF, O_RDWR) < 0) {
tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmalF),

2
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -314,7 +314,7 @@ int tsdbOpenFS(STsdb *pRepo) {
tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current);
tsdbGetRtnSnap(pRepo, &pRepo->rtn);
if (access(current, F_OK) == 0) {
if (taosCheckExistFile(current)) {
if (tsdbOpenFSFromCurrent(pRepo) < 0) {
tsdbError("vgId:%d failed to open FS since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;

15
source/dnode/vnode/src/tsdb/tsdbFile.c
View File

@ -443,25 +443,24 @@ int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) {
}
static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) {
struct stat dfstat;
SDFile df;
tsdbInitDFileEx(&df, pDFile);
if (access(TSDB_FILE_FULL_NAME(pDFile), F_OK) != 0) {
if (!taosCheckExistFile(TSDB_FILE_FULL_NAME(pDFile))) {
tsdbError("vgId:%d data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo),
TSDB_FILE_FULL_NAME(pDFile));
// pRepo->state |= TSDB_STATE_BAD_DATA;
TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD);
return 0;
}
if (stat(TSDB_FILE_FULL_NAME(&df), &dfstat) < 0) {
int64_t file_size = 0;
if (taosStatFile(TSDB_FILE_FULL_NAME(&df), &file_size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
if (pDFile->info.size < dfstat.st_size) {
if (pDFile->info.size < file_size) {
// if (tsdbOpenDFile(&df, O_WRONLY) < 0) {
if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) {
return -1;
@ -480,10 +479,10 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) {
tsdbCloseDFile(&df);
tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
dfstat.st_size, pDFile->info.size);
} else if (pDFile->info.size > dfstat.st_size) {
file_size, pDFile->info.size);
} else if (pDFile->info.size > file_size) {
tsdbError("vgId:%d data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), dfstat.st_size, pDFile->info.size);
REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), file_size, pDFile->info.size);
// pRepo->state |= TSDB_STATE_BAD_DATA;
TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD);
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;

1
source/libs/CMakeLists.txt
View File

@ -1,4 +1,3 @@
add_definitions("-D ALLOW_FORBID_FUNC")
add_subdirectory(transport)
add_subdirectory(sync)
add_subdirectory(tdb)

6
source/libs/catalog/test/catalogTests.cpp
View File

@ -14,9 +14,7 @@
*/
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#include "os.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
@ -24,8 +22,10 @@
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#pragma GCC diagnostic ignored "-Wformat"
#include <addr_any.h>
#include "addr_any.h"
#include "os.h"
#include "tglobal.h"
#include "catalog.h"
#include "stub.h"
#include "taos.h"

16
source/libs/executor/src/dataDispatcher.c
View File

@ -39,7 +39,7 @@ typedef struct SDataCacheEntry {
typedef struct SDataDispatchHandle {
SDataSinkHandle sink;
SDataSinkManager* pManager;
SDataBlockDescNode schema;
SDataBlockDescNode* pSchema;
STaosQueue* pDataBlocks;
SDataDispatchBuf nextOutput;
int32_t status;
@ -109,14 +109,14 @@ static void copyData(const SInputData* pInput, const SDataBlockDescNode* pSchema
// data format: SDataCacheEntry | col1_data col2_data ... | numOfTables | STableIdInfo STableIdInfo ...
static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) {
SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData;
pEntry->compressed = (int8_t)needCompress(pInput->pData, &(pHandle->schema));
pEntry->compressed = (int8_t)needCompress(pInput->pData, pHandle->pSchema);
pEntry->numOfRows = pInput->pData->info.rows;
pEntry->dataLen = 0;
pBuf->useSize = DATA_META_LENGTH(pInput->pTableRetrieveTsMap);
copyData(pInput, &pHandle->schema, pEntry->data, pEntry->compressed, &pEntry->dataLen);
copyData(pInput, pHandle->pSchema, pEntry->data, pEntry->compressed, &pEntry->dataLen);
if (0 == pEntry->compressed) {
pEntry->dataLen = pHandle->schema.resultRowSize * pInput->pData->info.rows;
pEntry->dataLen = pHandle->pSchema->resultRowSize * pInput->pData->info.rows;
}
pBuf->useSize += pEntry->dataLen;
// todo completed
@ -130,7 +130,7 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput,
return false;
}
pBuf->allocSize = DATA_META_LENGTH(pInput->pTableRetrieveTsMap) + pDispatcher->schema.resultRowSize * pInput->pData->info.rows;
pBuf->allocSize = DATA_META_LENGTH(pInput->pTableRetrieveTsMap) + pDispatcher->pSchema->resultRowSize * pInput->pData->info.rows;
pBuf->pData = malloc(pBuf->allocSize);
if (pBuf->pData == NULL) {
qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno));
@ -196,7 +196,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
if (NULL == pDispatcher->nextOutput.pData) {
assert(pDispatcher->queryEnd);
pOutput->useconds = pDispatcher->useconds;
pOutput->precision = pDispatcher->schema.precision;
pOutput->precision = pDispatcher->pSchema->precision;
return TSDB_CODE_SUCCESS;
}
SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData);
@ -208,7 +208,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
pthread_mutex_lock(&pDispatcher->mutex);
pOutput->queryEnd = pDispatcher->queryEnd;
pOutput->useconds = pDispatcher->useconds;
pOutput->precision = pDispatcher->schema.precision;
pOutput->precision = pDispatcher->pSchema->precision;
pthread_mutex_unlock(&pDispatcher->mutex);
return TSDB_CODE_SUCCESS;
}
@ -238,7 +238,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
dispatcher->sink.fGetData = getDataBlock;
dispatcher->sink.fDestroy = destroyDataSinker;
dispatcher->pManager = pManager;
dispatcher->schema = pDataSink->inputDataBlockDesc;
dispatcher->pSchema = pDataSink->pInputDataBlockDesc;
dispatcher->status = DS_BUF_EMPTY;
dispatcher->queryEnd = false;
dispatcher->pDataBlocks = taosOpenQueue();

45
source/libs/executor/src/executorimpl.c
View File

@ -315,8 +315,6 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO
}
SSDataBlock* createOutputBuf_rv(SArray* pExprInfo, int32_t numOfRows) {
const static int32_t minSize = 8;
size_t numOfOutput = taosArrayGetSize(pExprInfo);
SSDataBlock *res = calloc(1, sizeof(SSDataBlock));
@ -330,15 +328,28 @@ SSDataBlock* createOutputBuf_rv(SArray* pExprInfo, int32_t numOfRows) {
idata.info.type = pExpr->base.resSchema.type;
idata.info.bytes = pExpr->base.resSchema.bytes;
idata.info.colId = pExpr->base.resSchema.colId;
int32_t size = TMAX(idata.info.bytes * numOfRows, minSize);
idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform
taosArrayPush(res->pDataBlock, &idata);
}
blockDataEnsureCapacity(res, numOfRows);
return res;
}
SSDataBlock* createOutputBuf_rv1(SDataBlockDescNode* pNode) {
int32_t numOfCols = LIST_LENGTH(pNode->pSlots);
SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock));
pBlock->info.numOfCols = numOfCols;
pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
for(int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData idata = {{0}};
SSlotDescNode* pDescNode = nodesListGetNode(pNode->pSlots, i);
idata.info.type = pDescNode->dataType.type;
idata.info.bytes = pDescNode->dataType.bytes;
idata.info.slotId = pDescNode->slotId;
}
}
static bool isSelectivityWithTagsQuery(SqlFunctionCtx *pCtx, int32_t numOfOutput) {
return true;
// bool hasTags = false;
@ -2215,10 +2226,10 @@ static void destroyTsComp(STaskRuntimeEnv *pRuntimeEnv, STaskAttr *pQueryAttr) {
if (pQueryAttr->tsCompQuery && pRuntimeEnv->outputBuf && pRuntimeEnv->outputBuf->pDataBlock && taosArrayGetSize(pRuntimeEnv->outputBuf->pDataBlock) > 0) {
SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0);
if (pColInfoData) {
FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor
if (f) {
fclose(f);
*(FILE **)pColInfoData->pData = NULL;
TdFilePtr pFile = *(TdFilePtr *)pColInfoData->pData; // TODO refactor
if (pFile != NULL) {
taosCloseFile(&pFile);
*(TdFilePtr *)pColInfoData->pData = NULL;
}
}
}
@ -8041,6 +8052,10 @@ static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SRead
static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId);
SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) {
if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_PROJECT) { // ignore the project node
pPhyNode = nodesListGetNode(pPhyNode->pChildren, 0);
}
if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) {
if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pPhyNode)) {
SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode;
@ -8119,15 +8134,15 @@ static tsdbReaderT createDataReaderImpl(STableScanPhysiNode* pTableScanNode, STa
cond.twindow = pTableScanNode->scanRange;
cond.type = BLOCK_LOAD_OFFSET_SEQ_ORDER;
// cond.type = pTableScanNode->scanFlag;
for (int32_t i = 0; i < cond.numOfCols; ++i) {
// SExprInfo* pExprInfo = taosArrayGetP(pTableScanNode->scan.node.pTargets, i);
// assert(pExprInfo->pExpr->nodeType == TEXPR_COL_NODE);
STargetNode* pNode = (STargetNode*)nodesListGetNode(pTableScanNode->scan.pScanCols, i);
// SSchema* pSchema = pExprInfo->pExpr->pSchema;
// cond.colList[i].type = pSchema->type;
// cond.colList[i].bytes = pSchema->bytes;
// cond.colList[i].colId = pSchema->colId;
SColumnNode* pColNode = (SColumnNode*)pNode->pExpr;
cond.colList[i].type = pColNode->colType;
cond.colList[i].bytes = pColNode->node.resType.type;
cond.colList[i].colId = pColNode->colId;
}
return tsdbQueryTables(readHandle, &cond, pGroupInfo, queryId, taskId);

1232
source/libs/executor/test/executorTests.cpp
View File

File diff suppressed because it is too large Load Diff

2
source/libs/executor/test/sortTests.cpp
View File

@ -13,7 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <executorimpl.h>
#include <gtest/gtest.h>
#include <tglobal.h>
#include <tsort.h>
@ -26,6 +25,7 @@
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "executorimpl.h"
#include "executor.h"
#include "stub.h"
#include "taos.h"

59
source/libs/index/inc/indexInt.h
View File

@ -122,30 +122,53 @@ typedef struct TFileCacheKey {
int indexFlushCacheToTFile(SIndex* sIdx, void*);
int32_t indexSerialCacheKey(ICacheKey* key, char* buf);
// int32_t indexSerialKey(ICacheKey* key, char* buf);
// int32_t indexSerialTermKey(SIndexTerm* itm, char* buf);
#define indexFatal(...) \
do { \
if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); } \
#define indexFatal(...) \
do { \
if (sDebugFlag & DEBUG_FATAL) { \
taosPrintLog("index FATAL ", 255, __VA_ARGS__); \
} \
} while (0)
#define indexError(...) \
do { \
if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); } \
#define indexError(...) \
do { \
if (sDebugFlag & DEBUG_ERROR) { \
taosPrintLog("index ERROR ", 255, __VA_ARGS__); \
} \
} while (0)
#define indexWarn(...) \
do { \
if (sDebugFlag & DEBUG_WARN) { taosPrintLog("index WARN ", 255, __VA_ARGS__); } \
#define indexWarn(...) \
do { \
if (sDebugFlag & DEBUG_WARN) { \
taosPrintLog("index WARN ", 255, __VA_ARGS__); \
} \
} while (0)
#define indexInfo(...) \
do { \
if (sDebugFlag & DEBUG_INFO) { taosPrintLog("index ", 255, __VA_ARGS__); } \
#define indexInfo(...) \
do { \
if (sDebugFlag & DEBUG_INFO) { \
taosPrintLog("index ", 255, __VA_ARGS__); \
} \
} while (0)
#define indexDebug(...) \
do { \
if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); } \
#define indexDebug(...) \
do { \
if (sDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define indexTrace(...) \
do { \
if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); } \
#define indexTrace(...) \
do { \
if (sDebugFlag & DEBUG_TRACE) { \
taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0)
#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F)
#define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \
do { \
uint8_t oldTy = ty; \
ty = (ty >> 4) | exTy; \
ty = (ty << 4) | oldTy; \
} while (0)
#ifdef __cplusplus

32
source/libs/index/inc/index_comm.h Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_INDEX_COMM_H_
#define _TD_INDEX_COMM_H_
#ifdef __cplusplus
extern "C" {
#endif
extern char JSON_COLUMN[];
extern char JSON_VALUE_DELIM;
char* indexPackJsonData(SIndexTerm* itm);
#ifdef __cplusplus
}
#endif
#endif

26
source/libs/index/src/index.c
View File

@ -2,8 +2,8 @@
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
* it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
@ -30,6 +30,8 @@
void* indexQhandle = NULL;
static char JSON_COLUMN[] = "JSON";
void indexInit() {
// refactor later
indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index");
@ -63,6 +65,9 @@ static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch);
static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv);
static void indexMergeSameKey(SArray* result, TFileValue* tv);
// static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf);
// int32_t indexSerialKey(ICacheKey* key, char* buf);
int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
pthread_once(&isInit, indexInit);
SIndex* sIdx = calloc(1, sizeof(SIndex));
@ -148,7 +153,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
SIndexTerm* p = taosArrayGetP(fVals, i);
char buf[128] = {0};
ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)};
ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType};
int32_t sz = indexSerialCacheKey(&key, buf);
IndexCache** cache = taosHashGet(index->colObj, buf, sz);
@ -163,7 +168,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
SIndexTerm* p = taosArrayGetP(fVals, i);
char buf[128] = {0};
ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)};
ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType};
int32_t sz = indexSerialCacheKey(&key, buf);
IndexCache** cache = taosHashGet(index->colObj, buf, sz);
@ -330,8 +335,9 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result
IndexCache* cache = NULL;
char buf[128] = {0};
ICacheKey key = {.suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName)};
int32_t sz = indexSerialCacheKey(&key, buf);
ICacheKey key = {
.suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType};
int32_t sz = indexSerialCacheKey(&key, buf);
pthread_mutex_lock(&sIdx->mtx);
IndexCache** pCache = taosHashGet(sIdx->colObj, buf, sz);
@ -555,11 +561,17 @@ END:
}
int32_t indexSerialCacheKey(ICacheKey* key, char* buf) {
bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(key->colType, TSDB_DATA_TYPE_JSON);
char* p = buf;
SERIALIZE_MEM_TO_BUF(buf, key, suid);
SERIALIZE_VAR_TO_BUF(buf, '_', char);
// SERIALIZE_MEM_TO_BUF(buf, key, colType);
// SERIALIZE_VAR_TO_BUF(buf, '_', char);
SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName);
if (hasJson) {
SERIALIZE_STR_VAR_TO_BUF(buf, JSON_COLUMN, strlen(JSON_COLUMN));
} else {
SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName);
}
return buf - p;
}

34
source/libs/index/src/index_cache.c
View File

@ -14,6 +14,7 @@
*/
#include "index_cache.h"
#include "index_comm.h"
#include "index_util.h"
#include "tcompare.h"
#include "tsched.h"
@ -44,8 +45,9 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in
indexError("failed to create index cache");
return NULL;
};
cache->mem = indexInternalCacheCreate(type);
cache->colName = tstrdup(colName);
cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName);
cache->type = type;
cache->index = idx;
cache->version = 0;
@ -207,11 +209,11 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) {
}
}
}
int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
if (cache == NULL) {
return -1;
}
bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON);
IndexCache* pCache = cache;
indexCacheRef(pCache);
@ -222,8 +224,12 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
}
// set up key
ct->colType = term->colType;
ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1));
memcpy(ct->colVal, term->colVal, term->nColVal);
if (hasJson) {
ct->colVal = indexPackJsonData(term);
} else {
ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1));
memcpy(ct->colVal, term->colVal, term->nColVal);
}
ct->version = atomic_add_fetch_32(&pCache->version, 1);
// set value
ct->uid = uid;
@ -294,13 +300,22 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV
SIndexTerm* term = query->term;
EIndexQueryType qtype = query->qType;
CacheTerm ct = {.colVal = term->colVal, .version = atomic_load_32(&pCache->version)};
bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON);
char* p = term->colVal;
if (hasJson) {
p = indexPackJsonData(term);
}
CacheTerm ct = {.colVal = p, .version = atomic_load_32(&pCache->version)};
int ret = indexQueryMem(mem, &ct, qtype, result, s);
if (ret == 0 && *s != kTypeDeletion) {
// continue search in imm
ret = indexQueryMem(imm, &ct, qtype, result, s);
}
if (hasJson) {
tfree(p);
}
indexMemUnRef(mem);
indexMemUnRef(imm);
@ -367,6 +382,8 @@ static int32_t indexCacheTermCompare(const void* l, const void* r) {
}
static MemTable* indexInternalCacheCreate(int8_t type) {
type = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : type;
MemTable* tbl = calloc(1, sizeof(MemTable));
indexMemRef(tbl);
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
@ -389,9 +406,6 @@ static bool indexCacheIteratorNext(Iterate* itera) {
IterateValue* iv = &itera->val;
iterateValueDestroy(iv, false);
// IterateValue* iv = &itera->val;
// IterateValue tIterVal = {.colVal = NULL, .val = taosArrayInit(1, sizeof(uint64_t))};
bool next = tSkipListIterNext(iter);
if (next) {
SSkipListNode* node = tSkipListIterGet(iter);
@ -411,10 +425,6 @@ static bool indexCacheIteratorNext(Iterate* itera) {
taosArrayPush(iv->val, &ct->uid);
}
// IterateValue* iv = &itera->val;
// iterateValueDestroy(iv, true);
//*iv = tIterVal;
return next;
}

48
source/libs/index/src/index_comm.c Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "index.h"
#include "indexInt.h"
char JSON_COLUMN[] = "JSON";
char JSON_VALUE_DELIM = '&';
char* indexPackJsonData(SIndexTerm* itm) {
/*
* |<-----colname---->|<-----dataType---->|<--------colVal---------->|
* |<-----string----->|<-----uint8_t----->|<----depend on dataType-->|
*/
uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType);
int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1;
char* buf = (char*)calloc(1, sz);
char* p = buf;
memcpy(p, itm->colName, itm->nColName);
p += itm->nColName;
memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM));
p += sizeof(JSON_VALUE_DELIM);
memcpy(p, &ty, sizeof(ty));
p += sizeof(ty);
memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM));
p += sizeof(JSON_VALUE_DELIM);
memcpy(p, itm->colVal, itm->nColVal);
return buf;
}

18
source/libs/index/src/index_fst_counting_writer.c
View File

@ -63,9 +63,9 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off
}
static int writeCtxGetSize(WriterCtx* ctx) {
if (ctx->type == TFile) {
struct stat fstat;
stat(ctx->file.buf, &fstat);
return fstat.st_size;
int64_t file_size = 0;
taosStatFile(ctx->file.buf, &file_size, NULL);
return (int)file_size;
}
return 0;
}
@ -99,9 +99,9 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int
// ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.pFile = taosOpenFile(path, TD_FILE_READ);
struct stat fstat;
stat(path, &fstat);
ctx->file.size = fstat.st_size;
int64_t file_size = 0;
taosFStatFile(ctx->file.pFile, &file_size, NULL);
ctx->file.size = (int)file_size;
#ifdef USE_MMAP
ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size);
#endif
@ -142,8 +142,10 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) {
#endif
}
if (ctx->file.readOnly == false) {
struct stat fstat;
stat(ctx->file.buf, &fstat);
int64_t file_size = 0;
taosStatFile(ctx->file.buf, &file_size, NULL);
// struct stat fstat;
// stat(ctx->file.buf, &fstat);
// indexError("write file size: %d", (int)(fstat.st_size));
}
if (remove) { unlink(ctx->file.buf); }

44
source/libs/index/src/index_json.c Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "index.h"
#include "indexInt.h"
int tIndexJsonOpen(SIndexJsonOpts *opts, const char *path, SIndexJson **index) {
// handle
return indexOpen(opts, path, index);
}
int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) {
for (int i = 0; i < taosArrayGetSize(terms); i++) {
SIndexJsonTerm *p = taosArrayGetP(terms, i);
INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON);
}
return indexPut(index, terms, uid);
// handle put
}
int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *result) {
SArray *terms = tq->query;
for (int i = 0; i < taosArrayGetSize(terms); i++) {
SIndexJsonTerm *p = taosArrayGetP(terms, i);
INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON);
}
return indexSearch(index, tq, result);
// handle search
}
void tIndexJsonClose(SIndexJson *index) {
return indexClose(index);
// handle close
}

29
source/libs/index/src/index_tfile.c
View File

@ -15,6 +15,7 @@ p *
#include "index_tfile.h"
#include "index.h"
#include "index_comm.h"
#include "index_fst.h"
#include "index_fst_counting_writer.h"
#include "index_util.h"
@ -186,13 +187,20 @@ void tfileReaderDestroy(TFileReader* reader) {
int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result) {
SIndexTerm* term = query->term;
bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON);
EIndexQueryType qtype = query->qType;
int ret = -1;
// refactor to callback later
if (qtype == QUERY_TERM) {
uint64_t offset;
FstSlice key = fstSliceCreate(term->colVal, term->nColVal);
char* p = term->colVal;
uint64_t sz = term->nColVal;
if (hasJson) {
p = indexPackJsonData(term);
sz = strlen(p);
}
FstSlice key = fstSliceCreate(p, sz);
if (fstGet(reader->fst, &key, &offset)) {
indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex", term->suid, term->colName,
term->colVal);
@ -202,10 +210,17 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* resul
term->colVal);
}
fstSliceDestroy(&key);
if (hasJson) {
free(p);
}
} else if (qtype == QUERY_PREFIX) {
// handle later
//
} else {
} else if (qtype == QUERY_SUFFIX) {
// handle later
} else if (qtype == QUERY_REGEX) {
// handle later
} else if (qtype == QUERY_RANGE) {
// handle later
}
tfileReaderUnRef(reader);
@ -260,6 +275,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
__compar_fn_t fn;
int8_t colType = tw->header.colType;
colType = INDEX_TYPE_GET_TYPE(colType);
if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) {
fn = tfileStrCompare;
} else {
@ -557,6 +573,8 @@ static int tfileWriteHeader(TFileWriter* writer) {
static int tfileWriteData(TFileWriter* write, TFileValue* tval) {
TFileHeader* header = &write->header;
uint8_t colType = header->colType;
colType = INDEX_TYPE_GET_TYPE(colType);
if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) {
FstSlice key = fstSliceCreate((uint8_t*)(tval->colVal), (size_t)strlen(tval->colVal));
if (fstBuilderInsert(write->fb, key, tval->offset)) {
@ -586,11 +604,10 @@ static int tfileReaderLoadHeader(TFileReader* reader) {
int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0);
if (nread == -1) {
indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf),
errno, reader->ctx->file.buf);
indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf), errno,
reader->ctx->file.buf);
} else {
indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf),
reader->ctx->file.buf);
indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), reader->ctx->file.buf);
}
// assert(nread == sizeof(buf));
memcpy(&reader->header, buf, sizeof(buf));

18
source/libs/index/test/CMakeLists.txt
View File

@ -2,6 +2,7 @@ add_executable(indexTest "")
add_executable(fstTest "")
add_executable(fstUT "")
add_executable(UtilUT "")
add_executable(jsonUT "")
target_sources(indexTest
PRIVATE
@ -21,6 +22,10 @@ target_sources(UtilUT
"utilUT.cc"
)
target_sources(jsonUT
PRIVATE
"jsonUT.cc"
)
target_include_directories ( indexTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/index"
@ -43,6 +48,12 @@ target_include_directories ( UtilUT
"${CMAKE_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories (jsonUT
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries (indexTest
os
util
@ -73,6 +84,13 @@ target_link_libraries (UtilUT
index
)
target_link_libraries (jsonUT
os
util
common
gtest_main
index
)
#add_test(
# NAME index_test

25
source/libs/index/test/fstTest.cc
View File

@ -301,13 +301,18 @@ void validateTFile(char* arg) {
}
}
void iterTFileReader(char* path, char* ver) {
int version = atoi(ver);
TFileReader* reader = tfileReaderOpen(path, 0, version, "tag1");
Iterate* iter = tfileIteratorCreate(reader);
bool tn = iter ? iter->next(iter) : false;
int count = 0;
int termCount = 0;
void iterTFileReader(char* path, char* uid, char* colName, char* ver) {
// tfInit();
uint64_t suid = atoi(uid);
int version = atoi(ver);
TFileReader* reader = tfileReaderOpen(path, suid, version, colName);
Iterate* iter = tfileIteratorCreate(reader);
bool tn = iter ? iter->next(iter) : false;
int count = 0;
int termCount = 0;
while (tn == true) {
count++;
IterateValue* cv = iter->getValue(iter);
@ -323,9 +328,9 @@ void iterTFileReader(char* path, char* ver) {
int main(int argc, char* argv[]) {
// tool to check all kind of fst test
// if (argc > 1) { validateTFile(argv[1]); }
if (argc > 2) {
// opt
iterTFileReader(argv[1], argv[2]);
if (argc > 4) {
// path suid colName ver
iterTFileReader(argv[1], argv[2], argv[3], argv[4]);
}
// checkFstCheckIterator();
// checkFstLongTerm();

26
source/libs/index/test/fstUT.cc
View File

@ -213,21 +213,21 @@ class FstEnv : public ::testing::Test {
TEST_F(FstEnv, writeNormal) {
fst->CreateWriter();
std::string str("aa");
std::string str("11");
for (int i = 0; i < 10; i++) {
str[0] = 'a' + i;
str[0] = '1' + i;
str.resize(2);
assert(fst->Put(str, i) == true);
}
// order failed
assert(fst->Put("aa", 1) == false);
assert(fst->Put("11", 1) == false);
fst->DestroyWriter();
fst->CreateReader();
uint64_t val;
assert(fst->Get("a", &val) == false);
assert(fst->Get("aa", &val) == true);
assert(fst->Get("1", &val) == false);
assert(fst->Get("11", &val) == true);
assert(val == 0);
std::vector<uint64_t> rlt;
@ -235,3 +235,19 @@ TEST_F(FstEnv, writeNormal) {
assert(fst->Search(ctx, rlt) == true);
}
TEST_F(FstEnv, WriteMillonrRecord) {}
TEST_F(FstEnv, writeAbNormal) {
fst->CreateWriter();
std::string str1("voltage&\b&ab");
std::string str2("voltbge&\b&ab");
fst->Put(str1, 1);
fst->Put(str2, 2);
fst->DestroyWriter();
fst->CreateReader();
uint64_t val;
assert(fst->Get("1", &val) == false);
assert(fst->Get("voltage&\b&ab", &val) == true);
assert(val == 1);
}

135
source/libs/index/test/jsonUT.cc Normal file
View File

@ -0,0 +1,135 @@
#include <gtest/gtest.h>
#include <algorithm>
#include <iostream>
#include <string>
#include <thread>
#include <vector>
#include "index.h"
#include "indexInt.h"
#include "index_cache.h"
#include "index_fst.h"
#include "index_fst_counting_writer.h"
#include "index_fst_util.h"
#include "index_tfile.h"
#include "index_util.h"
#include "tglobal.h"
#include "tskiplist.h"
#include "tutil.h"
static std::string dir = "/tmp/json";
class JsonEnv : public ::testing::Test {
protected:
virtual void SetUp() {
taosRemoveDir(dir.c_str());
taosMkDir(dir.c_str());
printf("set up\n");
opts = indexOptsCreate();
int ret = tIndexJsonOpen(opts, dir.c_str(), &index);
assert(ret == 0);
}
virtual void TearDown() {
tIndexJsonClose(index);
indexOptsDestroy(opts);
printf("destory\n");
}
SIndexJsonOpts* opts;
SIndexJson* index;
};
TEST_F(JsonEnv, testWrite) {
{
std::string colName("test");
std::string colVal("ab");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 100; i++) {
tIndexJsonPut(index, terms, i);
}
indexMultiTermDestroy(terms);
}
{
std::string colName("voltage");
std::string colVal("ab1");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 100; i++) {
tIndexJsonPut(index, terms, i);
}
indexMultiTermDestroy(terms);
}
{
std::string colName("voltage");
std::string colVal("123");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 100; i++) {
tIndexJsonPut(index, terms, i);
}
indexMultiTermDestroy(terms);
}
{
std::string colName("test");
std::string colVal("ab");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_TERM);
tIndexJsonSearch(index, mq, result);
assert(100 == taosArrayGetSize(result));
indexMultiTermQueryDestroy(mq);
}
}
TEST_F(JsonEnv, testWriteMillonData) {
{
std::string colName("test");
std::string colVal("ab");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 100; i++) {
tIndexJsonPut(index, terms, i);
}
indexMultiTermDestroy(terms);
}
{
std::string colName("voltagefdadfa");
std::string colVal("abxxxxxxxxxxxx");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 1000000; i++) {
tIndexJsonPut(index, terms, i);
}
indexMultiTermDestroy(terms);
}
{
std::string colName("test");
std::string colVal("ab");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_TERM);
tIndexJsonSearch(index, mq, result);
assert(100 == taosArrayGetSize(result));
indexMultiTermQueryDestroy(mq);
}
}

287
source/libs/monitor/src/monitor.c
View File

@ -23,7 +23,7 @@
static SMonitor tsMonitor = {0};
int32_t monInit(const SMonCfg *pCfg) {
tsMonitor.logs = taosArrayInit(16, sizeof(SMonInfo));
tsMonitor.logs = taosArrayInit(16, sizeof(SMonLogItem));
if (tsMonitor.logs == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@ -44,7 +44,7 @@ void monCleanup() {
void monAddLogItem(SMonLogItem *pItem) {
taosWLockLatch(&tsMonitor.lock);
int32_t size = taosArrayGetSize(tsMonitor.logs);
if (size > tsMonitor.maxLogs) {
if (size >= tsMonitor.maxLogs) {
uInfo("too many logs for monitor");
} else {
taosArrayPush(tsMonitor.logs, pItem);
@ -54,7 +54,10 @@ void monAddLogItem(SMonLogItem *pItem) {
SMonInfo *monCreateMonitorInfo() {
SMonInfo *pMonitor = calloc(1, sizeof(SMonInfo));
if (pMonitor == NULL) return NULL;
if (pMonitor == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
taosWLockLatch(&tsMonitor.lock);
pMonitor->logs = taosArrayDup(tsMonitor.logs);
@ -77,41 +80,257 @@ void monCleanupMonitorInfo(SMonInfo *pMonitor) {
free(pMonitor);
}
void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo) {
SJson *pJson = pMonitor->pJson;
int64_t ms = taosGetTimestampMs();
char buf[40] = {0};
taosFormatUtcTime(buf, sizeof(buf), ms, TSDB_TIME_PRECISION_MILLI);
tjsonAddStringToObject(pJson, "ts", buf);
tjsonAddDoubleToObject(pJson, "dnode_id", pInfo->dnode_id);
tjsonAddStringToObject(pJson, "dnode_ep", pInfo->dnode_ep);
}
void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) {
SJson *pJson = tjsonCreateObject();
if (pJson == NULL) return;
if (tjsonAddItemToObject(pMonitor->pJson, "cluster_info", pJson) != 0) {
tjsonDelete(pJson);
return;
}
tjsonAddStringToObject(pJson, "first_ep", pInfo->first_ep);
tjsonAddDoubleToObject(pJson, "first_ep_dnode_id", pInfo->first_ep_dnode_id);
tjsonAddStringToObject(pJson, "version", pInfo->version);
tjsonAddDoubleToObject(pJson, "master_uptime", pInfo->master_uptime);
tjsonAddDoubleToObject(pJson, "monitor_interval", pInfo->monitor_interval);
tjsonAddDoubleToObject(pJson, "vgroups_total", pInfo->vgroups_total);
tjsonAddDoubleToObject(pJson, "vgroups_alive", pInfo->vgroups_alive);
tjsonAddDoubleToObject(pJson, "vnodes_total", pInfo->vnodes_total);
tjsonAddDoubleToObject(pJson, "vnodes_alive", pInfo->vnodes_alive);
tjsonAddDoubleToObject(pJson, "connections_total", pInfo->connections_total);
SJson *pDnodesJson = tjsonAddArrayToObject(pJson, "dnodes");
if (pDnodesJson == NULL) return;
for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) {
SJson *pDnodeJson = tjsonCreateObject();
if (pDnodeJson == NULL) continue;
SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i);
tjsonAddDoubleToObject(pDnodeJson, "dnode_id", pDnodeDesc->dnode_id);
tjsonAddStringToObject(pDnodeJson, "dnode_ep", pDnodeDesc->dnode_ep);
tjsonAddStringToObject(pDnodeJson, "status", pDnodeDesc->status);
if (tjsonAddItemToArray(pDnodesJson, pDnodeJson) != 0) tjsonDelete(pDnodeJson);
}
SJson *pMnodesJson = tjsonAddArrayToObject(pJson, "mnodes");
if (pMnodesJson == NULL) return;
for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) {
SJson *pMnodeJson = tjsonCreateObject();
if (pMnodeJson == NULL) continue;
SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->dnodes, i);
tjsonAddDoubleToObject(pMnodeJson, "mnode_id", pMnodeDesc->mnode_id);
tjsonAddStringToObject(pMnodeJson, "mnode_ep", pMnodeDesc->mnode_ep);
tjsonAddStringToObject(pMnodeJson, "role", pMnodeDesc->role);
if (tjsonAddItemToArray(pMnodesJson, pMnodeJson) != 0) tjsonDelete(pMnodeJson);
}
}
void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo) {
SJson *pJson = tjsonAddArrayToObject(pMonitor->pJson, "vgroup_infos");
if (pJson == NULL) return;
for (int32_t i = 0; i < taosArrayGetSize(pInfo->vgroups); ++i) {
SJson *pVgroupJson = tjsonCreateObject();
if (pVgroupJson == NULL) continue;
if (tjsonAddItemToArray(pJson, pVgroupJson) != 0) {
tjsonDelete(pVgroupJson);
continue;
}
SMonVgroupDesc *pVgroupDesc = taosArrayGet(pInfo->vgroups, i);
tjsonAddDoubleToObject(pVgroupJson, "vgroup_id", pVgroupDesc->vgroup_id);
tjsonAddStringToObject(pVgroupJson, "database_name", pVgroupDesc->database_name);
tjsonAddDoubleToObject(pVgroupJson, "tables_num", pVgroupDesc->tables_num);
tjsonAddStringToObject(pVgroupJson, "status", pVgroupDesc->status);
SJson *pVnodesJson = tjsonAddArrayToObject(pVgroupJson, "vnodes");
if (pVnodesJson == NULL) continue;
for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) {
SMonVnodeDesc *pVnodeDesc = &pVgroupDesc->vnodes[j];
if (pVnodeDesc->dnode_id <= 0) continue;
SJson *pVnodeJson = tjsonCreateObject();
if (pVnodeJson == NULL) continue;
tjsonAddDoubleToObject(pVnodeJson, "dnode_id", pVnodeDesc->dnode_id);
tjsonAddStringToObject(pVnodeJson, "vnode_role", pVnodeDesc->vnode_role);
if (tjsonAddItemToArray(pVnodesJson, pVnodeJson) != 0) tjsonDelete(pVnodeJson);
}
}
}
void monSetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo) {
SJson *pJson = tjsonCreateObject();
if (pJson == NULL) return;
if (tjsonAddItemToObject(pMonitor->pJson, "grant_info", pJson) != 0) {
tjsonDelete(pJson);
return;
}
tjsonAddDoubleToObject(pJson, "expire_time", pInfo->expire_time);
tjsonAddDoubleToObject(pJson, "timeseries_used", pInfo->timeseries_used);
tjsonAddDoubleToObject(pJson, "timeseries_total", pInfo->timeseries_total);
}
void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo) {
SJson *pJson = tjsonCreateObject();
if (pJson == NULL) return;
if (tjsonAddItemToObject(pMonitor->pJson, "dnode_info", pJson) != 0) {
tjsonDelete(pJson);
return;
}
tjsonAddDoubleToObject(pJson, "uptime", pInfo->uptime);
tjsonAddDoubleToObject(pJson, "cpu_engine", pInfo->cpu_engine);
tjsonAddDoubleToObject(pJson, "cpu_system", pInfo->cpu_system);
tjsonAddDoubleToObject(pJson, "cpu_cores", pInfo->cpu_cores);
tjsonAddDoubleToObject(pJson, "mem_engine", pInfo->mem_engine);
tjsonAddDoubleToObject(pJson, "mem_system", pInfo->mem_system);
tjsonAddDoubleToObject(pJson, "mem_total", pInfo->mem_total);
tjsonAddDoubleToObject(pJson, "disk_engine", pInfo->disk_engine);
tjsonAddDoubleToObject(pJson, "disk_used", pInfo->disk_used);
tjsonAddDoubleToObject(pJson, "disk_total", pInfo->disk_total);
tjsonAddDoubleToObject(pJson, "net_in", pInfo->net_in);
tjsonAddDoubleToObject(pJson, "net_out", pInfo->net_out);
tjsonAddDoubleToObject(pJson, "io_read", pInfo->io_read);
tjsonAddDoubleToObject(pJson, "io_write", pInfo->io_write);
tjsonAddDoubleToObject(pJson, "io_read_disk", pInfo->io_read_disk);
tjsonAddDoubleToObject(pJson, "io_write_disk", pInfo->io_write_disk);
tjsonAddDoubleToObject(pJson, "req_select", pInfo->req_select);
tjsonAddDoubleToObject(pJson, "req_select_rate", pInfo->req_select_rate);
tjsonAddDoubleToObject(pJson, "req_insert", pInfo->req_insert);
tjsonAddDoubleToObject(pJson, "req_insert_success", pInfo->req_insert_success);
tjsonAddDoubleToObject(pJson, "req_insert_rate", pInfo->req_insert_rate);
tjsonAddDoubleToObject(pJson, "req_insert_batch", pInfo->req_insert_batch);
tjsonAddDoubleToObject(pJson, "req_insert_batch_success", pInfo->req_insert_batch_success);
tjsonAddDoubleToObject(pJson, "req_insert_batch_rate", pInfo->req_insert_batch_rate);
tjsonAddDoubleToObject(pJson, "errors", pInfo->errors);
tjsonAddDoubleToObject(pJson, "vnodes_num", pInfo->vnodes_num);
tjsonAddDoubleToObject(pJson, "masters", pInfo->masters);
tjsonAddDoubleToObject(pJson, "has_mnode", pInfo->has_mnode);
}
void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) {
SJson *pJson = tjsonCreateObject();
if (pJson == NULL) return;
if (tjsonAddItemToObject(pMonitor->pJson, "disk_infos", pJson) != 0) {
tjsonDelete(pJson);
return;
}
SJson *pDatadirsJson = tjsonAddArrayToObject(pJson, "datadir");
if (pDatadirsJson == NULL) return;
for (int32_t i = 0; i < taosArrayGetSize(pInfo->datadirs); ++i) {
SJson *pDatadirJson = tjsonCreateObject();
if (pDatadirJson == NULL) continue;
SMonDiskDesc *pDatadirDesc = taosArrayGet(pInfo->datadirs, i);
if (tjsonAddStringToObject(pDatadirJson, "name", pDatadirDesc->name) != 0) tjsonDelete(pDatadirJson);
if (tjsonAddDoubleToObject(pDatadirJson, "level", pDatadirDesc->level) != 0) tjsonDelete(pDatadirJson);
if (tjsonAddDoubleToObject(pDatadirJson, "avail", pDatadirDesc->size.avail) != 0) tjsonDelete(pDatadirJson);
if (tjsonAddDoubleToObject(pDatadirJson, "used", pDatadirDesc->size.used) != 0) tjsonDelete(pDatadirJson);
if (tjsonAddDoubleToObject(pDatadirJson, "total", pDatadirDesc->size.total) != 0) tjsonDelete(pDatadirJson);
if (tjsonAddItemToArray(pDatadirsJson, pDatadirJson) != 0) tjsonDelete(pDatadirJson);
}
SJson *pLogdirJson = tjsonCreateObject();
if (pLogdirJson == NULL) return;
if (tjsonAddItemToObject(pJson, "logdir", pLogdirJson) != 0) return;
tjsonAddStringToObject(pLogdirJson, "name", pInfo->logdir.name);
tjsonAddDoubleToObject(pLogdirJson, "avail", pInfo->logdir.size.avail);
tjsonAddDoubleToObject(pLogdirJson, "used", pInfo->logdir.size.used);
tjsonAddDoubleToObject(pLogdirJson, "total", pInfo->logdir.size.total);
SJson *pTempdirJson = tjsonCreateObject();
if (pTempdirJson == NULL) return;
if (tjsonAddItemToObject(pJson, "tempdir", pTempdirJson) != 0) return;
tjsonAddStringToObject(pTempdirJson, "name", pInfo->tempdir.name);
tjsonAddDoubleToObject(pTempdirJson, "avail", pInfo->tempdir.size.avail);
tjsonAddDoubleToObject(pTempdirJson, "used", pInfo->tempdir.size.used);
tjsonAddDoubleToObject(pTempdirJson, "total", pInfo->tempdir.size.total);
}
static void monSetLogInfo(SMonInfo *pMonitor) {
SJson *pJson = tjsonCreateObject();
if (pJson == NULL) return;
if (tjsonAddItemToObject(pMonitor->pJson, "log_infos", pJson) != 0) {
tjsonDelete(pJson);
return;
}
SJson *pLogsJson = tjsonAddArrayToObject(pJson, "logs");
if (pLogsJson == NULL) return;
for (int32_t i = 0; i < taosArrayGetSize(pMonitor->logs); ++i) {
SJson *pLogJson = tjsonCreateObject();
if (pLogJson == NULL) continue;
SMonLogItem *pLogItem = taosArrayGet(pMonitor->logs, i);
char buf[40] = {0};
taosFormatUtcTime(buf, sizeof(buf), pLogItem->ts, TSDB_TIME_PRECISION_MILLI);
tjsonAddStringToObject(pLogJson, "ts", buf);
tjsonAddDoubleToObject(pLogJson, "level", pLogItem->level);
tjsonAddStringToObject(pLogJson, "content", pLogItem->content);
if (tjsonAddItemToArray(pLogsJson, pLogJson) != 0) tjsonDelete(pLogJson);
}
SJson *pSummaryJson = tjsonAddArrayToObject(pJson, "summary");
if (pSummaryJson == NULL) return;
SJson *pLogError = tjsonCreateObject();
if (pLogError == NULL) return;
tjsonAddStringToObject(pLogError, "level", "error");
tjsonAddDoubleToObject(pLogError, "total", 1);
if (tjsonAddItemToArray(pSummaryJson, pLogError) != 0) tjsonDelete(pLogError);
SJson *pLogInfo = tjsonCreateObject();
if (pLogInfo == NULL) return;
tjsonAddStringToObject(pLogInfo, "level", "info");
tjsonAddDoubleToObject(pLogInfo, "total", 1);
if (tjsonAddItemToArray(pSummaryJson, pLogInfo) != 0) tjsonDelete(pLogInfo);
SJson *pLogDebug = tjsonCreateObject();
if (pLogDebug == NULL) return;
tjsonAddStringToObject(pLogDebug, "level", "debug");
tjsonAddDoubleToObject(pLogDebug, "total", 1);
if (tjsonAddItemToArray(pSummaryJson, pLogDebug) != 0) tjsonDelete(pLogDebug);
SJson *pLogTrace = tjsonCreateObject();
if (pLogTrace == NULL) return;
tjsonAddStringToObject(pLogTrace, "level", "trace");
tjsonAddDoubleToObject(pLogTrace, "total", 1);
if (tjsonAddItemToArray(pSummaryJson, pLogTrace) != 0) tjsonDelete(pLogTrace);
}
void monSendReport(SMonInfo *pMonitor) {
monSetLogInfo(pMonitor);
char *pCont = tjsonToString(pMonitor->pJson);
if (pCont != NULL) {
taosSendHttpReport(tsMonitor.server, tsMonitor.port, pCont, strlen(pCont));
free(pCont);
}
}
void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo) {
SJson *pJson = pMonitor->pJson;
tjsonAddDoubleToObject(pJson, "dnode_id", pInfo->dnode_id);
tjsonAddStringToObject(pJson, "dnode_ep", pInfo->dnode_ep);
int64_t ms = taosGetTimestampMs();
char buf[40] = {0};
taosFormatUtcTime(buf, sizeof(buf), ms, TSDB_TIME_PRECISION_MILLI);
tjsonAddStringToObject(pJson, "ts", buf);
}
void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) {
}
void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo) {
}
void monSetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo) {
}
void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo) {
}
void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) {
}

255
source/libs/monitor/test/monTest.cpp
View File

@ -13,21 +13,262 @@
#include "os.h"
#include "monitor.h"
#include "tglobal.h"
class MonitorTest : public ::testing::Test {
protected:
static void SetUpTestSuite() { root = "/tmp/monTest"; }
static void TearDownTestSuite() {}
static void SetUpTestSuite() {
SMonCfg cfg;
cfg.maxLogs = 2;
cfg.port = 80;
cfg.server = "localhost";
monInit(&cfg);
}
static void TearDownTestSuite() { monCleanup(); }
public:
void SetUp() override {}
void TearDown() override {}
static const char *root;
void GetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo);
void GetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo);
void GetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo);
void GetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo);
void GetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo);
void GetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo);
void AddLogInfo1();
void AddLogInfo2();
};
const char *MonitorTest::root;
TEST_F(MonitorTest, 01_Open_Close) {
void MonitorTest::GetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo) {
pInfo->dnode_id = 1;
strcpy(pInfo->dnode_ep, "localhost");
}
void MonitorTest::GetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) {
strcpy(pInfo->first_ep, "localhost:6030");
pInfo->first_ep_dnode_id = 1;
strcpy(pInfo->version, "3.0.0.0");
pInfo->master_uptime = 1;
pInfo->monitor_interval = 2;
pInfo->vgroups_total = 3;
pInfo->vgroups_alive = 43;
pInfo->vnodes_total = 5;
pInfo->vnodes_alive = 6;
pInfo->connections_total = 7;
pInfo->dnodes = taosArrayInit(4, sizeof(SMonDnodeDesc));
SMonDnodeDesc d1 = {0};
d1.dnode_id = 1;
strcpy(d1.dnode_ep, "localhost:6030");
strcpy(d1.status, "ready");
taosArrayPush(pInfo->dnodes, &d1);
SMonDnodeDesc d2 = {0};
d2.dnode_id = 2;
strcpy(d2.dnode_ep, "localhost:7030");
strcpy(d2.status, "offline");
taosArrayPush(pInfo->dnodes, &d2);
pInfo->mnodes = taosArrayInit(4, sizeof(SMonMnodeDesc));
SMonMnodeDesc m1 = {0};
m1.mnode_id = 1;
strcpy(m1.mnode_ep, "localhost:6030");
strcpy(m1.role, "master");
taosArrayPush(pInfo->mnodes, &m1);
SMonMnodeDesc m2 = {0};
m2.mnode_id = 2;
strcpy(m2.mnode_ep, "localhost:7030");
strcpy(m2.role, "unsynced");
taosArrayPush(pInfo->mnodes, &m2);
}
void MonitorTest::GetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo) {
pInfo->vgroups = taosArrayInit(4, sizeof(SMonVgroupDesc));
SMonVgroupDesc vg1 = {0};
vg1.vgroup_id = 1;
strcpy(vg1.database_name, "d1");
vg1.tables_num = 4;
strcpy(vg1.status, "ready");
vg1.vnodes[0].dnode_id = 1;
strcpy(vg1.vnodes[0].vnode_role, "master");
vg1.vnodes[1].dnode_id = 2;
strcpy(vg1.vnodes[1].vnode_role, "slave");
taosArrayPush(pInfo->vgroups, &vg1);
SMonVgroupDesc vg2 = {0};
vg2.vgroup_id = 2;
strcpy(vg2.database_name, "d2");
vg2.tables_num = 5;
strcpy(vg2.status, "offline");
vg2.vnodes[0].dnode_id = 1;
strcpy(vg2.vnodes[0].vnode_role, "master");
vg2.vnodes[1].dnode_id = 2;
strcpy(vg2.vnodes[1].vnode_role, "unsynced");
taosArrayPush(pInfo->vgroups, &vg2);
SMonVgroupDesc vg3 = {0};
vg3.vgroup_id = 3;
strcpy(vg3.database_name, "d3");
vg3.tables_num = 6;
strcpy(vg3.status, "ready");
vg3.vnodes[0].dnode_id = 1;
strcpy(vg3.vnodes[0].vnode_role, "master");
taosArrayPush(pInfo->vgroups, &vg3);
}
void MonitorTest::GetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo) {
pInfo->expire_time = 1234567;
pInfo->timeseries_total = 234567;
pInfo->timeseries_used = 34567;
}
void MonitorTest::GetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo) {
pInfo->uptime = 1.2;
pInfo->cpu_engine = 2.1;
pInfo->cpu_system = 2.1;
pInfo->cpu_cores = 2;
pInfo->mem_engine = 3.1;
pInfo->mem_system = 3.2;
pInfo->mem_total = 3.3;
pInfo->disk_engine = 4.1;
pInfo->disk_used = 4.2;
pInfo->disk_total = 4.3;
pInfo->net_in = 5.1;
pInfo->net_out = 5.2;
pInfo->io_read = 6.1;
pInfo->io_write = 6.2;
pInfo->io_read_disk = 7.1;
pInfo->io_write_disk = 7.2;
pInfo->req_select = 8;
pInfo->req_select_rate = 8.1;
pInfo->req_insert = 9;
pInfo->req_insert_success = 10;
pInfo->req_insert_rate = 10.1;
pInfo->req_insert_batch = 11;
pInfo->req_insert_batch_success = 12;
pInfo->req_insert_batch_rate = 12.3;
pInfo->errors = 4;
pInfo->vnodes_num = 5;
pInfo->masters = 6;
pInfo->has_mnode = 1;
}
void MonitorTest::GetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) {
pInfo->datadirs = taosArrayInit(2, sizeof(SMonDiskDesc));
SMonDiskDesc d1 = {0};
strcpy(d1.name, "/t1/d1/d");
d1.level = 0;
d1.size.avail = 11;
d1.size.total = 12;
d1.size.used = 13;
taosArrayPush(pInfo->datadirs, &d1);
SMonDiskDesc d2 = {0};
strcpy(d2.name, "/t2d2/d");
d2.level = 2;
d2.size.avail = 21;
d2.size.total = 22;
d2.size.used = 23;
taosArrayPush(pInfo->datadirs, &d2);
SMonDiskDesc d3 = {0};
strcpy(d3.name, "/t3/d3/d");
d3.level = 3;
d3.size.avail = 31;
d3.size.total = 32;
d3.size.used = 33;
taosArrayPush(pInfo->datadirs, &d3);
strcpy(pInfo->logdir.name, "/log/dir/d");
pInfo->logdir.size.avail = 41;
pInfo->logdir.size.total = 42;
pInfo->logdir.size.used = 43;
strcpy(pInfo->tempdir.name, "/data/dir/d");
pInfo->tempdir.size.avail = 51;
pInfo->tempdir.size.total = 52;
pInfo->tempdir.size.used = 53;
}
void MonitorTest::AddLogInfo1() {
SMonLogItem log1 = {0};
log1.ts = taosGetTimestampMs();
log1.level = 1;
strcpy(log1.content, "1 -------------------------- a");
monAddLogItem(&log1);
SMonLogItem log2 = {0};
log2.ts = taosGetTimestampMs();
log2.level = 1;
strcpy(log2.content, "1 ------------------------ b");
monAddLogItem(&log2);
SMonLogItem log3 = {0};
log3.ts = taosGetTimestampMs();
log3.level = 1;
strcpy(log3.content, "1 ------- c");
monAddLogItem(&log3);
}
void MonitorTest::AddLogInfo2() {
SMonLogItem log1;
log1.ts = taosGetTimestampMs();
log1.level = 01;
strcpy(log1.content, "2 ------- a");
monAddLogItem(&log1);
SMonLogItem log2;
log2.ts = taosGetTimestampMs();
log2.level = 0;
strcpy(log2.content, "2 ------- b");
monAddLogItem(&log2);
}
TEST_F(MonitorTest, 01_Full) {
AddLogInfo1();
SMonInfo *pMonitor = monCreateMonitorInfo();
if (pMonitor == NULL) return;
SMonBasicInfo basicInfo = {0};
GetBasicInfo(pMonitor, &basicInfo);
monSetBasicInfo(pMonitor, &basicInfo);
SMonClusterInfo clusterInfo = {0};
SMonVgroupInfo vgroupInfo = {0};
SMonGrantInfo grantInfo = {0};
GetClusterInfo(pMonitor, &clusterInfo);
GetVgroupInfo(pMonitor, &vgroupInfo);
GetGrantInfo(pMonitor, &grantInfo);
monSetClusterInfo(pMonitor, &clusterInfo);
monSetVgroupInfo(pMonitor, &vgroupInfo);
monSetGrantInfo(pMonitor, &grantInfo);
SMonDnodeInfo dnodeInfo = {0};
GetDnodeInfo(pMonitor, &dnodeInfo);
monSetDnodeInfo(pMonitor, &dnodeInfo);
SMonDiskInfo diskInfo = {0};
GetDiskInfo(pMonitor, &diskInfo);
monSetDiskInfo(pMonitor, &diskInfo);
monSendReport(pMonitor);
monCleanupMonitorInfo(pMonitor);
taosArrayDestroy(clusterInfo.dnodes);
taosArrayDestroy(clusterInfo.mnodes);
taosArrayDestroy(vgroupInfo.vgroups);
taosArrayDestroy(diskInfo.datadirs);
}
TEST_F(MonitorTest, 02_Log) {
AddLogInfo2();
SMonInfo *pMonitor = monCreateMonitorInfo();
if (pMonitor == NULL) return;
monSendReport(pMonitor);
monCleanupMonitorInfo(pMonitor);
}

32
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "plannodes.h"
#include "querynodes.h"
#include "taos.h"
#include "taoserror.h"
@ -157,6 +158,29 @@ static SNode* groupingSetNodeCopy(const SGroupingSetNode* pSrc, SGroupingSetNode
return (SNode*)pDst;
}
static SNode* logicSubplanCopy(const SSubLogicPlan* pSrc, SSubLogicPlan* pDst) {
COPY_NODE_FIELD(pNode);
COPY_SCALAR_FIELD(subplanType);
return (SNode*)pDst;
}
static SNode* dataBlockDescCopy(const SDataBlockDescNode* pSrc, SDataBlockDescNode* pDst) {
COPY_SCALAR_FIELD(dataBlockId);
COPY_NODE_LIST_FIELD(pSlots);
COPY_SCALAR_FIELD(resultRowSize);
COPY_SCALAR_FIELD(precision);
return (SNode*)pDst;
}
static SNode* slotDescCopy(const SSlotDescNode* pSrc, SSlotDescNode* pDst) {
COPY_SCALAR_FIELD(slotId);
dataTypeCopy(&pSrc->dataType, &pDst->dataType);
COPY_SCALAR_FIELD(reserve);
COPY_SCALAR_FIELD(output);
COPY_SCALAR_FIELD(tag);
return (SNode*)pDst;
}
SNodeptr nodesCloneNode(const SNodeptr pNode) {
if (NULL == pNode) {
return NULL;
@ -187,9 +211,17 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) {
return groupingSetNodeCopy((const SGroupingSetNode*)pNode, (SGroupingSetNode*)pDst);
case QUERY_NODE_ORDER_BY_EXPR:
case QUERY_NODE_LIMIT:
break;
case QUERY_NODE_DATABLOCK_DESC:
return dataBlockDescCopy((const SDataBlockDescNode*)pNode, (SDataBlockDescNode*)pDst);
case QUERY_NODE_SLOT_DESC:
return slotDescCopy((const SSlotDescNode*)pNode, (SSlotDescNode*)pDst);
case QUERY_NODE_LOGIC_SUBPLAN:
return logicSubplanCopy((const SSubLogicPlan*)pNode, (SSubLogicPlan*)pDst);
default:
break;
}
printf("nodesCloneNode unknown node = %s\n", nodesNodeName(nodeType(pNode)));
return pDst;
}

251
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -24,7 +24,7 @@ static int32_t jsonToNode(const SJson* pJson, void* pObj);
static int32_t jsonToNodeObject(const SJson* pJson, const char* pName, SNode** pNode);
static int32_t makeNodeByJson(const SJson* pJson, SNode** pNode);
static char* nodeName(ENodeType type) {
const char* nodesNodeName(ENodeType type) {
switch (type) {
case QUERY_NODE_COLUMN:
return "Column";
@ -58,20 +58,32 @@ static char* nodeName(ENodeType type) {
return "NodeList";
case QUERY_NODE_FILL:
return "Fill";
case QUERY_NODE_TARGET:
return "Target";
case QUERY_NODE_RAW_EXPR:
return "RawExpr";
case QUERY_NODE_TARGET:
return "Target";
case QUERY_NODE_DATABLOCK_DESC:
return "TupleDesc";
case QUERY_NODE_SLOT_DESC:
return "SlotDesc";
case QUERY_NODE_COLUMN_DEF:
return "ColumnDef";
case QUERY_NODE_SET_OPERATOR:
return "SetOperator";
case QUERY_NODE_SELECT_STMT:
return "SelectStmt";
case QUERY_NODE_SHOW_STMT:
return "ShowStmt";
case QUERY_NODE_VNODE_MODIF_STMT:
return "VnodeModifStmt";
case QUERY_NODE_CREATE_DATABASE_STMT:
return "CreateDatabaseStmt";
case QUERY_NODE_CREATE_TABLE_STMT:
return "CreateTableStmt";
case QUERY_NODE_USE_DATABASE_STMT:
return "UseDatabaseStmt";
case QUERY_NODE_SHOW_DATABASES_STMT:
return "ShowDatabaseStmt";
case QUERY_NODE_SHOW_TABLES_STMT:
return "ShowTablesStmt";
case QUERY_NODE_LOGIC_PLAN_SCAN:
return "LogicScan";
case QUERY_NODE_LOGIC_PLAN_JOIN:
@ -80,16 +92,34 @@ static char* nodeName(ENodeType type) {
return "LogicAgg";
case QUERY_NODE_LOGIC_PLAN_PROJECT:
return "LogicProject";
case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF:
return "LogicVnodeModif";
case QUERY_NODE_LOGIC_SUBPLAN:
return "LogicSubplan";
case QUERY_NODE_LOGIC_PLAN:
return "LogicPlan";
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
return "PhysiTagScan";
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
return "PhysiTableScan";
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
return "PhysiTableSeqScan";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
return "PhysiSreamScan";
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return "PhysiProject";
case QUERY_NODE_PHYSICAL_PLAN_JOIN:
return "PhysiJoin";
case QUERY_NODE_PHYSICAL_PLAN_AGG:
return "PhysiAgg";
case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:
return "PhysiExchange";
case QUERY_NODE_PHYSICAL_PLAN_SORT:
return "PhysiSort";
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return "PhysiDispatch";
case QUERY_NODE_PHYSICAL_PLAN_INSERT:
return "PhysiInsert";
case QUERY_NODE_PHYSICAL_SUBPLAN:
return "PhysiSubplan";
case QUERY_NODE_PHYSICAL_PLAN:
@ -234,7 +264,7 @@ static const char* jkPhysiPlanChildren = "Children";
static int32_t physicPlanNodeToJson(const void* pObj, SJson* pJson) {
const SPhysiNode* pNode = (const SPhysiNode*)pObj;
int32_t code = tjsonAddObject(pJson, jkPhysiPlanOutputDataBlockDesc, nodeToJson, &pNode->outputDataBlockDesc);
int32_t code = tjsonAddObject(pJson, jkPhysiPlanOutputDataBlockDesc, nodeToJson, pNode->pOutputDataBlockDesc);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkPhysiPlanConditions, nodeToJson, pNode->pConditions);
}
@ -248,7 +278,7 @@ static int32_t physicPlanNodeToJson(const void* pObj, SJson* pJson) {
static int32_t jsonToPhysicPlanNode(const SJson* pJson, void* pObj) {
SPhysiNode* pNode = (SPhysiNode*)pObj;
int32_t code = tjsonToObject(pJson, jkPhysiPlanOutputDataBlockDesc, jsonToNode, &pNode->outputDataBlockDesc);
int32_t code = jsonToNodeObject(pJson, jkPhysiPlanOutputDataBlockDesc, (SNode**)&pNode->pOutputDataBlockDesc);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkPhysiPlanConditions, &pNode->pConditions);
}
@ -466,6 +496,26 @@ static int32_t jsonToPhysiAggNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkDataSinkInputDataBlockDesc = "InputDataBlockDesc";
static int32_t physicDataSinkNodeToJson(const void* pObj, SJson* pJson) {
const SDataSinkNode* pNode = (const SDataSinkNode*)pObj;
return tjsonAddObject(pJson, jkDataSinkInputDataBlockDesc, nodeToJson, pNode->pInputDataBlockDesc);
}
static int32_t jsonToPhysicDataSinkNode(const SJson* pJson, void* pObj) {
SDataSinkNode* pNode = (SDataSinkNode*)pObj;
return jsonToNodeObject(pJson, jkDataSinkInputDataBlockDesc, (SNode**)&pNode->pInputDataBlockDesc);
}
static int32_t physiDispatchNodeToJson(const void* pObj, SJson* pJson) {
return physicDataSinkNodeToJson(pObj, pJson);
}
static int32_t jsonToPhysiDispatchNode(const SJson* pJson, void* pObj) {
return jsonToPhysicDataSinkNode(pJson, pObj);
}
static const char* jkSubplanIdQueryId = "QueryId";
static const char* jkSubplanIdTemplateId = "TemplateId";
static const char* jkSubplanIdSubplanId = "SubplanId";
@ -833,41 +883,43 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkValueDuration, pNode->isDuration);
}
switch (pNode->node.resType.type) {
case TSDB_DATA_TYPE_NULL:
break;
case TSDB_DATA_TYPE_BOOL:
code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->datum.b);
break;
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->datum.i);
break;
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_UBIGINT:
code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->datum.u);
break;
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
code = tjsonAddDoubleToObject(pJson, jkValueDuration, pNode->datum.d);
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY:
code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->datum.p);
break;
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_BLOB:
// todo
default:
break;
if (TSDB_CODE_SUCCESS == code) {
switch (pNode->node.resType.type) {
case TSDB_DATA_TYPE_NULL:
break;
case TSDB_DATA_TYPE_BOOL:
code = tjsonAddIntegerToObject(pJson, jkValueDatum, pNode->datum.b);
break;
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
code = tjsonAddIntegerToObject(pJson, jkValueDatum, pNode->datum.i);
break;
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_UBIGINT:
code = tjsonAddIntegerToObject(pJson, jkValueDatum, pNode->datum.u);
break;
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
code = tjsonAddDoubleToObject(pJson, jkValueDatum, pNode->datum.d);
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY:
code = tjsonAddStringToObject(pJson, jkValueDatum, pNode->datum.p);
break;
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_BLOB:
// todo
default:
break;
}
}
return code;
@ -883,41 +935,43 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkValueDuration, &pNode->isDuration);
}
switch (pNode->node.resType.type) {
case TSDB_DATA_TYPE_NULL:
break;
case TSDB_DATA_TYPE_BOOL:
code = tjsonGetBoolValue(pJson, jkValueDuration, &pNode->datum.b);
break;
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
code = tjsonGetBigIntValue(pJson, jkValueDuration, &pNode->datum.i);
break;
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_UBIGINT:
code = tjsonGetUBigIntValue(pJson, jkValueDuration, &pNode->datum.u);
break;
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
code = tjsonGetDoubleValue(pJson, jkValueDuration, &pNode->datum.d);
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY:
code = tjsonDupStringValue(pJson, jkValueLiteral, &pNode->datum.p);
break;
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_BLOB:
// todo
default:
break;
if (TSDB_CODE_SUCCESS == code) {
switch (pNode->node.resType.type) {
case TSDB_DATA_TYPE_NULL:
break;
case TSDB_DATA_TYPE_BOOL:
code = tjsonGetBoolValue(pJson, jkValueDatum, &pNode->datum.b);
break;
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
code = tjsonGetBigIntValue(pJson, jkValueDatum, &pNode->datum.i);
break;
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_UBIGINT:
code = tjsonGetUBigIntValue(pJson, jkValueDatum, &pNode->datum.u);
break;
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
code = tjsonGetDoubleValue(pJson, jkValueDatum, &pNode->datum.d);
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY:
code = tjsonDupStringValue(pJson, jkValueDatum, &pNode->datum.p);
break;
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_BLOB:
// todo
default:
break;
}
}
return code;
@ -1249,23 +1303,29 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_STATE_WINDOW:
case QUERY_NODE_SESSION_WINDOW:
case QUERY_NODE_INTERVAL_WINDOW:
break;
case QUERY_NODE_NODE_LIST:
return nodeListNodeToJson(pObj, pJson);
case QUERY_NODE_FILL:
case QUERY_NODE_RAW_EXPR:
break;
case QUERY_NODE_TARGET:
return targetNodeToJson(pObj, pJson);
case QUERY_NODE_RAW_EXPR:
break;
case QUERY_NODE_DATABLOCK_DESC:
return dataBlockDescNodeToJson(pObj, pJson);
case QUERY_NODE_SLOT_DESC:
return slotDescNodeToJson(pObj, pJson);
case QUERY_NODE_COLUMN_DEF:
case QUERY_NODE_SET_OPERATOR:
break;
case QUERY_NODE_SELECT_STMT:
return selectStmtTojson(pObj, pJson);
case QUERY_NODE_SHOW_STMT:
case QUERY_NODE_VNODE_MODIF_STMT:
case QUERY_NODE_CREATE_DATABASE_STMT:
case QUERY_NODE_CREATE_TABLE_STMT:
case QUERY_NODE_USE_DATABASE_STMT:
case QUERY_NODE_SHOW_DATABASES_STMT:
case QUERY_NODE_SHOW_TABLES_STMT:
break;
case QUERY_NODE_LOGIC_PLAN_SCAN:
return logicScanNodeToJson(pObj, pJson);
@ -1275,16 +1335,29 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return logicAggNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_PROJECT:
return logicProjectNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF:
case QUERY_NODE_LOGIC_SUBPLAN:
case QUERY_NODE_LOGIC_PLAN:
break;
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
return physiTagScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
return physiTableScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
break;
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return physiProjectNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_JOIN:
return physiJoinNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_AGG:
return physiAggNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:
case QUERY_NODE_PHYSICAL_PLAN_SORT:
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return physiDispatchNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_INSERT:
break;
case QUERY_NODE_PHYSICAL_SUBPLAN:
return subplanToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN:
@ -1292,7 +1365,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
default:
break;
}
printf("================================ specificNodeToJson unknown node = %s\n", nodeName(nodeType(pObj)));
printf("================================ specificNodeToJson unknown node = %s\n", nodesNodeName(nodeType(pObj)));
return TSDB_CODE_SUCCESS;
}
@ -1334,8 +1407,6 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
// break;
// case QUERY_NODE_SELECT_STMT:
// return jsonToSelectStmt(pJson, pObj);
// case QUERY_NODE_SHOW_STMT:
// break;
// case QUERY_NODE_LOGIC_PLAN_SCAN:
// return jsonToLogicScanNode(pJson, pObj);
// case QUERY_NODE_LOGIC_PLAN_JOIN:
@ -1354,6 +1425,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToPhysiJoinNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_AGG:
return jsonToPhysiAggNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return jsonToPhysiDispatchNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_SUBPLAN:
return jsonToSubplan(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN:
@ -1361,6 +1434,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
default:
break;
}
printf("================================ jsonToSpecificNode unknown node = %s\n", nodesNodeName(nodeType(pObj)));
return TSDB_CODE_SUCCESS;
}
@ -1372,10 +1446,10 @@ static int32_t nodeToJson(const void* pObj, SJson* pJson) {
int32_t code = tjsonAddIntegerToObject(pJson, jkNodeType, pNode->type);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkNodeName, nodeName(pNode->type));
code = tjsonAddStringToObject(pJson, jkNodeName, nodesNodeName(pNode->type));
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, nodeName(pNode->type), specificNodeToJson, pNode);
code = tjsonAddObject(pJson, nodesNodeName(pNode->type), specificNodeToJson, pNode);
}
return code;
@ -1388,7 +1462,10 @@ static int32_t jsonToNode(const SJson* pJson, void* pObj) {
int32_t code = tjsonGetIntValue(pJson, jkNodeType, &val);
pNode->type = val;
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, nodeName(pNode->type), jsonToSpecificNode, pNode);
code = tjsonToObject(pJson, nodesNodeName(pNode->type), jsonToSpecificNode, pNode);
if (TSDB_CODE_SUCCESS != code) {
printf("%s toNode error\n", nodesNodeName(pNode->type));
}
}
return code;
@ -1411,7 +1488,7 @@ static int32_t makeNodeByJson(const SJson* pJson, SNode** pNode) {
static int32_t jsonToNodeObject(const SJson* pJson, const char* pName, SNode** pNode) {
SJson* pJsonNode = tjsonGetObjectItem(pJson, pName);
if (NULL == pJsonNode) {
return TSDB_CODE_FAILED;
return TSDB_CODE_SUCCESS;
}
return makeNodeByJson(pJsonNode, pNode);
}

54
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "cmdnodes.h"
#include "querynodes.h"
#include "plannodes.h"
#include "taos.h"
@ -65,12 +66,29 @@ SNodeptr nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SFillNode));
case QUERY_NODE_RAW_EXPR:
return makeNode(type, sizeof(SRawExprNode));
case QUERY_NODE_TARGET:
return makeNode(type, sizeof(STargetNode));
case QUERY_NODE_DATABLOCK_DESC:
return makeNode(type, sizeof(SDataBlockDescNode));
case QUERY_NODE_SLOT_DESC:
return makeNode(type, sizeof(SSlotDescNode));
case QUERY_NODE_COLUMN_DEF:
return makeNode(type, sizeof(SColumnDefNode));
case QUERY_NODE_SET_OPERATOR:
return makeNode(type, sizeof(SSetOperator));
case QUERY_NODE_SELECT_STMT:
return makeNode(type, sizeof(SSelectStmt));
// case QUERY_NODE_SHOW_STMT:
// return makeNode(type, sizeof(SShowStmt));
case QUERY_NODE_VNODE_MODIF_STMT:
return makeNode(type, sizeof(SVnodeModifOpStmt));
case QUERY_NODE_CREATE_DATABASE_STMT:
return makeNode(type, sizeof(SCreateDatabaseStmt));
case QUERY_NODE_CREATE_TABLE_STMT:
return makeNode(type, sizeof(SCreateTableStmt));
case QUERY_NODE_USE_DATABASE_STMT:
return makeNode(type, sizeof(SUseDatabaseStmt));
case QUERY_NODE_SHOW_DATABASES_STMT:
case QUERY_NODE_SHOW_TABLES_STMT:
return makeNode(type, sizeof(SNode));;
case QUERY_NODE_LOGIC_PLAN_SCAN:
return makeNode(type, sizeof(SScanLogicNode));
case QUERY_NODE_LOGIC_PLAN_JOIN:
@ -79,26 +97,34 @@ SNodeptr nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SAggLogicNode));
case QUERY_NODE_LOGIC_PLAN_PROJECT:
return makeNode(type, sizeof(SProjectLogicNode));
case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF:
return makeNode(type, sizeof(SVnodeModifLogicNode));
case QUERY_NODE_LOGIC_SUBPLAN:
return makeNode(type, sizeof(SSubLogicPlan));
case QUERY_NODE_LOGIC_PLAN:
return makeNode(type, sizeof(SQueryLogicPlan));
case QUERY_NODE_TARGET:
return makeNode(type, sizeof(STargetNode));
case QUERY_NODE_DATABLOCK_DESC:
return makeNode(type, sizeof(SDataBlockDescNode));
case QUERY_NODE_SLOT_DESC:
return makeNode(type, sizeof(SSlotDescNode));
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
return makeNode(type, sizeof(STagScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
return makeNode(type, sizeof(STableScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
return makeNode(type, sizeof(STableSeqScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
return makeNode(type, sizeof(SNode));
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return makeNode(type, sizeof(SProjectPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_JOIN:
return makeNode(type, sizeof(SJoinPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_AGG:
return makeNode(type, sizeof(SAggPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:
return makeNode(type, sizeof(SExchangePhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_SORT:
return makeNode(type, sizeof(SNode));
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return makeNode(type, sizeof(SDataDispatcherNode));
case QUERY_NODE_PHYSICAL_PLAN_INSERT:
return makeNode(type, sizeof(SDataInserterNode));
case QUERY_NODE_PHYSICAL_SUBPLAN:
return makeNode(type, sizeof(SSubplan));
case QUERY_NODE_PHYSICAL_PLAN:
@ -106,6 +132,7 @@ SNodeptr nodesMakeNode(ENodeType type) {
default:
break;
}
printf("================================ nodesMakeNode unknown node = %s\n", nodesNodeName(type));
return NULL;
}
@ -176,6 +203,17 @@ int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode) {
return TSDB_CODE_SUCCESS;
}
int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode) {
if (NULL == pNode) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = nodesListAppend(pList, pNode);
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode(pNode);
}
return code;
}
int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) {
if (NULL == pTarget || NULL == pSrc) {
return TSDB_CODE_SUCCESS;

6
source/libs/parser/inc/astCreateFuncs.h
View File

@ -116,8 +116,14 @@ typedef enum ETableOptionType {
STableOptions* createDefaultTableOptions(SAstCreateContext* pCxt);
STableOptions* setTableOption(SAstCreateContext* pCxt, STableOptions* pOptions, ETableOptionType type, const SToken* pVal);
SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment);
SDataType createDataType(uint8_t type);
SDataType createVarLenDataType(uint8_t type, const SToken* pLen);
SNode* createCreateTableStmt(SAstCreateContext* pCxt, bool ignoreExists, const STokenPair* pFullTableName, SNodeList* pCols, STableOptions* pOptions);
SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName);
SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type);
#ifdef __cplusplus
}
#endif

18
source/libs/parser/inc/new_sql.y
View File

@ -92,22 +92,25 @@ db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C).
db_options(A) ::= db_options(B) SINGLESTABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLESTABLE, &C); }
db_options(A) ::= db_options(B) STREAMMODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAMMODE, &C); }
/************************************************ create database *****************************************************/
cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A);}
/************************************************ create table *******************************************************/
cmd ::= CREATE TABLE exists_opt(A) full_table_name(B)
NK_LP column_def_list(C) NK_RP table_options(D). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, &B, C, D);}
%type full_table_name { STokenPair }
%destructor full_table_name { }
full_table_name(A) ::= NK_ID(B). { A = { .first = B, .second = nil_token}; }
full_table_name(A) ::= NK_ID(B) NK_DOT NK_ID(C). { A = { .first = B, .second = C}; }
full_table_name(A) ::= NK_ID(B). { STokenPair t = { .first = nil_token, .second = B }; A = t; }
full_table_name(A) ::= NK_ID(B) NK_DOT NK_ID(C). { STokenPair t = { .first = B, .second = C }; A = t; }
%type column_def_list { SNodeList* }
%destructor column_def_list { nodesDestroyList($$); }
column_def_list(A) ::= column_def(B). { A = createNodeList(pCxt, B); }
column_def_list(A) ::= column_def_list(B) NK_COMMA column_def(C). { A = addNodeToList(pCxt, B, C); }
column_def(A) ::= column_name(B) type_name(C). { A = createColumnDefNode(pCxt, B, C, NULL); }
column_def(A) ::= column_name(B) type_name(C) COMMENT NK_STRING(D). { A = createColumnDefNode(pCxt, B, C, &D); }
column_def(A) ::= column_name(B) type_name(C). { A = createColumnDefNode(pCxt, &B, C, NULL); }
column_def(A) ::= column_name(B) type_name(C) COMMENT NK_STRING(D). { A = createColumnDefNode(pCxt, &B, C, &D); }
%type type_name { SDataType }
%destructor type_name { }
@ -115,6 +118,7 @@ type_name(A) ::= BOOL.
type_name(A) ::= TINYINT. { A = createDataType(TSDB_DATA_TYPE_TINYINT); }
type_name(A) ::= SMALLINT. { A = createDataType(TSDB_DATA_TYPE_SMALLINT); }
type_name(A) ::= INT. { A = createDataType(TSDB_DATA_TYPE_INT); }
type_name(A) ::= INTEGER. { A = createDataType(TSDB_DATA_TYPE_INT); }
type_name(A) ::= BIGINT. { A = createDataType(TSDB_DATA_TYPE_BIGINT); }
type_name(A) ::= FLOAT. { A = createDataType(TSDB_DATA_TYPE_FLOAT); }
type_name(A) ::= DOUBLE. { A = createDataType(TSDB_DATA_TYPE_DOUBLE); }
@ -134,14 +138,16 @@ type_name(A) ::= DECIMAL.
type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); }
type_name(A) ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP. { A = createDataType(TSDB_DATA_TYPE_DECIMAL); }
%type table_options { SDatabaseOptions* }
%type table_options { STableOptions* }
%destructor table_options { tfree($$); }
table_options(A) ::= . { A = createDefaultTableOptions(pCxt);}
table_options(A) ::= table_options(B) COMMENT NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_COMMENT, &C); }
table_options(A) ::= table_options(B) KEEP NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_KEEP, &C); }
table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); }
//cmd ::= SHOW DATABASES. { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); }
/************************************************ show ***************************************************************/
cmd ::= SHOW DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); }
cmd ::= SHOW TABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT); }
/************************************************ select *************************************************************/
cmd ::= query_expression(A). { PARSER_TRACE; pCxt->pRootNode = A; }

61
source/libs/parser/src/astCreateFuncs.c
View File

@ -44,7 +44,7 @@ static SDatabaseOptions* setDbBlocks(SAstCreateContext* pCxt, SDatabaseOptions*
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_TOTAL_BLOCKS || val > TSDB_MAX_TOTAL_BLOCKS) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option totalBlocks: %" PRId64 " valid range: [%d, %d]", val, TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS);
"invalid db option totalBlocks: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS);
pCxt->valid = false;
return pOptions;
}
@ -56,7 +56,7 @@ static SDatabaseOptions* setDbCache(SAstCreateContext* pCxt, SDatabaseOptions* p
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option cacheBlockSize: %" PRId64 " valid range: [%d, %d]", val, TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE);
"invalid db option cacheBlockSize: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE);
pCxt->valid = false;
return pOptions;
}
@ -68,7 +68,7 @@ static SDatabaseOptions* setDbCacheLast(SAstCreateContext* pCxt, SDatabaseOption
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DB_CACHE_LAST_ROW || val > TSDB_MAX_DB_CACHE_LAST_ROW) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option cacheLast: %" PRId64 " valid range: [%d, %d]", val, TSDB_MIN_DB_CACHE_LAST_ROW, TSDB_MAX_DB_CACHE_LAST_ROW);
"invalid db option cacheLast: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_DB_CACHE_LAST_ROW, TSDB_MAX_DB_CACHE_LAST_ROW);
pCxt->valid = false;
return pOptions;
}
@ -80,7 +80,7 @@ static SDatabaseOptions* setDbComp(SAstCreateContext* pCxt, SDatabaseOptions* pO
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_COMP_LEVEL || val > TSDB_MAX_COMP_LEVEL) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option compression: %" PRId64 " valid range: [%d, %d]", val, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL);
"invalid db option compression: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL);
pCxt->valid = false;
return pOptions;
}
@ -92,7 +92,7 @@ static SDatabaseOptions* setDbDays(SAstCreateContext* pCxt, SDatabaseOptions* pO
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option daysPerFile: %" PRId64 " valid range: [%d, %d]", val, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE);
"invalid db option daysPerFile: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE);
pCxt->valid = false;
return pOptions;
}
@ -104,7 +104,7 @@ static SDatabaseOptions* setDbFsync(SAstCreateContext* pCxt, SDatabaseOptions* p
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option fsyncPeriod: %" PRId64 " valid range: [%d, %d]", val, TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD);
"invalid db option fsyncPeriod: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD);
pCxt->valid = false;
return pOptions;
}
@ -116,7 +116,7 @@ static SDatabaseOptions* setDbMaxRows(SAstCreateContext* pCxt, SDatabaseOptions*
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_MAX_ROW_FBLOCK || val > TSDB_MAX_MAX_ROW_FBLOCK) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option maxRowsPerBlock: %" PRId64 " valid range: [%d, %d]", val, TSDB_MIN_MAX_ROW_FBLOCK, TSDB_MAX_MAX_ROW_FBLOCK);
"invalid db option maxRowsPerBlock: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_MAX_ROW_FBLOCK, TSDB_MAX_MAX_ROW_FBLOCK);
pCxt->valid = false;
return pOptions;
}
@ -128,7 +128,7 @@ static SDatabaseOptions* setDbMinRows(SAstCreateContext* pCxt, SDatabaseOptions*
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_MIN_ROW_FBLOCK || val > TSDB_MAX_MIN_ROW_FBLOCK) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option minRowsPerBlock: %" PRId64 " valid range: [%d, %d]", val, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK);
"invalid db option minRowsPerBlock: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK);
pCxt->valid = false;
return pOptions;
}
@ -140,7 +140,7 @@ static SDatabaseOptions* setDbKeep(SAstCreateContext* pCxt, SDatabaseOptions* pO
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_KEEP || val > TSDB_MAX_KEEP) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option keep: %" PRId64 " valid range: [%d, %d]", val, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
"invalid db option keep: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
pCxt->valid = false;
return pOptions;
}
@ -168,7 +168,7 @@ static SDatabaseOptions* setDbQuorum(SAstCreateContext* pCxt, SDatabaseOptions*
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DB_QUORUM_OPTION || val > TSDB_MAX_DB_QUORUM_OPTION) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option quorum: %" PRId64 " valid range: [%d, %d]", val, TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION);
"invalid db option quorum: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION);
pCxt->valid = false;
return pOptions;
}
@ -180,7 +180,7 @@ static SDatabaseOptions* setDbReplica(SAstCreateContext* pCxt, SDatabaseOptions*
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DB_REPLICA_OPTION || val > TSDB_MAX_DB_REPLICA_OPTION) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option replications: %" PRId64 " valid range: [%d, %d]", val, TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION);
"invalid db option replications: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION);
pCxt->valid = false;
return pOptions;
}
@ -192,7 +192,7 @@ static SDatabaseOptions* setDbTtl(SAstCreateContext* pCxt, SDatabaseOptions* pOp
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DB_TTL_OPTION) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option ttl: %" PRId64 ", should be greater than or equal to %d", val, TSDB_MIN_DB_TTL_OPTION);
"invalid db option ttl: %"PRId64", should be greater than or equal to %d", val, TSDB_MIN_DB_TTL_OPTION);
pCxt->valid = false;
return pOptions;
}
@ -203,7 +203,7 @@ static SDatabaseOptions* setDbTtl(SAstCreateContext* pCxt, SDatabaseOptions* pOp
static SDatabaseOptions* setDbWal(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_WAL_LEVEL || val > TSDB_MAX_WAL_LEVEL) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option walLevel: %" PRId64 ", only 1-2 allowed", val);
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option walLevel: %"PRId64", only 1-2 allowed", val);
pCxt->valid = false;
return pOptions;
}
@ -215,7 +215,7 @@ static SDatabaseOptions* setDbVgroups(SAstCreateContext* pCxt, SDatabaseOptions*
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_VNODES_PER_DB || val > TSDB_MAX_VNODES_PER_DB) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option vgroups: %" PRId64 " valid range: [%d, %d]", val, TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB);
"invalid db option vgroups: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB);
pCxt->valid = false;
return pOptions;
}
@ -226,7 +226,7 @@ static SDatabaseOptions* setDbVgroups(SAstCreateContext* pCxt, SDatabaseOptions*
static SDatabaseOptions* setDbSingleStable(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DB_SINGLE_STABLE_OPTION || val > TSDB_MAX_DB_SINGLE_STABLE_OPTION) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option singleStable: %" PRId64 ", only 0-1 allowed", val);
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option singleStable: %"PRId64", only 0-1 allowed", val);
pCxt->valid = false;
return pOptions;
}
@ -237,7 +237,7 @@ static SDatabaseOptions* setDbSingleStable(SAstCreateContext* pCxt, SDatabaseOpt
static SDatabaseOptions* setDbStreamMode(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DB_STREAM_MODE_OPTION || val > TSDB_MAX_DB_STREAM_MODE_OPTION) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option streamMode: %" PRId64 ", only 0-1 allowed", val);
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option streamMode: %"PRId64", only 0-1 allowed", val);
pCxt->valid = false;
return pOptions;
}
@ -269,7 +269,7 @@ static STableOptions* setTableKeep(SAstCreateContext* pCxt, STableOptions* pOpti
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_KEEP || val > TSDB_MAX_KEEP) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid table option keep: %" PRId64 " valid range: [%d, %d]", val, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
"invalid table option keep: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
pCxt->valid = false;
return pOptions;
}
@ -281,7 +281,7 @@ static STableOptions* setTableTtl(SAstCreateContext* pCxt, STableOptions* pOptio
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DB_TTL_OPTION) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid table option ttl: %" PRId64 ", should be greater than or equal to %d", val, TSDB_MIN_DB_TTL_OPTION);
"invalid table option ttl: %"PRId64", should be greater than or equal to %d", val, TSDB_MIN_DB_TTL_OPTION);
pCxt->valid = false;
return pOptions;
}
@ -714,11 +714,21 @@ SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDat
strncpy(pCol->colName, pColName->z, pColName->n);
pCol->dataType = dataType;
if (NULL != pComment) {
strncpy(pCol->colName, pColName->z, pColName->n);
strncpy(pCol->comments, pComment->z, pComment->n);
}
return (SNode*)pCol;
}
SDataType createDataType(uint8_t type) {
SDataType dt = { .type = type, .precision = 0, .scale = 0, .bytes = tDataTypes[type].bytes };
return dt;
}
SDataType createVarLenDataType(uint8_t type, const SToken* pLen) {
SDataType dt = { .type = type, .precision = 0, .scale = 0, .bytes = tDataTypes[type].bytes };
return dt;
}
SNode* createCreateTableStmt(SAstCreateContext* pCxt,
bool ignoreExists, const STokenPair* pFullTableName, SNodeList* pCols, STableOptions* pOptions) {
SCreateTableStmt* pStmt = (SCreateTableStmt*)nodesMakeNode(QUERY_NODE_CREATE_TABLE_STMT);
@ -732,3 +742,16 @@ SNode* createCreateTableStmt(SAstCreateContext* pCxt,
pStmt->options = *pOptions;
return (SNode*)pStmt;
}
SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName) {
SUseDatabaseStmt* pStmt = (SUseDatabaseStmt*)nodesMakeNode(QUERY_NODE_USE_DATABASE_STMT);
CHECK_OUT_OF_MEM(pStmt);
strncpy(pStmt->dbName, pDbName->z, pDbName->n);
return (SNode*)pStmt;
}
SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type) {
SNode* pStmt = nodesMakeNode(type);;
CHECK_OUT_OF_MEM(pStmt);
return pStmt;
}

31
source/libs/parser/src/astParse.c
View File

@ -26,18 +26,20 @@ extern void NewParse(void*, int, SToken, void*);
extern void NewParseFree(void*, FFree);
extern void NewParseTrace(FILE*, char*);
static bool isCmd(const SNode* pRootNode) {
if (NULL == pRootNode) {
return true;
static void setQuery(SAstCreateContext* pCxt, SQuery* pQuery) {
pQuery->pRoot = pCxt->pRootNode;
ENodeType type = nodeType(pCxt->pRootNode);
if (QUERY_NODE_SELECT_STMT == type) {
pQuery->haveResultSet = true;
pQuery->directRpc = false;
} else if (QUERY_NODE_CREATE_TABLE_STMT == type) {
pQuery->haveResultSet = false;
pQuery->directRpc = false;
} else {
pQuery->haveResultSet = false;
pQuery->directRpc = true;
}
switch (nodeType(pRootNode)) {
case QUERY_NODE_SELECT_STMT:
case QUERY_NODE_CREATE_TABLE_STMT:
return false;
default:
break;
}
return true;
pQuery->msgType = (QUERY_NODE_CREATE_TABLE_STMT == type ? TDMT_VND_CREATE_TABLE : TDMT_VND_QUERY);
}
int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) {
@ -60,6 +62,10 @@ int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) {
case TK_COMMENT: {
break;
}
case TK_SEMI: {
NewParse(pParser, 0, t0, &cxt);
goto abort_parse;
}
case TK_QUESTION:
case TK_ILLEGAL: {
snprintf(cxt.pQueryCxt->pMsg, cxt.pQueryCxt->msgLen, "unrecognized token: \"%s\"", t0.z);
@ -89,8 +95,7 @@ abort_parse:
if (NULL == *pQuery) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pQuery)->isCmd = isCmd(cxt.pRootNode);
(*pQuery)->pRoot = cxt.pRootNode;
setQuery(&cxt, *pQuery);
}
return cxt.valid ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED;
}

233
source/libs/parser/src/astTranslate.c
View File

@ -561,6 +561,29 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect)
return TSDB_CODE_SUCCESS;
}
static int32_t setTableVgroupList(STranslateContext *pCxt, SName* name, SVgroupsInfo **pVgList) {
SArray* vgroupList = NULL;
int32_t code = catalogGetTableDistVgInfo(pCxt->pParseCxt->pCatalog, pCxt->pParseCxt->pTransporter, &(pCxt->pParseCxt->mgmtEpSet), name, &vgroupList);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
size_t vgroupNum = taosArrayGetSize(vgroupList);
SVgroupsInfo *vgList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupNum);
vgList->numOfVgroups = vgroupNum;
for (int32_t i = 0; i < vgroupNum; ++i) {
SVgroupInfo *vg = taosArrayGet(vgroupList, i);
vgList->vgroups[i] = *vg;
}
*pVgList = vgList;
taosArrayDestroy(vgroupList);
return TSDB_CODE_SUCCESS;
}
static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) {
int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pTable)) {
@ -572,6 +595,10 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) {
if (TSDB_CODE_SUCCESS != code) {
return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_TABLE_NOT_EXIST, pRealTable->table.tableName);
}
code = setTableVgroupList(pCxt, &name, &(pRealTable->pVgroupList));
if (TSDB_CODE_SUCCESS != code) {
return code;
}
code = addNamespace(pCxt, pRealTable);
break;
}
@ -829,7 +856,73 @@ static int32_t translateCreateDatabase(STranslateContext* pCxt, SCreateDatabaseS
return TSDB_CODE_SUCCESS;
}
static int32_t translateCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* pStmt) {
SName name = {0};
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
SUseDbReq usedbReq = {0};
tNameExtractFullName(&name, usedbReq.db);
pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo));
if (NULL== pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_MND_USE_DB;
pCxt->pCmdMsg->msgLen = tSerializeSUseDbReq(NULL, 0, &usedbReq);
pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen);
if (NULL== pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tSerializeSUseDbReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &usedbReq);
return TSDB_CODE_SUCCESS;
}
static int32_t translateShowDatabases(STranslateContext* pCxt) {
SShowReq showReq = { .type = TSDB_MGMT_TABLE_DB };
pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo));
if (NULL== pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_MND_SHOW;
pCxt->pCmdMsg->msgLen = tSerializeSShowReq(NULL, 0, &showReq);
pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen);
if (NULL== pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tSerializeSShowReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &showReq);
return TSDB_CODE_SUCCESS;
}
static int32_t translateShowTables(STranslateContext* pCxt) {
SName name = {0};
SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq));
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, strlen(pCxt->pParseCxt->db));
char dbFname[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(&name, dbFname);
SArray* array = NULL;
int32_t code = catalogGetDBVgInfo(pCxt->pParseCxt->pCatalog, pCxt->pParseCxt->pTransporter, &pCxt->pParseCxt->mgmtEpSet, dbFname, false, &array);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
SVgroupInfo* info = taosArrayGet(array, 0);
pShowReq->head.vgId = htonl(info->vgId);
pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo));
if (NULL== pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = info->epset;
pCxt->pCmdMsg->msgType = TDMT_VND_SHOW_TABLES;
pCxt->pCmdMsg->msgLen = sizeof(SVShowTablesReq);
pCxt->pCmdMsg->pMsg = pShowReq;
pCxt->pCmdMsg->pExtension = array;
return TSDB_CODE_SUCCESS;
}
@ -842,8 +935,14 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_CREATE_DATABASE_STMT:
code = translateCreateDatabase(pCxt, (SCreateDatabaseStmt*)pNode);
break;
case QUERY_NODE_CREATE_TABLE_STMT:
code = translateCreateTable(pCxt, (SCreateTableStmt*)pNode);
case QUERY_NODE_USE_DATABASE_STMT:
code = translateUseDatabase(pCxt, (SUseDatabaseStmt*)pNode);
break;
case QUERY_NODE_SHOW_DATABASES_STMT:
code = translateShowDatabases(pCxt);
break;
case QUERY_NODE_SHOW_TABLES_STMT:
code = translateShowTables(pCxt);
break;
default:
break;
@ -862,7 +961,7 @@ static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) {
return code;
}
int32_t setReslutSchema(STranslateContext* pCxt, SQuery* pQuery) {
static int32_t setReslutSchema(STranslateContext* pCxt, SQuery* pQuery) {
if (QUERY_NODE_SELECT_STMT == nodeType(pQuery->pRoot)) {
SSelectStmt* pSelect = (SSelectStmt*)pQuery->pRoot;
pQuery->numOfResCols = LIST_LENGTH(pSelect->pProjectionList);
@ -882,7 +981,7 @@ int32_t setReslutSchema(STranslateContext* pCxt, SQuery* pQuery) {
return TSDB_CODE_SUCCESS;
}
void destroyTranslateContext(STranslateContext* pCxt) {
static void destroyTranslateContext(STranslateContext* pCxt) {
taosArrayDestroy(pCxt->pNsLevel);
if (NULL != pCxt->pCmdMsg) {
tfree(pCxt->pCmdMsg->pMsg);
@ -890,6 +989,122 @@ void destroyTranslateContext(STranslateContext* pCxt) {
}
}
typedef struct SVgroupTablesBatch {
SVCreateTbBatchReq req;
SVgroupInfo info;
} SVgroupTablesBatch;
static void toSchema(const SColumnDefNode* pCol, int32_t colId, SSchema* pSchema) {
pSchema->colId = colId;
pSchema->type = pCol->dataType.type;
pSchema->bytes = pCol->dataType.bytes;
strcpy(pSchema->name, pCol->colName);
}
static int32_t doBuildSingleTableBatchReq(SName* pTableName, SNodeList* pColumns, SVgroupInfo* pVgroupInfo, SVgroupTablesBatch* pBatch) {
SVCreateTbReq req = {0};
req.type = TD_NORMAL_TABLE;
req.name = strdup(tNameGetTableName(pTableName));
req.ntbCfg.nCols = LIST_LENGTH(pColumns);
int32_t num = req.ntbCfg.nCols;
req.ntbCfg.pSchema = calloc(num, sizeof(SSchema));
SNode* pCol;
int32_t index = 0;
FOREACH(pCol, pColumns) {
toSchema((SColumnDefNode*)pCol, index + 1, req.ntbCfg.pSchema + index);
++index;
}
pBatch->info = *pVgroupInfo;
pBatch->req.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq));
if (pBatch->req.pArray == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
taosArrayPush(pBatch->req.pArray, &req);
return TSDB_CODE_SUCCESS;
}
static int32_t serializeVgroupTablesBatchImpl(SVgroupTablesBatch* pTbBatch, SArray* pBufArray) {
int tlen = sizeof(SMsgHead) + tSerializeSVCreateTbBatchReq(NULL, &(pTbBatch->req));
void* buf = malloc(tlen);
if (buf == NULL) {
// TODO: handle error
}
((SMsgHead*)buf)->vgId = htonl(pTbBatch->info.vgId);
((SMsgHead*)buf)->contLen = htonl(tlen);
void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tSerializeSVCreateTbBatchReq(&pBuf, &(pTbBatch->req));
SVgDataBlocks* pVgData = calloc(1, sizeof(SVgDataBlocks));
pVgData->vg = pTbBatch->info;
pVgData->pData = buf;
pVgData->size = tlen;
pVgData->numOfTables = (int32_t) taosArrayGetSize(pTbBatch->req.pArray);
taosArrayPush(pBufArray, &pVgData);
}
static void destroyCreateTbReqBatch(SVgroupTablesBatch* pTbBatch) {
size_t size = taosArrayGetSize(pTbBatch->req.pArray);
for(int32_t i = 0; i < size; ++i) {
SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i);
tfree(pTableReq->name);
if (pTableReq->type == TSDB_NORMAL_TABLE) {
tfree(pTableReq->ntbCfg.pSchema);
} else if (pTableReq->type == TSDB_CHILD_TABLE) {
tfree(pTableReq->ctbCfg.pTag);
} else {
assert(0);
}
}
taosArrayDestroy(pTbBatch->req.pArray);
}
static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
if (QUERY_NODE_CREATE_TABLE_STMT == nodeType(pQuery->pRoot)) {
SCreateTableStmt* pStmt = (SCreateTableStmt*)pQuery->pRoot;
SName tableName = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId };
if ('\0' == pStmt->dbName[0]) {
strcpy(tableName.dbname, pCxt->pParseCxt->db);
} else {
strcpy(tableName.dbname, pStmt->dbName);
}
strcpy(tableName.tname, pStmt->tableName);
SVgroupInfo info = {0};
catalogGetTableHashVgroup(pCxt->pParseCxt->pCatalog, pCxt->pParseCxt->pTransporter, &pCxt->pParseCxt->mgmtEpSet, &tableName, &info);
SVgroupTablesBatch tbatch = {0};
int32_t code = doBuildSingleTableBatchReq(&tableName, pStmt->pCols, &info, &tbatch);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
SArray* pBufArray = taosArrayInit(1, POINTER_BYTES);
if (pBufArray == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
serializeVgroupTablesBatchImpl(&tbatch, pBufArray);
destroyCreateTbReqBatch(&tbatch);
SVnodeModifOpStmt* pNewStmt = nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT);
pNewStmt->sqlNodeType = nodeType(pQuery->pRoot);
pNewStmt->pDataBlocks = pBufArray;
pQuery->sqlNodeType = nodeType(pQuery->pRoot);
nodesDestroyNode(pQuery->pRoot);
pQuery->pRoot = (SNode*)pNewStmt;
}
return TSDB_CODE_SUCCESS;
}
int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) {
STranslateContext cxt = {
.pParseCxt = pParseCxt,
@ -900,14 +1115,18 @@ int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) {
.currClause = 0
};
int32_t code = fmFuncMgtInit();
if (TSDB_CODE_SUCCESS == code) {
code = rewriteQuery(&cxt, pQuery);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateQuery(&cxt, pQuery->pRoot);
}
if (TSDB_CODE_SUCCESS == code) {
if (pQuery->isCmd) {
if (pQuery->directRpc) {
pQuery->pCmdMsg = cxt.pCmdMsg;
cxt.pCmdMsg = NULL;
} else {
}
if (pQuery->haveResultSet) {
code = setReslutSchema(&cxt, pQuery);
}
}

14
source/libs/parser/src/insertParser.c
View File

@ -159,12 +159,9 @@ static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pPar
}
static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) {
SName name = {0};
createSName(&name, pTname, pCxt->pComCxt, &pCxt->msg);
char tableName[TSDB_TABLE_FNAME_LEN] = {0};
tNameExtractFullName(&name, tableName);
SParseContext* pBasicCtx = pCxt->pComCxt;
SName name = {0};
createSName(&name, pTname, pBasicCtx, &pCxt->msg);
CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta));
SVgroupInfo vg;
CHECK_CODE(catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg));
@ -939,6 +936,13 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
*pQuery = calloc(1, sizeof(SQuery));
if (NULL == *pQuery) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pQuery)->directRpc = false;
(*pQuery)->haveResultSet = false;
(*pQuery)->msgType = TDMT_VND_SUBMIT;
(*pQuery)->pRoot = (SNode*)context.pOutput;
context.pOutput->payloadType = PAYLOAD_TYPE_KV;

2387
source/libs/parser/src/new_sql.c
View File

File diff suppressed because it is too large Load Diff

34
source/libs/parser/src/ttokenizer.c
View File

@ -30,18 +30,18 @@ typedef struct SKeyword {
// keywords in sql string
static SKeyword keywordTable[] = {
// {"ID", TK_ID},
// {"BOOL", TK_BOOL},
// {"TINYINT", TK_TINYINT},
// {"SMALLINT", TK_SMALLINT},
// {"INTEGER", TK_INTEGER},
// {"INT", TK_INTEGER},
// {"BIGINT", TK_BIGINT},
// {"FLOAT", TK_FLOAT},
// {"DOUBLE", TK_DOUBLE},
{"BOOL", TK_BOOL},
{"TINYINT", TK_TINYINT},
{"SMALLINT", TK_SMALLINT},
{"INTEGER", TK_INTEGER},
{"INT", TK_INTEGER},
{"BIGINT", TK_BIGINT},
{"FLOAT", TK_FLOAT},
{"DOUBLE", TK_DOUBLE},
// {"STRING", TK_STRING},
{"TIMESTAMP", TK_TIMESTAMP},
// {"BINARY", TK_BINARY},
// {"NCHAR", TK_NCHAR},
{"BINARY", TK_BINARY},
{"NCHAR", TK_NCHAR},
{"OR", TK_OR},
{"AND", TK_AND},
{"NOT", TK_NOT},
@ -74,8 +74,8 @@ static SKeyword keywordTable[] = {
// {"UMINUS", TK_UMINUS},
// {"UPLUS", TK_UPLUS},
// {"BITNOT", TK_BITNOT},
// {"SHOW", TK_SHOW},
// {"DATABASES", TK_DATABASES},
{"SHOW", TK_SHOW},
{"DATABASES", TK_DATABASES},
// {"MNODES", TK_MNODES},
// {"DNODES", TK_DNODES},
// {"ACCOUNTS", TK_ACCOUNTS},
@ -88,16 +88,16 @@ static SKeyword keywordTable[] = {
// {"SCORES", TK_SCORES},
// {"GRANTS", TK_GRANTS},
// {"DOT", TK_DOT},
// {"TABLES", TK_TABLES},
{"TABLES", TK_TABLES},
// {"STABLES", TK_STABLES},
{"VGROUPS", TK_VGROUPS},
// {"DROP", TK_DROP},
// {"TABLE", TK_TABLE},
{"TABLE", TK_TABLE},
{"DATABASE", TK_DATABASE},
// {"DNODE", TK_DNODE},
// {"USER", TK_USER},
// {"ACCOUNT", TK_ACCOUNT},
// {"USE", TK_USE},
{"USE", TK_USE},
// {"DESCRIBE", TK_DESCRIBE},
// {"SYNCDB", TK_SYNCDB},
// {"ALTER", TK_ALTER},
@ -309,7 +309,7 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
if (z[1] == '-') {
for (i = 2; z[i] && z[i] != '\n'; i++) {
}
*tokenId = TK_COMMENT;
*tokenId = TK_NK_COMMENT;
return i;
}
*tokenId = TK_MINUS;
@ -343,7 +343,7 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
for (i = 3; z[i] && (z[i] != '/' || z[i - 1] != '*'); i++) {
}
if (z[i]) i++;
*tokenId = TK_COMMENT;
*tokenId = TK_NK_COMMENT;
return i;
}
case '%': {

12
source/libs/parser/test/mockCatalog.cpp
View File

@ -13,19 +13,17 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mockCatalog.h"
#include <iostream>
#include "stub.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wformat"
#include "addr_any.h"
#include <addr_any.h>
#pragma GCC diagnostic pop
#include "mockCatalog.h"
namespace {
void generateTestT1(MockCatalogService* mcs) {
@ -60,6 +58,10 @@ int32_t __catalogGetTableHashVgroup(struct SCatalog* pCatalog, void *pRpc, const
return mockCatalogService->catalogGetTableHashVgroup(pTableName, vgInfo);
}
int32_t __catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) {
return mockCatalogService->catalogGetTableDistVgInfo(pTableName, pVgList);
}
void initMetaDataEnv() {
mockCatalogService.reset(new MockCatalogService());
@ -67,6 +69,8 @@ void initMetaDataEnv() {
stub.set(catalogGetHandle, __catalogGetHandle);
stub.set(catalogGetTableMeta, __catalogGetTableMeta);
stub.set(catalogGetTableHashVgroup, __catalogGetTableHashVgroup);
stub.set(catalogGetTableDistVgInfo, __catalogGetTableDistVgInfo);
{
AddrAny any("libcatalog.so");
std::map<std::string,void*> result;

17
source/libs/parser/test/mockCatalogService.cpp
View File

@ -126,6 +126,19 @@ public:
return 0;
}
int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const {
SVgroupInfo info = {0};
info.vgId = 1;
addEpIntoEpSet(&info.epset, "node1", 6030);
info.hashBegin = 0;
info.hashEnd = 1;
*pVgList = taosArrayInit(4, sizeof(SVgroupInfo));
taosArrayPush(*pVgList, &info);
return 0;
}
TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) {
builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags);
meta_[db][tbname] = builder_->table();
@ -313,4 +326,8 @@ int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableM
int32_t MockCatalogService::catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const {
return impl_->catalogGetTableHashVgroup(pTableName, vgInfo);
}
int32_t MockCatalogService::catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const {
return impl_->catalogGetTableDistVgInfo(pTableName, pVgList);
}

1
source/libs/parser/test/mockCatalogService.h
View File

@ -59,6 +59,7 @@ public:
int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const;
int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const;
int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const;
private:
std::unique_ptr<MockCatalogServiceImpl> impl_;

20
source/libs/parser/test/parserTest.cpp
View File

@ -696,3 +696,23 @@ TEST_F(ParserTest, selectSemanticError) {
bind("SELECT DISTINCT c2 FROM t1 WHERE c1 > 0 ORDER BY count(c2)");
ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION));
}
TEST_F(ParserTest, createDatabase) {
bind("create database wxy_db");
ASSERT_TRUE(run());
}
TEST_F(ParserTest, showDatabase) {
bind("show databases");
ASSERT_TRUE(run());
}
TEST_F(ParserTest, useDatabase) {
bind("use wxy_db");
ASSERT_TRUE(run());
}
TEST_F(ParserTest, createTable) {
bind("create table t1(ts timestamp, c1 int)");
ASSERT_TRUE(run());
}

11
source/libs/planner/inc/plannerInt.h
View File

@ -39,9 +39,18 @@ extern "C" {
} \
} while (0)
#define CHECK_CODE_EXT(exec) \
do { \
int32_t code = (exec); \
if (TSDB_CODE_SUCCESS != code) { \
pCxt->errCode = code; \
return code; \
} \
} while (0)
int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode);
int32_t optimize(SPlanContext* pCxt, SLogicNode* pLogicNode);
int32_t createPhysiPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SQueryPlan** pPlan);
int32_t createPhysiPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SQueryPlan** pPlan, SArray* pExecNodeList);
#ifdef __cplusplus
}

13
source/libs/planner/src/logicPlan.c
View File

@ -128,6 +128,7 @@ static SLogicNode* createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe
pScan->node.id = pCxt->planNodeId++;
pScan->pMeta = pRealTable->pMeta;
pScan->pVgroupList = pRealTable->pVgroupList;
// set columns to scan
SNodeList* pCols = NULL;
@ -348,10 +349,20 @@ static SLogicNode* createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p
return pRoot;
}
static SLogicNode* createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt) {
SVnodeModifLogicNode* pModif = (SVnodeModifLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIF);
CHECK_ALLOC(pModif, NULL);
pModif->pDataBlocks = pStmt->pDataBlocks;
pModif->msgType = (QUERY_NODE_CREATE_TABLE_STMT == pStmt->sqlNodeType ? TDMT_VND_CREATE_TABLE : TDMT_VND_SUBMIT);
return (SLogicNode*)pModif;
}
static SLogicNode* createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt) {
switch (nodeType(pStmt)) {
case QUERY_NODE_SELECT_STMT:
return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt);
return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt);
case QUERY_NODE_VNODE_MODIF_STMT:
return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt);
default:
break;
}

286
source/libs/planner/src/physicalPlan.c
View File

@ -27,6 +27,7 @@ typedef struct SPhysiPlanContext {
int32_t errCode;
int16_t nextDataBlockId;
SArray* pLocationHelper;
SArray* pExecNodeList;
} SPhysiPlanContext;
static int32_t getSlotKey(SNode* pNode, char* pKey) {
@ -153,17 +154,20 @@ static SNodeList* setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId
static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, ENodeType type) {
SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type);
if (NULL == pPhysiNode) {
CHECK_ALLOC(pPhysiNode, NULL);
pPhysiNode->pOutputDataBlockDesc = nodesMakeNode(QUERY_NODE_DATABLOCK_DESC);
if (NULL == pPhysiNode->pOutputDataBlockDesc) {
nodesDestroyNode(pPhysiNode);
return NULL;
}
pPhysiNode->outputDataBlockDesc.dataBlockId = pCxt->nextDataBlockId++;
pPhysiNode->outputDataBlockDesc.type = QUERY_NODE_DATABLOCK_DESC;
pPhysiNode->pOutputDataBlockDesc->dataBlockId = pCxt->nextDataBlockId++;
pPhysiNode->pOutputDataBlockDesc->type = QUERY_NODE_DATABLOCK_DESC;
return pPhysiNode;
}
static int32_t setConditionsSlotId(SPhysiPlanContext* pCxt, const SLogicNode* pLogicNode, SPhysiNode* pPhysiNode) {
if (NULL != pLogicNode->pConditions) {
pPhysiNode->pConditions = setNodeSlotId(pCxt, pPhysiNode->outputDataBlockDesc.dataBlockId, -1, pLogicNode->pConditions);
pPhysiNode->pConditions = setNodeSlotId(pCxt, pPhysiNode->pOutputDataBlockDesc->dataBlockId, -1, pLogicNode->pConditions);
CHECK_ALLOC(pPhysiNode->pConditions, TSDB_CODE_OUT_OF_MEMORY);
}
return TSDB_CODE_SUCCESS;
@ -182,17 +186,47 @@ static int32_t setSlotOutput(SPhysiPlanContext* pCxt, SNodeList* pTargets, SData
return TSDB_CODE_SUCCESS;
}
static SNodeptr createPrimaryKeyCol(SPhysiPlanContext* pCxt, uint64_t tableId) {
SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN);
CHECK_ALLOC(pCol, NULL);
pCol->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP;
pCol->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
pCol->tableId = tableId;
pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
pCol->colType = COLUMN_TYPE_COLUMN;
return pCol;
}
static int32_t addPrimaryKeyCol(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhysiNode) {
if (NULL == pScanPhysiNode->pScanCols) {
pScanPhysiNode->pScanCols = nodesMakeList();
CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY);
CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, createPrimaryKeyCol(pCxt, pScanPhysiNode->uid)));
return TSDB_CODE_SUCCESS;
}
SNode* pNode;
FOREACH(pNode, pScanPhysiNode->pScanCols) {
if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) {
return TSDB_CODE_SUCCESS;
}
}
CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, createPrimaryKeyCol(pCxt, pScanPhysiNode->uid)));
return TSDB_CODE_SUCCESS;
}
static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) {
if (NULL != pScanLogicNode->pScanCols) {
pScanPhysiNode->pScanCols = nodesCloneList(pScanLogicNode->pScanCols);
CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY);
}
CHECK_CODE(addPrimaryKeyCol(pCxt, pScanPhysiNode), TSDB_CODE_OUT_OF_MEMORY);
// Data block describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t
CHECK_CODE(addDataBlockDesc(pCxt, pScanPhysiNode->pScanCols, &pScanPhysiNode->node.outputDataBlockDesc), TSDB_CODE_OUT_OF_MEMORY);
CHECK_CODE(addDataBlockDesc(pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc), TSDB_CODE_OUT_OF_MEMORY);
CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode), TSDB_CODE_OUT_OF_MEMORY);
CHECK_CODE(setSlotOutput(pCxt, pScanLogicNode->node.pTargets, &pScanPhysiNode->node.outputDataBlockDesc), TSDB_CODE_OUT_OF_MEMORY);
CHECK_CODE(setSlotOutput(pCxt, pScanLogicNode->node.pTargets, pScanPhysiNode->node.pOutputDataBlockDesc), TSDB_CODE_OUT_OF_MEMORY);
pScanPhysiNode->uid = pScanLogicNode->pMeta->uid;
pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType;
@ -203,6 +237,11 @@ static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanL
return TSDB_CODE_SUCCESS;
}
static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAddr) {
pNodeAddr->nodeId = vg->vgId;
pNodeAddr->epset = vg->epset;
}
static SPhysiNode* createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) {
STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN);
CHECK_ALLOC(pTagScan, NULL);
@ -210,21 +249,23 @@ static SPhysiNode* createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNod
return (SPhysiNode*)pTagScan;
}
static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) {
static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) {
STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
CHECK_ALLOC(pTableScan, NULL);
CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan);
pTableScan->scanFlag = pScanLogicNode->scanFlag;
pTableScan->scanRange = pScanLogicNode->scanRange;
vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode);
return (SPhysiNode*)pTableScan;
}
static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) {
static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) {
switch (pScanLogicNode->scanType) {
case SCAN_TYPE_TAG:
return createTagScanPhysiNode(pCxt, pScanLogicNode);
case SCAN_TYPE_TABLE:
return createTableScanPhysiNode(pCxt, pScanLogicNode);
return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode);
case SCAN_TYPE_STABLE:
case SCAN_TYPE_STREAM:
break;
@ -276,18 +317,18 @@ static SPhysiNode* createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChil
SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_JOIN);
CHECK_ALLOC(pJoin, NULL);
SDataBlockDescNode* pLeftDesc = &((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputDataBlockDesc;
SDataBlockDescNode* pRightDesc = &((SPhysiNode*)nodesListGetNode(pChildren, 1))->outputDataBlockDesc;
SDataBlockDescNode* pLeftDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc;
SDataBlockDescNode* pRightDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 1))->pOutputDataBlockDesc;
pJoin->pOnConditions = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pOnConditions);
CHECK_ALLOC(pJoin->pOnConditions, (SPhysiNode*)pJoin);
pJoin->pTargets = createJoinOutputCols(pCxt, pLeftDesc, pRightDesc);
CHECK_ALLOC(pJoin->pTargets, (SPhysiNode*)pJoin);
CHECK_CODE(addDataBlockDesc(pCxt, pJoin->pTargets, &pJoin->node.outputDataBlockDesc), (SPhysiNode*)pJoin);
CHECK_CODE(addDataBlockDesc(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc), (SPhysiNode*)pJoin);
CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin), (SPhysiNode*)pJoin);
CHECK_CODE(setSlotOutput(pCxt, pJoinLogicNode->node.pTargets, &pJoin->node.outputDataBlockDesc), (SPhysiNode*)pJoin);
CHECK_CODE(setSlotOutput(pCxt, pJoinLogicNode->node.pTargets, pJoin->node.pOutputDataBlockDesc), (SPhysiNode*)pJoin);
return (SPhysiNode*)pJoin;
}
@ -385,8 +426,8 @@ static SPhysiNode* createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild
CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pGroupKeys, &pPrecalcExprs, &pGroupKeys), (SPhysiNode*)pAgg);
CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pAggFuncs, &pPrecalcExprs, &pAggFuncs), (SPhysiNode*)pAgg);
SDataBlockDescNode* pChildTupe = &(((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputDataBlockDesc);
// push down expression to outputDataBlockDesc of child node
SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc);
// push down expression to pOutputDataBlockDesc of child node
if (NULL != pPrecalcExprs) {
pAgg->pExprs = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs);
CHECK_ALLOC(pAgg->pExprs, (SPhysiNode*)pAgg);
@ -396,18 +437,18 @@ static SPhysiNode* createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild
if (NULL != pGroupKeys) {
pAgg->pGroupKeys = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pGroupKeys);
CHECK_ALLOC(pAgg->pGroupKeys, (SPhysiNode*)pAgg);
CHECK_CODE(addDataBlockDesc(pCxt, pAgg->pGroupKeys, &pAgg->node.outputDataBlockDesc), (SPhysiNode*)pAgg);
CHECK_CODE(addDataBlockDesc(pCxt, pAgg->pGroupKeys, pAgg->node.pOutputDataBlockDesc), (SPhysiNode*)pAgg);
}
if (NULL != pAggFuncs) {
pAgg->pAggFuncs = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pAggFuncs);
CHECK_ALLOC(pAgg->pAggFuncs, (SPhysiNode*)pAgg);
CHECK_CODE(addDataBlockDesc(pCxt, pAgg->pAggFuncs, &pAgg->node.outputDataBlockDesc), (SPhysiNode*)pAgg);
CHECK_CODE(addDataBlockDesc(pCxt, pAgg->pAggFuncs, pAgg->node.pOutputDataBlockDesc), (SPhysiNode*)pAgg);
}
CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pAggLogicNode, (SPhysiNode*)pAgg), (SPhysiNode*)pAgg);
CHECK_CODE(setSlotOutput(pCxt, pAggLogicNode->node.pTargets, &pAgg->node.outputDataBlockDesc), (SPhysiNode*)pAgg);
CHECK_CODE(setSlotOutput(pCxt, pAggLogicNode->node.pTargets, pAgg->node.pOutputDataBlockDesc), (SPhysiNode*)pAgg);
return (SPhysiNode*)pAgg;
}
@ -416,22 +457,22 @@ static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC
SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_PROJECT);
CHECK_ALLOC(pProject, NULL);
pProject->pProjections = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputDataBlockDesc.dataBlockId, -1, pProjectLogicNode->pProjections);
pProject->pProjections = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc->dataBlockId, -1, pProjectLogicNode->pProjections);
CHECK_ALLOC(pProject->pProjections, (SPhysiNode*)pProject);
CHECK_CODE(addDataBlockDesc(pCxt, pProject->pProjections, &pProject->node.outputDataBlockDesc), (SPhysiNode*)pProject);
CHECK_CODE(addDataBlockDesc(pCxt, pProject->pProjections, pProject->node.pOutputDataBlockDesc), (SPhysiNode*)pProject);
CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pProjectLogicNode, (SPhysiNode*)pProject), (SPhysiNode*)pProject);
return (SPhysiNode*)pProject;
}
static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPlan) {
static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SLogicNode* pLogicPlan) {
SNodeList* pChildren = nodesMakeList();
CHECK_ALLOC(pChildren, NULL);
SNode* pLogicChild;
FOREACH(pLogicChild, pLogicPlan->pChildren) {
SNode* pChildPhyNode = (SNode*)createPhysiNode(pCxt, (SLogicNode*)pLogicChild);
SNode* pChildPhyNode = (SNode*)createPhysiNode(pCxt, pSubplan, (SLogicNode*)pLogicChild);
if (TSDB_CODE_SUCCESS != nodesListAppend(pChildren, pChildPhyNode)) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
nodesDestroyList(pChildren);
@ -442,7 +483,7 @@ static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPl
SPhysiNode* pPhyNode = NULL;
switch (nodeType(pLogicPlan)) {
case QUERY_NODE_LOGIC_PLAN_SCAN:
pPhyNode = createScanPhysiNode(pCxt, (SScanLogicNode*)pLogicPlan);
pPhyNode = createScanPhysiNode(pCxt, pSubplan, (SScanLogicNode*)pLogicPlan);
break;
case QUERY_NODE_LOGIC_PLAN_JOIN:
pPhyNode = createJoinPhysiNode(pCxt, pChildren, (SJoinLogicNode*)pLogicPlan);
@ -466,91 +507,184 @@ static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPl
return pPhyNode;
}
static SDataSinkNode* createDataInserter(SPhysiPlanContext* pCxt, SVgDataBlocks* pBlocks) {
SDataInserterNode* pInserter = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT);
CHECK_ALLOC(pInserter, NULL);
pInserter->numOfTables = pBlocks->numOfTables;
pInserter->size = pBlocks->size;
TSWAP(pInserter->pData, pBlocks->pData, char*);
return (SDataSinkNode*)pInserter;
}
static SDataSinkNode* createDataDispatcher(SPhysiPlanContext* pCxt, const SPhysiNode* pRoot) {
SDataDispatcherNode* pDispatcher = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_DISPATCH);
CHECK_ALLOC(pDispatcher, NULL);
pDispatcher->sink.pInputDataBlockDesc = nodesCloneNode(pRoot->pOutputDataBlockDesc);
CHECK_ALLOC(pDispatcher->sink.pInputDataBlockDesc, (SDataSinkNode*)pDispatcher);
return (SDataSinkNode*)pDispatcher;
}
static SSubplan* createPhysiSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLogicSubplan) {
SSubplan* pSubplan = (SSubplan*)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
CHECK_ALLOC(pSubplan, NULL);
pSubplan->pNode = createPhysiNode(pCxt, pLogicSubplan->pNode);
if (SUBPLAN_TYPE_MODIFY == pLogicSubplan->subplanType) {
SVnodeModifLogicNode* pModif = (SVnodeModifLogicNode*)pLogicSubplan->pNode;
pSubplan->pDataSink = createDataInserter(pCxt, pModif->pVgDataBlocks);
pSubplan->msgType = pModif->msgType;
pSubplan->execNode.epset = pModif->pVgDataBlocks->vg.epset;
taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode);
} else {
pSubplan->pNode = createPhysiNode(pCxt, pSubplan, pLogicSubplan->pNode);
pSubplan->pDataSink = createDataDispatcher(pCxt, pSubplan->pNode);
pSubplan->msgType = TDMT_VND_QUERY;
}
pSubplan->subplanType = pLogicSubplan->subplanType;
return pSubplan;
}
static int32_t strictListAppend(SNodeList* pList, SNodeptr pNode) {
if (NULL == pNode) {
return TSDB_CODE_OUT_OF_MEMORY;
static int32_t splitLogicPlan(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubLogicPlan** pSubLogicPlan) {
*pSubLogicPlan = (SSubLogicPlan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN);
CHECK_ALLOC(*pSubLogicPlan, TSDB_CODE_OUT_OF_MEMORY);
// todo pSubplan->pNode = nodesCloneNode(pLogicNode);
(*pSubLogicPlan)->pNode = pLogicNode;
if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIF == nodeType(pLogicNode)) {
(*pSubLogicPlan)->subplanType = SUBPLAN_TYPE_MODIFY;
}
int32_t code = nodesListAppend(pList, pNode);
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode(pNode);
}
return code;
// todo split
return TSDB_CODE_SUCCESS;
}
static SQueryLogicPlan* createRawQueryLogicPlan(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode) {
static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNodeptr pSubplan, int32_t level, SNodeList* pSubplans) {
SNodeListNode* pGroup;
if (level >= LIST_LENGTH(pSubplans)) {
pGroup = nodesMakeNode(QUERY_NODE_NODE_LIST);
CHECK_ALLOC(pGroup, TSDB_CODE_OUT_OF_MEMORY);
CHECK_CODE(nodesListStrictAppend(pSubplans, pGroup), TSDB_CODE_OUT_OF_MEMORY);
} else {
pGroup = nodesListGetNode(pSubplans, level);
}
if (NULL == pGroup->pNodeList) {
pGroup->pNodeList = nodesMakeList();
CHECK_ALLOC(pGroup->pNodeList, TSDB_CODE_OUT_OF_MEMORY);
}
CHECK_CODE(nodesListStrictAppend(pGroup->pNodeList, pSubplan), TSDB_CODE_OUT_OF_MEMORY);
}
SSubLogicPlan* singleCloneSubLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pSrc, int32_t level) {
SSubLogicPlan* pDst = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN);
CHECK_ALLOC(pDst, NULL);
// todo pDst->pNode = nodesCloneNode(pSrc->pNode);
pDst->pNode = pSrc->pNode;
if (NULL == pDst->pNode) {
nodesDestroyNode(pDst);
return NULL;
}
pDst->subplanType = pSrc->subplanType;
pDst->level = level;
return pDst;
}
static int32_t doScaleOut(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SQueryLogicPlan* pLogicPlan) {
if (SUBPLAN_TYPE_MODIFY == pSubplan->subplanType) {
SVnodeModifLogicNode* pNode = (SVnodeModifLogicNode*)pSubplan->pNode;
size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks);
for (int32_t i = 0; i < numOfVgroups; ++i) {
SSubLogicPlan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level);
CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY);
SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i);
((SVnodeModifLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = blocks;
CHECK_CODE_EXT(pushSubplan(pCxt, pNewSubplan, level, pLogicPlan->pSubplans));
}
} else {
SSubLogicPlan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level);
CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY);
CHECK_CODE_EXT(pushSubplan(pCxt, pNewSubplan, level, pLogicPlan->pSubplans));
}
SNode* pChild;
FOREACH(pChild, pSubplan->pChildren) {
CHECK_CODE_EXT(doScaleOut(pCxt, (SSubLogicPlan*)pChild, level + 1, pLogicPlan));
}
return TSDB_CODE_SUCCESS;
}
static SQueryLogicPlan* makeQueryLogicPlan(SPhysiPlanContext* pCxt) {
SQueryLogicPlan* pLogicPlan = (SQueryLogicPlan*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN);
CHECK_ALLOC(pLogicPlan, NULL);
pLogicPlan->pSubplans = nodesMakeList();
CHECK_ALLOC(pLogicPlan->pSubplans, pLogicPlan);
SNodeListNode* pTopSubplans = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST);
CHECK_ALLOC(pTopSubplans, pLogicPlan);
CHECK_CODE(strictListAppend(pLogicPlan->pSubplans, pTopSubplans), pLogicPlan);
pTopSubplans->pNodeList = nodesMakeList();
CHECK_ALLOC(pTopSubplans->pNodeList, pLogicPlan);
SSubLogicPlan* pSubplan = (SSubLogicPlan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN);
CHECK_ALLOC(pSubplan, pLogicPlan);
CHECK_CODE(strictListAppend(pTopSubplans->pNodeList, pSubplan), pLogicPlan);
pSubplan->pNode = pLogicNode;
CHECK_ALLOC(pSubplan->pNode, pLogicPlan);
if (NULL == pLogicPlan->pSubplans) {
nodesDestroyNode(pLogicPlan);
return NULL;
}
return pLogicPlan;
}
static int32_t splitLogicPlan(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SQueryLogicPlan** pLogicPlan) {
SQueryLogicPlan* pPlan = createRawQueryLogicPlan(pCxt, pLogicNode);
if (TSDB_CODE_SUCCESS != pCxt->errCode) {
nodesDestroyNode((SNode*)pPlan);
return pCxt->errCode;
static int32_t scaleOutLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pRootSubLogicPlan, SQueryLogicPlan** pLogicPlan) {
*pLogicPlan = makeQueryLogicPlan(pCxt);
CHECK_ALLOC(*pLogicPlan, TSDB_CODE_OUT_OF_MEMORY);
return doScaleOut(pCxt, pRootSubLogicPlan, 0, *pLogicPlan);
}
typedef struct SBuildPhysiSubplanCxt {
int32_t errCode;
SQueryPlan* pQueryPlan;
SPhysiPlanContext* pPhyCxt;
} SBuildPhysiSubplanCxt;
static EDealRes doBuildPhysiSubplan(SNode* pNode, void* pContext) {
SBuildPhysiSubplanCxt* pCxt = (SBuildPhysiSubplanCxt*)pContext;
if (QUERY_NODE_LOGIC_SUBPLAN == nodeType(pNode)) {
SSubplan* pSubplan = createPhysiSubplan(pCxt->pPhyCxt, (SSubLogicPlan*)pNode);
CHECK_ALLOC(pSubplan, DEAL_RES_ERROR);
CHECK_CODE(pushSubplan(pCxt->pPhyCxt, pSubplan, ((SSubLogicPlan*)pNode)->level, pCxt->pQueryPlan->pSubplans), DEAL_RES_ERROR);
++(pCxt->pQueryPlan->numOfSubplans);
return DEAL_RES_IGNORE_CHILD;
}
// todo split
*pLogicPlan = pPlan;
return TSDB_CODE_SUCCESS;
return DEAL_RES_CONTINUE;
}
static SQueryPlan* makeQueryPhysiPlan(SPhysiPlanContext* pCxt) {
SQueryPlan* pPlan = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
CHECK_ALLOC(pPlan, NULL);
pPlan->pSubplans = nodesMakeList();
if (NULL == pPlan->pSubplans) {
nodesDestroyNode(pPlan);
return NULL;
}
pPlan->queryId = pCxt->pPlanCxt->queryId;
return pPlan;
}
static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan) {
SQueryPlan* pQueryPlan = (SQueryPlan*)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN);
CHECK_ALLOC(pQueryPlan, TSDB_CODE_OUT_OF_MEMORY);
*pPlan = pQueryPlan;
pQueryPlan->queryId = pCxt->pPlanCxt->queryId;
pQueryPlan->pSubplans = nodesMakeList();
CHECK_ALLOC(pQueryPlan->pSubplans, TSDB_CODE_OUT_OF_MEMORY);
SNode* pNode;
FOREACH(pNode, pLogicPlan->pSubplans) {
SNodeListNode* pLevelSubplans = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST);
CHECK_ALLOC(pLevelSubplans, TSDB_CODE_OUT_OF_MEMORY);
CHECK_CODE(strictListAppend(pQueryPlan->pSubplans, pLevelSubplans), TSDB_CODE_OUT_OF_MEMORY);
pLevelSubplans->pNodeList = nodesMakeList();
CHECK_ALLOC(pLevelSubplans->pNodeList, TSDB_CODE_OUT_OF_MEMORY);
SNode* pLogicSubplan;
FOREACH(pLogicSubplan, ((SNodeListNode*)pNode)->pNodeList) {
CHECK_CODE(strictListAppend(pLevelSubplans->pNodeList,
createPhysiSubplan(pCxt, (SSubLogicPlan*)pLogicSubplan)), TSDB_CODE_OUT_OF_MEMORY);
++(pQueryPlan->numOfSubplans);
}
SBuildPhysiSubplanCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pQueryPlan = makeQueryPhysiPlan(pCxt), .pPhyCxt = pCxt };
CHECK_ALLOC(cxt.pQueryPlan, TSDB_CODE_OUT_OF_MEMORY);
nodesWalkList(pLogicPlan->pSubplans, doBuildPhysiSubplan, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyNode(cxt.pQueryPlan);
return cxt.errCode;
}
*pPlan = cxt.pQueryPlan;
return TSDB_CODE_SUCCESS;
}
int32_t createPhysiPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SQueryPlan** pPlan) {
int32_t createPhysiPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SQueryPlan** pPlan, SArray* pExecNodeList) {
SPhysiPlanContext cxt = {
.pPlanCxt = pCxt,
.errCode = TSDB_CODE_SUCCESS,
.nextDataBlockId = 0,
.pLocationHelper = taosArrayInit(32, POINTER_BYTES)
.pLocationHelper = taosArrayInit(32, POINTER_BYTES),
.pExecNodeList = pExecNodeList
};
if (NULL == cxt.pLocationHelper) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SQueryLogicPlan* pLogicPlan;
int32_t code = splitLogicPlan(&cxt, pLogicNode, &pLogicPlan);
// todo scale out
SSubLogicPlan* pSubLogicPlan;
int32_t code = splitLogicPlan(&cxt, pLogicNode, &pSubLogicPlan);
if (TSDB_CODE_SUCCESS == code) {
code = scaleOutLogicPlan(&cxt, pSubLogicPlan, &pLogicPlan);
}
// todo maping
if (TSDB_CODE_SUCCESS == code) {
code = buildPhysiPlan(&cxt, pLogicPlan, pPlan);

11
source/libs/planner/src/planner.c
View File

@ -21,14 +21,14 @@ int32_t optimize(SPlanContext* pCxt, SLogicNode* pLogicNode) {
return TSDB_CODE_SUCCESS;
}
int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan) {
int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNodeList) {
SLogicNode* pLogicNode = NULL;
int32_t code = createLogicPlan(pCxt, &pLogicNode);
if (TSDB_CODE_SUCCESS == code) {
code = optimize(pCxt, pLogicNode);
}
if (TSDB_CODE_SUCCESS == code) {
code = createPhysiPlan(pCxt, pLogicNode, pPlan);
code = createPhysiPlan(pCxt, pLogicNode, pPlan, pExecNodeList);
}
return code;
}
@ -38,6 +38,13 @@ void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstrea
}
int32_t qSubPlanToString(const SSubplan* subplan, char** str, int32_t* len) {
if (SUBPLAN_TYPE_MODIFY == subplan->subplanType) {
SDataInserterNode* insert = (SDataInserterNode*)subplan->pDataSink;
*len = insert->size;
*str = insert->pData;
insert->pData = NULL;
return TSDB_CODE_SUCCESS;
}
return nodesNodeToString((const SNode*)subplan, false, str, len);
}

10
source/libs/planner/test/plannerTest.cpp
View File

@ -71,7 +71,7 @@ protected:
if (TEST_PHYSICAL_PLAN == target) {
SQueryPlan* pPlan = nullptr;
code = createPhysiPlan(&cxt, pLogicPlan, &pPlan);
code = createPhysiPlan(&cxt, pLogicPlan, &pPlan, NULL);
if (code != TSDB_CODE_SUCCESS) {
cout << "sql:[" << cxt_.pSql << "] physical plan code:" << code << ", strerror:" << tstrerror(code) << endl;
return false;
@ -109,6 +109,14 @@ private:
cout << "sql:[" << cxt_.pSql << "] toString code:" << code << ", strerror:" << tstrerror(code) << endl;
return string();
}
SNode* pNode;
code = nodesStringToNode(pStr, &pNode);
if (code != TSDB_CODE_SUCCESS) {
tfree(pStr);
cout << "sql:[" << cxt_.pSql << "] toObject code:" << code << ", strerror:" << tstrerror(code) << endl;
return string();
}
nodesDestroyNode(pNode);
string str(pStr);
tfree(pStr);
return str;

4
source/libs/qworker/test/qworkerTests.cpp
View File

@ -14,7 +14,6 @@
*/
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic push
@ -26,9 +25,11 @@
#pragma GCC diagnostic ignored "-Wformat"
#pragma GCC diagnostic ignored "-Wint-to-pointer-cast"
#pragma GCC diagnostic ignored "-Wpointer-arith"
#include <addr_any.h>
#include "os.h"
#include "tglobal.h"
#include "taos.h"
#include "tdef.h"
#include "tvariant.h"
@ -37,7 +38,6 @@
#include "planner.h"
#include "qworker.h"
#include "stub.h"
#include "addr_any.h"
#include "executor.h"
#include "dataSinkMgt.h"

4
source/libs/scalar/test/filter/filterTests.cpp
View File

@ -14,7 +14,6 @@
*/
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic push
@ -26,15 +25,16 @@
#pragma GCC diagnostic ignored "-Wformat"
#pragma GCC diagnostic ignored "-Wint-to-pointer-cast"
#pragma GCC diagnostic ignored "-Wpointer-arith"
#include <addr_any.h>
#include "os.h"
#include "tglobal.h"
#include "taos.h"
#include "tdef.h"
#include "tvariant.h"
#include "tep.h"
#include "stub.h"
#include "addr_any.h"
#include "scalar.h"
#include "nodes.h"
#include "tlog.h"

38
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -14,7 +14,6 @@
*/
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic push
@ -26,21 +25,30 @@
#pragma GCC diagnostic ignored "-Wformat"
#pragma GCC diagnostic ignored "-Wint-to-pointer-cast"
#pragma GCC diagnostic ignored "-Wpointer-arith"
#include <addr_any.h>
#include "os.h"
#include "tglobal.h"
#include "taos.h"
#include "tdef.h"
#include "tvariant.h"
#include "tep.h"
#include "stub.h"
#include "addr_any.h"
#include "scalar.h"
#include "nodes.h"
#include "tlog.h"
namespace {
SColumnInfo createColumnInfo(int32_t colId, int32_t type, int32_t bytes) {
SColumnInfo info = {0};
info.colId = colId;
info.type = type;
info.bytes = bytes;
return info;
}
int64_t scltLeftV = 21, scltRightV = 10;
double scltLeftVd = 21.0, scltRightVd = 10.0;
@ -914,7 +922,7 @@ TEST(columnTest, smallint_value_add_int_column) {
SArray *blockList = taosArrayInit(2, POINTER_BYTES);
taosArrayPush(blockList, &src);
SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)};
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo);
scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
@ -954,7 +962,7 @@ TEST(columnTest, bigint_column_multi_binary_column) {
SArray *blockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(blockList, &src);
SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)};
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo);
scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
@ -992,7 +1000,7 @@ TEST(columnTest, smallint_column_and_binary_column) {
SArray *blockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(blockList, &src);
SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BIGINT, .bytes = sizeof(int64_t)};
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t));
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo);
scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
@ -1025,7 +1033,7 @@ TEST(columnTest, smallint_column_or_float_column) {
SArray *blockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(blockList, &src);
SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BIGINT, .bytes = sizeof(int64_t)};
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t));
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo);
scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
@ -1058,7 +1066,7 @@ TEST(columnTest, smallint_column_or_double_value) {
SArray *blockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(blockList, &src);
SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BIGINT, .bytes = sizeof(int64_t)};
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t));
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo);
scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
@ -1091,7 +1099,7 @@ TEST(columnTest, smallint_column_greater_double_value) {
SArray *blockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(blockList, &src);
SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_BOOL, sizeof(bool));
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo);
scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
@ -1131,7 +1139,7 @@ TEST(columnTest, int_column_in_double_list) {
SArray *blockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(blockList, &src);
SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_BOOL, sizeof(bool));
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo);
scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
@ -1190,7 +1198,7 @@ TEST(columnTest, binary_column_in_binary_list) {
SArray *blockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(blockList, &src);
SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_BOOL, sizeof(bool));
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo);
scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
@ -1234,7 +1242,7 @@ TEST(columnTest, binary_column_like_binary) {
SArray *blockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(blockList, &src);
SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_BOOL, sizeof(bool));
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo);
scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
@ -1276,7 +1284,7 @@ TEST(columnTest, binary_column_is_true) {
SArray *blockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(blockList, &src);
SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_BOOL, sizeof(bool));
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo);
scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
@ -1320,7 +1328,7 @@ TEST(columnTest, binary_column_is_null) {
SArray *blockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(blockList, &src);
SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_BOOL, sizeof(bool));
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo);
scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
@ -1363,7 +1371,7 @@ TEST(columnTest, binary_column_is_not_null) {
SArray *blockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(blockList, &src);
SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_BOOL, sizeof(bool));
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo);
scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
@ -1405,7 +1413,7 @@ TEST(columnTest, greater_and_lower) {
SArray *blockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(blockList, &src);
SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_BOOL, sizeof(bool));
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo);
scltMakeTargetNode(&logicNode, dataBlockId, slotId, logicNode);

26
source/libs/scheduler/test/schedulerTests.cpp
View File

@ -14,19 +14,8 @@
*/
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#include "os.h"
#include "taos.h"
#include "tdef.h"
#include "tvariant.h"
#include "catalog.h"
#include "scheduler.h"
#include "tep.h"
#include "trpc.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
@ -34,10 +23,21 @@
#pragma GCC diagnostic ignored "-Wsign-compare"
#pragma GCC diagnostic ignored "-Wreturn-type"
#pragma GCC diagnostic ignored "-Wformat"
#include <addr_any.h>
#include "os.h"
#include "tglobal.h"
#include "taos.h"
#include "tdef.h"
#include "tvariant.h"
#include "catalog.h"
#include "scheduler.h"
#include "tep.h"
#include "trpc.h"
#include "schedulerInt.h"
#include "stub.h"
#include "addr_any.h"
#include "tref.h"
namespace {
@ -542,6 +542,8 @@ TEST(queryTest, normalCase) {
int64_t job = 0;
SQueryPlan dag;
memset(&dag, 0, sizeof(dag));
SArray *qnodeList = taosArrayInit(1, sizeof(SEp));
SEp qnodeAddr = {0};

72
source/libs/sync/inc/syncInt.h
View File

@ -25,6 +25,7 @@ extern "C" {
#include <stdlib.h>
#include "sync.h"
#include "taosdef.h"
#include "tglobal.h"
#include "tlog.h"
#include "ttimer.h"
@ -91,31 +92,61 @@ typedef struct SyncAppendEntriesReply SyncAppendEntriesReply;
struct SSyncEnv;
typedef struct SSyncEnv SSyncEnv;
struct SRaftStore;
typedef struct SRaftStore SRaftStore;
struct SVotesGranted;
typedef struct SVotesGranted SVotesGranted;
struct SVotesResponded;
typedef struct SVotesResponded SVotesResponded;
typedef struct SRaftId {
SyncNodeId addr; // typedef uint64_t SyncNodeId;
SyncGroupId vgId; // typedef int32_t SyncGroupId;
} SRaftId;
typedef struct SSyncNode {
// init by SSyncInfo
SyncGroupId vgId;
SSyncCfg syncCfg;
char path[TSDB_FILENAME_LEN];
SSyncFSM* pFsm;
// passed from outside
void* rpcClient;
void* rpcClient;
int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg);
// init internal
SNodeInfo me;
int32_t peersNum;
SNodeInfo peers[TSDB_MAX_REPLICA];
// raft algorithm
SSyncFSM* pFsm;
SRaftId raftId;
SRaftId peersId[TSDB_MAX_REPLICA];
int32_t replicaNum;
int32_t quorum;
// life cycle
int32_t refCount;
int64_t rid;
SNodeInfo me;
SNodeInfo peers[TSDB_MAX_REPLICA];
int32_t peersNum;
// tla+ server vars
ESyncState state;
SRaftStore* pRaftStore;
ESyncRole role;
SRaftId raftId;
// tla+ candidate vars
SVotesGranted* pVotesGranted;
SVotesResponded* pVotesResponded;
// tla+ leader vars
SHashObj* pNextIndex;
SHashObj* pMatchIndex;
// tla+ log vars
SSyncLogStore* pLogStore;
SyncIndex commitIndex;
// timer
tmr_h pPingTimer;
int32_t pingTimerMS;
uint8_t pingTimerStart;
@ -136,32 +167,21 @@ typedef struct SSyncNode {
// callback
int32_t (*FpOnPing)(SSyncNode* ths, SyncPing* pMsg);
int32_t (*FpOnPingReply)(SSyncNode* ths, SyncPingReply* pMsg);
int32_t (*FpOnRequestVote)(SSyncNode* ths, SyncRequestVote* pMsg);
int32_t (*FpOnRequestVoteReply)(SSyncNode* ths, SyncRequestVoteReply* pMsg);
int32_t (*FpOnAppendEntries)(SSyncNode* ths, SyncAppendEntries* pMsg);
int32_t (*FpOnAppendEntriesReply)(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
} SSyncNode;
SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo);
void syncNodeClose(SSyncNode* pSyncNode);
void syncNodePingAll(SSyncNode* pSyncNode);
void syncNodePingPeers(SSyncNode* pSyncNode);
void syncNodePingSelf(SSyncNode* pSyncNode);
int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode);
int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode);
void syncNodeClose(SSyncNode* pSyncNode);
void syncNodePingAll(SSyncNode* pSyncNode);
void syncNodePingPeers(SSyncNode* pSyncNode);
void syncNodePingSelf(SSyncNode* pSyncNode);
int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode);
int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode);
#ifdef __cplusplus
}

182
source/libs/sync/inc/syncMessage.h
View File

@ -28,30 +28,25 @@ extern "C" {
#include "syncRaftEntry.h"
#include "taosdef.h"
// encode as uint64
// encode as uint32
typedef enum ESyncMessageType {
SYNC_PING = 101,
SYNC_PING_REPLY = 103,
SYNC_CLIENT_REQUEST,
SYNC_CLIENT_REQUEST_REPLY,
SYNC_REQUEST_VOTE,
SYNC_REQUEST_VOTE_REPLY,
SYNC_APPEND_ENTRIES,
SYNC_APPEND_ENTRIES_REPLY,
SYNC_CLIENT_REQUEST = 105,
SYNC_CLIENT_REQUEST_REPLY = 107,
SYNC_REQUEST_VOTE = 109,
SYNC_REQUEST_VOTE_REPLY = 111,
SYNC_APPEND_ENTRIES = 113,
SYNC_APPEND_ENTRIES_REPLY = 115,
} ESyncMessageType;
/*
typedef struct SRaftId {
SyncNodeId addr; // typedef uint64_t SyncNodeId;
SyncGroupId vgId; // typedef int32_t SyncGroupId;
} SRaftId;
*/
// ---------------------------------------------
typedef struct SyncPing {
uint32_t bytes;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
uint32_t dataLen;
char data[];
} SyncPing;
@ -59,28 +54,22 @@ typedef struct SyncPing {
#define SYNC_PING_FIX_LEN (sizeof(uint32_t) + sizeof(uint32_t) + sizeof(SRaftId) + sizeof(SRaftId) + sizeof(uint32_t))
SyncPing* syncPingBuild(uint32_t dataLen);
void syncPingDestroy(SyncPing* pMsg);
void syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen);
void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg);
void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg);
void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg);
cJSON* syncPing2Json(const SyncPing* pMsg);
void syncPingDestroy(SyncPing* pMsg);
void syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen);
void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg);
void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg);
void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg);
cJSON* syncPing2Json(const SyncPing* pMsg);
SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str);
SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId);
// ---------------------------------------------
typedef struct SyncPingReply {
uint32_t bytes;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
uint32_t dataLen;
char data[];
} SyncPingReply;
@ -89,74 +78,117 @@ typedef struct SyncPingReply {
(sizeof(uint32_t) + sizeof(uint32_t) + sizeof(SRaftId) + sizeof(SRaftId) + sizeof(uint32_t))
SyncPingReply* syncPingReplyBuild(uint32_t dataLen);
void syncPingReplyDestroy(SyncPingReply* pMsg);
void syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen);
void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg);
void syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg);
void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg);
cJSON* syncPingReply2Json(const SyncPingReply* pMsg);
void syncPingReplyDestroy(SyncPingReply* pMsg);
void syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen);
void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg);
void syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg);
void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg);
cJSON* syncPingReply2Json(const SyncPingReply* pMsg);
SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str);
SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId);
// ---------------------------------------------
typedef struct SyncClientRequest {
ESyncMessageType msgType;
char* data;
uint32_t dataLen;
int64_t seqNum;
bool isWeak;
uint32_t bytes;
uint32_t msgType;
int64_t seqNum;
bool isWeak;
uint32_t dataLen;
char data[];
} SyncClientRequest;
// ---------------------------------------------
typedef struct SyncClientRequestReply {
ESyncMessageType msgType;
int32_t errCode;
SSyncBuffer* pErrMsg;
SSyncBuffer* pLeaderHint;
uint32_t bytes;
uint32_t msgType;
int32_t errCode;
SRaftId leaderHint;
} SyncClientRequestReply;
// ---------------------------------------------
typedef struct SyncRequestVote {
ESyncMessageType msgType;
SyncTerm currentTerm;
SyncNodeId nodeId;
SyncGroupId vgId;
SyncIndex lastLogIndex;
SyncTerm lastLogTerm;
uint32_t bytes;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
SyncTerm currentTerm;
SyncIndex lastLogIndex;
SyncTerm lastLogTerm;
} SyncRequestVote;
SyncRequestVote* syncRequestVoteBuild();
void syncRequestVoteDestroy(SyncRequestVote* pMsg);
void syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen);
void syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg);
void syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg);
void syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg);
cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg);
// ---------------------------------------------
typedef struct SyncRequestVoteReply {
ESyncMessageType msgType;
SyncTerm currentTerm;
SyncNodeId nodeId;
SyncGroupId vgId;
bool voteGranted;
uint32_t bytes;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
SyncTerm term;
bool voteGranted;
} SyncRequestVoteReply;
SyncRequestVoteReply* SyncRequestVoteReplyBuild();
void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg);
void syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen);
void syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg);
void syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg);
void syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg);
cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg);
// ---------------------------------------------
typedef struct SyncAppendEntries {
ESyncMessageType msgType;
SyncTerm currentTerm;
SyncNodeId nodeId;
SyncIndex prevLogIndex;
SyncTerm prevLogTerm;
int32_t entryCount;
SSyncRaftEntry* logEntries;
SyncIndex commitIndex;
uint32_t bytes;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
SyncIndex prevLogIndex;
SyncTerm prevLogTerm;
SyncIndex commitIndex;
uint32_t dataLen;
char data[];
} SyncAppendEntries;
#define SYNC_APPEND_ENTRIES_FIX_LEN \
(sizeof(uint32_t) + sizeof(uint32_t) + sizeof(SRaftId) + sizeof(SRaftId) + sizeof(SyncIndex) + sizeof(SyncTerm) + \
sizeof(SyncIndex) + sizeof(uint32_t))
SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen);
void syncAppendEntriesDestroy(SyncAppendEntries* pMsg);
void syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen);
void syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg);
void syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg);
void syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg);
cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg);
// ---------------------------------------------
typedef struct SyncAppendEntriesReply {
ESyncMessageType msgType;
SyncTerm currentTerm;
SyncNodeId nodeId;
bool success;
SyncIndex matchIndex;
uint32_t bytes;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
bool success;
SyncIndex matchIndex;
} SyncAppendEntriesReply;
SyncAppendEntriesReply* syncAppendEntriesReplyBuild();
void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg);
void syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen);
void syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg);
void syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg);
void syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg);
cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg);
#ifdef __cplusplus
}
#endif

6
source/libs/sync/inc/syncVoteMgr.h
View File

@ -26,6 +26,12 @@ extern "C" {
#include "syncInt.h"
#include "taosdef.h"
typedef struct SVotesGranted {
} SVotesGranted;
typedef struct SVotesResponded {
} SVotesResponded;
#ifdef __cplusplus
}
#endif

14
source/libs/sync/src/syncIO.c
View File

@ -211,23 +211,17 @@ static void *syncIOConsumerFunc(void *param) {
if (pRpcMsg->msgType == SYNC_PING) {
if (io->FpOnSyncPing != NULL) {
SyncPing *pSyncMsg;
SRpcMsg tmpRpcMsg;
memcpy(&tmpRpcMsg, pRpcMsg, sizeof(SRpcMsg));
pSyncMsg = syncPingBuild(tmpRpcMsg.contLen);
pSyncMsg = syncPingBuild(pRpcMsg->contLen);
syncPingFromRpcMsg(pRpcMsg, pSyncMsg);
// memcpy(pSyncMsg, tmpRpcMsg.pCont, tmpRpcMsg.contLen);
io->FpOnSyncPing(io->pSyncNode, pSyncMsg);
}
} else if (pRpcMsg->msgType == SYNC_PING_REPLY) {
SyncPingReply *pSyncMsg = syncPingReplyBuild(pRpcMsg->contLen);
syncPingReplyFromRpcMsg(pRpcMsg, pSyncMsg);
if (io->FpOnSyncPingReply != NULL) {
SyncPingReply *pSyncMsg;
pSyncMsg = syncPingReplyBuild(pRpcMsg->contLen);
syncPingReplyFromRpcMsg(pRpcMsg, pSyncMsg);
io->FpOnSyncPingReply(io->pSyncNode, pSyncMsg);
}
} else {

12
source/libs/sync/src/syncMain.c
View File

@ -88,7 +88,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
}
}
pSyncNode->role = TAOS_SYNC_STATE_FOLLOWER;
pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER;
syncUtilnodeInfo2raftId(&pSyncNode->me, pSyncNode->vgId, &pSyncNode->raftId);
pSyncNode->pPingTimer = NULL;
@ -171,16 +171,6 @@ static int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, Syn
SRpcMsg rpcMsg;
syncPing2RpcMsg(pMsg, &rpcMsg);
/*
SRpcMsg rpcMsg;
rpcMsg.contLen = 64;
rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
snprintf((char*)rpcMsg.pCont, rpcMsg.contLen, "%s", "xxxxxxxxxxxxxx");
rpcMsg.handle = NULL;
rpcMsg.msgType = 1;
*/
syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg);
{

421
source/libs/sync/src/syncMessage.c
View File

@ -60,12 +60,15 @@ void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg) {
}
cJSON* syncPing2Json(const SyncPing* pMsg) {
char u64buf[128];
cJSON* pRoot = cJSON_CreateObject();
cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
cJSON* pSrcId = cJSON_CreateObject();
cJSON_AddNumberToObject(pSrcId, "addr", pMsg->srcId.addr);
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr);
cJSON_AddStringToObject(pSrcId, "addr", u64buf);
{
uint64_t u64 = pMsg->srcId.addr;
cJSON* pTmp = pSrcId;
@ -79,7 +82,8 @@ cJSON* syncPing2Json(const SyncPing* pMsg) {
cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
cJSON* pDestId = cJSON_CreateObject();
cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr);
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr);
cJSON_AddStringToObject(pDestId, "addr", u64buf);
{
uint64_t u64 = pMsg->destId.addr;
cJSON* pTmp = pDestId;
@ -154,12 +158,15 @@ void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg) {
}
cJSON* syncPingReply2Json(const SyncPingReply* pMsg) {
char u64buf[128];
cJSON* pRoot = cJSON_CreateObject();
cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
cJSON* pSrcId = cJSON_CreateObject();
cJSON_AddNumberToObject(pSrcId, "addr", pMsg->srcId.addr);
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr);
cJSON_AddStringToObject(pSrcId, "addr", u64buf);
{
uint64_t u64 = pMsg->srcId.addr;
cJSON* pTmp = pSrcId;
@ -173,7 +180,8 @@ cJSON* syncPingReply2Json(const SyncPingReply* pMsg) {
cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
cJSON* pDestId = cJSON_CreateObject();
cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr);
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr);
cJSON_AddStringToObject(pDestId, "addr", u64buf);
{
uint64_t u64 = pMsg->destId.addr;
cJSON* pTmp = pDestId;
@ -208,72 +216,345 @@ SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId)
return pMsg;
}
#if 0
void syncPingSerialize(const SyncPing* pMsg, char** ppBuf, uint32_t* bufLen) {
*bufLen = sizeof(SyncPing) + pMsg->dataLen;
*ppBuf = (char*)malloc(*bufLen);
void* pStart = *ppBuf;
uint32_t allBytes = *bufLen;
int len = 0;
len = taosEncodeFixedU32(&pStart, pMsg->msgType);
allBytes -= len;
assert(len > 0);
pStart += len;
len = taosEncodeFixedU64(&pStart, pMsg->srcId.addr);
allBytes -= len;
assert(len > 0);
pStart += len;
len = taosEncodeFixedI32(&pStart, pMsg->srcId.vgId);
allBytes -= len;
assert(len > 0);
pStart += len;
len = taosEncodeFixedU64(&pStart, pMsg->destId.addr);
allBytes -= len;
assert(len > 0);
pStart += len;
len = taosEncodeFixedI32(&pStart, pMsg->destId.vgId);
allBytes -= len;
assert(len > 0);
pStart += len;
len = taosEncodeFixedU32(&pStart, pMsg->dataLen);
allBytes -= len;
assert(len > 0);
pStart += len;
memcpy(pStart, pMsg->data, pMsg->dataLen);
allBytes -= pMsg->dataLen;
assert(allBytes == 0);
// ---- message process SyncRequestVote----
SyncRequestVote* syncRequestVoteBuild() {
uint32_t bytes = sizeof(SyncRequestVote);
SyncRequestVote* pMsg = malloc(bytes);
memset(pMsg, 0, bytes);
pMsg->bytes = bytes;
pMsg->msgType = SYNC_REQUEST_VOTE;
}
void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg) {
void* pStart = (void*)buf;
uint64_t u64;
int32_t i32;
uint32_t u32;
pStart = taosDecodeFixedU64(pStart, &u64);
pMsg->msgType = u64;
pStart = taosDecodeFixedU64(pStart, &u64);
pMsg->srcId.addr = u64;
pStart = taosDecodeFixedI32(pStart, &i32);
pMsg->srcId.vgId = i32;
pStart = taosDecodeFixedU64(pStart, &u64);
pMsg->destId.addr = u64;
pStart = taosDecodeFixedI32(pStart, &i32);
pMsg->destId.vgId = i32;
pStart = taosDecodeFixedU32(pStart, &u32);
pMsg->dataLen = u32;
void syncRequestVoteDestroy(SyncRequestVote* pMsg) {
if (pMsg != NULL) {
free(pMsg);
}
}
#endif
void syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen) {
assert(pMsg->bytes <= bufLen);
memcpy(buf, pMsg, pMsg->bytes);
}
void syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg) {
memcpy(pMsg, buf, len);
assert(len == pMsg->bytes);
}
void syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg) {
memset(pRpcMsg, 0, sizeof(*pRpcMsg));
pRpcMsg->msgType = pMsg->msgType;
pRpcMsg->contLen = pMsg->bytes;
pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
syncRequestVoteSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
}
void syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg) {
syncRequestVoteDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
}
cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg) {
char u64buf[128];
cJSON* pRoot = cJSON_CreateObject();
cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
cJSON* pSrcId = cJSON_CreateObject();
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr);
cJSON_AddStringToObject(pSrcId, "addr", u64buf);
{
uint64_t u64 = pMsg->srcId.addr;
cJSON* pTmp = pSrcId;
char host[128];
uint16_t port;
syncUtilU642Addr(u64, host, sizeof(host), &port);
cJSON_AddStringToObject(pTmp, "addr_host", host);
cJSON_AddNumberToObject(pTmp, "addr_port", port);
}
cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId);
cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
cJSON* pDestId = cJSON_CreateObject();
cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr);
{
uint64_t u64 = pMsg->destId.addr;
cJSON* pTmp = pDestId;
char host[128];
uint16_t port;
syncUtilU642Addr(u64, host, sizeof(host), &port);
cJSON_AddStringToObject(pTmp, "addr_host", host);
cJSON_AddNumberToObject(pTmp, "addr_port", port);
}
cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
cJSON_AddItemToObject(pRoot, "destId", pDestId);
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->currentTerm);
cJSON_AddStringToObject(pRoot, "currentTerm", u64buf);
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogIndex);
cJSON_AddStringToObject(pRoot, "lastLogIndex", u64buf);
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogTerm);
cJSON_AddStringToObject(pRoot, "lastLogTerm", u64buf);
cJSON* pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SyncRequestVote", pRoot);
return pJson;
}
// ---- message process SyncRequestVoteReply----
SyncRequestVoteReply* SyncRequestVoteReplyBuild() {
uint32_t bytes = sizeof(SyncRequestVoteReply);
SyncRequestVoteReply* pMsg = malloc(bytes);
memset(pMsg, 0, bytes);
pMsg->bytes = bytes;
pMsg->msgType = SYNC_REQUEST_VOTE_REPLY;
}
void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg) {
if (pMsg != NULL) {
free(pMsg);
}
}
void syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen) {
assert(pMsg->bytes <= bufLen);
memcpy(buf, pMsg, pMsg->bytes);
}
void syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg) {
memcpy(pMsg, buf, len);
assert(len == pMsg->bytes);
}
void syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg) {
memset(pRpcMsg, 0, sizeof(*pRpcMsg));
pRpcMsg->msgType = pMsg->msgType;
pRpcMsg->contLen = pMsg->bytes;
pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
syncRequestVoteReplySerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
}
void syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg) {
syncRequestVoteReplyDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
}
cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg) {
char u64buf[128];
cJSON* pRoot = cJSON_CreateObject();
cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
cJSON* pSrcId = cJSON_CreateObject();
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr);
cJSON_AddStringToObject(pSrcId, "addr", u64buf);
{
uint64_t u64 = pMsg->srcId.addr;
cJSON* pTmp = pSrcId;
char host[128];
uint16_t port;
syncUtilU642Addr(u64, host, sizeof(host), &port);
cJSON_AddStringToObject(pTmp, "addr_host", host);
cJSON_AddNumberToObject(pTmp, "addr_port", port);
}
cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId);
cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
cJSON* pDestId = cJSON_CreateObject();
cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr);
{
uint64_t u64 = pMsg->destId.addr;
cJSON* pTmp = pDestId;
char host[128];
uint16_t port;
syncUtilU642Addr(u64, host, sizeof(host), &port);
cJSON_AddStringToObject(pTmp, "addr_host", host);
cJSON_AddNumberToObject(pTmp, "addr_port", port);
}
cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
cJSON_AddItemToObject(pRoot, "destId", pDestId);
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term);
cJSON_AddStringToObject(pRoot, "term", u64buf);
cJSON_AddNumberToObject(pRoot, "vote_granted", pMsg->voteGranted);
cJSON* pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SyncRequestVoteReply", pRoot);
return pJson;
}
// ---- message process SyncAppendEntries----
SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen) {
uint32_t bytes = SYNC_APPEND_ENTRIES_FIX_LEN + dataLen;
SyncAppendEntries* pMsg = malloc(bytes);
memset(pMsg, 0, bytes);
pMsg->bytes = bytes;
pMsg->msgType = SYNC_APPEND_ENTRIES;
pMsg->dataLen = dataLen;
}
void syncAppendEntriesDestroy(SyncAppendEntries* pMsg) {
if (pMsg != NULL) {
free(pMsg);
}
}
void syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen) {
assert(pMsg->bytes <= bufLen);
memcpy(buf, pMsg, pMsg->bytes);
}
void syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg) {
memcpy(pMsg, buf, len);
assert(len == pMsg->bytes);
assert(pMsg->bytes == SYNC_APPEND_ENTRIES_FIX_LEN + pMsg->dataLen);
}
void syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg) {
memset(pRpcMsg, 0, sizeof(*pRpcMsg));
pRpcMsg->msgType = pMsg->msgType;
pRpcMsg->contLen = pMsg->bytes;
pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
syncAppendEntriesSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
}
void syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg) {
syncAppendEntriesDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
}
cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) {
char u64buf[128];
cJSON* pRoot = cJSON_CreateObject();
cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
cJSON* pSrcId = cJSON_CreateObject();
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr);
cJSON_AddStringToObject(pSrcId, "addr", u64buf);
{
uint64_t u64 = pMsg->srcId.addr;
cJSON* pTmp = pSrcId;
char host[128];
uint16_t port;
syncUtilU642Addr(u64, host, sizeof(host), &port);
cJSON_AddStringToObject(pTmp, "addr_host", host);
cJSON_AddNumberToObject(pTmp, "addr_port", port);
}
cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId);
cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
cJSON* pDestId = cJSON_CreateObject();
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr);
cJSON_AddStringToObject(pDestId, "addr", u64buf);
{
uint64_t u64 = pMsg->destId.addr;
cJSON* pTmp = pDestId;
char host[128];
uint16_t port;
syncUtilU642Addr(u64, host, sizeof(host), &port);
cJSON_AddStringToObject(pTmp, "addr_host", host);
cJSON_AddNumberToObject(pTmp, "addr_port", port);
}
cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
cJSON_AddItemToObject(pRoot, "destId", pDestId);
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogIndex);
cJSON_AddStringToObject(pRoot, "pre_log_index", u64buf);
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogTerm);
cJSON_AddStringToObject(pRoot, "pre_log_term", u64buf);
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->commitIndex);
cJSON_AddStringToObject(pRoot, "commit_index", u64buf);
cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen);
cJSON_AddStringToObject(pRoot, "data", pMsg->data);
cJSON* pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SyncAppendEntries", pRoot);
return pJson;
}
// ---- message process SyncAppendEntriesReply----
SyncAppendEntriesReply* syncAppendEntriesReplyBuild() {
uint32_t bytes = sizeof(SyncAppendEntriesReply);
SyncAppendEntriesReply* pMsg = malloc(bytes);
memset(pMsg, 0, bytes);
pMsg->bytes = bytes;
pMsg->msgType = SYNC_APPEND_ENTRIES_REPLY;
}
void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg) {
if (pMsg != NULL) {
free(pMsg);
}
}
void syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen) {
assert(pMsg->bytes <= bufLen);
memcpy(buf, pMsg, pMsg->bytes);
}
void syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg) {
memcpy(pMsg, buf, len);
assert(len == pMsg->bytes);
}
void syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg) {
memset(pRpcMsg, 0, sizeof(*pRpcMsg));
pRpcMsg->msgType = pMsg->msgType;
pRpcMsg->contLen = pMsg->bytes;
pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
syncAppendEntriesReplySerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
}
void syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg) {
syncAppendEntriesReplyDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
}
cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) {
char u64buf[128];
cJSON* pRoot = cJSON_CreateObject();
cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
cJSON* pSrcId = cJSON_CreateObject();
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr);
cJSON_AddStringToObject(pSrcId, "addr", u64buf);
{
uint64_t u64 = pMsg->srcId.addr;
cJSON* pTmp = pSrcId;
char host[128];
uint16_t port;
syncUtilU642Addr(u64, host, sizeof(host), &port);
cJSON_AddStringToObject(pTmp, "addr_host", host);
cJSON_AddNumberToObject(pTmp, "addr_port", port);
}
cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId);
cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
cJSON* pDestId = cJSON_CreateObject();
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr);
cJSON_AddStringToObject(pDestId, "addr", u64buf);
{
uint64_t u64 = pMsg->destId.addr;
cJSON* pTmp = pDestId;
char host[128];
uint16_t port;
syncUtilU642Addr(u64, host, sizeof(host), &port);
cJSON_AddStringToObject(pTmp, "addr_host", host);
cJSON_AddNumberToObject(pTmp, "addr_port", port);
}
cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
cJSON_AddItemToObject(pRoot, "destId", pDestId);
cJSON_AddNumberToObject(pRoot, "success", pMsg->success);
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->matchIndex);
cJSON_AddStringToObject(pRoot, "match_index", u64buf);
cJSON* pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SyncAppendEntriesReply", pRoot);
return pJson;
}

371
source/libs/sync/test/syncEncodeTest.cpp
View File

@ -3,6 +3,7 @@
#include "syncIO.h"
#include "syncInt.h"
#include "syncMessage.h"
#include "syncUtil.h"
void logTest() {
sTrace("--- sync log test: trace");
@ -14,6 +15,7 @@ void logTest() {
}
#define PING_MSG_LEN 20
#define APPEND_ENTRIES_VALUE_LEN 32
void test1() {
sTrace("test1: ---- syncPingSerialize, syncPingDeserialize");
@ -21,16 +23,16 @@ void test1() {
char msg[PING_MSG_LEN];
snprintf(msg, sizeof(msg), "%s", "test ping");
SyncPing* pMsg = syncPingBuild(PING_MSG_LEN);
pMsg->srcId.addr = 1;
pMsg->srcId.vgId = 2;
pMsg->destId.addr = 3;
pMsg->destId.vgId = 4;
pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1111);
pMsg->srcId.vgId = 100;
pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 2222);
pMsg->destId.vgId = 100;
memcpy(pMsg->data, msg, PING_MSG_LEN);
{
cJSON* pJson = syncPing2Json(pMsg);
char* serialized = cJSON_Print(pJson);
printf("SyncPing: \n%s\n\n", serialized);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
@ -45,7 +47,7 @@ void test1() {
{
cJSON* pJson = syncPing2Json(pMsg2);
char* serialized = cJSON_Print(pJson);
printf("SyncPing2: \n%s\n\n", serialized);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
@ -61,16 +63,16 @@ void test2() {
char msg[PING_MSG_LEN];
snprintf(msg, sizeof(msg), "%s", "hello raft");
SyncPing* pMsg = syncPingBuild(PING_MSG_LEN);
pMsg->srcId.addr = 100;
pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 3333);
pMsg->srcId.vgId = 200;
pMsg->destId.addr = 300;
pMsg->destId.vgId = 400;
pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 4444);
pMsg->destId.vgId = 200;
memcpy(pMsg->data, msg, PING_MSG_LEN);
{
cJSON* pJson = syncPing2Json(pMsg);
char* serialized = cJSON_Print(pJson);
printf("SyncPing: \n%s\n\n", serialized);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
@ -84,7 +86,7 @@ void test2() {
{
cJSON* pJson = syncPing2Json(pMsg2);
char* serialized = cJSON_Print(pJson);
printf("SyncPing2: \n%s\n\n", serialized);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
@ -99,16 +101,16 @@ void test3() {
char msg[PING_MSG_LEN];
snprintf(msg, sizeof(msg), "%s", "test ping");
SyncPingReply* pMsg = syncPingReplyBuild(PING_MSG_LEN);
pMsg->srcId.addr = 19;
pMsg->srcId.vgId = 29;
pMsg->destId.addr = 39;
pMsg->destId.vgId = 49;
pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 5555);
pMsg->srcId.vgId = 100;
pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 6666);
pMsg->destId.vgId = 100;
memcpy(pMsg->data, msg, PING_MSG_LEN);
{
cJSON* pJson = syncPingReply2Json(pMsg);
char* serialized = cJSON_Print(pJson);
printf("SyncPingReply: \n%s\n\n", serialized);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
@ -123,7 +125,7 @@ void test3() {
{
cJSON* pJson = syncPingReply2Json(pMsg2);
char* serialized = cJSON_Print(pJson);
printf("SyncPingReply2: \n%s\n\n", serialized);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
@ -139,16 +141,16 @@ void test4() {
char msg[PING_MSG_LEN];
snprintf(msg, sizeof(msg), "%s", "hello raft");
SyncPingReply* pMsg = syncPingReplyBuild(PING_MSG_LEN);
pMsg->srcId.addr = 66;
pMsg->srcId.vgId = 77;
pMsg->destId.addr = 88;
pMsg->destId.vgId = 99;
pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 7777);
pMsg->srcId.vgId = 100;
pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 8888);
pMsg->destId.vgId = 100;
memcpy(pMsg->data, msg, PING_MSG_LEN);
{
cJSON* pJson = syncPingReply2Json(pMsg);
char* serialized = cJSON_Print(pJson);
printf("SyncPingReply: \n%s\n\n", serialized);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
@ -162,7 +164,7 @@ void test4() {
{
cJSON* pJson = syncPingReply2Json(pMsg2);
char* serialized = cJSON_Print(pJson);
printf("SyncPingReply2: \n%s\n\n", serialized);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
@ -170,6 +172,321 @@ void test4() {
syncPingReplyDestroy(pMsg);
syncPingReplyDestroy(pMsg2);
}
void test5() {
sTrace("test5: ---- syncRequestVoteSerialize, syncRequestVoteDeserialize");
SyncRequestVote* pMsg = syncRequestVoteBuild();
pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
pMsg->srcId.vgId = 100;
pMsg->destId.addr = syncUtilAddr2U64("8.8.8.8", 5678);
pMsg->destId.vgId = 100;
pMsg->currentTerm = 20;
pMsg->lastLogIndex = 21;
pMsg->lastLogTerm = 22;
{
cJSON* pJson = syncRequestVote2Json(pMsg);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
uint32_t bufLen = pMsg->bytes;
char* buf = (char*)malloc(bufLen);
syncRequestVoteSerialize(pMsg, buf, bufLen);
SyncRequestVote* pMsg2 = (SyncRequestVote*)malloc(pMsg->bytes);
syncRequestVoteDeserialize(buf, bufLen, pMsg2);
{
cJSON* pJson = syncRequestVote2Json(pMsg2);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
syncRequestVoteDestroy(pMsg);
syncRequestVoteDestroy(pMsg2);
free(buf);
}
void test6() {
sTrace("test6: ---- syncRequestVote2RpcMsg, syncRequestVoteFromRpcMsg");
SyncRequestVote* pMsg = syncRequestVoteBuild();
pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
pMsg->srcId.vgId = 100;
pMsg->destId.addr = syncUtilAddr2U64("8.8.8.8", 5678);
pMsg->destId.vgId = 100;
pMsg->currentTerm = 20;
pMsg->lastLogIndex = 21;
pMsg->lastLogTerm = 22;
{
cJSON* pJson = syncRequestVote2Json(pMsg);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
SRpcMsg rpcMsg;
syncRequestVote2RpcMsg(pMsg, &rpcMsg);
SyncRequestVote* pMsg2 = (SyncRequestVote*)malloc(pMsg->bytes);
syncRequestVoteFromRpcMsg(&rpcMsg, pMsg2);
rpcFreeCont(rpcMsg.pCont);
{
cJSON* pJson = syncRequestVote2Json(pMsg2);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
syncRequestVoteDestroy(pMsg);
syncRequestVoteDestroy(pMsg2);
}
void test7() {
sTrace("test7: ---- syncRequestVoteReplySerialize, syncRequestVoteReplyDeserialize");
SyncRequestVoteReply* pMsg = SyncRequestVoteReplyBuild();
pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
pMsg->srcId.vgId = 100;
pMsg->destId.addr = syncUtilAddr2U64("8.8.8.8", 5678);
pMsg->destId.vgId = 100;
pMsg->term = 20;
pMsg->voteGranted = 1;
{
cJSON* pJson = syncRequestVoteReply2Json(pMsg);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
uint32_t bufLen = pMsg->bytes;
char* buf = (char*)malloc(bufLen);
syncRequestVoteReplySerialize(pMsg, buf, bufLen);
SyncRequestVoteReply* pMsg2 = (SyncRequestVoteReply*)malloc(pMsg->bytes);
syncRequestVoteReplyDeserialize(buf, bufLen, pMsg2);
{
cJSON* pJson = syncRequestVoteReply2Json(pMsg2);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
syncRequestVoteReplyDestroy(pMsg);
syncRequestVoteReplyDestroy(pMsg2);
free(buf);
}
void test8() {
sTrace("test8: ---- syncRequestVoteReply2RpcMsg, syncRequestVoteReplyFromRpcMsg");
SyncRequestVoteReply* pMsg = SyncRequestVoteReplyBuild();
pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
pMsg->srcId.vgId = 100;
pMsg->destId.addr = syncUtilAddr2U64("8.8.8.8", 5678);
pMsg->destId.vgId = 100;
pMsg->term = 20;
pMsg->voteGranted = 1;
{
cJSON* pJson = syncRequestVoteReply2Json(pMsg);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
SRpcMsg rpcMsg;
syncRequestVoteReply2RpcMsg(pMsg, &rpcMsg);
SyncRequestVoteReply* pMsg2 = (SyncRequestVoteReply*)malloc(pMsg->bytes);
syncRequestVoteReplyFromRpcMsg(&rpcMsg, pMsg2);
rpcFreeCont(rpcMsg.pCont);
{
cJSON* pJson = syncRequestVoteReply2Json(pMsg2);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
syncRequestVoteReplyDestroy(pMsg);
syncRequestVoteReplyDestroy(pMsg2);
}
void test9() {
sTrace("test9: ---- syncAppendEntriesSerialize, syncAppendEntriesDeserialize");
char msg[APPEND_ENTRIES_VALUE_LEN];
snprintf(msg, sizeof(msg), "%s", "test value");
SyncAppendEntries* pMsg = syncAppendEntriesBuild(APPEND_ENTRIES_VALUE_LEN);
pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1111);
pMsg->srcId.vgId = 100;
pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 2222);
pMsg->destId.vgId = 100;
pMsg->prevLogIndex = 55;
pMsg->prevLogTerm = 66;
pMsg->commitIndex = 77;
memcpy(pMsg->data, msg, APPEND_ENTRIES_VALUE_LEN);
{
cJSON* pJson = syncAppendEntries2Json(pMsg);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
uint32_t bufLen = pMsg->bytes;
char* buf = (char*)malloc(bufLen);
syncAppendEntriesSerialize(pMsg, buf, bufLen);
SyncAppendEntries* pMsg2 = (SyncAppendEntries*)malloc(pMsg->bytes);
syncAppendEntriesDeserialize(buf, bufLen, pMsg2);
{
cJSON* pJson = syncAppendEntries2Json(pMsg2);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
syncAppendEntriesDestroy(pMsg);
syncAppendEntriesDestroy(pMsg2);
free(buf);
}
void test10() {
sTrace("test10: ---- syncAppendEntries2RpcMsg, syncAppendEntriesFromRpcMsg");
char msg[APPEND_ENTRIES_VALUE_LEN];
snprintf(msg, sizeof(msg), "%s", "test value");
SyncAppendEntries* pMsg = syncAppendEntriesBuild(APPEND_ENTRIES_VALUE_LEN);
pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1111);
pMsg->srcId.vgId = 100;
pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 2222);
pMsg->destId.vgId = 100;
pMsg->prevLogIndex = 55;
pMsg->prevLogTerm = 66;
pMsg->commitIndex = 77;
memcpy(pMsg->data, msg, APPEND_ENTRIES_VALUE_LEN);
{
cJSON* pJson = syncAppendEntries2Json(pMsg);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
SRpcMsg rpcMsg;
syncAppendEntries2RpcMsg(pMsg, &rpcMsg);
SyncAppendEntries* pMsg2 = (SyncAppendEntries*)malloc(pMsg->bytes);
syncAppendEntriesFromRpcMsg(&rpcMsg, pMsg2);
rpcFreeCont(rpcMsg.pCont);
{
cJSON* pJson = syncAppendEntries2Json(pMsg2);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
syncAppendEntriesDestroy(pMsg);
syncAppendEntriesDestroy(pMsg2);
}
void test11() {
sTrace("test11: ---- syncAppendEntriesReplySerialize, syncAppendEntriesReplyDeserialize");
SyncAppendEntriesReply* pMsg = syncAppendEntriesReplyBuild();
pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1111);
pMsg->srcId.vgId = 100;
pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 2222);
pMsg->destId.vgId = 100;
pMsg->success = 1;
pMsg->matchIndex = 23;
{
cJSON* pJson = syncAppendEntriesReply2Json(pMsg);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
uint32_t bufLen = pMsg->bytes;
char* buf = (char*)malloc(bufLen);
syncAppendEntriesReplySerialize(pMsg, buf, bufLen);
SyncAppendEntriesReply* pMsg2 = (SyncAppendEntriesReply*)malloc(pMsg->bytes);
syncAppendEntriesReplyDeserialize(buf, bufLen, pMsg2);
{
cJSON* pJson = syncAppendEntriesReply2Json(pMsg2);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
syncAppendEntriesReplyDestroy(pMsg);
syncAppendEntriesReplyDestroy(pMsg2);
free(buf);
}
void test12() {
sTrace("test12: ---- syncAppendEntriesReply2RpcMsg, syncAppendEntriesReplyFromRpcMsg");
SyncAppendEntriesReply* pMsg = syncAppendEntriesReplyBuild();
pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1111);
pMsg->srcId.vgId = 100;
pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 2222);
pMsg->destId.vgId = 100;
pMsg->success = 1;
pMsg->matchIndex = 23;
{
cJSON* pJson = syncAppendEntriesReply2Json(pMsg);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
SRpcMsg rpcMsg;
syncAppendEntriesReply2RpcMsg(pMsg, &rpcMsg);
SyncAppendEntriesReply* pMsg2 = (SyncAppendEntriesReply*)malloc(pMsg->bytes);
syncAppendEntriesReplyFromRpcMsg(&rpcMsg, pMsg2);
rpcFreeCont(rpcMsg.pCont);
{
cJSON* pJson = syncAppendEntriesReply2Json(pMsg2);
char* serialized = cJSON_Print(pJson);
printf("\n%s\n\n", serialized);
free(serialized);
cJSON_Delete(pJson);
}
syncAppendEntriesReplyDestroy(pMsg);
syncAppendEntriesReplyDestroy(pMsg2);
}
int main() {
// taosInitLog((char*)"syncPingTest.log", 100000, 10);
tsAsyncLog = 0;
@ -179,6 +496,14 @@ int main() {
test2();
test3();
test4();
test5();
test6();
test7();
test8();
test9();
test10();
test11();
test12();
return 0;
}

2
source/libs/sync/test/syncIOSendMsgClientTest.cpp
View File

@ -1,5 +1,5 @@
#include <stdio.h>
#include "gtest/gtest.h"
#include <gtest/gtest.h>
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftStore.h"

2
source/libs/sync/test/syncIOSendMsgServerTest.cpp
View File

@ -1,5 +1,5 @@
#include <stdio.h>
#include "gtest/gtest.h"
#include <gtest/gtest.h>
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftStore.h"

2
source/libs/sync/test/syncIOSendMsgTest.cpp
View File

@ -1,5 +1,5 @@
#include <stdio.h>
#include "gtest/gtest.h"
#include <gtest/gtest.h>
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftStore.h"

2
source/libs/sync/test/syncIOTickPingTest.cpp
View File

@ -1,5 +1,5 @@
#include <stdio.h>
#include "gtest/gtest.h"
#include <gtest/gtest.h>
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftStore.h"

2
source/libs/sync/test/syncIOTickQTest.cpp
View File

@ -1,5 +1,5 @@
#include <stdio.h>
#include "gtest/gtest.h"
#include <gtest/gtest.h>
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftStore.h"

31
source/libs/sync/test/syncPingTest.cpp
View File

@ -13,7 +13,9 @@ void logTest() {
sFatal("--- sync log test: fatal");
}
SSyncNode* doSync() {
uint16_t ports[3] = {7010, 7110, 7210};
SSyncNode* doSync(int myIndex) {
SSyncFSM* pFsm;
SSyncInfo syncInfo;
@ -24,18 +26,18 @@ SSyncNode* doSync() {
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping");
SSyncCfg* pCfg = &syncInfo.syncCfg;
pCfg->myIndex = 0;
pCfg->replicaNum = 2;
pCfg->myIndex = myIndex;
pCfg->replicaNum = 3;
pCfg->nodeInfo[0].nodePort = 7010;
pCfg->nodeInfo[0].nodePort = ports[0];
snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1");
// taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
pCfg->nodeInfo[1].nodePort = 7110;
pCfg->nodeInfo[1].nodePort = ports[1];
snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1");
// taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn);
pCfg->nodeInfo[2].nodePort = 7210;
pCfg->nodeInfo[2].nodePort = ports[2];
snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1");
// taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn);
@ -53,20 +55,25 @@ void timerPingAll(void* param, void* tmrId) {
syncNodePingAll(pSyncNode);
}
int main() {
int main(int argc, char** argv) {
// taosInitLog((char*)"syncPingTest.log", 100000, 10);
tsAsyncLog = 0;
sDebugFlag = 143 + 64;
logTest();
int32_t ret = syncIOStart((char*)"127.0.0.1", 7010);
int myIndex = 0;
if (argc >= 2) {
myIndex = atoi(argv[1]);
}
int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
assert(ret == 0);
ret = syncEnvStart();
assert(ret == 0);
SSyncNode* pSyncNode = doSync();
SSyncNode* pSyncNode = doSync(myIndex);
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
@ -74,9 +81,9 @@ int main() {
assert(ret == 0);
/*
taosMsleep(10000);
ret = syncNodeStopPingTimer(pSyncNode);
assert(ret == 0);
taosMsleep(10000);
ret = syncNodeStopPingTimer(pSyncNode);
assert(ret == 0);
*/
while (1) {

4
source/libs/sync/test/syncRaftStoreTest.cpp
View File

@ -1,6 +1,6 @@
#include "syncRaftStore.h"
#include <stdio.h>
#include "gtest/gtest.h"
#include <gtest/gtest.h>
#include "syncRaftStore.h"
#include "syncIO.h"
#include "syncInt.h"

2
source/libs/sync/test/syncTest.cpp
View File

@ -1,5 +1,5 @@
#include <stdio.h>
#include "gtest/gtest.h"
#include <gtest/gtest.h>
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftStore.h"

2
source/libs/tdb/src/db/tdb.c
View File

@ -90,7 +90,7 @@ int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv) {
// get page file from the env, if not opened yet, open it
pPgFile = NULL;
snprintf(dbfname, 128, "%s/%s", tdbEnvGetRootDir(pEnv), fname);
fileExist = (tdbCheckFileAccess(fname, TDB_F_OK) == 0);
fileExist = taosCheckExistFile(fname);
if (fileExist) {
tdbGnrtFileID(dbfname, fileid, false);
pPgFile = tdbEnvGetPageFile(pEnv, fileid);

43
source/libs/tdb/src/db/tdbUtil.c
View File

@ -16,16 +16,16 @@
#include "tdbInt.h"
int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) {
struct stat statbuf;
int64_t stDev = 0, stIno = 0;
if (stat(fname, &statbuf) < 0) {
if (taosDevInoFile(fname, &stDev, &stIno) < 0) {
return -1;
}
memset(fileid, 0, TDB_FILE_ID_LEN);
((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino;
((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev;
((uint64_t *)fileid)[0] = stDev;
((uint64_t *)fileid)[1] = stIno;
if (unique) {
((uint64_t *)fileid)[2] = rand();
}
@ -33,35 +33,34 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) {
return 0;
}
int tdbCheckFileAccess(const char *pathname, int mode) {
int flags = 0;
// int tdbCheckFileAccess(const char *pathname, int mode) {
// int flags = 0;
if (mode & TDB_F_OK) {
flags |= F_OK;
}
// if (mode & TDB_F_OK) {
// flags |= F_OK;
// }
if (mode & TDB_R_OK) {
flags |= R_OK;
}
// if (mode & TDB_R_OK) {
// flags |= R_OK;
// }
if (mode & TDB_W_OK) {
flags |= W_OK;
}
// if (mode & TDB_W_OK) {
// flags |= W_OK;
// }
return access(pathname, flags);
}
// return access(pathname, flags);
// }
int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize) {
struct stat st;
int ret;
ret = stat(fname, &st);
int64_t file_size = 0;
ret = taosStatFile(fname, &file_size, NULL);
if (ret != 0) {
return -1;
}
ASSERT(st.st_size % pgSize == 0);
ASSERT(file_size % pgSize == 0);
*pSize = st.st_size / pgSize;
*pSize = file_size / pgSize;
return 0;
}

8
source/libs/tdb/src/inc/tdbUtil.h
View File

@ -30,10 +30,10 @@ extern "C" {
int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique);
#define TDB_F_OK 0x1
#define TDB_R_OK 0x2
#define TDB_W_OK 0x4
int tdbCheckFileAccess(const char *pathname, int mode);
// #define TDB_F_OK 0x1
// #define TDB_R_OK 0x2
// #define TDB_W_OK 0x4
// int tdbCheckFileAccess(const char *pathname, int mode);
int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize);

2
source/libs/tdb/test/tdbTest.cpp
View File

@ -1,4 +1,4 @@
#include "gtest/gtest.h"
#include <gtest/gtest.h>
#include "tdb.h"

2
source/libs/tfs/CMakeLists.txt
View File

@ -6,7 +6,7 @@ target_include_directories(
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(tfs os util common)
target_link_libraries(tfs os util common monitor)
if(${BUILD_TEST})
add_subdirectory(test)

32
source/libs/tfs/src/tfs.c
View File

@ -389,7 +389,6 @@ static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg) {
static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) {
char dirName[TSDB_FILENAME_LEN] = "\0";
struct stat pstat;
if (pCfg->level < 0 || pCfg->level >= TFS_MAX_TIERS) {
fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level);
@ -422,19 +421,13 @@ static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) {
return -1;
}
if (access(dirName, W_OK | R_OK | F_OK) != 0) {
if (!taosCheckAccessFile(dirName, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) {
fError("failed to mount %s to FS since no R/W access rights", pCfg->dir);
terrno = TSDB_CODE_FS_INVLD_CFG;
return -1;
}
if (stat(dirName, &pstat) < 0) {
fError("failed to mount %s to FS since %s", pCfg->dir, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
if (!S_ISDIR(pstat.st_mode)) {
if (!taosIsDir(dirName)) {
fError("failed to mount %s to FS since not a directory", pCfg->dir);
terrno = TSDB_CODE_FS_INVLD_CFG;
return -1;
@ -544,3 +537,24 @@ static STfsDisk *tfsNextDisk(STfs *pTfs, SDiskIter *pIter) {
return pDisk;
}
int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo) {
pInfo->datadirs = taosArrayInit(32, sizeof(SMonDiskDesc));
if (pInfo->datadirs == NULL) return -1;
tfsUpdateSize(pTfs);
tfsLock(pTfs);
for (int32_t level = 0; level < pTfs->nlevel; level++) {
STfsTier *pTier = &pTfs->tiers[level];
for (int32_t disk = 0; disk < pTier->ndisk; ++disk) {
STfsDisk *pDisk = pTier->disks[disk];
SMonDiskDesc dinfo = {0};
dinfo.size = pDisk->size;
dinfo.level = pDisk->level;
tstrncpy(dinfo.name, pDisk->path, sizeof(dinfo.name));
taosArrayPush(pInfo->datadirs, &dinfo);
}
}
tfsUnLock(pTfs);
}

98
source/libs/tfs/test/tfsTest.cpp
View File

@ -119,29 +119,29 @@ TEST_F(TfsTest, 03_Dir) {
char p1[] = "p1";
char ap1[128] = {0};
snprintf(ap1, 128, "%s%s%s", root, TD_DIRSEP, p1);
EXPECT_NE(taosDirExist(ap1), 0);
EXPECT_NE(taosDirExist(ap1), 1);
EXPECT_EQ(tfsMkdir(pTfs, p1), 0);
EXPECT_EQ(taosDirExist(ap1), 0);
EXPECT_EQ(taosDirExist(ap1), 1);
char p2[] = "p2";
char ap2[128] = {0};
snprintf(ap2, 128, "%s%s%s", root, TD_DIRSEP, p2);
SDiskID did = {0};
EXPECT_NE(taosDirExist(ap2), 0);
EXPECT_NE(taosDirExist(ap2), 1);
EXPECT_EQ(tfsMkdirAt(pTfs, p2, did), 0);
EXPECT_EQ(taosDirExist(ap2), 0);
EXPECT_EQ(taosDirExist(ap2), 1);
char p3[] = "p3/p2/p1/p0";
char ap3[128] = {0};
snprintf(ap3, 128, "%s%s%s", root, TD_DIRSEP, p3);
EXPECT_NE(taosDirExist(ap3), 0);
EXPECT_NE(taosDirExist(ap3), 1);
EXPECT_NE(tfsMkdir(pTfs, p3), 0);
EXPECT_NE(tfsMkdirAt(pTfs, p3, did), 0);
EXPECT_EQ(tfsMkdirRecurAt(pTfs, p3, did), 0);
EXPECT_EQ(taosDirExist(ap3), 0);
EXPECT_EQ(taosDirExist(ap3), 1);
EXPECT_EQ(tfsRmdir(pTfs, p3), 0);
EXPECT_NE(taosDirExist(ap3), 0);
EXPECT_NE(taosDirExist(ap3), 1);
char p45[] = "p5";
char p44[] = "p4";
@ -149,12 +149,12 @@ TEST_F(TfsTest, 03_Dir) {
char ap4[128] = {0};
snprintf(ap4, 128, "%s%s%s", root, TD_DIRSEP, p4);
EXPECT_NE(taosDirExist(ap4), 0);
EXPECT_NE(taosDirExist(ap4), 1);
EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0);
EXPECT_EQ(taosDirExist(ap4), 0);
EXPECT_EQ(taosDirExist(ap4), 1);
EXPECT_EQ(tfsRename(pTfs, p44, p45), 0);
EXPECT_EQ(tfsRmdir(pTfs, p4), 0);
EXPECT_NE(taosDirExist(ap4), 0);
EXPECT_NE(taosDirExist(ap4), 1);
tfsClose(pTfs);
}
@ -251,9 +251,9 @@ TEST_F(TfsTest, 04_File) {
char af2[128] = {0};
snprintf(af2, 128, "%s%s%s", root, TD_DIRSEP, n2);
EXPECT_EQ(taosDirExist(af2), 0);
EXPECT_EQ(taosDirExist(af2), 1);
tfsRemoveFile(&f2);
EXPECT_NE(taosDirExist(af2), 0);
EXPECT_NE(taosDirExist(af2), 1);
{
STfsDir *pDir = tfsOpendir(pTfs, "t3");
@ -529,35 +529,35 @@ TEST_F(TfsTest, 05_MultiDisk) {
snprintf(ap22, 128, "%s%s%s", root22, TD_DIRSEP, p1);
char ap23[128] = {0};
snprintf(ap23, 128, "%s%s%s", root23, TD_DIRSEP, p1);
EXPECT_NE(taosDirExist(ap00), 0);
EXPECT_NE(taosDirExist(ap01), 0);
EXPECT_NE(taosDirExist(ap10), 0);
EXPECT_NE(taosDirExist(ap11), 0);
EXPECT_NE(taosDirExist(ap12), 0);
EXPECT_NE(taosDirExist(ap20), 0);
EXPECT_NE(taosDirExist(ap21), 0);
EXPECT_NE(taosDirExist(ap22), 0);
EXPECT_NE(taosDirExist(ap23), 0);
EXPECT_NE(taosDirExist(ap00), 1);
EXPECT_NE(taosDirExist(ap01), 1);
EXPECT_NE(taosDirExist(ap10), 1);
EXPECT_NE(taosDirExist(ap11), 1);
EXPECT_NE(taosDirExist(ap12), 1);
EXPECT_NE(taosDirExist(ap20), 1);
EXPECT_NE(taosDirExist(ap21), 1);
EXPECT_NE(taosDirExist(ap22), 1);
EXPECT_NE(taosDirExist(ap23), 1);
EXPECT_EQ(tfsMkdir(pTfs, p1), 0);
EXPECT_EQ(taosDirExist(ap00), 0);
EXPECT_EQ(taosDirExist(ap01), 0);
EXPECT_EQ(taosDirExist(ap10), 0);
EXPECT_EQ(taosDirExist(ap11), 0);
EXPECT_EQ(taosDirExist(ap12), 0);
EXPECT_EQ(taosDirExist(ap20), 0);
EXPECT_EQ(taosDirExist(ap21), 0);
EXPECT_EQ(taosDirExist(ap22), 0);
EXPECT_EQ(taosDirExist(ap23), 0);
EXPECT_EQ(taosDirExist(ap00), 1);
EXPECT_EQ(taosDirExist(ap01), 1);
EXPECT_EQ(taosDirExist(ap10), 1);
EXPECT_EQ(taosDirExist(ap11), 1);
EXPECT_EQ(taosDirExist(ap12), 1);
EXPECT_EQ(taosDirExist(ap20), 1);
EXPECT_EQ(taosDirExist(ap21), 1);
EXPECT_EQ(taosDirExist(ap22), 1);
EXPECT_EQ(taosDirExist(ap23), 1);
EXPECT_EQ(tfsRmdir(pTfs, p1), 0);
EXPECT_NE(taosDirExist(ap00), 0);
EXPECT_NE(taosDirExist(ap01), 0);
EXPECT_NE(taosDirExist(ap10), 0);
EXPECT_NE(taosDirExist(ap11), 0);
EXPECT_NE(taosDirExist(ap12), 0);
EXPECT_NE(taosDirExist(ap20), 0);
EXPECT_NE(taosDirExist(ap21), 0);
EXPECT_NE(taosDirExist(ap22), 0);
EXPECT_NE(taosDirExist(ap23), 0);
EXPECT_NE(taosDirExist(ap00), 1);
EXPECT_NE(taosDirExist(ap01), 1);
EXPECT_NE(taosDirExist(ap10), 1);
EXPECT_NE(taosDirExist(ap11), 1);
EXPECT_NE(taosDirExist(ap12), 1);
EXPECT_NE(taosDirExist(ap20), 1);
EXPECT_NE(taosDirExist(ap21), 1);
EXPECT_NE(taosDirExist(ap22), 1);
EXPECT_NE(taosDirExist(ap23), 1);
char p2[] = "p2";
char _ap21[128] = {0};
@ -565,22 +565,22 @@ TEST_F(TfsTest, 05_MultiDisk) {
SDiskID did = {0};
did.level = 2;
did.id = 1;
EXPECT_NE(taosDirExist(_ap21), 0);
EXPECT_NE(taosDirExist(_ap21), 1);
EXPECT_EQ(tfsMkdirAt(pTfs, p2, did), 0);
EXPECT_EQ(taosDirExist(_ap21), 0);
EXPECT_EQ(taosDirExist(_ap21), 1);
char p3[] = "p3/p2/p1/p0";
char _ap12[128] = {0};
snprintf(_ap12, 128, "%s%s%s", root12, TD_DIRSEP, p3);
did.level = 1;
did.id = 2;
EXPECT_NE(taosDirExist(_ap12), 0);
EXPECT_NE(taosDirExist(_ap12), 1);
EXPECT_NE(tfsMkdir(pTfs, p3), 0);
EXPECT_NE(tfsMkdirAt(pTfs, p3, did), 0);
EXPECT_EQ(tfsMkdirRecurAt(pTfs, p3, did), 0);
EXPECT_EQ(taosDirExist(_ap12), 0);
EXPECT_EQ(taosDirExist(_ap12), 1);
EXPECT_EQ(tfsRmdir(pTfs, p3), 0);
EXPECT_NE(taosDirExist(_ap12), 0);
EXPECT_NE(taosDirExist(_ap12), 1);
char p45[] = "p5";
char p44[] = "p4";
@ -590,12 +590,12 @@ TEST_F(TfsTest, 05_MultiDisk) {
did.level = 2;
did.id = 2;
EXPECT_NE(taosDirExist(_ap22), 0);
EXPECT_NE(taosDirExist(_ap22), 1);
EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0);
EXPECT_EQ(taosDirExist(_ap22), 0);
EXPECT_EQ(taosDirExist(_ap22), 1);
EXPECT_EQ(tfsRename(pTfs, p44, p45), 0);
EXPECT_EQ(tfsRmdir(pTfs, p4), 0);
EXPECT_NE(taosDirExist(_ap22), 0);
EXPECT_NE(taosDirExist(_ap22), 1);
}
//------------- File -----------------//
@ -660,7 +660,7 @@ TEST_F(TfsTest, 05_MultiDisk) {
char af2[128] = {0};
snprintf(af2, 128, "%s%s%s", root23, TD_DIRSEP, n2);
EXPECT_EQ(taosDirExist(af2), 0);
EXPECT_EQ(taosDirExist(af2), 1);
tfsRemoveFile(&f2);
{
@ -678,7 +678,7 @@ TEST_F(TfsTest, 05_MultiDisk) {
tfsClosedir(pDir);
}
EXPECT_NE(taosDirExist(af2), 0);
EXPECT_NE(taosDirExist(af2), 1);
EXPECT_GT(tfsCopyFile(&f1, &f2), 0);
{

20
source/libs/transport/src/transSrv.c
View File

@ -286,15 +286,17 @@ void uvOnWriteCb(uv_write_t* req, int status) {
transClearBuffer(&conn->readBuf);
if (status == 0) {
tTrace("server conn %p data already was written on stream", conn);
assert(taosArrayGetSize(conn->srvMsgs) >= 1);
SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0);
taosArrayRemove(conn->srvMsgs, 0);
destroySmsg(msg);
if (conn->srvMsgs != NULL) {
assert(taosArrayGetSize(conn->srvMsgs) >= 1);
SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0);
taosArrayRemove(conn->srvMsgs, 0);
destroySmsg(msg);
// send second data, just use for push
if (taosArrayGetSize(conn->srvMsgs) > 0) {
msg = (SSrvMsg*)taosArrayGetP(conn->srvMsgs, 0);
uvStartSendRespInternal(msg);
// send second data, just use for push
if (taosArrayGetSize(conn->srvMsgs) > 0) {
msg = (SSrvMsg*)taosArrayGetP(conn->srvMsgs, 0);
uvStartSendRespInternal(msg);
}
}
} else {
tError("server conn %p failed to write data, %s", conn, uv_err_name(status));
@ -615,7 +617,7 @@ static void destroyConn(SSrvConn* conn, bool clear) {
SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, i);
destroySmsg(msg);
}
taosArrayDestroy(conn->srvMsgs);
conn->srvMsgs = taosArrayDestroy(conn->srvMsgs);
QUEUE_REMOVE(&conn->queue);
if (clear) {

22
source/libs/wal/src/walMeta.c
View File

@ -64,10 +64,10 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) {
char fnameStr[WAL_FILE_LEN];
walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr);
struct stat statbuf;
stat(fnameStr, &statbuf);
int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size);
pLastFileInfo->fileSize = statbuf.st_size;
int64_t file_size = 0;
taosStatFile(fnameStr, &file_size, NULL);
int readSize = TMIN(WAL_MAX_SIZE + 2, file_size);
pLastFileInfo->fileSize = file_size;
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pFile == NULL) {
@ -177,11 +177,11 @@ int walCheckAndRepairMeta(SWal* pWal) {
SWalFileInfo *pLastFileInfo = taosArrayGet(pWal->fileInfoSet, newSz-1);
char fnameStr[WAL_FILE_LEN];
walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr);
struct stat statbuf;
stat(fnameStr, &statbuf);
int64_t file_size = 0;
taosStatFile(fnameStr, &file_size, NULL);
if (oldSz != newSz || pLastFileInfo->fileSize != statbuf.st_size) {
pLastFileInfo->fileSize = statbuf.st_size;
if (oldSz != newSz || pLastFileInfo->fileSize != file_size) {
pLastFileInfo->fileSize = file_size;
pWal->vers.lastVer = walScanLogGetLastVer(pWal);
((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = pWal->vers.lastVer;
ASSERT(pWal->vers.lastVer != -1);
@ -395,9 +395,9 @@ int walLoadMeta(SWal* pWal) {
char fnameStr[WAL_FILE_LEN];
walBuildMetaName(pWal, metaVer, fnameStr);
// read metafile
struct stat statbuf;
stat(fnameStr, &statbuf);
int size = statbuf.st_size;
int64_t file_size = 0;
taosStatFile(fnameStr, &file_size, NULL);
int size = (int)file_size;
char* buf = malloc(size + 5);
if (buf == NULL) {
terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;

2
source/os/src/osDir.c
View File

@ -60,7 +60,7 @@ void taosRemoveDir(const char *dirname) {
//printf("dir:%s is removed\n", dirname);
}
int32_t taosDirExist(char *dirname) { return access(dirname, F_OK); }
bool taosDirExist(char *dirname) { return taosCheckExistFile(dirname); }
int32_t taosMkDir(const char *dirname) {
int32_t code = mkdir(dirname, 0755);

Some files were not shown because too many files have changed in this diff Show More