xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/shm

This commit is contained in:
Shengliang Guan 2022-03-20 00:30:24 +08:00
parent fd7f61a069 a4ea6c7191
commit 916e569229
93 changed files with 7494 additions and 4296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
include/common/tmsg.h
View File

@ -111,15 +111,16 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_MAX, TSDB_MGMT_TABLE_MAX,
} EShowType; } EShowType;
#define TSDB_ALTER_TABLE_ADD_TAG 1 #define TSDB_ALTER_TABLE_ADD_TAG 1
#define TSDB_ALTER_TABLE_DROP_TAG 2 #define TSDB_ALTER_TABLE_DROP_TAG 2
#define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3 #define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3
#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 #define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4
#define TSDB_ALTER_TABLE_ADD_COLUMN 5 #define TSDB_ALTER_TABLE_ADD_COLUMN 5
#define TSDB_ALTER_TABLE_DROP_COLUMN 6 #define TSDB_ALTER_TABLE_DROP_COLUMN 6
#define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES 7 #define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES 7
#define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8 #define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8
#define TSDB_ALTER_TABLE_UPDATE_OPTIONS 9
#define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME 10
#define TSDB_FILL_NONE 0 #define TSDB_FILL_NONE 0
#define TSDB_FILL_NULL 1 #define TSDB_FILL_NULL 1
@ -2053,27 +2054,19 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) {
buf = taosDecodeFixedI64(buf, &pSma->sliding); buf = taosDecodeFixedI64(buf, &pSma->sliding);
if (pSma->exprLen > 0) { if (pSma->exprLen > 0) {
pSma->expr = (char*)calloc(pSma->exprLen, 1); if ((buf = taosDecodeString(buf, &pSma->expr)) == NULL) {
if (pSma->expr != NULL) {
buf = taosDecodeStringTo(buf, pSma->expr);
} else {
tdDestroyTSma(pSma); tdDestroyTSma(pSma);
return NULL; return NULL;
} }
} else { } else {
pSma->expr = NULL; pSma->expr = NULL;
} }
if (pSma->tagsFilterLen > 0) { if (pSma->tagsFilterLen > 0) {
pSma->tagsFilter = (char*)calloc(pSma->tagsFilterLen, 1); if ((buf = taosDecodeString(buf, &pSma->tagsFilter)) == NULL) {
if (pSma->tagsFilter != NULL) {
buf = taosDecodeStringTo(buf, pSma->tagsFilter);
} else {
tdDestroyTSma(pSma); tdDestroyTSma(pSma);
return NULL; return NULL;
} }
} else { } else {
pSma->tagsFilter = NULL; pSma->tagsFilter = NULL;
} }

263
include/common/ttokendef.h
View File

@ -34,128 +34,147 @@
#define TK_NK_REM 16 #define TK_NK_REM 16
#define TK_NK_CONCAT 17 #define TK_NK_CONCAT 17
#define TK_CREATE 18 #define TK_CREATE 18
#define TK_USER 19 #define TK_ACCOUNT 19
#define TK_PASS 20 #define TK_NK_ID 20
#define TK_NK_STRING 21 #define TK_PASS 21
#define TK_ALTER 22 #define TK_NK_STRING 22
#define TK_PRIVILEGE 23 #define TK_ALTER 23
#define TK_DROP 24 #define TK_PPS 24
#define TK_SHOW 25 #define TK_TSERIES 25
#define TK_USERS 26 #define TK_STORAGE 26
#define TK_DNODE 27 #define TK_STREAMS 27
#define TK_PORT 28 #define TK_QTIME 28
#define TK_NK_INTEGER 29 #define TK_DBS 29
#define TK_DNODES 30 #define TK_USERS 30
#define TK_NK_ID 31 #define TK_CONNS 31
#define TK_NK_IPTOKEN 32 #define TK_STATE 32
#define TK_QNODE 33 #define TK_USER 33
#define TK_ON 34 #define TK_PRIVILEGE 34
#define TK_QNODES 35 #define TK_DROP 35
#define TK_DATABASE 36 #define TK_SHOW 36
#define TK_DATABASES 37 #define TK_DNODE 37
#define TK_USE 38 #define TK_PORT 38
#define TK_IF 39 #define TK_NK_INTEGER 39
#define TK_NOT 40 #define TK_DNODES 40
#define TK_EXISTS 41 #define TK_NK_IPTOKEN 41
#define TK_BLOCKS 42 #define TK_LOCAL 42
#define TK_CACHE 43 #define TK_QNODE 43
#define TK_CACHELAST 44 #define TK_ON 44
#define TK_COMP 45 #define TK_QNODES 45
#define TK_DAYS 46 #define TK_DATABASE 46
#define TK_FSYNC 47 #define TK_DATABASES 47
#define TK_MAXROWS 48 #define TK_USE 48
#define TK_MINROWS 49 #define TK_IF 49
#define TK_KEEP 50 #define TK_NOT 50
#define TK_PRECISION 51 #define TK_EXISTS 51
#define TK_QUORUM 52 #define TK_BLOCKS 52
#define TK_REPLICA 53 #define TK_CACHE 53
#define TK_TTL 54 #define TK_CACHELAST 54
#define TK_WAL 55 #define TK_COMP 55
#define TK_VGROUPS 56 #define TK_DAYS 56
#define TK_SINGLE_STABLE 57 #define TK_FSYNC 57
#define TK_STREAM_MODE 58 #define TK_MAXROWS 58
#define TK_TABLE 59 #define TK_MINROWS 59
#define TK_NK_LP 60 #define TK_KEEP 60
#define TK_NK_RP 61 #define TK_PRECISION 61
#define TK_STABLE 62 #define TK_QUORUM 62
#define TK_TABLES 63 #define TK_REPLICA 63
#define TK_STABLES 64 #define TK_TTL 64
#define TK_USING 65 #define TK_WAL 65
#define TK_TAGS 66 #define TK_VGROUPS 66
#define TK_NK_DOT 67 #define TK_SINGLE_STABLE 67
#define TK_NK_COMMA 68 #define TK_STREAM_MODE 68
#define TK_COMMENT 69 #define TK_RETENTIONS 69
#define TK_BOOL 70 #define TK_FILE_FACTOR 70
#define TK_TINYINT 71 #define TK_NK_FLOAT 71
#define TK_SMALLINT 72 #define TK_TABLE 72
#define TK_INT 73 #define TK_NK_LP 73
#define TK_INTEGER 74 #define TK_NK_RP 74
#define TK_BIGINT 75 #define TK_STABLE 75
#define TK_FLOAT 76 #define TK_TABLES 76
#define TK_DOUBLE 77 #define TK_STABLES 77
#define TK_BINARY 78 #define TK_ADD 78
#define TK_TIMESTAMP 79 #define TK_COLUMN 79
#define TK_NCHAR 80 #define TK_MODIFY 80
#define TK_UNSIGNED 81 #define TK_RENAME 81
#define TK_JSON 82 #define TK_TAG 82
#define TK_VARCHAR 83 #define TK_SET 83
#define TK_MEDIUMBLOB 84 #define TK_NK_EQ 84
#define TK_BLOB 85 #define TK_USING 85
#define TK_VARBINARY 86 #define TK_TAGS 86
#define TK_DECIMAL 87 #define TK_NK_DOT 87
#define TK_SMA 88 #define TK_NK_COMMA 88
#define TK_INDEX 89 #define TK_COMMENT 89
#define TK_FULLTEXT 90 #define TK_BOOL 90
#define TK_FUNCTION 91 #define TK_TINYINT 91
#define TK_INTERVAL 92 #define TK_SMALLINT 92
#define TK_TOPIC 93 #define TK_INT 93
#define TK_AS 94 #define TK_INTEGER 94
#define TK_MNODES 95 #define TK_BIGINT 95
#define TK_NK_FLOAT 96 #define TK_FLOAT 96
#define TK_NK_BOOL 97 #define TK_DOUBLE 97
#define TK_NK_VARIABLE 98 #define TK_BINARY 98
#define TK_BETWEEN 99 #define TK_TIMESTAMP 99
#define TK_IS 100 #define TK_NCHAR 100
#define TK_NULL 101 #define TK_UNSIGNED 101
#define TK_NK_LT 102 #define TK_JSON 102
#define TK_NK_GT 103 #define TK_VARCHAR 103
#define TK_NK_LE 104 #define TK_MEDIUMBLOB 104
#define TK_NK_GE 105 #define TK_BLOB 105
#define TK_NK_NE 106 #define TK_VARBINARY 106
#define TK_NK_EQ 107 #define TK_DECIMAL 107
#define TK_LIKE 108 #define TK_SMA 108
#define TK_MATCH 109 #define TK_ROLLUP 109
#define TK_NMATCH 110 #define TK_INDEX 110
#define TK_IN 111 #define TK_FULLTEXT 111
#define TK_FROM 112 #define TK_FUNCTION 112
#define TK_JOIN 113 #define TK_INTERVAL 113
#define TK_INNER 114 #define TK_TOPIC 114
#define TK_SELECT 115 #define TK_AS 115
#define TK_DISTINCT 116 #define TK_MNODES 116
#define TK_WHERE 117 #define TK_NK_BOOL 117
#define TK_PARTITION 118 #define TK_NK_VARIABLE 118
#define TK_BY 119 #define TK_BETWEEN 119
#define TK_SESSION 120 #define TK_IS 120
#define TK_STATE_WINDOW 121 #define TK_NULL 121
#define TK_SLIDING 122 #define TK_NK_LT 122
#define TK_FILL 123 #define TK_NK_GT 123
#define TK_VALUE 124 #define TK_NK_LE 124
#define TK_NONE 125 #define TK_NK_GE 125
#define TK_PREV 126 #define TK_NK_NE 126
#define TK_LINEAR 127 #define TK_LIKE 127
#define TK_NEXT 128 #define TK_MATCH 128
#define TK_GROUP 129 #define TK_NMATCH 129
#define TK_HAVING 130 #define TK_IN 130
#define TK_ORDER 131 #define TK_FROM 131
#define TK_SLIMIT 132 #define TK_JOIN 132
#define TK_SOFFSET 133 #define TK_INNER 133
#define TK_LIMIT 134 #define TK_SELECT 134
#define TK_OFFSET 135 #define TK_DISTINCT 135
#define TK_ASC 136 #define TK_WHERE 136
#define TK_DESC 137 #define TK_PARTITION 137
#define TK_NULLS 138 #define TK_BY 138
#define TK_FIRST 139 #define TK_SESSION 139
#define TK_LAST 140 #define TK_STATE_WINDOW 140
#define TK_SLIDING 141
#define TK_FILL 142
#define TK_VALUE 143
#define TK_NONE 144
#define TK_PREV 145
#define TK_LINEAR 146
#define TK_NEXT 147
#define TK_GROUP 148
#define TK_HAVING 149
#define TK_ORDER 150
#define TK_SLIMIT 151
#define TK_SOFFSET 152
#define TK_LIMIT 153
#define TK_OFFSET 154
#define TK_ASC 155
#define TK_DESC 156
#define TK_NULLS 157
#define TK_FIRST 158
#define TK_LAST 159
#define TK_NK_SPACE 300 #define TK_NK_SPACE 300
#define TK_NK_COMMENT 301 #define TK_NK_COMMENT 301

2
include/common/ttypes.h
View File

@ -27,7 +27,7 @@ extern "C" {
typedef int32_t VarDataOffsetT; typedef int32_t VarDataOffsetT;
typedef uint32_t TDRowLenT; typedef uint32_t TDRowLenT;
typedef uint8_t TDRowValT; typedef uint8_t TDRowValT;
typedef uint16_t col_id_t; typedef int16_t col_id_t;
typedef int8_t col_type_t; typedef int8_t col_type_t;
#pragma pack(push, 1) #pragma pack(push, 1)

25
include/libs/nodes/cmdnodes.h
View File

@ -127,6 +127,18 @@ typedef struct SDropSuperTableStmt {
bool ignoreNotExists; bool ignoreNotExists;
} SDropSuperTableStmt; } SDropSuperTableStmt;
typedef struct SAlterTableStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN];
int8_t alterType;
char colName[TSDB_COL_NAME_LEN];
char newColName[TSDB_COL_NAME_LEN];
STableOptions* pOptions;
SDataType dataType;
SValueNode* pVal;
} SAlterTableStmt;
typedef struct SCreateUserStmt { typedef struct SCreateUserStmt {
ENodeType type; ENodeType type;
char useName[TSDB_USER_LEN]; char useName[TSDB_USER_LEN];
@ -158,6 +170,13 @@ typedef struct SDropDnodeStmt {
int32_t port; int32_t port;
} SDropDnodeStmt; } SDropDnodeStmt;
typedef struct SAlterDnodeStmt {
ENodeType type;
int32_t dnodeId;
char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_DNODE_VALUE_LEN];
} SAlterDnodeStmt;
typedef struct SShowStmt { typedef struct SShowStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
@ -215,6 +234,12 @@ typedef struct SDropTopicStmt {
bool ignoreNotExists; bool ignoreNotExists;
} SDropTopicStmt; } SDropTopicStmt;
typedef struct SAlterLocalStmt {
ENodeType type;
char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_DNODE_VALUE_LEN];
} SAlterLocalStmt;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

4
include/libs/nodes/nodes.h
View File

@ -85,6 +85,7 @@ typedef enum ENodeType {
QUERY_NODE_DROP_TABLE_CLAUSE, QUERY_NODE_DROP_TABLE_CLAUSE,
QUERY_NODE_DROP_TABLE_STMT, QUERY_NODE_DROP_TABLE_STMT,
QUERY_NODE_DROP_SUPER_TABLE_STMT, QUERY_NODE_DROP_SUPER_TABLE_STMT,
QUERY_NODE_ALTER_TABLE_STMT,
QUERY_NODE_SHOW_TABLES_STMT, // temp QUERY_NODE_SHOW_TABLES_STMT, // temp
QUERY_NODE_SHOW_STABLES_STMT, QUERY_NODE_SHOW_STABLES_STMT,
QUERY_NODE_CREATE_USER_STMT, QUERY_NODE_CREATE_USER_STMT,
@ -94,6 +95,7 @@ typedef enum ENodeType {
QUERY_NODE_USE_DATABASE_STMT, QUERY_NODE_USE_DATABASE_STMT,
QUERY_NODE_CREATE_DNODE_STMT, QUERY_NODE_CREATE_DNODE_STMT,
QUERY_NODE_DROP_DNODE_STMT, QUERY_NODE_DROP_DNODE_STMT,
QUERY_NODE_ALTER_DNODE_STMT,
QUERY_NODE_SHOW_DNODES_STMT, QUERY_NODE_SHOW_DNODES_STMT,
QUERY_NODE_SHOW_VGROUPS_STMT, QUERY_NODE_SHOW_VGROUPS_STMT,
QUERY_NODE_SHOW_MNODES_STMT, QUERY_NODE_SHOW_MNODES_STMT,
@ -104,6 +106,7 @@ typedef enum ENodeType {
QUERY_NODE_DROP_QNODE_STMT, QUERY_NODE_DROP_QNODE_STMT,
QUERY_NODE_CREATE_TOPIC_STMT, QUERY_NODE_CREATE_TOPIC_STMT,
QUERY_NODE_DROP_TOPIC_STMT, QUERY_NODE_DROP_TOPIC_STMT,
QUERY_NODE_ALTER_LOCAL_STMT,
// logic plan node // logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN, QUERY_NODE_LOGIC_PLAN_SCAN,
@ -163,6 +166,7 @@ SNodeList* nodesMakeList();
int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode); int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode);
int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode); int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode);
int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc); int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
SListCell* nodesListErase(SNodeList* pList, SListCell* pCell); SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
SNodeptr nodesListGetNode(SNodeList* pList, int32_t index); SNodeptr nodesListGetNode(SNodeList* pList, int32_t index);
void nodesDestroyList(SNodeList* pList); void nodesDestroyList(SNodeList* pList);

5
include/libs/nodes/plannodes.h
View File

@ -26,7 +26,6 @@ extern "C" {
typedef struct SLogicNode { typedef struct SLogicNode {
ENodeType type; ENodeType type;
int32_t id;
SNodeList* pTargets; // SColumnNode SNodeList* pTargets; // SColumnNode
SNode* pConditions; SNode* pConditions;
SNodeList* pChildren; SNodeList* pChildren;
@ -37,6 +36,7 @@ typedef enum EScanType {
SCAN_TYPE_TAG, SCAN_TYPE_TAG,
SCAN_TYPE_TABLE, SCAN_TYPE_TABLE,
SCAN_TYPE_STABLE, SCAN_TYPE_STABLE,
SCAN_TYPE_TOPIC,
SCAN_TYPE_STREAM SCAN_TYPE_STREAM
} EScanType; } EScanType;
@ -155,7 +155,7 @@ typedef struct SPhysiNode {
} SPhysiNode; } SPhysiNode;
typedef struct SScanPhysiNode { typedef struct SScanPhysiNode {
SPhysiNode node; SPhysiNode node;
SNodeList* pScanCols; SNodeList* pScanCols;
uint64_t uid; // unique id of the table uint64_t uid; // unique id of the table
int8_t tableType; int8_t tableType;
@ -167,6 +167,7 @@ typedef struct SScanPhysiNode {
typedef SScanPhysiNode SSystemTableScanPhysiNode; typedef SScanPhysiNode SSystemTableScanPhysiNode;
typedef SScanPhysiNode STagScanPhysiNode; typedef SScanPhysiNode STagScanPhysiNode;
typedef SScanPhysiNode SStreamScanPhysiNode;
typedef struct STableScanPhysiNode { typedef struct STableScanPhysiNode {
SScanPhysiNode scan; SScanPhysiNode scan;

4
include/libs/nodes/querynodes.h
View File

@ -23,6 +23,10 @@ extern "C" {
#include "nodes.h" #include "nodes.h"
#include "tmsg.h" #include "tmsg.h"
#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags)
#define TABLE_META_SIZE(pMeta) (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema)))
#define VGROUPS_INFO_SIZE(pInfo) (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo)))
typedef struct SRawExprNode { typedef struct SRawExprNode {
ENodeType nodeType; ENodeType nodeType;
char* p; char* p;

1
include/libs/parser/parser.h
View File

@ -26,6 +26,7 @@ typedef struct SParseContext {
uint64_t requestId; uint64_t requestId;
int32_t acctId; int32_t acctId;
const char *db; const char *db;
bool topicQuery;
void *pTransporter; void *pTransporter;
SEpSet mgmtEpSet; SEpSet mgmtEpSet;
const char *pSql; // sql string const char *pSql; // sql string

1
include/libs/planner/planner.h
View File

@ -26,6 +26,7 @@ typedef struct SPlanContext {
uint64_t queryId; uint64_t queryId;
int32_t acctId; int32_t acctId;
SNode* pAstRoot; SNode* pAstRoot;
bool topicQuery;
bool streamQuery; bool streamQuery;
} SPlanContext; } SPlanContext;

44
include/libs/transport/trpc.h
View File

@ -38,13 +38,13 @@ typedef struct SRpcConnInfo {
typedef struct SRpcMsg { typedef struct SRpcMsg {
tmsg_t msgType; tmsg_t msgType;
tmsg_t expectMsgType;
void * pCont; void * pCont;
int contLen; int contLen;
int32_t code; int32_t code;
void * handle; // rpc handle returned to app void * handle; // rpc handle returned to app
void * ahandle; // app handle set by client void * ahandle; // app handle set by client
int noResp; // has response or not(default 0 indicate resp); int noResp; // has response or not(default 0, 0: resp, 1: no resp);
int persistHandle; // persist handle or not
} SRpcMsg; } SRpcMsg;
@ -77,18 +77,19 @@ typedef struct SRpcInit {
// call back to retrieve the client auth info, for server app only // call back to retrieve the client auth info, for server app only
int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
// call back to keep conn or not
bool (*pfp)(void *parent, tmsg_t msgType);
// to support Send messages multiple times on a link
void *(*mfp)(void *parent, tmsg_t msgType);
// call back to handle except when query/fetch in progress
bool (*efp)(void *parent, tmsg_t msgType);
void *parent; void *parent;
} SRpcInit; } SRpcInit;
typedef struct {
void * val;
int32_t len;
void (*free)(void *arg);
} SRpcCtxVal;
typedef struct {
SHashObj *args;
} SRpcCtx;
int32_t rpcInit(); int32_t rpcInit();
void rpcCleanup(); void rpcCleanup();
void * rpcOpen(const SRpcInit *pRpc); void * rpcOpen(const SRpcInit *pRpc);
@ -97,16 +98,17 @@ void * rpcMallocCont(int contLen);
void rpcFreeCont(void *pCont); void rpcFreeCont(void *pCont);
void * rpcReallocCont(void *ptr, int contLen); void * rpcReallocCont(void *ptr, int contLen);
void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
void rpcSendResponse(const SRpcMsg *pMsg); void rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx);
void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet);
int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
int rpcReportProgress(void *pConn, char *pCont, int contLen);
void rpcCancelRequest(int64_t rid);
void rpcSendResponse(const SRpcMsg *pMsg);
void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet);
int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
int rpcReportProgress(void *pConn, char *pCont, int contLen);
void rpcCancelRequest(int64_t rid);
void rpcRegisterBrokenLinkArg(SRpcMsg *msg);
// just release client conn to rpc instance, no close sock // just release client conn to rpc instance, no close sock
void rpcReleaseHandle(void *handle, int8_t type); void rpcReleaseHandle(void *handle, int8_t type); //
void rpcRefHandle(void *handle, int8_t type); void rpcRefHandle(void *handle, int8_t type);
void rpcUnrefHandle(void *handle, int8_t type); void rpcUnrefHandle(void *handle, int8_t type);

1
include/util/taoserror.h
View File

@ -463,6 +463,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_PASSWD_EMPTY TAOS_DEF_ERROR_CODE(0, 0x2611) #define TSDB_CODE_PAR_PASSWD_EMPTY TAOS_DEF_ERROR_CODE(0, 0x2611)
#define TSDB_CODE_PAR_INVALID_PORT TAOS_DEF_ERROR_CODE(0, 0x2612) #define TSDB_CODE_PAR_INVALID_PORT TAOS_DEF_ERROR_CODE(0, 0x2612)
#define TSDB_CODE_PAR_INVALID_ENDPOINT TAOS_DEF_ERROR_CODE(0, 0x2613) #define TSDB_CODE_PAR_INVALID_ENDPOINT TAOS_DEF_ERROR_CODE(0, 0x2613)
#define TSDB_CODE_PAR_EXPRIE_STATEMENT TAOS_DEF_ERROR_CODE(0, 0x2614)
#ifdef __cplusplus #ifdef __cplusplus
} }

10
include/util/tjson.h
View File

@ -22,6 +22,14 @@
extern "C" { extern "C" {
#endif #endif
#define tjsonGetNumberValue(pJson, pName, val) \
({ \
uint64_t _tmp = 0; \
int32_t _code = tjsonGetUBigIntValue(pJson, pName, &_tmp); \
val = _tmp; \
_code; \
})
typedef void SJson; typedef void SJson;
SJson* tjsonCreateObject(); SJson* tjsonCreateObject();
@ -44,6 +52,7 @@ int32_t tjsonGetIntValue(const SJson* pJson, const char* pName, int32_t* pVal);
int32_t tjsonGetSmallIntValue(const SJson* pJson, const char* pName, int16_t* pVal); int32_t tjsonGetSmallIntValue(const SJson* pJson, const char* pName, int16_t* pVal);
int32_t tjsonGetTinyIntValue(const SJson* pJson, const char* pName, int8_t* pVal); int32_t tjsonGetTinyIntValue(const SJson* pJson, const char* pName, int8_t* pVal);
int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pVal); int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pVal);
int32_t tjsonGetUIntValue(const SJson* pJson, const char* pName, uint32_t* pVal);
int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal); int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal);
int32_t tjsonGetBoolValue(const SJson* pJson, const char* pName, bool* pVal); int32_t tjsonGetBoolValue(const SJson* pJson, const char* pName, bool* pVal);
int32_t tjsonGetDoubleValue(const SJson* pJson, const char* pName, double* pVal); int32_t tjsonGetDoubleValue(const SJson* pJson, const char* pName, double* pVal);
@ -60,6 +69,7 @@ int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void*
typedef int32_t (*FToObject)(const SJson* pJson, void* pObj); typedef int32_t (*FToObject)(const SJson* pJson, void* pObj);
int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj); int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj);
int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, void** pObj, int32_t objSize);
int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize); int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize);
char* tjsonToString(const SJson* pJson); char* tjsonToString(const SJson* pJson);

3
source/client/inc/clientInt.h
View File

@ -179,6 +179,7 @@ typedef struct SRequestObj {
uint64_t requestId; uint64_t requestId;
int32_t type; // request type int32_t type; // request type
STscObj* pTscObj; STscObj* pTscObj;
char* pDb;
char* sqlstr; // sql string char* sqlstr; // sql string
int32_t sqlLen; int32_t sqlLen;
int64_t self; int64_t self;
@ -229,7 +230,7 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t
int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery); int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery);
int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList); int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);
// --- heartbeat // --- heartbeat

3
source/client/src/clientEnv.c
View File

@ -90,7 +90,6 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) {
rpcInit.label = "TSC"; rpcInit.label = "TSC";
rpcInit.numOfThreads = numOfThread; rpcInit.numOfThreads = numOfThread;
rpcInit.cfp = processMsgFromServer; rpcInit.cfp = processMsgFromServer;
rpcInit.pfp = persistConnForSpecificMsg;
rpcInit.sessions = tsMaxConnections; rpcInit.sessions = tsMaxConnections;
rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.user = (char *)user; rpcInit.user = (char *)user;
@ -150,6 +149,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty
return NULL; return NULL;
} }
pRequest->pDb = getDbOfConnection(pObj);
pRequest->requestId = generateRequestId(); pRequest->requestId = generateRequestId();
pRequest->metric.start = taosGetTimestampMs(); pRequest->metric.start = taosGetTimestampMs();
@ -180,6 +180,7 @@ static void doDestroyRequest(void *p) {
tfree(pRequest->msgBuf); tfree(pRequest->msgBuf);
tfree(pRequest->sqlstr); tfree(pRequest->sqlstr);
tfree(pRequest->pInfo); tfree(pRequest->pInfo);
tfree(pRequest->pDb);
doFreeReqResultInfo(&pRequest->body.resInfo); doFreeReqResultInfo(&pRequest->body.resInfo);
qDestroyQueryPlan(pRequest->body.pDag); qDestroyQueryPlan(pRequest->body.pDag);

9
source/client/src/clientImpl.c
View File

@ -137,13 +137,14 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj*
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) { int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) {
STscObj* pTscObj = pRequest->pTscObj; STscObj* pTscObj = pRequest->pTscObj;
SParseContext cxt = { SParseContext cxt = {
.requestId = pRequest->requestId, .requestId = pRequest->requestId,
.acctId = pTscObj->acctId, .acctId = pTscObj->acctId,
.db = getDbOfConnection(pTscObj), .db = pRequest->pDb,
.topicQuery = topicQuery,
.pSql = pRequest->sqlstr, .pSql = pRequest->sqlstr,
.sqlLen = pRequest->sqlLen, .sqlLen = pRequest->sqlLen,
.pMsg = pRequest->msgBuf, .pMsg = pRequest->msgBuf,
@ -154,7 +155,6 @@ int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) {
cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog); int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tfree(cxt.db);
return code; return code;
} }
@ -163,7 +163,6 @@ int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) {
setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols); setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols);
} }
tfree(cxt.db);
return code; return code;
} }
@ -249,7 +248,7 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) {
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return); CHECK_CODE_GOTO(parseSql(pRequest, false, &pQuery), _return);
if (pQuery->directRpc) { if (pQuery->directRpc) {
CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return); CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return);

30
source/client/src/tmq.c
View File

@ -482,38 +482,24 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
} }
tscDebug("start to create topic, %s", topicName); tscDebug("start to create topic, %s", topicName);
#if 0
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return);
pQueryNode->streamQuery = true; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, true, &pQueryNode), _return);
// todo check for invalid sql statement and return with error code // todo check for invalid sql statement and return with error code
SSchema* schema = NULL; CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &pStr, NULL), _return);
int32_t numOfCols = 0;
CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, NULL), _return);
pStr = qQueryPlanToString(pRequest->body.pDag);
if (pStr == NULL) {
goto _return;
}
/*printf("%s\n", pStr);*/ /*printf("%s\n", pStr);*/
// The topic should be related to a database that the queried table is belonged to. SName name = { .acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T };
SName name = {0}; strcpy(name.dbname, pRequest->pDb);
char dbName[TSDB_DB_FNAME_LEN] = {0}; strcpy(name.tname, topicName);
// tNameGetFullDbName(&((SQueryStmtInfo*)pQueryNode)->pTableMetaInfo[0]->name, dbName);
tNameFromString(&name, dbName, T_NAME_ACCT | T_NAME_DB);
tNameFromString(&name, topicName, T_NAME_TABLE);
SCMCreateTopicReq req = { SCMCreateTopicReq req = {
.igExists = 1, .igExists = 1,
.physicalPlan = (char*)pStr, .ast = (char*)pStr,
.sql = (char*)sql, .sql = (char*)sql,
.logicalPlan = (char*)"no logic plan",
}; };
tNameExtractFullName(&name, req.name); tNameExtractFullName(&name, req.name);
@ -536,7 +522,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
tsem_wait(&pRequest->body.rspSem); tsem_wait(&pRequest->body.rspSem);
#endif
_return: _return:
qDestroyQuery(pQueryNode); qDestroyQuery(pQueryNode);
/*if (sendInfo != NULL) {*/ /*if (sendInfo != NULL) {*/

10
source/common/src/tdatablock.c
View File

@ -1072,6 +1072,10 @@ void blockDataClearup(SSDataBlock* pDataBlock) {
} }
int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) { int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) {
if (0 == numOfRows) {
return TSDB_CODE_SUCCESS;
}
if (IS_VAR_DATA_TYPE(pColumn->info.type)) { if (IS_VAR_DATA_TYPE(pColumn->info.type)) {
char* tmp = realloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); char* tmp = realloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows);
if (tmp == NULL) { if (tmp == NULL) {
@ -1092,7 +1096,7 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo
pColumn->nullbitmap = tmp; pColumn->nullbitmap = tmp;
memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows)); memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows));
assert(pColumn->info.bytes);
tmp = realloc(pColumn->pData, numOfRows * pColumn->info.bytes); tmp = realloc(pColumn->pData, numOfRows * pColumn->info.bytes);
if (tmp == NULL) { if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -1137,7 +1141,7 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
taosArrayDestroy(pBlock->pDataBlock); taosArrayDestroy(pBlock->pDataBlock);
tfree(pBlock->pBlockAgg); tfree(pBlock->pBlockAgg);
tfree(pBlock); // tfree(pBlock);
return NULL; return NULL;
} }
@ -1190,7 +1194,7 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
} }
int32_t len = colDataGetLength(pColData, rows); int32_t len = colDataGetLength(pColData, rows);
taosEncodeFixedI32(buf, len); tlen += taosEncodeFixedI32(buf, len);
tlen += taosEncodeBinary(buf, pColData->pData, len); tlen += taosEncodeBinary(buf, pColData->pData, len);
} }

2
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -246,7 +246,7 @@ static int32_t mndGetPlanString(SCMCreateTopicReq *pCreate, char **pStr) {
SQueryPlan* pPlan = NULL; SQueryPlan* pPlan = NULL;
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
SPlanContext cxt = { .pAstRoot = pAst, .streamQuery = true }; SPlanContext cxt = { .pAstRoot = pAst, .topicQuery = true };
code = qCreateQueryPlan(&cxt, &pPlan, NULL); code = qCreateQueryPlan(&cxt, &pPlan, NULL);
} }

5
source/dnode/vnode/src/inc/tsdbDBDef.h
View File

@ -26,8 +26,9 @@ typedef struct SDBFile SDBFile;
typedef DB_ENV* TDBEnv; typedef DB_ENV* TDBEnv;
struct SDBFile { struct SDBFile {
DB* pDB; int32_t fid;
char* path; DB* pDB;
char* path;
}; };
int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile* pDBF); int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile* pDBF);

1
source/dnode/vnode/src/inc/tsdbDef.h
View File

@ -63,6 +63,7 @@ struct STsdb {
#define REPO_ID(r) ((r)->vgId) #define REPO_ID(r) ((r)->vgId)
#define REPO_CFG(r) (&(r)->config) #define REPO_CFG(r) (&(r)->config)
#define REPO_FS(r) (r)->fs #define REPO_FS(r) (r)->fs
#define REPO_TFS(r) (r)->pTfs
#define IS_REPO_LOCKED(r) (r)->repoLocked #define IS_REPO_LOCKED(r) (r)->repoLocked
#define REPO_SMA_ENV(r, t) ((TSDB_SMA_TYPE_ROLLUP == (t)) ? (r)->pRSmaEnv : (r)->pTSmaEnv) #define REPO_SMA_ENV(r, t) ((TSDB_SMA_TYPE_ROLLUP == (t)) ? (r)->pRSmaEnv : (r)->pTSmaEnv)

24
source/dnode/vnode/src/inc/tsdbFS.h
View File

@ -42,23 +42,23 @@ typedef struct {
typedef struct { typedef struct {
STsdbFSMeta meta; // FS meta STsdbFSMeta meta; // FS meta
SArray * df; // data file array SArray * df; // data file array
SArray * sf; // sma data file array v2(t|r)1900.index_name_1 SArray * sf; // sma data file array v2f1900.index_name_1
} SFSStatus; } SFSStatus;
/** /**
* @brief Directory structure of .tsma data files. * @brief Directory structure of .tsma data files.
* *
* /vnode2/tsdb $ tree .sma/ * /vnode2/tsdb $ tree tsma/
* .sma/ * tsma/
* ├── v2t100.index_name_1 * ├── v2f100.index_name_1
* ├── v2t101.index_name_1 * ├── v2f101.index_name_1
* ├── v2t102.index_name_1 * ├── v2f102.index_name_1
* ├── v2t1900.index_name_3 * ├── v2f1900.index_name_3
* ├── v2t1901.index_name_3 * ├── v2f1901.index_name_3
* ├── v2t1902.index_name_3 * ├── v2f1902.index_name_3
* ├── v2t200.index_name_2 * ├── v2f200.index_name_2
* ├── v2t201.index_name_2 * ├── v2f201.index_name_2
* └── v2t202.index_name_2 * └── v2f202.index_name_2
* *
* 0 directories, 9 files * 0 directories, 9 files
*/ */

6
source/dnode/vnode/src/inc/tsdbSma.h
View File

@ -21,12 +21,14 @@ typedef struct SSmaEnv SSmaEnv;
struct SSmaEnv { struct SSmaEnv {
pthread_rwlock_t lock; pthread_rwlock_t lock;
TDBEnv dbEnv; SDiskID did;
char * path; TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level?
char * path; // relative path
SSmaStat * pStat; SSmaStat * pStat;
}; };
#define SMA_ENV_LOCK(env) ((env)->lock) #define SMA_ENV_LOCK(env) ((env)->lock)
#define SMA_ENV_DID(env) ((env)->did)
#define SMA_ENV_ENV(env) ((env)->dbEnv) #define SMA_ENV_ENV(env) ((env)->dbEnv)
#define SMA_ENV_PATH(env) ((env)->path) #define SMA_ENV_PATH(env) ((env)->path)
#define SMA_ENV_STAT(env) ((env)->pStat) #define SMA_ENV_STAT(env) ((env)->pStat)

2
source/dnode/vnode/src/meta/metaBDBImpl.c
View File

@ -884,7 +884,7 @@ const char *metaSmaCursorNext(SMSmaCursor *pCur) {
STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
STSmaWrapper *pSW = NULL; STSmaWrapper *pSW = NULL;
pSW = calloc(sizeof(*pSW), 1); pSW = calloc(1, sizeof(*pSW));
if (pSW == NULL) { if (pSW == NULL) {
return NULL; return NULL;
} }

8
source/dnode/vnode/src/tq/tqRead.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tdatablock.h"
#include "vnode.h" #include "vnode.h"
STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
@ -128,10 +129,13 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
int j = 0; int j = 0;
for (int32_t i = 0; i < colNumNeed; i++) { for (int32_t i = 0; i < colNumNeed; i++) {
int32_t colId = *(int32_t*)taosArrayGet(pHandle->pColIdList, i); int16_t colId = *(int16_t*)taosArrayGet(pHandle->pColIdList, i);
while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) { while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) {
j++; j++;
} }
if (j >= pSchemaWrapper->nCols) {
continue;
}
SSchema* pColSchema = &pSchemaWrapper->pSchema[j]; SSchema* pColSchema = &pSchemaWrapper->pSchema[j];
SColumnInfoData colInfo = {0}; SColumnInfoData colInfo = {0};
int sz = numOfRows * pColSchema->bytes; int sz = numOfRows * pColSchema->bytes;
@ -145,6 +149,8 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
taosArrayDestroy(pArray); taosArrayDestroy(pArray);
return NULL; return NULL;
} }
blockDataEnsureColumnCapacity(&colInfo, numOfRows);
taosArrayPush(pArray, &colInfo); taosArrayPush(pArray, &colInfo);
} }

138
source/dnode/vnode/src/tsdb/tsdbSma.c
View File

@ -67,7 +67,7 @@ typedef struct {
*/ */
int8_t state; // ETsdbSmaStat int8_t state; // ETsdbSmaStat
SHashObj *expiredWindows; // key: skey of time window, value: N/A SHashObj *expiredWindows; // key: skey of time window, value: N/A
STSma * pSma; STSma * pSma; // cache schema
} SSmaStatItem; } SSmaStatItem;
struct SSmaStat { struct SSmaStat {
@ -81,8 +81,8 @@ struct SSmaStat {
static int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg); static int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg);
static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat); static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat);
static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat); static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat);
static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path); static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did);
static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv); static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv);
static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey); static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey);
static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat); static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat);
static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat); static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat);
@ -102,8 +102,8 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p
static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen); static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen);
static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision); static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision);
static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel); static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel);
static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid); static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int64_t indexUid, int32_t fid);
static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey); static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey);
static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey);
static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]); static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]);
static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg); static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg);
@ -111,10 +111,11 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg);
// implementation // implementation
static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) { static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) {
snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s", vgId, TSDB_SMA_DNAME[smaType]); snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%stsdb%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TD_DIRSEP,
TSDB_SMA_DNAME[smaType]);
} }
static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) { static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) {
SSmaEnv *pEnv = NULL; SSmaEnv *pEnv = NULL;
pEnv = (SSmaEnv *)calloc(1, sizeof(SSmaEnv)); pEnv = (SSmaEnv *)calloc(1, sizeof(SSmaEnv));
@ -137,12 +138,16 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) {
return NULL; return NULL;
} }
pEnv->did = did;
if (tsdbInitSmaStat(&pEnv->pStat) != TSDB_CODE_SUCCESS) { if (tsdbInitSmaStat(&pEnv->pStat) != TSDB_CODE_SUCCESS) {
tsdbFreeSmaEnv(pEnv); tsdbFreeSmaEnv(pEnv);
return NULL; return NULL;
} }
if (tsdbOpenBDBEnv(&pEnv->dbEnv, pEnv->path) != TSDB_CODE_SUCCESS) { char aname[TSDB_FILENAME_LEN] = {0};
tfsAbsoluteName(pTsdb->pTfs, did, path, aname);
if (tsdbOpenBDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) {
tsdbFreeSmaEnv(pEnv); tsdbFreeSmaEnv(pEnv);
return NULL; return NULL;
} }
@ -150,18 +155,14 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) {
return pEnv; return pEnv;
} }
static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv) { static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv) {
if (!pEnv) { if (!pEnv) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
if (*pEnv) {
return TSDB_CODE_SUCCESS;
}
if (*pEnv == NULL) { if (*pEnv == NULL) {
if ((*pEnv = tsdbNewSmaEnv(pTsdb, path)) == NULL) { if ((*pEnv = tsdbNewSmaEnv(pTsdb, path, did)) == NULL) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
} }
@ -260,10 +261,15 @@ static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) {
int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) { int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) {
if (pSmaStat) { if (pSmaStat) {
// TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
SSmaStatItem *item = taosHashIterate(pSmaStat->smaStatItems, NULL); void *item = taosHashIterate(pSmaStat->smaStatItems, NULL);
while (item != NULL) { while (item != NULL) {
tfree(item->pSma); SSmaStatItem *pItem = *(SSmaStatItem **)item;
taosHashCleanup(item->expiredWindows); if (pItem != NULL) {
tdDestroyTSma(pItem->pSma);
tfree(pItem->pSma);
taosHashCleanup(pItem->expiredWindows);
tfree(pItem);
}
item = taosHashIterate(pSmaStat->smaStatItems, item); item = taosHashIterate(pSmaStat->smaStatItems, item);
} }
taosHashCleanup(pSmaStat->smaStatItems); taosHashCleanup(pSmaStat->smaStatItems);
@ -292,9 +298,9 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) {
// init sma env // init sma env
tsdbLockRepo(pTsdb); tsdbLockRepo(pTsdb);
if (pTsdb->pTSmaEnv == NULL) { pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&pTsdb->pTSmaEnv) : atomic_load_ptr(&pTsdb->pRSmaEnv);
if (pEnv == NULL) {
char rname[TSDB_FILENAME_LEN] = {0}; char rname[TSDB_FILENAME_LEN] = {0};
char aname[TSDB_FILENAME_LEN * 2 + 32] = {0}; // TODO: make TMPNAME_LEN public as TSDB_FILENAME_LEN?
SDiskID did = {0}; SDiskID did = {0};
tfsAllocDisk(pTsdb->pTfs, TFS_PRIMARY_LEVEL, &did); tfsAllocDisk(pTsdb->pTfs, TFS_PRIMARY_LEVEL, &did);
@ -303,23 +309,19 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
tsdbGetSmaDir(REPO_ID(pTsdb), smaType, rname); tsdbGetSmaDir(REPO_ID(pTsdb), smaType, rname);
tfsAbsoluteName(pTsdb->pTfs, did, rname, aname);
if (tfsMkdirRecurAt(pTsdb->pTfs, rname, did) != TSDB_CODE_SUCCESS) { if (tfsMkdirRecurAt(pTsdb->pTfs, rname, did) != TSDB_CODE_SUCCESS) {
tsdbUnlockRepo(pTsdb); tsdbUnlockRepo(pTsdb);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
if (tsdbInitSmaEnv(pTsdb, aname, &pEnv) != TSDB_CODE_SUCCESS) { if (tsdbInitSmaEnv(pTsdb, rname, did, &pEnv) != TSDB_CODE_SUCCESS) {
tsdbUnlockRepo(pTsdb); tsdbUnlockRepo(pTsdb);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&pTsdb->pTSmaEnv, pEnv)
atomic_store_ptr(&pTsdb->pTSmaEnv, pEnv); : atomic_store_ptr(&pTsdb->pRSmaEnv, pEnv);
} else {
atomic_store_ptr(&pTsdb->pRSmaEnv, pEnv);
}
} }
tsdbUnlockRepo(pTsdb); tsdbUnlockRepo(pTsdb);
@ -340,11 +342,6 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
if (tsdbCheckAndInitSmaEnv(pTsdb, smaType) != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TDB_INIT_FAILED;
return TSDB_CODE_FAILED;
}
// TODO: decode the msg from Stream Computing module => start // TODO: decode the msg from Stream Computing module => start
int64_t indexUid = SMA_TEST_INDEX_UID; int64_t indexUid = SMA_TEST_INDEX_UID;
const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10; const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10;
@ -355,12 +352,19 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg) {
} }
// TODO: decode the msg <= end // TODO: decode the msg <= end
if (tsdbCheckAndInitSmaEnv(pTsdb, smaType) != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TDB_INIT_FAILED;
return TSDB_CODE_FAILED;
}
SSmaEnv * pEnv = REPO_SMA_ENV(pTsdb, smaType); SSmaEnv * pEnv = REPO_SMA_ENV(pTsdb, smaType);
SSmaStat *pStat = SMA_ENV_STAT(pEnv); SSmaStat *pStat = SMA_ENV_STAT(pEnv);
SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv);
TASSERT(pEnv != NULL && pStat != NULL && pItemsHash != NULL);
tsdbRefSmaStat(pTsdb, pStat); tsdbRefSmaStat(pTsdb, pStat);
SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid));
if (pItem == NULL) { if (pItem == NULL) {
pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED); // TODO use the real state pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED); // TODO use the real state
if (pItem == NULL) { if (pItem == NULL) {
@ -421,9 +425,9 @@ static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t ind
tsdbRefSmaStat(pTsdb, pStat); tsdbRefSmaStat(pTsdb, pStat);
if (pStat && pStat->smaStatItems) { if (pStat && pStat->smaStatItems) {
pItem = *(SSmaStatItem **)taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid)); pItem = taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid));
} }
if (pItem != NULL) { if ((pItem != NULL) && ((pItem = *(SSmaStatItem **)pItem) != NULL)) {
// pItem resides in hash buffer all the time unless drop sma index // pItem resides in hash buffer all the time unless drop sma index
// TODO: multithread protect // TODO: multithread protect
if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) {
@ -494,7 +498,7 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
* @brief Insert TSma data blocks to DB File build by B+Tree * @brief Insert TSma data blocks to DB File build by B+Tree
* *
* @param pSmaH * @param pSmaH
* @param smaKey * @param smaKey tableUid-colId-skeyOfWindow(8-2-8)
* @param keyLen * @param keyLen
* @param pData * @param pData
* @param dataLen * @param dataLen
@ -502,12 +506,11 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
*/ */
static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) { static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) {
SDBFile *pDBFile = &pSmaH->dFile; SDBFile *pDBFile = &pSmaH->dFile;
// TODO: insert sma data blocks into B+Tree
tsdbDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d", tsdbDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d",
REPO_ID(pSmaH->pTsdb), pDBFile->path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), REPO_ID(pSmaH->pTsdb), pDBFile->path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8),
*(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen); *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen);
// TODO: insert sma data blocks into B+Tree(TDB)
if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) { if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -564,34 +567,34 @@ static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit
return interval / 1e3; return interval / 1e3;
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second
return interval / 1e6; return interval / 1e6;
} else { } else { // ms
return interval; return interval;
} }
break; break;
case TSDB_TIME_PRECISION_MICRO: case TSDB_TIME_PRECISION_MICRO:
if (TIME_UNIT_MICROSECOND == intervalUnit) { // us if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
return interval; return interval;
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns
return interval / 1e3; return interval / 1e3;
} else { } else { // ms
return interval * 1e3; return interval * 1e3;
} }
break; break;
case TSDB_TIME_PRECISION_NANO: case TSDB_TIME_PRECISION_NANO:
if (TIME_UNIT_MICROSECOND == intervalUnit) { if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
return interval * 1e3; return interval * 1e3;
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns
return interval; return interval;
} else { } else { // ms
return interval * 1e6; return interval * 1e6;
} }
break; break;
default: // ms default: // ms
if (TIME_UNIT_MICROSECOND == intervalUnit) { // us if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
return interval / 1e3; return interval / 1e3;
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns
return interval / 1e6; return interval / 1e6;
} else { } else { // ms
return interval; return interval;
} }
break; break;
@ -660,12 +663,15 @@ static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH) {
} }
} }
static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid) { static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int64_t indexUid, int32_t fid) {
STsdb *pTsdb = pSmaH->pTsdb; STsdb *pTsdb = pSmaH->pTsdb;
ASSERT(pSmaH->dFile.path == NULL && pSmaH->dFile.pDB == NULL); ASSERT(pSmaH->dFile.path == NULL && pSmaH->dFile.pDB == NULL);
pSmaH->dFile.fid = fid;
char tSmaFile[TSDB_FILENAME_LEN] = {0}; char tSmaFile[TSDB_FILENAME_LEN] = {0};
snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pTsdb), fid); snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid);
pSmaH->dFile.path = strdup(tSmaFile); pSmaH->dFile.path = strdup(tSmaFile);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -704,8 +710,9 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe
static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
STsdbCfg * pCfg = REPO_CFG(pTsdb); STsdbCfg * pCfg = REPO_CFG(pTsdb);
STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; STSmaDataWrapper *pData = (STSmaDataWrapper *)msg;
SSmaEnv * pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv);
if (!pTsdb->pTSmaEnv) { if (pEnv == NULL) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb));
return terrno; return terrno;
@ -723,6 +730,17 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
int64_t indexUid = SMA_TEST_INDEX_UID;
char rPath[TSDB_FILENAME_LEN] = {0};
char aPath[TSDB_FILENAME_LEN] = {0};
snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid);
tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath);
if (!taosCheckExistFile(aPath)) {
if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED;
}
}
// Step 1: Judge the storage level and days // Step 1: Judge the storage level and days
int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit);
int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel);
@ -731,7 +749,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
// Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file
// - Set and open the DFile or the B+Tree file // - Set and open the DFile or the B+Tree file
// TODO: tsdbStartTSmaCommit(); // TODO: tsdbStartTSmaCommit();
tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid); tsdbSetTSmaDataFile(&tSmaH, pData, indexUid, fid);
if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) { if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) {
tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb),
tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno));
@ -818,13 +836,16 @@ static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interv
* @brief Init of tSma FS * @brief Init of tSma FS
* *
* @param pReadH * @param pReadH
* @param indexUid
* @param skey * @param skey
* @return int32_t * @return int32_t
*/ */
static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey) { static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) {
int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pSmaH->pTsdb)->precision)); STsdb *pTsdb = pSmaH->pTsdb;
int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pTsdb)->precision));
char tSmaFile[TSDB_FILENAME_LEN] = {0}; char tSmaFile[TSDB_FILENAME_LEN] = {0};
snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pSmaH->pTsdb), fid); snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid);
pSmaH->dFile.path = strdup(tSmaFile); pSmaH->dFile.path = strdup(tSmaFile);
pSmaH->smaFsIter.iter = 0; pSmaH->smaFsIter.iter = 0;
pSmaH->smaFsIter.fid = fid; pSmaH->smaFsIter.fid = fid;
@ -883,15 +904,17 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) {
static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval,
int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey,
int32_t nMaxResult) { int32_t nMaxResult) {
if (!pTsdb->pTSmaEnv) { SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv);
if (!pEnv) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
tsdbWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); tsdbWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", REPO_ID(pTsdb));
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
tsdbRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); tsdbRefSmaStat(pTsdb, SMA_ENV_STAT(pEnv));
SSmaStatItem *pItem = *(SSmaStatItem **)taosHashGet(SMA_ENV_STAT_ITEMS(pTsdb->pTSmaEnv), &indexUid, sizeof(indexUid)); SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid));
if (pItem == NULL) { if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL)) {
// Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if
// it's NULL. // it's NULL.
tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv));
@ -922,11 +945,12 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_
tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv));
#endif #endif
STSmaReadH tReadH = {0}; STSmaReadH tReadH = {0};
tsdbInitTSmaReadH(&tReadH, pTsdb, interval, intervalUnit); tsdbInitTSmaReadH(&tReadH, pTsdb, interval, intervalUnit);
tsdbCloseDBF(&tReadH.dFile); tsdbCloseDBF(&tReadH.dFile);
tsdbInitTSmaFile(&tReadH, querySKey); tsdbInitTSmaFile(&tReadH, indexUid, querySKey);
if (tsdbOpenDBF(SMA_ENV_ENV(pTsdb->pTSmaEnv), &tReadH.dFile) != 0) { if (tsdbOpenDBF(SMA_ENV_ENV(pTsdb->pTSmaEnv), &tReadH.dFile) != 0) {
tsdbWarn("vgId:%d open DBF %s failed since %s", REPO_ID(pTsdb), tReadH.dFile.path, tstrerror(terrno)); tsdbWarn("vgId:%d open DBF %s failed since %s", REPO_ID(pTsdb), tReadH.dFile.path, tstrerror(terrno));
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;

3
source/dnode/vnode/src/vnd/vnodeWrite.c
View File

@ -146,6 +146,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
return -1; return -1;
} }
// record current timezone of server side
tstrncpy(vCreateSmaReq.tSma.timezone, tsTimezone, TD_TIMEZONE_LEN);
if (metaCreateTSma(pVnode->pMeta, &vCreateSmaReq) < 0) { if (metaCreateTSma(pVnode->pMeta, &vCreateSmaReq) < 0) {
// TODO: handle error // TODO: handle error
tdDestroyTSma(&vCreateSmaReq.tSma); tdDestroyTSma(&vCreateSmaReq.tSma);

48
source/dnode/vnode/test/tsdbSmaTest.cpp
View File

@ -33,6 +33,7 @@ int main(int argc, char **argv) {
return RUN_ALL_TESTS(); return RUN_ALL_TESTS();
} }
#if 1
TEST(testCase, tSma_Meta_Encode_Decode_Test) { TEST(testCase, tSma_Meta_Encode_Decode_Test) {
// encode // encode
STSma tSma = {0}; STSma tSma = {0};
@ -49,7 +50,7 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) {
STSmaWrapper tSmaWrapper = {.number = 1, .tSma = &tSma}; STSmaWrapper tSmaWrapper = {.number = 1, .tSma = &tSma};
uint32_t bufLen = tEncodeTSmaWrapper(NULL, &tSmaWrapper); uint32_t bufLen = tEncodeTSmaWrapper(NULL, &tSmaWrapper);
void *buf = calloc(bufLen, 1); void *buf = calloc(1, bufLen);
ASSERT_NE(buf, nullptr); ASSERT_NE(buf, nullptr);
STSmaWrapper *pSW = (STSmaWrapper *)buf; STSmaWrapper *pSW = (STSmaWrapper *)buf;
@ -84,9 +85,11 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) {
} }
// resource release // resource release
tfree(pSW);
tdDestroyTSma(&tSma); tdDestroyTSma(&tSma);
tdDestroyTSmaWrapper(&dstTSmaWrapper); tdDestroyTSmaWrapper(&dstTSmaWrapper);
} }
#endif
#if 1 #if 1
TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
@ -113,7 +116,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
tSma.tableUid = tbUid; tSma.tableUid = tbUid;
tSma.exprLen = strlen(expr); tSma.exprLen = strlen(expr);
tSma.expr = (char *)calloc(tSma.exprLen + 1, 1); tSma.expr = (char *)calloc(1, tSma.exprLen + 1);
ASSERT_NE(tSma.expr, nullptr); ASSERT_NE(tSma.expr, nullptr);
tstrncpy(tSma.expr, expr, tSma.exprLen + 1); tstrncpy(tSma.expr, expr, tSma.exprLen + 1);
@ -251,12 +254,12 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
tSma.tableUid = tbUid; tSma.tableUid = tbUid;
tSma.exprLen = strlen(expr); tSma.exprLen = strlen(expr);
tSma.expr = (char *)calloc(tSma.exprLen + 1, 1); tSma.expr = (char *)calloc(1, tSma.exprLen + 1);
ASSERT_NE(tSma.expr, nullptr); ASSERT_NE(tSma.expr, nullptr);
tstrncpy(tSma.expr, expr, tSma.exprLen + 1); tstrncpy(tSma.expr, expr, tSma.exprLen + 1);
tSma.tagsFilterLen = strlen(tagsFilter); tSma.tagsFilterLen = strlen(tagsFilter);
tSma.tagsFilter = (char *)calloc(tSma.tagsFilterLen + 1, 1); tSma.tagsFilter = (char *)calloc(1, tSma.tagsFilterLen + 1);
ASSERT_NE(tSma.tagsFilter, nullptr); ASSERT_NE(tSma.tagsFilter, nullptr);
tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1); tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1);
@ -273,20 +276,20 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
// step 2: insert data // step 2: insert data
STSmaDataWrapper *pSmaData = NULL; STSmaDataWrapper *pSmaData = NULL;
STsdb tsdb = {0}; STsdb * pTsdb = (STsdb *)calloc(1, sizeof(STsdb));
STsdbCfg * pCfg = &tsdb.config; STsdbCfg * pCfg = &pTsdb->config;
tsdb.pMeta = pMeta; pTsdb->pMeta = pMeta;
tsdb.vgId = 2; pTsdb->vgId = 2;
tsdb.config.daysPerFile = 10; // default days is 10 pTsdb->config.daysPerFile = 10; // default days is 10
tsdb.config.keep1 = 30; pTsdb->config.keep1 = 30;
tsdb.config.keep2 = 90; pTsdb->config.keep2 = 90;
tsdb.config.keep = 365; pTsdb->config.keep = 365;
tsdb.config.precision = TSDB_TIME_PRECISION_MILLI; pTsdb->config.precision = TSDB_TIME_PRECISION_MILLI;
tsdb.config.update = TD_ROW_OVERWRITE_UPDATE; pTsdb->config.update = TD_ROW_OVERWRITE_UPDATE;
tsdb.config.compression = TWO_STAGE_COMP; pTsdb->config.compression = TWO_STAGE_COMP;
switch (tsdb.config.precision) { switch (pTsdb->config.precision) {
case TSDB_TIME_PRECISION_MILLI: case TSDB_TIME_PRECISION_MILLI:
skey1 *= 1e3; skey1 *= 1e3;
break; break;
@ -304,12 +307,12 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
SDiskCfg pDisks = {.level = 0, .primary = 1}; SDiskCfg pDisks = {.level = 0, .primary = 1};
strncpy(pDisks.dir, "/var/lib/taos", TSDB_FILENAME_LEN); strncpy(pDisks.dir, "/var/lib/taos", TSDB_FILENAME_LEN);
int32_t numOfDisks = 1; int32_t numOfDisks = 1;
tsdb.pTfs = tfsOpen(&pDisks, numOfDisks); pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks);
ASSERT_NE(tsdb.pTfs, nullptr); ASSERT_NE(pTsdb->pTfs, nullptr);
char *msg = (char *)calloc(1, 100); char *msg = (char *)calloc(1, 100);
ASSERT_NE(msg, nullptr); ASSERT_NE(msg, nullptr);
ASSERT_EQ(tsdbUpdateSmaWindow(&tsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0); ASSERT_EQ(tsdbUpdateSmaWindow(pTsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0);
// init // init
int32_t allocCnt = 0; int32_t allocCnt = 0;
@ -367,13 +370,13 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
ASSERT_GE(bufSize, pSmaData->dataLen); ASSERT_GE(bufSize, pSmaData->dataLen);
// execute // execute
ASSERT_EQ(tsdbInsertTSmaData(&tsdb, (char *)pSmaData), TSDB_CODE_SUCCESS); ASSERT_EQ(tsdbInsertTSmaData(pTsdb, (char *)pSmaData), TSDB_CODE_SUCCESS);
// step 3: query // step 3: query
uint32_t checkDataCnt = 0; uint32_t checkDataCnt = 0;
for (int32_t t = 0; t < numOfTables; ++t) { for (int32_t t = 0; t < numOfTables; ++t) {
for (col_id_t c = 0; c < numOfCols; ++c) { for (col_id_t c = 0; c < numOfCols; ++c) {
ASSERT_EQ(tsdbGetTSmaData(&tsdb, NULL, indexUid1, interval1, intervalUnit1, tbUid + t, ASSERT_EQ(tsdbGetTSmaData(pTsdb, NULL, indexUid1, interval1, intervalUnit1, tbUid + t,
c + PRIMARYKEY_TIMESTAMP_COL_ID, skey1, 1), c + PRIMARYKEY_TIMESTAMP_COL_ID, skey1, 1),
TSDB_CODE_SUCCESS); TSDB_CODE_SUCCESS);
++checkDataCnt; ++checkDataCnt;
@ -383,9 +386,12 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
printf("%s:%d The sma data check count for insert and query is %" PRIu32 "\n", __FILE__, __LINE__, checkDataCnt); printf("%s:%d The sma data check count for insert and query is %" PRIu32 "\n", __FILE__, __LINE__, checkDataCnt);
// release data // release data
tfree(msg);
taosTZfree(buf); taosTZfree(buf);
// release meta // release meta
tdDestroyTSma(&tSma); tdDestroyTSma(&tSma);
tfsClose(pTsdb->pTfs);
tsdbClose(pTsdb);
metaClose(pMeta); metaClose(pMeta);
} }
#endif #endif

2
source/libs/CMakeLists.txt
View File

@ -1,6 +1,6 @@
add_subdirectory(transport) add_subdirectory(transport)
add_subdirectory(sync) add_subdirectory(sync)
add_subdirectory(tdb) # add_subdirectory(tdb)
add_subdirectory(index) add_subdirectory(index)
add_subdirectory(wal) add_subdirectory(wal)
add_subdirectory(parser) add_subdirectory(parser)

5
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -190,7 +190,6 @@ static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) {
} }
static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) { static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) {
COPY_SCALAR_FIELD(id);
CLONE_NODE_LIST_FIELD(pTargets); CLONE_NODE_LIST_FIELD(pTargets);
CLONE_NODE_FIELD(pConditions); CLONE_NODE_FIELD(pConditions);
CLONE_NODE_LIST_FIELD(pChildren); CLONE_NODE_LIST_FIELD(pChildren);
@ -198,7 +197,7 @@ static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) {
} }
static STableMeta* tableMetaClone(const STableMeta* pSrc) { static STableMeta* tableMetaClone(const STableMeta* pSrc) {
int32_t len = sizeof(STableMeta) + (pSrc->tableInfo.numOfTags + pSrc->tableInfo.numOfColumns) * sizeof(SSchema); int32_t len = TABLE_META_SIZE(pSrc);
STableMeta* pDst = malloc(len); STableMeta* pDst = malloc(len);
if (NULL == pDst) { if (NULL == pDst) {
return NULL; return NULL;
@ -208,7 +207,7 @@ static STableMeta* tableMetaClone(const STableMeta* pSrc) {
} }
static SVgroupsInfo* vgroupsInfoClone(const SVgroupsInfo* pSrc) { static SVgroupsInfo* vgroupsInfoClone(const SVgroupsInfo* pSrc) {
int32_t len = sizeof(SVgroupsInfo) + pSrc->numOfVgroups * sizeof(SVgroupInfo); int32_t len = VGROUPS_INFO_SIZE(pSrc);
SVgroupsInfo* pDst = malloc(len); SVgroupsInfo* pDst = malloc(len);
if (NULL == pDst) { if (NULL == pDst) {
return NULL; return NULL;

494
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "cmdnodes.h"
#include "nodesUtil.h" #include "nodesUtil.h"
#include "plannodes.h" #include "plannodes.h"
#include "querynodes.h" #include "querynodes.h"
@ -85,6 +86,8 @@ const char* nodesNodeName(ENodeType type) {
return "ShowDatabaseStmt"; return "ShowDatabaseStmt";
case QUERY_NODE_SHOW_TABLES_STMT: case QUERY_NODE_SHOW_TABLES_STMT:
return "ShowTablesStmt"; return "ShowTablesStmt";
case QUERY_NODE_CREATE_TOPIC_STMT:
return "CreateTopicStmt";
case QUERY_NODE_LOGIC_PLAN_SCAN: case QUERY_NODE_LOGIC_PLAN_SCAN:
return "LogicScan"; return "LogicScan";
case QUERY_NODE_LOGIC_PLAN_JOIN: case QUERY_NODE_LOGIC_PLAN_JOIN:
@ -179,21 +182,151 @@ static int32_t jsonToNodeList(const SJson* pJson, const char* pName, SNodeList**
return jsonToNodeListImpl(tjsonGetObjectItem(pJson, pName), pList); return jsonToNodeListImpl(tjsonGetObjectItem(pJson, pName), pList);
} }
static const char* jkTableMetaUid = "TableMetaUid"; static const char* jkTableComInfoNumOfTags = "NumOfTags";
static const char* jkTableMetaSuid = "TableMetaSuid"; static const char* jkTableComInfoPrecision = "Precision";
static const char* jkTableComInfoNumOfColumns = "NumOfColumns";
static const char* jkTableComInfoRowSize = "RowSize";
static int32_t tableComInfoToJson(const void* pObj, SJson* pJson) {
const STableComInfo* pNode = (const STableComInfo*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkTableComInfoNumOfTags, pNode->numOfTags);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableComInfoPrecision, pNode->precision);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableComInfoRowSize, pNode->rowSize);
}
return code;
}
static int32_t jsonToTableComInfo(const SJson* pJson, void* pObj) {
STableComInfo* pNode = (STableComInfo*)pObj;
int32_t code = tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize);
}
return code;
}
static const char* jkSchemaType = "Type";
static const char* jkSchemaColId = "ColId";
static const char* jkSchemaBytes = "bytes";
static const char* jkSchemaName = "Name";
static int32_t schemaToJson(const void* pObj, SJson* pJson) {
const SSchema* pNode = (const SSchema*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkSchemaType, pNode->type);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkSchemaColId, pNode->colId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkSchemaBytes, pNode->bytes);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkSchemaName, pNode->name);
}
return code;
}
static int32_t jsonToSchema(const SJson* pJson, void* pObj) {
SSchema* pNode = (SSchema*)pObj;
int32_t code = tjsonGetNumberValue(pJson, jkSchemaType, pNode->type);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkSchemaName, pNode->name);
}
return code;
}
static const char* jkTableMetaVgId = "VgId";
static const char* jkTableMetaTableType = "TableType";
static const char* jkTableMetaUid = "Uid";
static const char* jkTableMetaSuid = "Suid";
static const char* jkTableMetaSversion = "Sversion";
static const char* jkTableMetaTversion = "Tversion";
static const char* jkTableMetaComInfo = "ComInfo";
static const char* jkTableMetaColSchemas = "ColSchemas";
static int32_t tableMetaToJson(const void* pObj, SJson* pJson) { static int32_t tableMetaToJson(const void* pObj, SJson* pJson) {
const STableMeta* pNode = (const STableMeta*)pObj; const STableMeta* pNode = (const STableMeta*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkTableMetaUid, pNode->uid); int32_t code = tjsonAddIntegerToObject(pJson, jkTableMetaVgId, pNode->vgId);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableMetaTableType, pNode->tableType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableMetaUid, pNode->uid);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableMetaSuid, pNode->suid); code = tjsonAddIntegerToObject(pJson, jkTableMetaSuid, pNode->suid);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableMetaSversion, pNode->sversion);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableMetaTversion, pNode->tversion);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkTableMetaComInfo, tableComInfoToJson, &pNode->tableInfo);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddArray(pJson, jkTableMetaColSchemas, schemaToJson, pNode->schema, sizeof(SSchema), TABLE_TOTAL_COL_NUM(pNode));
}
return code;
}
static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) {
STableMeta* pNode = (STableMeta*)pObj;
int32_t code = tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, jkTableMetaComInfo, jsonToTableComInfo, &pNode->tableInfo);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToArray(pJson, jkTableMetaColSchemas, jsonToSchema, pNode->schema, sizeof(SSchema));
}
return code; return code;
} }
static const char* jkLogicPlanId = "Id";
static const char* jkLogicPlanTargets = "Targets"; static const char* jkLogicPlanTargets = "Targets";
static const char* jkLogicPlanConditions = "Conditions"; static const char* jkLogicPlanConditions = "Conditions";
static const char* jkLogicPlanChildren = "Children"; static const char* jkLogicPlanChildren = "Children";
@ -201,10 +334,7 @@ static const char* jkLogicPlanChildren = "Children";
static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) { static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) {
const SLogicNode* pNode = (const SLogicNode*)pObj; const SLogicNode* pNode = (const SLogicNode*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkLogicPlanId, pNode->id); int32_t code = nodeListToJson(pJson, jkLogicPlanTargets, pNode->pTargets);
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkLogicPlanTargets, pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkLogicPlanConditions, nodeToJson, pNode->pConditions); code = tjsonAddObject(pJson, jkLogicPlanConditions, nodeToJson, pNode->pConditions);
} }
@ -215,7 +345,22 @@ static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) {
return code; return code;
} }
static int32_t jsonToLogicPlanNode(const SJson* pJson, void* pObj) {
SLogicNode* pNode = (SLogicNode*)pObj;
int32_t code = jsonToNodeList(pJson, jkLogicPlanTargets, &pNode->pTargets);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkLogicPlanConditions, &pNode->pConditions);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkLogicPlanChildren, &pNode->pChildren);
}
return code;
}
static const char* jkScanLogicPlanScanCols = "ScanCols"; static const char* jkScanLogicPlanScanCols = "ScanCols";
static const char* jkScanLogicPlanTableMetaSize = "TableMetaSize";
static const char* jkScanLogicPlanTableMeta = "TableMeta"; static const char* jkScanLogicPlanTableMeta = "TableMeta";
static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
@ -225,6 +370,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkScanLogicPlanScanCols, pNode->pScanCols); code = nodeListToJson(pJson, jkScanLogicPlanScanCols, pNode->pScanCols);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, TABLE_META_SIZE(pNode->pMeta));
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkScanLogicPlanTableMeta, tableMetaToJson, pNode->pMeta); code = tjsonAddObject(pJson, jkScanLogicPlanTableMeta, tableMetaToJson, pNode->pMeta);
} }
@ -232,6 +380,24 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
return code; return code;
} }
static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
SScanLogicNode* pNode = (SScanLogicNode*)pObj;
int32_t objSize = 0;
int32_t code = jsonToLogicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkScanLogicPlanScanCols, &pNode->pScanCols);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkScanLogicPlanTableMetaSize, &objSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonMakeObject(pJson, jkScanLogicPlanTableMeta, jsonToTableMeta, (void**)&pNode->pMeta, objSize);
}
return code;
}
static const char* jkProjectLogicPlanProjections = "Projections"; static const char* jkProjectLogicPlanProjections = "Projections";
static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) { static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
@ -245,6 +411,17 @@ static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
return code; return code;
} }
static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) {
SProjectLogicNode* pNode = (SProjectLogicNode*)pObj;
int32_t code = jsonToLogicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkProjectLogicPlanProjections, &pNode->pProjections);
}
return code;
}
static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanJoinType = "JoinType";
static const char* jkJoinLogicPlanOnConditions = "OnConditions"; static const char* jkJoinLogicPlanOnConditions = "OnConditions";
@ -445,6 +622,14 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
return code; return code;
} }
static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) {
return physiScanNodeToJson(pObj, pJson);
}
static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) {
return jsonToPhysiScanNode(pJson, pObj);
}
static const char* jkProjectPhysiPlanProjections = "Projections"; static const char* jkProjectPhysiPlanProjections = "Projections";
static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) { static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
@ -1273,6 +1458,193 @@ static int32_t jsonToFunctionNode(const SJson* pJson, void* pObj) {
return code; return code;
} }
static const char* jkTableDbName = "DbName";
static const char* jkTableTableName = "tableName";
static const char* jkTableTableAlias = "tableAlias";
static int32_t tableNodeToJson(const void* pObj, SJson* pJson) {
const STableNode* pNode = (const STableNode*)pObj;
int32_t code = exprNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkTableDbName, pNode->dbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkTableTableName, pNode->tableName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkTableTableAlias, pNode->tableAlias);
}
return code;
}
static int32_t jsonToTableNode(const SJson* pJson, void* pObj) {
STableNode* pNode = (STableNode*)pObj;
int32_t code = jsonToExprNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkTableDbName, pNode->dbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkTableTableName, pNode->tableName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkTableTableAlias, pNode->tableAlias);
}
return code;
}
static const char* jkEpSetInUse = "InUse";
static const char* jkEpSetNumOfEps = "NumOfEps";
static const char* jkEpSetEps = "Eps";
static int32_t epSetToJson(const void* pObj, SJson* pJson) {
const SEpSet* pNode = (const SEpSet*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkEpSetInUse, pNode->inUse);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkEpSetNumOfEps, pNode->numOfEps);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddArray(pJson, jkEpSetEps, epToJson, pNode->eps, sizeof(SEp), pNode->numOfEps);
}
return code;
}
static int32_t jsonToEpSet(const SJson* pJson, void* pObj) {
SEpSet* pNode = (SEpSet*)pObj;
int32_t code = tjsonGetTinyIntValue(pJson, jkEpSetInUse, &pNode->inUse);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkEpSetNumOfEps, &pNode->numOfEps);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToArray(pJson, jkEpSetEps, jsonToEp, pNode->eps, sizeof(SEp));
}
return code;
}
static const char* jkVgroupInfoVgId = "VgId";
static const char* jkVgroupInfoHashBegin = "HashBegin";
static const char* jkVgroupInfoHashEnd = "HashEnd";
static const char* jkVgroupInfoEpSet = "EpSet";
static const char* jkVgroupInfoNumOfTable = "NumOfTable";
static int32_t vgroupInfoToJson(const void* pObj, SJson* pJson) {
const SVgroupInfo* pNode = (const SVgroupInfo*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupInfoVgId, pNode->vgId);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashBegin, pNode->hashBegin);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashEnd, pNode->hashEnd);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkVgroupInfoEpSet, epSetToJson, &pNode->epSet);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkVgroupInfoNumOfTable, pNode->numOfTable);
}
return code;
}
static int32_t jsonToVgroupInfo(const SJson* pJson, void* pObj) {
SVgroupInfo* pNode = (SVgroupInfo*)pObj;
int32_t code = tjsonGetIntValue(pJson, jkVgroupInfoVgId, &pNode->vgId);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetUIntValue(pJson, jkVgroupInfoHashBegin, &pNode->hashBegin);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetUIntValue(pJson, jkVgroupInfoHashEnd, &pNode->hashEnd);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, jkVgroupInfoEpSet, jsonToEpSet, &pNode->epSet);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkVgroupInfoNumOfTable, &pNode->numOfTable);
}
return code;
}
static const char* jkVgroupsInfoNum = "Num";
static const char* jkVgroupsInfoVgroups = "Vgroups";
static int32_t vgroupsInfoToJson(const void* pObj, SJson* pJson) {
const SVgroupsInfo* pNode = (const SVgroupsInfo*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupsInfoNum, pNode->numOfVgroups);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddArray(pJson, jkVgroupsInfoVgroups, vgroupInfoToJson, pNode->vgroups, sizeof(SVgroupInfo), pNode->numOfVgroups);
}
return code;
}
static int32_t jsonToVgroupsInfo(const SJson* pJson, void* pObj) {
SVgroupsInfo* pNode = (SVgroupsInfo*)pObj;
int32_t code = tjsonGetIntValue(pJson, jkVgroupsInfoNum, &pNode->numOfVgroups);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToArray(pJson, jkVgroupsInfoVgroups, jsonToVgroupInfo, pNode->vgroups, sizeof(SVgroupInfo));
}
return code;
}
static const char* jkRealTableMetaSize = "MetaSize";
static const char* jkRealTableMeta = "Meta";
static const char* jkRealTableVgroupsInfoSize = "VgroupsInfoSize";
static const char* jkRealTableVgroupsInfo = "VgroupsInfo";
static int32_t realTableNodeToJson(const void* pObj, SJson* pJson) {
const SRealTableNode* pNode = (const SRealTableNode*)pObj;
int32_t code = tableNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkRealTableMetaSize, TABLE_META_SIZE(pNode->pMeta));
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkRealTableMeta, tableMetaToJson, pNode->pMeta);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkRealTableVgroupsInfoSize, VGROUPS_INFO_SIZE(pNode->pVgroupList));
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkRealTableVgroupsInfo, vgroupsInfoToJson, pNode->pVgroupList);
}
return code;
}
static int32_t jsonToRealTableNode(const SJson* pJson, void* pObj) {
SRealTableNode* pNode = (SRealTableNode*)pObj;
int32_t objSize = 0;
int32_t code = jsonToTableNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkRealTableMetaSize, &objSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonMakeObject(pJson, jkRealTableMeta, jsonToTableMeta, (void**)&pNode->pMeta, objSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkRealTableVgroupsInfoSize, &objSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonMakeObject(pJson, jkRealTableVgroupsInfo, jsonToVgroupsInfo, (void**)&pNode->pVgroupList, objSize);
}
return code;
}
static const char* jkGroupingSetType = "GroupingSetType"; static const char* jkGroupingSetType = "GroupingSetType";
static const char* jkGroupingSetParameter = "Parameters"; static const char* jkGroupingSetParameter = "Parameters";
@ -1464,7 +1836,7 @@ static const char* jkSelectStmtSlimit = "Slimit";
static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { static int32_t selectStmtTojson(const void* pObj, SJson* pJson) {
const SSelectStmt* pNode = (const SSelectStmt*)pObj; const SSelectStmt* pNode = (const SSelectStmt*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct); int32_t code = tjsonAddBoolToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkSelectStmtProjections, pNode->pProjectionList); code = nodeListToJson(pJson, jkSelectStmtProjections, pNode->pProjectionList);
} }
@ -1499,6 +1871,83 @@ static int32_t selectStmtTojson(const void* pObj, SJson* pJson) {
return code; return code;
} }
static int32_t jsonToSelectStmt(const SJson* pJson, void* pObj) {
SSelectStmt* pNode = (SSelectStmt*)pObj;
int32_t code = tjsonGetBoolValue(pJson, jkSelectStmtDistinct, &pNode->isDistinct);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSelectStmtProjections, &pNode->pProjectionList);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtFrom, &pNode->pFromTable);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtWhere, &pNode->pWhere);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSelectStmtPartitionBy, &pNode->pPartitionByList);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtWindow, &pNode->pWindow);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSelectStmtGroupBy, &pNode->pGroupByList);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtHaving, &pNode->pHaving);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSelectStmtOrderBy, &pNode->pOrderByList);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtLimit, &pNode->pLimit);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtSlimit, &pNode->pSlimit);
}
return code;
}
static const char* jkCreateTopicStmtTopicName = "TopicName";
static const char* jkCreateTopicStmtSubscribeDbName = "SubscribeDbName";
static const char* jkCreateTopicStmtIgnoreExists = "IgnoreExists";
static const char* jkCreateTopicStmtQuery = "Query";
static int32_t createTopicStmtToJson(const void* pObj, SJson* pJson) {
const SCreateTopicStmt* pNode = (const SCreateTopicStmt*)pObj;
int32_t code = tjsonAddStringToObject(pJson, jkCreateTopicStmtTopicName, pNode->topicName);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subscribeDbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkCreateTopicStmtIgnoreExists, pNode->ignoreExists);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkCreateTopicStmtQuery, nodeToJson, pNode->pQuery);
}
return code;
}
static int32_t jsonToCreateTopicStmt(const SJson* pJson, void* pObj) {
SCreateTopicStmt* pNode = (SCreateTopicStmt*)pObj;
int32_t code = tjsonGetStringValue(pJson, jkCreateTopicStmtTopicName, pNode->topicName);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subscribeDbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkCreateTopicStmtIgnoreExists, &pNode->ignoreExists);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkCreateTopicStmtQuery, &pNode->pQuery);
}
return code;
}
static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
switch (nodeType(pObj)) { switch (nodeType(pObj)) {
case QUERY_NODE_COLUMN: case QUERY_NODE_COLUMN:
@ -1512,6 +1961,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_FUNCTION: case QUERY_NODE_FUNCTION:
return functionNodeToJson(pObj, pJson); return functionNodeToJson(pObj, pJson);
case QUERY_NODE_REAL_TABLE: case QUERY_NODE_REAL_TABLE:
return realTableNodeToJson(pObj, pJson);
case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_TEMP_TABLE:
case QUERY_NODE_JOIN_TABLE: case QUERY_NODE_JOIN_TABLE:
break; break;
@ -1549,6 +1999,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_SHOW_DATABASES_STMT: case QUERY_NODE_SHOW_DATABASES_STMT:
case QUERY_NODE_SHOW_TABLES_STMT: case QUERY_NODE_SHOW_TABLES_STMT:
break; break;
case QUERY_NODE_CREATE_TOPIC_STMT:
return createTopicStmtToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_SCAN: case QUERY_NODE_LOGIC_PLAN_SCAN:
return logicScanNodeToJson(pObj, pJson); return logicScanNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_JOIN: case QUERY_NODE_LOGIC_PLAN_JOIN:
@ -1565,9 +2017,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return physiTagScanNodeToJson(pObj, pJson); return physiTagScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
return physiTableScanNodeToJson(pObj, pJson); return physiTableScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
break; return physiStreamScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_PROJECT: case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return physiProjectNodeToJson(pObj, pJson); return physiProjectNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_JOIN: case QUERY_NODE_PHYSICAL_PLAN_JOIN:
@ -1589,6 +2040,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN: case QUERY_NODE_PHYSICAL_PLAN:
return planToJson(pObj, pJson); return planToJson(pObj, pJson);
default: default:
assert(0);
break; break;
} }
nodesWarn("specificNodeToJson unknown node = %s", nodesNodeName(nodeType(pObj))); nodesWarn("specificNodeToJson unknown node = %s", nodesNodeName(nodeType(pObj)));
@ -1607,7 +2059,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToLogicConditionNode(pJson, pObj); return jsonToLogicConditionNode(pJson, pObj);
case QUERY_NODE_FUNCTION: case QUERY_NODE_FUNCTION:
return jsonToFunctionNode(pJson, pObj); return jsonToFunctionNode(pJson, pObj);
// case QUERY_NODE_REAL_TABLE: case QUERY_NODE_REAL_TABLE:
return jsonToRealTableNode(pJson, pObj);
// case QUERY_NODE_TEMP_TABLE: // case QUERY_NODE_TEMP_TABLE:
// case QUERY_NODE_JOIN_TABLE: // case QUERY_NODE_JOIN_TABLE:
// break; // break;
@ -1633,20 +2086,24 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToDownstreamSourceNode(pJson, pObj); return jsonToDownstreamSourceNode(pJson, pObj);
// case QUERY_NODE_SET_OPERATOR: // case QUERY_NODE_SET_OPERATOR:
// break; // break;
// case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SELECT_STMT:
// return jsonToSelectStmt(pJson, pObj); return jsonToSelectStmt(pJson, pObj);
// case QUERY_NODE_LOGIC_PLAN_SCAN: case QUERY_NODE_CREATE_TOPIC_STMT:
// return jsonToLogicScanNode(pJson, pObj); return jsonToCreateTopicStmt(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_SCAN:
return jsonToLogicScanNode(pJson, pObj);
// case QUERY_NODE_LOGIC_PLAN_JOIN: // case QUERY_NODE_LOGIC_PLAN_JOIN:
// return jsonToLogicJoinNode(pJson, pObj); // return jsonToLogicJoinNode(pJson, pObj);
// case QUERY_NODE_LOGIC_PLAN_AGG: // case QUERY_NODE_LOGIC_PLAN_AGG:
// return jsonToLogicAggNode(pJson, pObj); // return jsonToLogicAggNode(pJson, pObj);
// case QUERY_NODE_LOGIC_PLAN_PROJECT: case QUERY_NODE_LOGIC_PLAN_PROJECT:
// return jsonToLogicProjectNode(pJson, pObj); return jsonToLogicProjectNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
return jsonToPhysiTagScanNode(pJson, pObj); return jsonToPhysiTagScanNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
return jsonToPhysiTableScanNode(pJson, pObj); return jsonToPhysiTableScanNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
return jsonToPhysiStreamScanNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_PROJECT: case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return jsonToPhysiProjectNode(pJson, pObj); return jsonToPhysiProjectNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_JOIN: case QUERY_NODE_PHYSICAL_PLAN_JOIN:
@ -1768,6 +2225,7 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode) {
int32_t code = makeNodeByJson(pJson, pNode); int32_t code = makeNodeByJson(pJson, pNode);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode(*pNode); nodesDestroyNode(*pNode);
*pNode = NULL;
terrno = code; terrno = code;
return code; return code;
} }

17
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -125,6 +125,8 @@ SNodeptr nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SCreateDnodeStmt)); return makeNode(type, sizeof(SCreateDnodeStmt));
case QUERY_NODE_DROP_DNODE_STMT: case QUERY_NODE_DROP_DNODE_STMT:
return makeNode(type, sizeof(SDropDnodeStmt)); return makeNode(type, sizeof(SDropDnodeStmt));
case QUERY_NODE_ALTER_DNODE_STMT:
return makeNode(type, sizeof(SAlterDnodeStmt));
case QUERY_NODE_SHOW_DNODES_STMT: case QUERY_NODE_SHOW_DNODES_STMT:
return makeNode(type, sizeof(SShowStmt)); return makeNode(type, sizeof(SShowStmt));
case QUERY_NODE_SHOW_VGROUPS_STMT: case QUERY_NODE_SHOW_VGROUPS_STMT:
@ -168,7 +170,7 @@ SNodeptr nodesMakeNode(ENodeType type) {
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
return makeNode(type, sizeof(STableSeqScanPhysiNode)); return makeNode(type, sizeof(STableSeqScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
return makeNode(type, sizeof(SNode)); return makeNode(type, sizeof(SStreamScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_PROJECT: case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return makeNode(type, sizeof(SProjectPhysiNode)); return makeNode(type, sizeof(SProjectPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_JOIN: case QUERY_NODE_PHYSICAL_PLAN_JOIN:
@ -357,6 +359,17 @@ int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc) {
if (NULL == pSrc) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = nodesListAppendList(pTarget, pSrc);
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pSrc);
}
return code;
}
SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) {
if (NULL == pCell->pPrev) { if (NULL == pCell->pPrev) {
pList->pHead = pCell->pNext; pList->pHead = pCell->pNext;
@ -571,7 +584,7 @@ typedef struct SCollectFuncsCxt {
static EDealRes collectFuncs(SNode* pNode, void* pContext) { static EDealRes collectFuncs(SNode* pNode, void* pContext) {
SCollectFuncsCxt* pCxt = (SCollectFuncsCxt*)pContext; SCollectFuncsCxt* pCxt = (SCollectFuncsCxt*)pContext;
if (QUERY_NODE_FUNCTION == nodeType(pNode) && pCxt->classifier(((SFunctionNode*)pNode)->funcId)) { if (QUERY_NODE_FUNCTION == nodeType(pNode) && pCxt->classifier(((SFunctionNode*)pNode)->funcId)) {
pCxt->errCode = nodesListAppend(pCxt->pFuncs, pNode); pCxt->errCode = nodesListStrictAppend(pCxt->pFuncs, nodesCloneNode(pNode));
return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR);
} }
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;

21
source/libs/parser/inc/parAst.h
View File

@ -50,8 +50,10 @@ typedef enum EDatabaseOptionType {
DB_OPTION_TTL, DB_OPTION_TTL,
DB_OPTION_WAL, DB_OPTION_WAL,
DB_OPTION_VGROUPS, DB_OPTION_VGROUPS,
DB_OPTION_SINGLESTABLE, DB_OPTION_SINGLE_STABLE,
DB_OPTION_STREAMMODE, DB_OPTION_STREAM_MODE,
DB_OPTION_RETENTIONS,
DB_OPTION_FILE_FACTOR,
DB_OPTION_MAX DB_OPTION_MAX
} EDatabaseOptionType; } EDatabaseOptionType;
@ -65,6 +67,11 @@ typedef enum ETableOptionType {
TABLE_OPTION_MAX TABLE_OPTION_MAX
} ETableOptionType; } ETableOptionType;
typedef struct SAlterOption {
int32_t type;
SToken val;
} SAlterOption;
extern SToken nil_token; extern SToken nil_token;
void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt); void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt);
@ -110,13 +117,16 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr
SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight); SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight);
SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt); SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt);
SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt);
SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal); SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal);
SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pDbName, SNode* pOptions); SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pDbName, SNode* pOptions);
SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pDbName); SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pDbName);
SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName, SNode* pOptions); SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName, SNode* pOptions);
SNode* createDefaultTableOptions(SAstCreateContext* pCxt); SNode* createDefaultTableOptions(SAstCreateContext* pCxt);
SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt);
SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal); SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal);
SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pSma); SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pSma);
SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs);
SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment); SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment);
SDataType createDataType(uint8_t type); SDataType createDataType(uint8_t type);
SDataType createVarLenDataType(uint8_t type, const SToken* pLen); SDataType createVarLenDataType(uint8_t type, const SToken* pLen);
@ -126,6 +136,11 @@ SNode* createCreateMultiTableStmt(SAstCreateContext* pCxt, SNodeList* pSubTables
SNode* createDropTableClause(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable); SNode* createDropTableClause(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable);
SNode* createDropTableStmt(SAstCreateContext* pCxt, SNodeList* pTables); SNode* createDropTableStmt(SAstCreateContext* pCxt, SNodeList* pTables);
SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable); SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable);
SNode* createAlterTableOption(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions);
SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName, SDataType dataType);
SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName);
SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pOldColName, const SToken* pNewColName);
SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const SToken* pTagName, SNode* pVal);
SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName); SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName);
SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDbName); SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDbName);
SNode* createCreateUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, const SToken* pPassword); SNode* createCreateUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, const SToken* pPassword);
@ -133,6 +148,7 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, int
SNode* createDropUserStmt(SAstCreateContext* pCxt, const SToken* pUserName); SNode* createDropUserStmt(SAstCreateContext* pCxt, const SToken* pUserName);
SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort); SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort);
SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode); SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode);
SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue);
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions); SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions);
SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding); SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding);
SNode* createDropIndexStmt(SAstCreateContext* pCxt, const SToken* pIndexName, const SToken* pTableName); SNode* createDropIndexStmt(SAstCreateContext* pCxt, const SToken* pIndexName, const SToken* pTableName);
@ -140,6 +156,7 @@ SNode* createCreateQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId);
SNode* createDropQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId); SNode* createDropQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId);
SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName); SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName);
SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName); SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName);
SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue);
#ifdef __cplusplus #ifdef __cplusplus
} }

129
source/libs/parser/inc/sql.y
View File

@ -21,13 +21,15 @@
#include "parAst.h" #include "parAst.h"
} }
%syntax_error { %syntax_error {
if(TOKEN.z) { if (pCxt->valid) {
generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); if(TOKEN.z) {
} else { generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z);
generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL); } else {
generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCOMPLETE_SQL);
}
pCxt->valid = false;
} }
pCxt->valid = false;
} }
%left OR. %left OR.
@ -41,6 +43,41 @@
%left NK_CONCAT. %left NK_CONCAT.
//%right NK_BITNOT. //%right NK_BITNOT.
/************************************************ create/alter account *****************************************/
cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); }
cmd ::= ALTER ACCOUNT NK_ID alter_account_options. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); }
%type account_options { int32_t }
%destructor account_options { }
account_options ::= . { }
account_options ::= account_options PPS literal. { }
account_options ::= account_options TSERIES literal. { }
account_options ::= account_options STORAGE literal. { }
account_options ::= account_options STREAMS literal. { }
account_options ::= account_options QTIME literal. { }
account_options ::= account_options DBS literal. { }
account_options ::= account_options USERS literal. { }
account_options ::= account_options CONNS literal. { }
account_options ::= account_options STATE literal. { }
%type alter_account_options { int32_t }
%destructor alter_account_options { }
alter_account_options ::= alter_account_option. { }
alter_account_options ::= alter_account_options alter_account_option. { }
%type alter_account_option { int32_t }
%destructor alter_account_option { }
alter_account_option ::= PASS literal. { }
alter_account_option ::= PPS literal. { }
alter_account_option ::= TSERIES literal. { }
alter_account_option ::= STORAGE literal. { }
alter_account_option ::= STREAMS literal. { }
alter_account_option ::= QTIME literal. { }
alter_account_option ::= DBS literal. { }
alter_account_option ::= USERS literal. { }
alter_account_option ::= CONNS literal. { }
alter_account_option ::= STATE literal. { }
/************************************************ create/alter/drop/show user *****************************************/ /************************************************ create/alter/drop/show user *****************************************/
cmd ::= CREATE USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B); } cmd ::= CREATE USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B); }
cmd ::= ALTER USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B); } cmd ::= ALTER USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B); }
@ -48,12 +85,16 @@ cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B).
cmd ::= DROP USER user_name(A). { pCxt->pRootNode = createDropUserStmt(pCxt, &A); } cmd ::= DROP USER user_name(A). { pCxt->pRootNode = createDropUserStmt(pCxt, &A); }
cmd ::= SHOW USERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL); } cmd ::= SHOW USERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL); }
/************************************************ create/drop/show dnode **********************************************/ /************************************************ create/drop/alter/show dnode ****************************************/
cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); } cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); }
cmd ::= CREATE DNODE dnode_host_name(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B); } cmd ::= CREATE DNODE dnode_host_name(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B); }
cmd ::= DROP DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); } cmd ::= DROP DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); }
cmd ::= DROP DNODE dnode_endpoint(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); } cmd ::= DROP DNODE dnode_endpoint(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); }
cmd ::= SHOW DNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL); } cmd ::= SHOW DNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL); }
cmd ::= ALTER DNODE NK_INTEGER(A) NK_STRING(B). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &A, &B, NULL); }
cmd ::= ALTER DNODE NK_INTEGER(A) NK_STRING(B) NK_STRING(C). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &A, &B, &C); }
cmd ::= ALTER ALL DNODES NK_STRING(A). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &A, NULL); }
cmd ::= ALTER ALL DNODES NK_STRING(A) NK_STRING(B). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &A, &B); }
%type dnode_endpoint { SToken } %type dnode_endpoint { SToken }
%destructor dnode_endpoint { } %destructor dnode_endpoint { }
@ -64,9 +105,13 @@ dnode_endpoint(A) ::= NK_STRING(B).
dnode_host_name(A) ::= NK_ID(B). { A = B; } dnode_host_name(A) ::= NK_ID(B). { A = B; }
dnode_host_name(A) ::= NK_IPTOKEN(B). { A = B; } dnode_host_name(A) ::= NK_IPTOKEN(B). { A = B; }
/************************************************ alter local *********************************************************/
cmd ::= ALTER LOCAL NK_STRING(A). { pCxt->pRootNode = createAlterLocalStmt(pCxt, &A, NULL); }
cmd ::= ALTER LOCAL NK_STRING(A) NK_STRING(B). { pCxt->pRootNode = createAlterLocalStmt(pCxt, &A, &B); }
/************************************************ create/drop qnode ***************************************************/ /************************************************ create/drop qnode ***************************************************/
cmd ::= CREATE QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createCreateQnodeStmt(pCxt, &A); } cmd ::= CREATE QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createCreateQnodeStmt(pCxt, &A); }
cmd ::= DROP QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropQnodeStmt(pCxt, &A); } cmd ::= DROP QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropQnodeStmt(pCxt, &A); }
cmd ::= SHOW QNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT, NULL); } cmd ::= SHOW QNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT, NULL); }
/************************************************ create/drop/show/use database ***************************************/ /************************************************ create/drop/show/use database ***************************************/
@ -74,7 +119,7 @@ cmd ::= CREATE DATABASE not_exists_opt(A) db_name(B) db_options(C).
cmd ::= DROP DATABASE exists_opt(A) db_name(B). { pCxt->pRootNode = createDropDatabaseStmt(pCxt, A, &B); } cmd ::= DROP DATABASE exists_opt(A) db_name(B). { pCxt->pRootNode = createDropDatabaseStmt(pCxt, A, &B); }
cmd ::= SHOW DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL); } cmd ::= SHOW DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL); }
cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A); } cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A); }
cmd ::= ALTER DATABASE db_name(A) db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); } cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); }
%type not_exists_opt { bool } %type not_exists_opt { bool }
%destructor not_exists_opt { } %destructor not_exists_opt { }
@ -102,20 +147,55 @@ db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C).
db_options(A) ::= db_options(B) TTL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TTL, &C); } db_options(A) ::= db_options(B) TTL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TTL, &C); }
db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL, &C); } db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL, &C); }
db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); } db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); }
db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLESTABLE, &C); } db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLE_STABLE, &C); }
db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAMMODE, &C); } db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAM_MODE, &C); }
db_options(A) ::= db_options(B) RETENTIONS NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_RETENTIONS, &C); }
db_options(A) ::= db_options(B) FILE_FACTOR NK_FLOAT(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_FILE_FACTOR, &C); }
alter_db_options(A) ::= alter_db_option(B). { A = createDefaultAlterDatabaseOptions(pCxt); A = setDatabaseOption(pCxt, A, B.type, &B.val); }
alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setDatabaseOption(pCxt, B, C.type, &C.val); }
%type alter_db_option { SAlterOption }
%destructor alter_db_option { }
alter_db_option(A) ::= BLOCKS NK_INTEGER(B). { A.type = DB_OPTION_BLOCKS; A.val = B; }
alter_db_option(A) ::= FSYNC NK_INTEGER(B). { A.type = DB_OPTION_FSYNC; A.val = B; }
alter_db_option(A) ::= KEEP NK_INTEGER(B). { A.type = DB_OPTION_KEEP; A.val = B; }
alter_db_option(A) ::= WAL NK_INTEGER(B). { A.type = DB_OPTION_WAL; A.val = B; }
alter_db_option(A) ::= QUORUM NK_INTEGER(B). { A.type = DB_OPTION_QUORUM; A.val = B; }
alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.val = B; }
/************************************************ create/drop/show table/stable ***************************************/ /************************************************ create/drop/show table/stable ***************************************/
cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B) cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B)
NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E);} NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); }
cmd ::= CREATE TABLE multi_create_clause(A). { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, A);} cmd ::= CREATE TABLE multi_create_clause(A). { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, A); }
cmd ::= CREATE STABLE not_exists_opt(A) full_table_name(B) cmd ::= CREATE STABLE not_exists_opt(A) full_table_name(B)
NK_LP column_def_list(C) NK_RP tags_def(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E);} NK_LP column_def_list(C) NK_RP tags_def(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); }
cmd ::= DROP TABLE multi_drop_clause(A). { pCxt->pRootNode = createDropTableStmt(pCxt, A); } cmd ::= DROP TABLE multi_drop_clause(A). { pCxt->pRootNode = createDropTableStmt(pCxt, A); }
cmd ::= DROP STABLE exists_opt(A) full_table_name(B). { pCxt->pRootNode = createDropSuperTableStmt(pCxt, A, B); } cmd ::= DROP STABLE exists_opt(A) full_table_name(B). { pCxt->pRootNode = createDropSuperTableStmt(pCxt, A, B); }
cmd ::= SHOW TABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, NULL); } cmd ::= SHOW TABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, NULL); }
cmd ::= SHOW STABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, NULL); } cmd ::= SHOW STABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, NULL); }
cmd ::= ALTER TABLE alter_table_clause(A). { pCxt->pRootNode = A; }
cmd ::= ALTER STABLE alter_table_clause(A). { pCxt->pRootNode = A; }
alter_table_clause(A) ::= full_table_name(B) alter_table_options(C). { A = createAlterTableOption(pCxt, B, C); }
alter_table_clause(A) ::=
full_table_name(B) ADD COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_ADD_COLUMN, &C, D); }
alter_table_clause(A) ::= full_table_name(B) DROP COLUMN column_name(C). { A = createAlterTableDropCol(pCxt, B, TSDB_ALTER_TABLE_DROP_COLUMN, &C); }
alter_table_clause(A) ::=
full_table_name(B) MODIFY COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &C, D); }
alter_table_clause(A) ::=
full_table_name(B) RENAME COLUMN column_name(C) column_name(D). { A = createAlterTableRenameCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &C, &D); }
alter_table_clause(A) ::=
full_table_name(B) ADD TAG column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_ADD_TAG, &C, D); }
alter_table_clause(A) ::= full_table_name(B) DROP TAG column_name(C). { A = createAlterTableDropCol(pCxt, B, TSDB_ALTER_TABLE_DROP_TAG, &C); }
alter_table_clause(A) ::=
full_table_name(B) MODIFY TAG column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &C, D); }
alter_table_clause(A) ::=
full_table_name(B) RENAME TAG column_name(C) column_name(D). { A = createAlterTableRenameCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &C, &D); }
alter_table_clause(A) ::=
full_table_name(B) SET TAG column_name(C) NK_EQ literal(D). { A = createAlterTableSetTag(pCxt, B, &C, D); }
%type multi_create_clause { SNodeList* } %type multi_create_clause { SNodeList* }
%destructor multi_create_clause { nodesDestroyList($$); } %destructor multi_create_clause { nodesDestroyList($$); }
multi_create_clause(A) ::= create_subtable_clause(B). { A = createNodeList(pCxt, B); } multi_create_clause(A) ::= create_subtable_clause(B). { A = createNodeList(pCxt, B); }
@ -183,11 +263,21 @@ tags_def_opt(A) ::= tags_def(B).
%destructor tags_def { nodesDestroyList($$); } %destructor tags_def { nodesDestroyList($$); }
tags_def(A) ::= TAGS NK_LP column_def_list(B) NK_RP. { A = B; } tags_def(A) ::= TAGS NK_LP column_def_list(B) NK_RP. { A = B; }
table_options(A) ::= . { A = createDefaultTableOptions(pCxt);} table_options(A) ::= . { A = createDefaultTableOptions(pCxt); }
table_options(A) ::= table_options(B) COMMENT NK_STRING(C). { A = setTableOption(pCxt, B, TABLE_OPTION_COMMENT, &C); } table_options(A) ::= table_options(B) COMMENT NK_STRING(C). { A = setTableOption(pCxt, B, TABLE_OPTION_COMMENT, &C); }
table_options(A) ::= table_options(B) KEEP NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_KEEP, &C); } table_options(A) ::= table_options(B) KEEP NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_KEEP, &C); }
table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); } table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); }
table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableSmaOption(pCxt, B, C); } table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableSmaOption(pCxt, B, C); }
table_options(A) ::= table_options(B) ROLLUP NK_LP func_name_list(C) NK_RP. { A = setTableRollupOption(pCxt, B, C); }
alter_table_options(A) ::= alter_table_option(B). { A = createDefaultAlterTableOptions(pCxt); A = setTableOption(pCxt, A, B.type, &B.val); }
alter_table_options(A) ::= alter_table_options(B) alter_table_option(C). { A = setTableOption(pCxt, B, C.type, &C.val); }
%type alter_table_option { SAlterOption }
%destructor alter_table_option { }
alter_table_option(A) ::= COMMENT NK_STRING(B). { A.type = TABLE_OPTION_COMMENT; A.val = B; }
alter_table_option(A) ::= KEEP NK_INTEGER(B). { A.type = TABLE_OPTION_KEEP; A.val = B; }
alter_table_option(A) ::= TTL NK_INTEGER(B). { A.type = TABLE_OPTION_TTL; A.val = B; }
%type col_name_list { SNodeList* } %type col_name_list { SNodeList* }
%destructor col_name_list { nodesDestroyList($$); } %destructor col_name_list { nodesDestroyList($$); }
@ -196,6 +286,13 @@ col_name_list(A) ::= col_name_list(B) NK_COMMA col_name(C).
col_name(A) ::= column_name(B). { A = createColumnNode(pCxt, NULL, &B); } col_name(A) ::= column_name(B). { A = createColumnNode(pCxt, NULL, &B); }
%type func_name_list { SNodeList* }
%destructor func_name_list { nodesDestroyList($$); }
func_name_list(A) ::= func_name(B). { A = createNodeList(pCxt, B); }
func_name_list(A) ::= func_name_list(B) NK_COMMA col_name(C). { A = addNodeToList(pCxt, B, C); }
func_name(A) ::= function_name(B). { A = createFunctionNode(pCxt, &B, NULL); }
/************************************************ create index ********************************************************/ /************************************************ create index ********************************************************/
cmd ::= CREATE SMA INDEX index_name(A) ON table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, &A, &B, NULL, C); } cmd ::= CREATE SMA INDEX index_name(A) ON table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, &A, &B, NULL, C); }
cmd ::= CREATE FULLTEXT INDEX cmd ::= CREATE FULLTEXT INDEX
@ -546,7 +643,7 @@ query_expression_body(A) ::=
query_primary(A) ::= query_specification(B). { A = B; } query_primary(A) ::= query_specification(B). { A = B; }
//query_primary(A) ::= //query_primary(A) ::=
// NK_LP query_expression_body(B) // NK_LP query_expression_body(B)
// order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP. { A = B;} // order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP. { A = B; }
%type order_by_clause_opt { SNodeList* } %type order_by_clause_opt { SNodeList* }
%destructor order_by_clause_opt { nodesDestroyList($$); } %destructor order_by_clause_opt { nodesDestroyList($$); }

116
source/libs/parser/src/parAstCreater.c
View File

@ -246,6 +246,16 @@ static SDatabaseOptions* setDbStreamMode(SAstCreateContext* pCxt, SDatabaseOptio
return pOptions; return pOptions;
} }
static SDatabaseOptions* setDbRetentions(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
// todo
return pOptions;
}
static SDatabaseOptions* setDbFileFactor(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
// todo
return pOptions;
}
static void initSetDatabaseOptionFp() { static void initSetDatabaseOptionFp() {
setDbOptionFuncs[DB_OPTION_BLOCKS] = setDbBlocks; setDbOptionFuncs[DB_OPTION_BLOCKS] = setDbBlocks;
setDbOptionFuncs[DB_OPTION_CACHE] = setDbCache; setDbOptionFuncs[DB_OPTION_CACHE] = setDbCache;
@ -262,8 +272,10 @@ static void initSetDatabaseOptionFp() {
setDbOptionFuncs[DB_OPTION_TTL] = setDbTtl; setDbOptionFuncs[DB_OPTION_TTL] = setDbTtl;
setDbOptionFuncs[DB_OPTION_WAL] = setDbWal; setDbOptionFuncs[DB_OPTION_WAL] = setDbWal;
setDbOptionFuncs[DB_OPTION_VGROUPS] = setDbVgroups; setDbOptionFuncs[DB_OPTION_VGROUPS] = setDbVgroups;
setDbOptionFuncs[DB_OPTION_SINGLESTABLE] = setDbSingleStable; setDbOptionFuncs[DB_OPTION_SINGLE_STABLE] = setDbSingleStable;
setDbOptionFuncs[DB_OPTION_STREAMMODE] = setDbStreamMode; setDbOptionFuncs[DB_OPTION_STREAM_MODE] = setDbStreamMode;
setDbOptionFuncs[DB_OPTION_RETENTIONS] = setDbRetentions;
setDbOptionFuncs[DB_OPTION_FILE_FACTOR] = setDbFileFactor;
} }
static STableOptions* setTableKeep(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) { static STableOptions* setTableKeep(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) {
@ -772,6 +784,29 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
return (SNode*)pOptions; return (SNode*)pOptions;
} }
SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt) {
SDatabaseOptions* pOptions = nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions);
pOptions->numOfBlocks = -1;
pOptions->cacheBlockSize = -1;
pOptions->cachelast = -1;
pOptions->compressionLevel = -1;
pOptions->daysPerFile = -1;
pOptions->fsyncPeriod = -1;
pOptions->maxRowsPerBlock = -1;
pOptions->minRowsPerBlock = -1;
pOptions->keep = -1;
pOptions->precision = -1;
pOptions->quorum = -1;
pOptions->replica = -1;
pOptions->ttl = -1;
pOptions->walLevel = -1;
pOptions->numOfVgroups = -1;
pOptions->singleStable = -1;
pOptions->streamMode = -1;
return (SNode*)pOptions;
}
SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal) { SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal) {
return (SNode*)setDbOptionFuncs[type](pCxt, (SDatabaseOptions*)pOptions, pVal); return (SNode*)setDbOptionFuncs[type](pCxt, (SDatabaseOptions*)pOptions, pVal);
} }
@ -818,6 +853,14 @@ SNode* createDefaultTableOptions(SAstCreateContext* pCxt) {
return (SNode*)pOptions; return (SNode*)pOptions;
} }
SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt) {
STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions);
pOptions->keep = -1;
pOptions->ttl = -1;
return (SNode*)pOptions;
}
SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal) { SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal) {
return (SNode*)setTableOptionFuncs[type](pCxt, (STableOptions*)pOptions, pVal); return (SNode*)setTableOptionFuncs[type](pCxt, (STableOptions*)pOptions, pVal);
} }
@ -827,6 +870,11 @@ SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pS
return pOptions; return pOptions;
} }
SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs) {
// todo
return pOptions;
}
SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment) { SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment) {
SColumnDefNode* pCol = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF); SColumnDefNode* pCol = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF);
CHECK_OUT_OF_MEM(pCol); CHECK_OUT_OF_MEM(pCol);
@ -912,6 +960,49 @@ SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, S
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createAlterTableOption(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions) {
SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = TSDB_ALTER_TABLE_UPDATE_OPTIONS;
pStmt->pOptions = (STableOptions*)pOptions;
return (SNode*)pStmt;
}
SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName, SDataType dataType) {
SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = alterType;
strncpy(pStmt->colName, pColName->z, pColName->n);
pStmt->dataType = dataType;
return (SNode*)pStmt;
}
SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName) {
SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = alterType;
strncpy(pStmt->colName, pColName->z, pColName->n);
return (SNode*)pStmt;
}
SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pOldColName, const SToken* pNewColName) {
SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = alterType;
strncpy(pStmt->colName, pOldColName->z, pOldColName->n);
strncpy(pStmt->newColName, pNewColName->z, pNewColName->n);
return (SNode*)pStmt;
}
SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const SToken* pTagName, SNode* pVal) {
SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = TSDB_ALTER_TABLE_UPDATE_TAG_VAL;
strncpy(pStmt->colName, pTagName->z, pTagName->n);
pStmt->pVal = (SValueNode*)pVal;
return (SNode*)pStmt;
}
SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName) { SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName) {
SUseDatabaseStmt* pStmt = (SUseDatabaseStmt*)nodesMakeNode(QUERY_NODE_USE_DATABASE_STMT); SUseDatabaseStmt* pStmt = (SUseDatabaseStmt*)nodesMakeNode(QUERY_NODE_USE_DATABASE_STMT);
CHECK_OUT_OF_MEM(pStmt); CHECK_OUT_OF_MEM(pStmt);
@ -1009,6 +1100,17 @@ SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode) {
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue) {
SAlterDnodeStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_DNODE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->dnodeId = strtol(pDnode->z, NULL, 10);
trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config));
if (NULL != pValue) {
trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value));
}
return (SNode*)pStmt;
}
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions) { SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions) {
if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) { if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) {
return NULL; return NULL;
@ -1077,3 +1179,13 @@ SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const
pStmt->ignoreNotExists = ignoreNotExists; pStmt->ignoreNotExists = ignoreNotExists;
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue) {
SAlterLocalStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_LOCAL_STMT);
CHECK_OUT_OF_MEM(pStmt);
trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config));
if (NULL != pValue) {
trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value));
}
return (SNode*)pStmt;
}

2
source/libs/parser/src/parInsert.c
View File

@ -391,7 +391,7 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha
} }
// Remove quotation marks // Remove quotation marks
if (TSDB_DATA_TYPE_BINARY == type) { if (TK_NK_STRING == pToken->type) {
if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) {
return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z);
} }

2
source/libs/parser/src/parTokenizer.c
View File

@ -28,6 +28,7 @@ typedef struct SKeyword {
// keywords in sql string // keywords in sql string
static SKeyword keywordTable[] = { static SKeyword keywordTable[] = {
{"ACCOUNT", TK_ACCOUNT},
{"ALL", TK_ALL}, {"ALL", TK_ALL},
{"ALTER", TK_ALTER}, {"ALTER", TK_ALTER},
{"AND", TK_AND}, {"AND", TK_AND},
@ -168,7 +169,6 @@ static SKeyword keywordTable[] = {
// {"SCORES", TK_SCORES}, // {"SCORES", TK_SCORES},
// {"GRANTS", TK_GRANTS}, // {"GRANTS", TK_GRANTS},
// {"DOT", TK_DOT}, // {"DOT", TK_DOT},
// {"ACCOUNT", TK_ACCOUNT},
// {"DESCRIBE", TK_DESCRIBE}, // {"DESCRIBE", TK_DESCRIBE},
// {"SYNCDB", TK_SYNCDB}, // {"SYNCDB", TK_SYNCDB},
// {"LOCAL", TK_LOCAL}, // {"LOCAL", TK_LOCAL},

120
source/libs/parser/src/parTranslater.c
View File

@ -126,7 +126,7 @@ static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SCol
static int32_t createColumnNodeByTable(STranslateContext* pCxt, const STableNode* pTable, SNodeList* pList) { static int32_t createColumnNodeByTable(STranslateContext* pCxt, const STableNode* pTable, SNodeList* pList) {
if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) {
const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta;
int32_t nums = pMeta->tableInfo.numOfColumns + ((TSDB_SUPER_TABLE == pMeta->tableType)? pMeta->tableInfo.numOfTags:0); int32_t nums = pMeta->tableInfo.numOfColumns + ((TSDB_SUPER_TABLE == pMeta->tableType) ? pMeta->tableInfo.numOfTags : 0);
for (int32_t i = 0; i < nums; ++i) { for (int32_t i = 0; i < nums; ++i) {
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) { if (NULL == pCol) {
@ -499,6 +499,10 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect)
} }
static int32_t setTableVgroupList(SParseContext* pCxt, SName* name, SRealTableNode* pRealTable) { static int32_t setTableVgroupList(SParseContext* pCxt, SName* name, SRealTableNode* pRealTable) {
if (pCxt->topicQuery) {
return TSDB_CODE_SUCCESS;
}
if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) { if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) {
SArray* vgroupList = NULL; SArray* vgroupList = NULL;
int32_t code = catalogGetTableDistVgInfo(pCxt->pCatalog, pCxt->pTransporter, &pCxt->mgmtEpSet, name, &vgroupList); int32_t code = catalogGetTableDistVgInfo(pCxt->pCatalog, pCxt->pTransporter, &pCxt->mgmtEpSet, name, &vgroupList);
@ -962,6 +966,73 @@ static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableS
return doTranslateDropSuperTable(pCxt, &tableName, pStmt->ignoreNotExists); return doTranslateDropSuperTable(pCxt, &tableName, pStmt->ignoreNotExists);
} }
static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAltertbReq* pAlterReq) {
pAlterReq->pFields = taosArrayInit(2, sizeof(TAOS_FIELD));
if (NULL == pAlterReq->pFields) {
return TSDB_CODE_OUT_OF_MEMORY;
}
switch (pStmt->alterType) {
case TSDB_ALTER_TABLE_ADD_TAG:
case TSDB_ALTER_TABLE_DROP_TAG:
case TSDB_ALTER_TABLE_ADD_COLUMN:
case TSDB_ALTER_TABLE_DROP_COLUMN:
case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: {
TAOS_FIELD field = { .type = pStmt->dataType.type, .bytes = pStmt->dataType.bytes };
strcpy(field.name, pStmt->colName);
taosArrayPush(pAlterReq->pFields, &field);
break;
}
case TSDB_ALTER_TABLE_UPDATE_TAG_NAME:
case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: {
TAOS_FIELD oldField = {0};
strcpy(oldField.name, pStmt->colName);
taosArrayPush(pAlterReq->pFields, &oldField);
TAOS_FIELD newField = {0};
strcpy(oldField.name, pStmt->newColName);
taosArrayPush(pAlterReq->pFields, &newField);
break;
}
default:
break;
}
return TSDB_CODE_SUCCESS;
}
static int32_t translateAlterTable(STranslateContext* pCxt, SAlterTableStmt* pStmt) {
SMAltertbReq alterReq = {0};
SName tableName = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId };
strcpy(tableName.dbname, pStmt->dbName);
strcpy(tableName.tname, pStmt->tableName);
tNameExtractFullName(&tableName, alterReq.name);
alterReq.alterType = pStmt->alterType;
alterReq.numOfFields = 1;
if (TSDB_ALTER_TABLE_UPDATE_OPTIONS == pStmt->alterType) {
// todo
} else {
if (TSDB_CODE_SUCCESS != setAlterTableField(pStmt, &alterReq)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo));
if (NULL == pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_STB;
pCxt->pCmdMsg->msgLen = tSerializeSMAlterStbReq(NULL, 0, &alterReq);
pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tSerializeSMAlterStbReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &alterReq);
return TSDB_CODE_SUCCESS;
}
static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* pStmt) { static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* pStmt) {
SName name = {0}; SName name = {0};
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
@ -1099,6 +1170,28 @@ static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pStmt) {
SMCfgDnodeReq cfgReq = {0};
cfgReq.dnodeId = pStmt->dnodeId;
strcpy(cfgReq.config, pStmt->config);
strcpy(cfgReq.value, pStmt->value);
pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo));
if (NULL == pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_MND_CONFIG_DNODE;
pCxt->pCmdMsg->msgLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq);
pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tSerializeSMCfgDnodeReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &cfgReq);
return TSDB_CODE_SUCCESS;
}
static int32_t nodeTypeToShowType(ENodeType nt) { static int32_t nodeTypeToShowType(ENodeType nt) {
switch (nt) { switch (nt) {
case QUERY_NODE_SHOW_DATABASES_STMT: case QUERY_NODE_SHOW_DATABASES_STMT:
@ -1300,6 +1393,7 @@ static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* p
SCMCreateTopicReq createReq = {0}; SCMCreateTopicReq createReq = {0};
if (NULL != pStmt->pQuery) { if (NULL != pStmt->pQuery) {
pCxt->pParseCxt->topicQuery = true;
int32_t code = translateQuery(pCxt, pStmt->pQuery); int32_t code = translateQuery(pCxt, pStmt->pQuery);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = nodesNodeToString(pStmt->pQuery, false, &createReq.ast, NULL); code = nodesNodeToString(pStmt->pQuery, false, &createReq.ast, NULL);
@ -1364,6 +1458,11 @@ static int32_t translateDropTopic(STranslateContext* pCxt, SDropTopicStmt* pStmt
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pStmt) {
// todo
return TSDB_CODE_SUCCESS;
}
static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pNode)) { switch (nodeType(pNode)) {
@ -1388,6 +1487,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_DROP_SUPER_TABLE_STMT: case QUERY_NODE_DROP_SUPER_TABLE_STMT:
code = translateDropSuperTable(pCxt, (SDropSuperTableStmt*)pNode); code = translateDropSuperTable(pCxt, (SDropSuperTableStmt*)pNode);
break; break;
case QUERY_NODE_ALTER_TABLE_STMT:
code = translateAlterTable(pCxt, (SAlterTableStmt*)pNode);
break;
case QUERY_NODE_CREATE_USER_STMT: case QUERY_NODE_CREATE_USER_STMT:
code = translateCreateUser(pCxt, (SCreateUserStmt*)pNode); code = translateCreateUser(pCxt, (SCreateUserStmt*)pNode);
break; break;
@ -1406,6 +1508,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_DROP_DNODE_STMT: case QUERY_NODE_DROP_DNODE_STMT:
code = translateDropDnode(pCxt, (SDropDnodeStmt*)pNode); code = translateDropDnode(pCxt, (SDropDnodeStmt*)pNode);
break; break;
case QUERY_NODE_ALTER_DNODE_STMT:
code = translateAlterDnode(pCxt, (SAlterDnodeStmt*)pNode);
break;
case QUERY_NODE_SHOW_DATABASES_STMT: case QUERY_NODE_SHOW_DATABASES_STMT:
case QUERY_NODE_SHOW_STABLES_STMT: case QUERY_NODE_SHOW_STABLES_STMT:
case QUERY_NODE_SHOW_USERS_STMT: case QUERY_NODE_SHOW_USERS_STMT:
@ -1436,6 +1541,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_DROP_TOPIC_STMT: case QUERY_NODE_DROP_TOPIC_STMT:
code = translateDropTopic(pCxt, (SDropTopicStmt*)pNode); code = translateDropTopic(pCxt, (SDropTopicStmt*)pNode);
break; break;
case QUERY_NODE_ALTER_LOCAL_STMT:
code = translateAlterLocal(pCxt, (SAlterLocalStmt*)pNode);
break;
default: default:
break; break;
} }
@ -1855,6 +1963,11 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery)
return rewriteToVnodeModifOpStmt(pQuery, pBufArray); return rewriteToVnodeModifOpStmt(pQuery, pBufArray);
} }
static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) {
// todo
return TSDB_CODE_SUCCESS;
}
static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pQuery->pRoot)) { switch (nodeType(pQuery->pRoot)) {
@ -1866,6 +1979,11 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_CREATE_MULTI_TABLE_STMT: case QUERY_NODE_CREATE_MULTI_TABLE_STMT:
code = rewriteCreateMultiTable(pCxt, pQuery); code = rewriteCreateMultiTable(pCxt, pQuery);
break; break;
case QUERY_NODE_ALTER_TABLE_STMT:
if (TSDB_ALTER_TABLE_UPDATE_TAG_VAL == ((SAlterTableStmt*)pQuery->pRoot)->alterType) {
code = rewriteAlterTable(pCxt, pQuery);
}
break;
default: default:
break; break;
} }

14
source/libs/parser/src/parUtil.c
View File

@ -46,17 +46,19 @@ static char* getSyntaxErrFormat(int32_t errCode) {
case TSDB_CODE_PAR_NOT_SINGLE_GROUP: case TSDB_CODE_PAR_NOT_SINGLE_GROUP:
return "Not a single-group group function"; return "Not a single-group group function";
case TSDB_CODE_PAR_TAGS_NOT_MATCHED: case TSDB_CODE_PAR_TAGS_NOT_MATCHED:
return "tags number not matched"; return "Tags number not matched";
case TSDB_CODE_PAR_INVALID_TAG_NAME: case TSDB_CODE_PAR_INVALID_TAG_NAME:
return "invalid tag name : %s"; return "Invalid tag name : %s";
case TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG: case TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG:
return "name or password too long"; return "Name or password too long";
case TSDB_CODE_PAR_PASSWD_EMPTY: case TSDB_CODE_PAR_PASSWD_EMPTY:
return "password can not be empty"; return "Password can not be empty";
case TSDB_CODE_PAR_INVALID_PORT: case TSDB_CODE_PAR_INVALID_PORT:
return "port should be an integer that is less than 65535 and greater than 0"; return "Port should be an integer that is less than 65535 and greater than 0";
case TSDB_CODE_PAR_INVALID_ENDPOINT: case TSDB_CODE_PAR_INVALID_ENDPOINT:
return "endpoint should be in the format of 'fqdn:port'"; return "Endpoint should be in the format of 'fqdn:port'";
case TSDB_CODE_PAR_EXPRIE_STATEMENT:
return "This statement is no longer supported";
case TSDB_CODE_OUT_OF_MEMORY: case TSDB_CODE_OUT_OF_MEMORY:
return "Out of memory"; return "Out of memory";
default: default:

3760
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

17
source/libs/parser/test/parserAstTest.cpp
View File

@ -302,6 +302,13 @@ TEST_F(ParserTest, createUser) {
ASSERT_TRUE(run()); ASSERT_TRUE(run());
} }
TEST_F(ParserTest, alterAccount) {
setDatabase("root", "test");
bind("alter account ac_wxy pass '123456'");
ASSERT_TRUE(run(TSDB_CODE_PAR_EXPRIE_STATEMENT));
}
TEST_F(ParserTest, createDnode) { TEST_F(ParserTest, createDnode) {
setDatabase("root", "test"); setDatabase("root", "test");
@ -312,6 +319,16 @@ TEST_F(ParserTest, createDnode) {
ASSERT_TRUE(run()); ASSERT_TRUE(run());
} }
TEST_F(ParserTest, alterDnode) {
setDatabase("root", "test");
bind("alter dnode 1 'resetLog'");
ASSERT_TRUE(run());
bind("alter dnode 1 'debugFlag' '134'");
ASSERT_TRUE(run());
}
TEST_F(ParserTest, createDatabase) { TEST_F(ParserTest, createDatabase) {
setDatabase("root", "test"); setDatabase("root", "test");

474
source/libs/planner/src/planLogicCreater.c
View File

@ -18,13 +18,13 @@
#include "functionMgt.h" #include "functionMgt.h"
typedef struct SLogicPlanContext { typedef struct SLogicPlanContext {
int32_t errCode; SPlanContext* pPlanCxt;
int32_t planNodeId;
int32_t acctId;
} SLogicPlanContext; } SLogicPlanContext;
static SLogicNode* createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt); typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, SSelectStmt*, SLogicNode**);
static SLogicNode* createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable);
static int32_t doCreateLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode);
static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogicNode** pLogicNode);
typedef struct SRewriteExprCxt { typedef struct SRewriteExprCxt {
int32_t errCode; int32_t errCode;
@ -66,7 +66,6 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) {
} }
typedef struct SNameExprCxt { typedef struct SNameExprCxt {
int32_t planNodeId;
int32_t rewriteId; int32_t rewriteId;
} SNameExprCxt; } SNameExprCxt;
@ -76,7 +75,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) {
case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_LOGIC_CONDITION:
case QUERY_NODE_FUNCTION: { case QUERY_NODE_FUNCTION: {
SNameExprCxt* pCxt = (SNameExprCxt*)pContext; SNameExprCxt* pCxt = (SNameExprCxt*)pContext;
sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d_%d", pCxt->planNodeId, pCxt->rewriteId++); sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d", pCxt->rewriteId++);
return DEAL_RES_IGNORE_CHILD; return DEAL_RES_IGNORE_CHILD;
} }
default: default:
@ -86,131 +85,194 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
static int32_t rewriteExpr(int32_t planNodeId, int32_t rewriteId, SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) {
SNameExprCxt nameCxt = { .planNodeId = planNodeId, .rewriteId = rewriteId }; static int32_t rewriteId = 1;
SNameExprCxt nameCxt = { .rewriteId = rewriteId };
nodesWalkList(pExprs, doNameExpr, &nameCxt); nodesWalkList(pExprs, doNameExpr, &nameCxt);
SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs };
nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt);
return cxt.errCode; return cxt.errCode;
} }
static SLogicNode* pushLogicNode(SLogicPlanContext* pCxt, SLogicNode* pRoot, SLogicNode* pNode) { static int32_t pushLogicNode(SLogicPlanContext* pCxt, SLogicNode** pOldRoot, SLogicNode* pNewRoot) {
if (TSDB_CODE_SUCCESS != pCxt->errCode) { if (NULL == pNewRoot->pChildren) {
goto error; pNewRoot->pChildren = nodesMakeList();
} if (NULL == pNewRoot->pChildren) {
return TSDB_CODE_OUT_OF_MEMORY;
if (NULL == pRoot) {
return pNode;
}
if (NULL == pNode) {
return pRoot;
}
if (NULL == pNode->pChildren) {
pNode->pChildren = nodesMakeList();
if (NULL == pNode->pChildren) {
goto error;
} }
} }
if (TSDB_CODE_SUCCESS != nodesListAppend(pNode->pChildren, (SNode*)pRoot)) { if (TSDB_CODE_SUCCESS != nodesListAppend(pNewRoot->pChildren, (SNode*)*pOldRoot)) {
goto error; return TSDB_CODE_OUT_OF_MEMORY;
} }
pRoot->pParent = pNode;
return pNode; (*pOldRoot)->pParent = pNewRoot;
error: *pOldRoot = pNewRoot;
nodesDestroyNode((SNode*)pNode);
return pRoot; return TSDB_CODE_SUCCESS;
} }
static SLogicNode* createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable) { static int32_t createChildLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, FCreateLogicNode func, SLogicNode** pRoot) {
SLogicNode* pNode = NULL;
int32_t code = func(pCxt, pSelect, &pNode);
if (TSDB_CODE_SUCCESS == code && NULL != pNode) {
code = pushLogicNode(pCxt, pRoot, pNode);
}
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode(pNode);
}
return code;
}
static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, SLogicNode** pLogicNode) {
SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN);
CHECK_ALLOC(pScan, NULL); if (NULL == pScan) {
pScan->node.id = pCxt->planNodeId++; return TSDB_CODE_OUT_OF_MEMORY;
}
TSWAP(pScan->pMeta, pRealTable->pMeta, STableMeta*); TSWAP(pScan->pMeta, pRealTable->pMeta, STableMeta*);
TSWAP(pScan->pVgroupList, pRealTable->pVgroupList, SVgroupsInfo*); TSWAP(pScan->pVgroupList, pRealTable->pVgroupList, SVgroupsInfo*);
pScan->scanType = pCxt->pPlanCxt->topicQuery ? SCAN_TYPE_TOPIC : SCAN_TYPE_TABLE;
// set columns to scan
SNodeList* pCols = NULL;
CHECK_CODE(nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols), (SLogicNode*)pScan);
if (NULL != pCols) {
pScan->pScanCols = nodesCloneList(pCols);
CHECK_ALLOC(pScan->pScanCols, (SLogicNode*)pScan);
}
// set output
if (NULL != pCols) {
pScan->node.pTargets = nodesCloneList(pCols);
CHECK_ALLOC(pScan->node.pTargets, (SLogicNode*)pScan);
}
pScan->scanType = SCAN_TYPE_TABLE;
pScan->scanFlag = MAIN_SCAN; pScan->scanFlag = MAIN_SCAN;
pScan->scanRange = TSWINDOW_INITIALIZER; pScan->scanRange = TSWINDOW_INITIALIZER;
pScan->tableName.type = TSDB_TABLE_NAME_T; pScan->tableName.type = TSDB_TABLE_NAME_T;
pScan->tableName.acctId = pCxt->acctId; pScan->tableName.acctId = pCxt->pPlanCxt->acctId;
strcpy(pScan->tableName.dbname, pRealTable->table.dbName); strcpy(pScan->tableName.dbname, pRealTable->table.dbName);
strcpy(pScan->tableName.tname, pRealTable->table.tableName); strcpy(pScan->tableName.tname, pRealTable->table.tableName);
return (SLogicNode*)pScan; // set columns to scan
} SNodeList* pCols = NULL;
int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols);
static SLogicNode* createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable) { if (TSDB_CODE_SUCCESS == code && NULL != pCols) {
SLogicNode* pRoot = createQueryLogicNode(pCxt, pTable->pSubquery); pScan->pScanCols = nodesCloneList(pCols);
CHECK_ALLOC(pRoot, NULL); if (NULL == pScan) {
SNode* pNode; code = TSDB_CODE_OUT_OF_MEMORY;
FOREACH(pNode, pRoot->pTargets) { }
strcpy(((SColumnNode*)pNode)->tableAlias, pTable->table.tableAlias);
} }
return pRoot;
// set output
if (TSDB_CODE_SUCCESS == code && NULL != pCols) {
pScan->node.pTargets = nodesCloneList(pCols);
if (NULL == pScan) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pScan;
} else {
nodesDestroyNode(pScan);
}
return code;
} }
static SLogicNode* createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SJoinTableNode* pJoinTable) { static int32_t createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable, SLogicNode** pLogicNode) {
int32_t code = createQueryLogicNode(pCxt, pTable->pSubquery, pLogicNode);
if (TSDB_CODE_SUCCESS == code) {
SNode* pNode;
FOREACH(pNode, (*pLogicNode)->pTargets) {
strcpy(((SColumnNode*)pNode)->tableAlias, pTable->table.tableAlias);
}
}
return code;
}
static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SJoinTableNode* pJoinTable, SLogicNode** pLogicNode) {
SJoinLogicNode* pJoin = (SJoinLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_JOIN); SJoinLogicNode* pJoin = (SJoinLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_JOIN);
CHECK_ALLOC(pJoin, NULL); if (NULL == pJoin) {
pJoin->node.id = pCxt->planNodeId++; return TSDB_CODE_OUT_OF_MEMORY;
}
pJoin->joinType = pJoinTable->joinType; pJoin->joinType = pJoinTable->joinType;
int32_t code = TSDB_CODE_SUCCESS;
// set left and right node // set left and right node
pJoin->node.pChildren = nodesMakeList(); pJoin->node.pChildren = nodesMakeList();
CHECK_ALLOC(pJoin->node.pChildren, (SLogicNode*)pJoin); if (NULL == pJoin->node.pChildren) {
SLogicNode* pLeft = createLogicNodeByTable(pCxt, pSelect, pJoinTable->pLeft); code = TSDB_CODE_OUT_OF_MEMORY;
CHECK_ALLOC(pLeft, (SLogicNode*)pJoin); }
CHECK_CODE(nodesListAppend(pJoin->node.pChildren, (SNode*)pLeft), (SLogicNode*)pJoin);
SLogicNode* pRight = createLogicNodeByTable(pCxt, pSelect, pJoinTable->pRight); SLogicNode* pLeft = NULL;
CHECK_ALLOC(pRight, (SLogicNode*)pJoin); if (TSDB_CODE_SUCCESS == code) {
CHECK_CODE(nodesListAppend(pJoin->node.pChildren, (SNode*)pRight), (SLogicNode*)pJoin); code = doCreateLogicNodeByTable(pCxt, pSelect, pJoinTable->pLeft, &pLeft);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppend(pJoin->node.pChildren, (SNode*)pLeft);
}
}
SLogicNode* pRight = NULL;
if (TSDB_CODE_SUCCESS == code) {
code = doCreateLogicNodeByTable(pCxt, pSelect, pJoinTable->pRight, &pRight);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppend(pJoin->node.pChildren, (SNode*)pRight);
}
}
// set on conditions // set on conditions
if (NULL != pJoinTable->pOnCond) { if (TSDB_CODE_SUCCESS == code && NULL != pJoinTable->pOnCond) {
pJoin->pOnConditions = nodesCloneNode(pJoinTable->pOnCond); pJoin->pOnConditions = nodesCloneNode(pJoinTable->pOnCond);
CHECK_ALLOC(pJoin->pOnConditions, (SLogicNode*)pJoin); if (NULL == pJoin->pOnConditions) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} }
// set the output // set the output
pJoin->node.pTargets = nodesCloneList(pLeft->pTargets); if (TSDB_CODE_SUCCESS == code) {
CHECK_ALLOC(pJoin->node.pTargets, (SLogicNode*)pJoin); pJoin->node.pTargets = nodesCloneList(pLeft->pTargets);
SNodeList* pTargets = nodesCloneList(pRight->pTargets); if (NULL == pJoin->pOnConditions) {
CHECK_ALLOC(pTargets, (SLogicNode*)pJoin); code = TSDB_CODE_OUT_OF_MEMORY;
nodesListAppendList(pJoin->node.pTargets, pTargets); }
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppendList(pJoin->node.pTargets, nodesCloneList(pRight->pTargets));
}
}
return (SLogicNode*)pJoin; if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pJoin;
} else {
nodesDestroyNode(pJoin);
}
return code;
} }
static SLogicNode* createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable) { static int32_t doCreateLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode) {
switch (nodeType(pTable)) { switch (nodeType(pTable)) {
case QUERY_NODE_REAL_TABLE: case QUERY_NODE_REAL_TABLE:
return createScanLogicNode(pCxt, pSelect, (SRealTableNode*)pTable); return createScanLogicNode(pCxt, pSelect, (SRealTableNode*)pTable, pLogicNode);
case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_TEMP_TABLE:
return createSubqueryLogicNode(pCxt, pSelect, (STempTableNode*)pTable); return createSubqueryLogicNode(pCxt, pSelect, (STempTableNode*)pTable, pLogicNode);
case QUERY_NODE_JOIN_TABLE: case QUERY_NODE_JOIN_TABLE:
return createJoinLogicNode(pCxt, pSelect, (SJoinTableNode*)pTable); return createJoinLogicNode(pCxt, pSelect, (SJoinTableNode*)pTable, pLogicNode);
default: default:
break; break;
} }
return NULL; return TSDB_CODE_FAILED;
}
static int32_t createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode) {
SLogicNode* pNode = NULL;
int32_t code = doCreateLogicNodeByTable(pCxt, pSelect, pTable, &pNode);
if (TSDB_CODE_SUCCESS == code) {
pNode->pConditions = nodesCloneNode(pSelect->pWhere);
if (NULL != pSelect->pWhere && NULL == pNode->pConditions) {
nodesDestroyNode(pNode);
return TSDB_CODE_OUT_OF_MEMORY;
}
*pLogicNode = pNode;
}
return code;
}
static SColumnNode* createColumnByExpr(SExprNode* pExpr) {
SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
return NULL;
}
pCol->node.resType = pExpr->resType;
strcpy(pCol->colName, pExpr->aliasName);
return pCol;
} }
typedef struct SCreateColumnCxt { typedef struct SCreateColumnCxt {
@ -245,197 +307,241 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
static SNodeList* createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pExprs) { static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pList) {
SCreateColumnCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pList = nodesMakeList() }; SCreateColumnCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pList = (NULL == *pList ? nodesMakeList() : *pList) };
CHECK_ALLOC(cxt.pList, NULL); if (NULL == cxt.pList) {
return TSDB_CODE_OUT_OF_MEMORY;
}
nodesWalkList(pExprs, doCreateColumn, &cxt); nodesWalkList(pExprs, doCreateColumn, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) { if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyList(cxt.pList); nodesDestroyList(cxt.pList);
return NULL; return cxt.errCode;
} }
return cxt.pList; if (NULL == *pList) {
*pList = cxt.pList;
}
return cxt.errCode;
} }
static SLogicNode* createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
SNodeList* pAggFuncs = NULL; SNodeList* pAggFuncs = NULL;
CHECK_CODE(nodesCollectFuncs(pSelect, fmIsAggFunc, &pAggFuncs), NULL); int32_t code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pAggFuncs);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
if (NULL == pAggFuncs && NULL == pSelect->pGroupByList) { if (NULL == pAggFuncs && NULL == pSelect->pGroupByList) {
return NULL; return TSDB_CODE_SUCCESS;
} }
SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG); SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG);
CHECK_ALLOC(pAgg, NULL); if (NULL == pAgg) {
pAgg->node.id = pCxt->planNodeId++; return TSDB_CODE_OUT_OF_MEMORY;
}
// set grouyp keys, agg funcs and having conditions // set grouyp keys, agg funcs and having conditions
if (NULL != pSelect->pGroupByList) { if (NULL != pSelect->pGroupByList) {
pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList); pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList);
CHECK_ALLOC(pAgg->pGroupKeys, (SLogicNode*)pAgg); if (NULL == pAgg->pGroupKeys) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} }
if (NULL != pAggFuncs) {
if (TSDB_CODE_SUCCESS == code && NULL != pAggFuncs) {
pAgg->pAggFuncs = nodesCloneList(pAggFuncs); pAgg->pAggFuncs = nodesCloneList(pAggFuncs);
CHECK_ALLOC(pAgg->pAggFuncs, (SLogicNode*)pAgg); if (NULL == pAgg->pAggFuncs) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} }
// rewrite the expression in subsequent clauses // rewrite the expression in subsequent clauses
CHECK_CODE(rewriteExpr(pAgg->node.id, 1, pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg); if (TSDB_CODE_SUCCESS == code) {
CHECK_CODE(rewriteExpr(pAgg->node.id, 1 + LIST_LENGTH(pAgg->pGroupKeys), pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg); code = rewriteExpr(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY);
}
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExpr(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY);
}
if (NULL != pSelect->pHaving) { if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) {
pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving); pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving);
CHECK_ALLOC(pAgg->node.pConditions, (SLogicNode*)pAgg); if (NULL == pAgg->node.pConditions) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} }
// set the output // set the output
pAgg->node.pTargets = nodesMakeList(); if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pGroupKeys) {
CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg); code = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys, &pAgg->node.pTargets);
if (NULL != pAgg->pGroupKeys) {
SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys);
CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg);
nodesListAppendList(pAgg->node.pTargets, pTargets);
} }
if (NULL != pAgg->pAggFuncs) { if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pAggFuncs) {
SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs); code = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs, &pAgg->node.pTargets);
CHECK_ALLOC(pTargets, (SLogicNode*)pAgg);
nodesListAppendList(pAgg->node.pTargets, pTargets);
} }
return (SLogicNode*)pAgg; if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pAgg;
} else {
nodesDestroyNode(pAgg);
}
return code;
} }
static SLogicNode* createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SIntervalWindowNode* pInterval, SSelectStmt* pSelect) { static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SIntervalWindowNode* pInterval, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW); SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW);
CHECK_ALLOC(pWindow, NULL); if (NULL == pWindow) {
pWindow->node.id = pCxt->planNodeId++; return TSDB_CODE_OUT_OF_MEMORY;
}
pWindow->winType = WINDOW_TYPE_INTERVAL; pWindow->winType = WINDOW_TYPE_INTERVAL;
//SValueNode* pIntervalNode = (SValueNode*)((SRawExprNode*)(pInterval->pInterval))->pNode; pWindow->interval = ((SValueNode*)pInterval->pInterval)->datum.i;
SValueNode* pIntervalNode = (SValueNode*)(pInterval->pInterval); pWindow->intervalUnit = ((SValueNode*)pInterval->pInterval)->unit;
pWindow->interval = pIntervalNode->datum.i;
pWindow->offset = (NULL != pInterval->pOffset ? ((SValueNode*)pInterval->pOffset)->datum.i : 0); pWindow->offset = (NULL != pInterval->pOffset ? ((SValueNode*)pInterval->pOffset)->datum.i : 0);
pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval); pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval);
pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit);
int32_t code = TSDB_CODE_SUCCESS;
if (NULL != pInterval->pFill) { if (NULL != pInterval->pFill) {
pWindow->pFill = nodesCloneNode(pInterval->pFill); pWindow->pFill = nodesCloneNode(pInterval->pFill);
CHECK_ALLOC(pWindow->pFill, (SLogicNode*)pWindow); if (NULL == pWindow->pFill) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} }
SNodeList* pFuncs = NULL; if (TSDB_CODE_SUCCESS == code) {
CHECK_CODE(nodesCollectFuncs(pSelect, fmIsAggFunc, &pFuncs), NULL); code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pWindow->pFuncs);
if (NULL != pFuncs) {
pWindow->pFuncs = nodesCloneList(pFuncs);
CHECK_ALLOC(pWindow->pFuncs, (SLogicNode*)pWindow);
} }
CHECK_CODE(rewriteExpr(pWindow->node.id, 1, pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW), (SLogicNode*)pWindow); if (TSDB_CODE_SUCCESS == code) {
code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW);
}
pWindow->node.pTargets = createColumnByRewriteExps(pCxt, pWindow->pFuncs); if (TSDB_CODE_SUCCESS == code) {
CHECK_ALLOC(pWindow->node.pTargets, (SLogicNode*)pWindow); code = createColumnByRewriteExps(pCxt, pWindow->pFuncs, &pWindow->node.pTargets);
}
return (SLogicNode*)pWindow; if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pWindow;
} else {
nodesDestroyNode(pWindow);
}
return code;
} }
static SLogicNode* createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
if (NULL == pSelect->pWindow) { if (NULL == pSelect->pWindow) {
return NULL; return TSDB_CODE_SUCCESS;
} }
switch (nodeType(pSelect->pWindow)) { switch (nodeType(pSelect->pWindow)) {
case QUERY_NODE_INTERVAL_WINDOW: case QUERY_NODE_INTERVAL_WINDOW:
return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect); return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect, pLogicNode);
default: default:
break; break;
} }
return NULL; return TSDB_CODE_FAILED;
} }
static SNodeList* createColumnByProjections(SLogicPlanContext* pCxt, SNodeList* pExprs) { static int32_t createColumnByProjections(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pCols) {
SNodeList* pList = nodesMakeList(); SNodeList* pList = nodesMakeList();
CHECK_ALLOC(pList, NULL); if (NULL == pList) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SNode* pNode; SNode* pNode;
FOREACH(pNode, pExprs) { FOREACH(pNode, pExprs) {
SExprNode* pExpr = (SExprNode*)pNode; if (TSDB_CODE_SUCCESS != nodesListAppend(pList, createColumnByExpr((SExprNode*)pNode))) {
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); nodesDestroyList(pList);
if (NULL == pCol) { return TSDB_CODE_OUT_OF_MEMORY;
goto error;
}
pCol->node.resType = pExpr->resType;
strcpy(pCol->colName, pExpr->aliasName);
if (TSDB_CODE_SUCCESS != nodesListAppend(pList, (SNode*)pCol)) {
goto error;
} }
} }
return pList;
error: *pCols = pList;
nodesDestroyList(pList); return TSDB_CODE_SUCCESS;
return NULL;
} }
static SLogicNode* createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
SProjectLogicNode* pProject = (SProjectLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PROJECT); SProjectLogicNode* pProject = (SProjectLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PROJECT);
CHECK_ALLOC(pProject, NULL); if (NULL == pProject) {
pProject->node.id = pCxt->planNodeId++; return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = TSDB_CODE_SUCCESS;
pProject->pProjections = nodesCloneList(pSelect->pProjectionList); pProject->pProjections = nodesCloneList(pSelect->pProjectionList);
if (NULL == pProject->pProjections) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
pProject->node.pTargets = createColumnByProjections(pCxt,pSelect->pProjectionList); if (TSDB_CODE_SUCCESS == code) {
CHECK_ALLOC(pProject->node.pTargets, (SLogicNode*)pProject); code = createColumnByProjections(pCxt,pSelect->pProjectionList, &pProject->node.pTargets);
}
return (SLogicNode*)pProject; if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pProject;
} else {
nodesDestroyNode(pProject);
}
return code;
} }
static SLogicNode* createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
SLogicNode* pRoot = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable); SLogicNode* pRoot = NULL;
if (TSDB_CODE_SUCCESS == pCxt->errCode && NULL != pSelect->pWhere) { int32_t code = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable, &pRoot);
pRoot->pConditions = nodesCloneNode(pSelect->pWhere); if (TSDB_CODE_SUCCESS == code) {
CHECK_ALLOC(pRoot->pConditions, pRoot); code = createChildLogicNode(pCxt, pSelect, createWindowLogicNode, &pRoot);
} }
if (TSDB_CODE_SUCCESS == pCxt->errCode) { if (TSDB_CODE_SUCCESS == code) {
pRoot = pushLogicNode(pCxt, pRoot, createWindowLogicNode(pCxt, pSelect)); code = createChildLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot);
} }
if (TSDB_CODE_SUCCESS == pCxt->errCode) { if (TSDB_CODE_SUCCESS == code) {
pRoot = pushLogicNode(pCxt, pRoot, createAggLogicNode(pCxt, pSelect)); code = createChildLogicNode(pCxt, pSelect, createProjectLogicNode, &pRoot);
} }
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pRoot = pushLogicNode(pCxt, pRoot, createProjectLogicNode(pCxt, pSelect)); if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = pRoot;
} else {
nodesDestroyNode(pRoot);
} }
return pRoot;
return code;
} }
static int32_t getMsgType(ENodeType sqlType) { static int32_t getMsgType(ENodeType sqlType) {
return (QUERY_NODE_CREATE_TABLE_STMT == sqlType || QUERY_NODE_CREATE_MULTI_TABLE_STMT == sqlType) ? TDMT_VND_CREATE_TABLE : TDMT_VND_SUBMIT; return (QUERY_NODE_CREATE_TABLE_STMT == sqlType || QUERY_NODE_CREATE_MULTI_TABLE_STMT == sqlType) ? TDMT_VND_CREATE_TABLE : TDMT_VND_SUBMIT;
} }
static SLogicNode* createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt, SLogicNode** pLogicNode) {
SVnodeModifLogicNode* pModif = (SVnodeModifLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIF); SVnodeModifLogicNode* pModif = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIF);
CHECK_ALLOC(pModif, NULL); if (NULL == pModif) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pModif->pDataBlocks = pStmt->pDataBlocks; pModif->pDataBlocks = pStmt->pDataBlocks;
pModif->msgType = getMsgType(pStmt->sqlNodeType); pModif->msgType = getMsgType(pStmt->sqlNodeType);
return (SLogicNode*)pModif; *pLogicNode = (SLogicNode*)pModif;
return TSDB_CODE_SUCCESS;
} }
static SLogicNode* createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt) { static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogicNode** pLogicNode) {
switch (nodeType(pStmt)) { switch (nodeType(pStmt)) {
case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SELECT_STMT:
return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt); return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt, pLogicNode);
case QUERY_NODE_VNODE_MODIF_STMT: case QUERY_NODE_VNODE_MODIF_STMT:
return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt); return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt, pLogicNode);
default: default:
break; break;
} }
return NULL; // to avoid compiler error return TSDB_CODE_FAILED;
} }
int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode) { int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode) {
SLogicPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .planNodeId = 1, .acctId = pCxt->acctId }; SLogicPlanContext cxt = { .pPlanCxt = pCxt };
SLogicNode* pRoot = createQueryLogicNode(&cxt, pCxt->pAstRoot); int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, pLogicNode);
if (TSDB_CODE_SUCCESS != cxt.errCode) { if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode((SNode*)pRoot); return code;
return cxt.errCode;
} }
*pLogicNode = pRoot;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

34
source/libs/planner/src/planPhysiCreater.c
View File

@ -259,15 +259,22 @@ static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* p
return (SPhysiNode*)pTableScan; return (SPhysiNode*)pTableScan;
} }
static SPhysiNode* createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) {
SStreamScanPhysiNode* pTableScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
CHECK_ALLOC(pTableScan, NULL);
CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan);
return (SPhysiNode*)pTableScan;
}
static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) { static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) {
switch (pScanLogicNode->scanType) { switch (pScanLogicNode->scanType) {
case SCAN_TYPE_TAG: case SCAN_TYPE_TAG:
return createTagScanPhysiNode(pCxt, pScanLogicNode); return createTagScanPhysiNode(pCxt, pScanLogicNode);
case SCAN_TYPE_TABLE: case SCAN_TYPE_TABLE:
return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode); return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode);
case SCAN_TYPE_STABLE: case SCAN_TYPE_TOPIC:
case SCAN_TYPE_STREAM: case SCAN_TYPE_STREAM:
break; return createStreamScanPhysiNode(pCxt, pSubplan, pScanLogicNode);
default: default:
break; break;
} }
@ -466,11 +473,20 @@ static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC
} }
static SPhysiNode* createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode) { static SPhysiNode* createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode) {
SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); if (pCxt->pPlanCxt->streamQuery) {
CHECK_ALLOC(pExchange, NULL); SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pExchange->node.pOutputDataBlockDesc), (SPhysiNode*)pExchange); CHECK_ALLOC(pScan, NULL);
pExchange->srcGroupId = pExchangeLogicNode->srcGroupId; pScan->pScanCols = nodesCloneList(pExchangeLogicNode->node.pTargets);
return (SPhysiNode*)pExchange; CHECK_ALLOC(pScan->pScanCols, (SPhysiNode*)pScan);
CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pScan->node.pOutputDataBlockDesc), (SPhysiNode*)pScan);
return (SPhysiNode*)pScan;
} else {
SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE);
CHECK_ALLOC(pExchange, NULL);
CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pExchange->node.pOutputDataBlockDesc), (SPhysiNode*)pExchange);
pExchange->srcGroupId = pExchangeLogicNode->srcGroupId;
return (SPhysiNode*)pExchange;
}
} }
static SPhysiNode* createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode) { static SPhysiNode* createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode) {
@ -608,7 +624,9 @@ static SSubplan* createPhysiSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLog
taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode);
} else { } else {
pSubplan->pNode = createPhysiNode(pCxt, pSubplan, pLogicSubplan->pNode); pSubplan->pNode = createPhysiNode(pCxt, pSubplan, pLogicSubplan->pNode);
pSubplan->pDataSink = createDataDispatcher(pCxt, pSubplan->pNode); if (!pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) {
pSubplan->pDataSink = createDataDispatcher(pCxt, pSubplan->pNode);
}
pSubplan->msgType = TDMT_VND_QUERY; pSubplan->msgType = TDMT_VND_QUERY;
} }
return pSubplan; return pSubplan;

3
source/libs/planner/src/planSpliter.c
View File

@ -44,7 +44,8 @@ typedef struct SStsInfo {
} SStsInfo; } SStsInfo;
static SLogicNode* stsMatchByNode(SLogicNode* pNode) { static SLogicNode* stsMatchByNode(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->pMeta->tableType) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->pMeta->tableType &&
SCAN_TYPE_TOPIC != ((SScanLogicNode*)pNode)->scanType) {
return pNode; return pNode;
} }
SNode* pChild; SNode* pChild;

20
source/libs/planner/test/plannerTest.cpp
View File

@ -17,6 +17,7 @@
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include "cmdnodes.h"
#include "parser.h" #include "parser.h"
#include "planInt.h" #include "planInt.h"
@ -56,7 +57,8 @@ protected:
const string syntaxTreeStr = toString(query_->pRoot, false); const string syntaxTreeStr = toString(query_->pRoot, false);
SLogicNode* pLogicPlan = nullptr; SLogicNode* pLogicPlan = nullptr;
SPlanContext cxt = { .queryId = 1, .acctId = 0, .pAstRoot = query_->pRoot }; SPlanContext cxt = { .queryId = 1, .acctId = 0 };
setPlanContext(query_, &cxt);
code = createLogicPlan(&cxt, &pLogicPlan); code = createLogicPlan(&cxt, &pLogicPlan);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
cout << "sql:[" << cxt_.pSql << "] logic plan code:" << code << ", strerror:" << tstrerror(code) << endl; cout << "sql:[" << cxt_.pSql << "] logic plan code:" << code << ", strerror:" << tstrerror(code) << endl;
@ -94,6 +96,15 @@ protected:
private: private:
static const int max_err_len = 1024; static const int max_err_len = 1024;
void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) {
if (QUERY_NODE_CREATE_TOPIC_STMT == nodeType(pQuery->pRoot)) {
pCxt->pAstRoot = ((SCreateTopicStmt*)pQuery->pRoot)->pQuery;
pCxt->topicQuery = true;
} else {
pCxt->pAstRoot = pQuery->pRoot;
}
}
void reset() { void reset() {
memset(&cxt_, 0, sizeof(cxt_)); memset(&cxt_, 0, sizeof(cxt_));
memset(errMagBuf_, 0, max_err_len); memset(errMagBuf_, 0, max_err_len);
@ -173,3 +184,10 @@ TEST_F(PlannerTest, interval) {
bind("SELECT count(*) FROM t1 interval(10s)"); bind("SELECT count(*) FROM t1 interval(10s)");
ASSERT_TRUE(run()); ASSERT_TRUE(run());
} }
TEST_F(PlannerTest, createTopic) {
setDatabase("root", "test");
bind("create topic tp as SELECT * FROM st1");
ASSERT_TRUE(run());
}

4
source/libs/qcom/src/queryUtil.c
View File

@ -155,6 +155,10 @@ int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransp
.ahandle = (void*)pInfo, .ahandle = (void*)pInfo,
.handle = pInfo->msgInfo.handle, .handle = pInfo->msgInfo.handle,
.code = 0}; .code = 0};
if (pInfo->msgType == TDMT_VND_QUERY || pInfo->msgType == TDMT_VND_FETCH ||
pInfo->msgType == TDMT_VND_QUERY_CONTINUE) {
rpcMsg.persistHandle = 1;
}
assert(pInfo->fp != NULL); assert(pInfo->fp != NULL);

5
source/libs/sync/src/syncAppendEntries.c
View File

@ -87,7 +87,10 @@
// //
int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
int32_t ret = 0; int32_t ret = 0;
syncAppendEntriesLog2("==syncNodeOnAppendEntriesCb==", pMsg);
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesCb== term:%lu", ths->pRaftStore->currentTerm);
syncAppendEntriesLog2(logBuf, pMsg);
if (pMsg->term > ths->pRaftStore->currentTerm) { if (pMsg->term > ths->pRaftStore->currentTerm) {
syncNodeUpdateTerm(ths, pMsg->term); syncNodeUpdateTerm(ths, pMsg->term);

5
source/libs/sync/src/syncAppendEntriesReply.c
View File

@ -37,7 +37,10 @@
// //
int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) {
int32_t ret = 0; int32_t ret = 0;
syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplyCb==", pMsg);
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesReplyCb== term:%lu", ths->pRaftStore->currentTerm);
syncAppendEntriesReplyLog2(logBuf, pMsg);
if (pMsg->term < ths->pRaftStore->currentTerm) { if (pMsg->term < ths->pRaftStore->currentTerm) {
sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term,

29
source/libs/sync/src/syncCommit.c
View File

@ -18,6 +18,7 @@
#include "syncInt.h" #include "syncInt.h"
#include "syncRaftLog.h" #include "syncRaftLog.h"
#include "syncRaftStore.h" #include "syncRaftStore.h"
#include "syncUtil.h"
// \* Leader i advances its commitIndex. // \* Leader i advances its commitIndex.
// \* This is done as a separate step from handling AppendEntries responses, // \* This is done as a separate step from handling AppendEntries responses,
@ -49,8 +50,11 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
// update commit index // update commit index
SyncIndex newCommitIndex = pSyncNode->commitIndex; SyncIndex newCommitIndex = pSyncNode->commitIndex;
for (SyncIndex index = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore); index > pSyncNode->commitIndex; for (SyncIndex index = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore); index > pSyncNode->commitIndex;
++index) { --index) {
if (syncAgree(pSyncNode, index)) { bool agree = syncAgree(pSyncNode, index);
sTrace("syncMaybeAdvanceCommitIndex syncAgree:%d, index:%ld, pSyncNode->commitIndex:%ld", agree, index,
pSyncNode->commitIndex);
if (agree) {
// term // term
SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index); SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index);
assert(pEntry != NULL); assert(pEntry != NULL);
@ -68,6 +72,8 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
SyncIndex beginIndex = pSyncNode->commitIndex + 1; SyncIndex beginIndex = pSyncNode->commitIndex + 1;
SyncIndex endIndex = newCommitIndex; SyncIndex endIndex = newCommitIndex;
sTrace("syncMaybeAdvanceCommitIndex sync commit %ld", newCommitIndex);
// update commit index // update commit index
pSyncNode->commitIndex = newCommitIndex; pSyncNode->commitIndex = newCommitIndex;
@ -97,14 +103,19 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
} }
bool syncAgreeIndex(SSyncNode* pSyncNode, SRaftId* pRaftId, SyncIndex index) { bool syncAgreeIndex(SSyncNode* pSyncNode, SRaftId* pRaftId, SyncIndex index) {
SyncIndex matchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, pRaftId); // I am leader, I agree
if (syncUtilSameId(pRaftId, &(pSyncNode->myRaftId)) && pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
// b for debug return true;
bool b = false;
if (matchIndex >= index) {
b = true;
} }
return b;
// follower agree
SyncIndex matchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, pRaftId);
if (matchIndex >= index) {
return true;
}
// not agree
return false;
} }
bool syncAgree(SSyncNode* pSyncNode, SyncIndex index) { bool syncAgree(SSyncNode* pSyncNode, SyncIndex index) {

26
source/libs/sync/src/syncMain.c
View File

@ -60,11 +60,14 @@ int64_t syncStart(const SSyncInfo* pSyncInfo) {
int32_t ret = 0; int32_t ret = 0;
SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo);
assert(pSyncNode != NULL); assert(pSyncNode != NULL);
// todo : return ref id
return ret; return ret;
} }
void syncStop(int64_t rid) { void syncStop(int64_t rid) {
SSyncNode* pSyncNode = NULL; // get pointer from rid // todo : get pointer from rid
SSyncNode* pSyncNode = NULL;
syncNodeClose(pSyncNode); syncNodeClose(pSyncNode);
} }
@ -74,8 +77,10 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) {
} }
int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) {
int32_t ret = 0; int32_t ret = 0;
SSyncNode* pSyncNode = NULL; // get pointer from rid
// todo : get pointer from rid
SSyncNode* pSyncNode = NULL;
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, 0, isWeak); SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, 0, isWeak);
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
@ -86,13 +91,14 @@ int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) {
} else { } else {
sTrace("syncForwardToPeer not leader, %s", syncUtilState2String(pSyncNode->state)); sTrace("syncForwardToPeer not leader, %s", syncUtilState2String(pSyncNode->state));
ret = -1; // need define err code !! ret = -1; // todo : need define err code !!
} }
return ret; return ret;
} }
ESyncState syncGetMyRole(int64_t rid) { ESyncState syncGetMyRole(int64_t rid) {
SSyncNode* pSyncNode = NULL; // get pointer from rid // todo : get pointer from rid
SSyncNode* pSyncNode = NULL;
return pSyncNode->state; return pSyncNode->state;
} }
@ -195,7 +201,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
// init TLA+ log vars // init TLA+ log vars
pSyncNode->pLogStore = logStoreCreate(pSyncNode); pSyncNode->pLogStore = logStoreCreate(pSyncNode);
assert(pSyncNode->pLogStore != NULL); assert(pSyncNode->pLogStore != NULL);
pSyncNode->commitIndex = 0; pSyncNode->commitIndex = SYNC_INDEX_INVALID;
// init ping timer // init ping timer
pSyncNode->pPingTimer = NULL; pSyncNode->pPingTimer = NULL;
@ -774,9 +780,6 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg
if (ths->state == TAOS_SYNC_STATE_LEADER) { if (ths->state == TAOS_SYNC_STATE_LEADER) {
ths->pLogStore->appendEntry(ths->pLogStore, pEntry); ths->pLogStore->appendEntry(ths->pLogStore, pEntry);
// only myself, maybe commit
syncMaybeAdvanceCommitIndex(ths);
// start replicate right now! // start replicate right now!
syncNodeReplicate(ths); syncNodeReplicate(ths);
@ -791,6 +794,9 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg
} }
rpcFreeCont(rpcMsg.pCont); rpcFreeCont(rpcMsg.pCont);
// only myself, maybe commit
syncMaybeAdvanceCommitIndex(ths);
} else { } else {
// pre commit // pre commit
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
@ -798,7 +804,7 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg
if (ths->pFsm != NULL) { if (ths->pFsm != NULL) {
if (ths->pFsm->FpPreCommitCb != NULL) { if (ths->pFsm->FpPreCommitCb != NULL) {
ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, -1); ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, -2);
} }
} }
rpcFreeCont(rpcMsg.pCont); rpcFreeCont(rpcMsg.pCont);

5
source/libs/sync/src/syncRequestVote.c
View File

@ -43,7 +43,10 @@
// //
int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) {
int32_t ret = 0; int32_t ret = 0;
syncRequestVoteLog2("==syncNodeOnRequestVoteCb==", pMsg);
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteCb== term:%lu", ths->pRaftStore->currentTerm);
syncRequestVoteLog2(logBuf, pMsg);
if (pMsg->term > ths->pRaftStore->currentTerm) { if (pMsg->term > ths->pRaftStore->currentTerm) {
syncNodeUpdateTerm(ths, pMsg->term); syncNodeUpdateTerm(ths, pMsg->term);

5
source/libs/sync/src/syncRequestVoteReply.c
View File

@ -38,7 +38,10 @@
// //
int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) {
int32_t ret = 0; int32_t ret = 0;
syncRequestVoteReplyLog2("==syncNodeOnRequestVoteReplyCb==", pMsg);
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplyCb== term:%lu", ths->pRaftStore->currentTerm);
syncRequestVoteReplyLog2(logBuf, pMsg);
if (pMsg->term < ths->pRaftStore->currentTerm) { if (pMsg->term < ths->pRaftStore->currentTerm) {
sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term,

14
source/libs/sync/test/CMakeLists.txt
View File

@ -30,6 +30,7 @@ add_executable(syncPingSelfTest "")
add_executable(syncElectTest "") add_executable(syncElectTest "")
add_executable(syncEncodeTest "") add_executable(syncEncodeTest "")
add_executable(syncWriteTest "") add_executable(syncWriteTest "")
add_executable(syncReplicateTest "")
target_sources(syncTest target_sources(syncTest
@ -160,6 +161,10 @@ target_sources(syncWriteTest
PRIVATE PRIVATE
"syncWriteTest.cpp" "syncWriteTest.cpp"
) )
target_sources(syncReplicateTest
PRIVATE
"syncReplicateTest.cpp"
)
target_include_directories(syncTest target_include_directories(syncTest
@ -327,6 +332,11 @@ target_include_directories(syncWriteTest
"${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc" "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
) )
target_include_directories(syncReplicateTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries(syncTest target_link_libraries(syncTest
@ -457,6 +467,10 @@ target_link_libraries(syncWriteTest
sync sync
gtest_main gtest_main
) )
target_link_libraries(syncReplicateTest
sync
gtest_main
)
enable_testing() enable_testing()

5
source/libs/sync/test/syncElectTest.cpp
View File

@ -116,10 +116,9 @@ int main(int argc, char** argv) {
//--------------------------- //---------------------------
while (1) { while (1) {
sTrace("while 1 sleep, state: %d, %s, electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", sTrace("elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d",
gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->electTimerLogicClock, gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, gSyncNode->electTimerLogicClock,
gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS); gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS);
taosMsleep(1000);
} }
return 0; return 0;

188
source/libs/sync/test/syncReplicateTest.cpp Normal file
View File

@ -0,0 +1,188 @@
#include <gtest/gtest.h>
#include <stdio.h>
#include "syncEnv.h"
#include "syncIO.h"
#include "syncInt.h"
#include "syncMessage.h"
#include "syncRaftEntry.h"
#include "syncRaftLog.h"
#include "syncRaftStore.h"
#include "syncUtil.h"
void logTest() {
sTrace("--- sync log test: trace");
sDebug("--- sync log test: debug");
sInfo("--- sync log test: info");
sWarn("--- sync log test: warn");
sError("--- sync log test: error");
sFatal("--- sync log test: fatal");
}
uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
int32_t replicaNum = 3;
int32_t myIndex = 0;
SRaftId ids[TSDB_MAX_REPLICA];
SSyncInfo syncInfo;
SSyncFSM * pFsm;
SWal * pWal;
SSyncNode *gSyncNode;
void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) {
printf("==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code);
syncRpcMsgPrint2((char *)"==CommitCb==", (SRpcMsg *)pBuf);
}
void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) {
printf("==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code);
syncRpcMsgPrint2((char *)"==PreCommitCb==", (SRpcMsg *)pBuf);
}
void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) {
printf("==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code);
syncRpcMsgPrint2((char *)"==RollBackCb==", (SRpcMsg *)pBuf);
}
void initFsm() {
pFsm = (SSyncFSM *)malloc(sizeof(SSyncFSM));
pFsm->FpCommitCb = CommitCb;
pFsm->FpPreCommitCb = PreCommitCb;
pFsm->FpRollBackCb = RollBackCb;
}
SSyncNode *syncNodeInit() {
syncInfo.vgId = 1234;
syncInfo.rpcClient = gSyncIO->clientRpc;
syncInfo.FpSendMsg = syncIOSendMsg;
syncInfo.queue = gSyncIO->pMsgQ;
syncInfo.FpEqMsg = syncIOEqMsg;
syncInfo.pFsm = pFsm;
snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate_test_%d", myIndex);
int code = walInit();
assert(code == 0);
SWalCfg walCfg;
memset(&walCfg, 0, sizeof(SWalCfg));
walCfg.vgId = syncInfo.vgId;
walCfg.fsyncPeriod = 1000;
walCfg.retentionPeriod = 1000;
walCfg.rollPeriod = 1000;
walCfg.retentionSize = 1000;
walCfg.segSize = 1000;
walCfg.level = TAOS_WAL_FSYNC;
char tmpdir[128];
snprintf(tmpdir, sizeof(tmpdir), "./replicate_test_wal_%d", myIndex);
pWal = walOpen(tmpdir, &walCfg);
assert(pWal != NULL);
syncInfo.pWal = pWal;
SSyncCfg *pCfg = &syncInfo.syncCfg;
pCfg->myIndex = myIndex;
pCfg->replicaNum = replicaNum;
for (int i = 0; i < replicaNum; ++i) {
pCfg->nodeInfo[i].nodePort = ports[i];
snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
// taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
}
SSyncNode *pSyncNode = syncNodeOpen(&syncInfo);
assert(pSyncNode != NULL);
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout;
gSyncIO->pSyncNode = pSyncNode;
return pSyncNode;
}
SSyncNode *syncInitTest() { return syncNodeInit(); }
void initRaftId(SSyncNode *pSyncNode) {
for (int i = 0; i < replicaNum; ++i) {
ids[i] = pSyncNode->replicasId[i];
char *s = syncUtilRaftId2Str(&ids[i]);
printf("raftId[%d] : %s\n", i, s);
free(s);
}
}
SRpcMsg *step0(int i) {
SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg));
memset(pMsg, 0, sizeof(SRpcMsg));
pMsg->msgType = 9999;
pMsg->contLen = 128;
pMsg->pCont = malloc(pMsg->contLen);
snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i);
return pMsg;
}
SyncClientRequest *step1(const SRpcMsg *pMsg) {
SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true);
return pRetMsg;
}
int main(int argc, char **argv) {
// taosInitLog((char *)"syncTest.log", 100000, 10);
tsAsyncLog = 0;
sDebugFlag = 143 + 64;
void logTest();
myIndex = 0;
if (argc >= 2) {
myIndex = atoi(argv[1]);
}
int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]);
assert(ret == 0);
ret = syncEnvStart();
assert(ret == 0);
taosRemoveDir("./wal_test");
initFsm();
gSyncNode = syncInitTest();
assert(gSyncNode != NULL);
syncNodePrint2((char *)"", gSyncNode);
initRaftId(gSyncNode);
for (int i = 0; i < 30; ++i) {
// step0
SRpcMsg *pMsg0 = step0(i);
syncRpcMsgPrint2((char *)"==step0==", pMsg0);
// step1
SyncClientRequest *pMsg1 = step1(pMsg0);
syncClientRequestPrint2((char *)"==step1==", pMsg1);
SyncClientRequest *pSyncClientRequest = pMsg1;
SRpcMsg rpcMsg;
syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg);
gSyncNode->FpEqMsg(gSyncNode->queue, &rpcMsg);
taosMsleep(1000);
sTrace("replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d",
gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, gSyncNode->electTimerLogicClock,
gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS);
}
while (1) {
sTrace("replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d",
gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, gSyncNode->electTimerLogicClock,
gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS);
taosMsleep(1000);
}
return 0;
}

22
source/libs/tdb/CMakeLists.txt
View File

@ -1,10 +1,17 @@
# tdb
set(TDB_SUBDIRS "db") add_library(tdb "")
foreach(TDB_SUBDIR ${TDB_SUBDIRS}) target_sources(tdb
aux_source_directory("src/${TDB_SUBDIR}" TDB_SRC) PRIVATE
endforeach() "src/db/tdbPCache.c"
"src/db/tdbPager.c"
add_library(tdb STATIC ${TDB_SRC}) "src/db/tdbUtil.c"
"src/db/tdbBtree.c"
"src/db/tdbDb.c"
"src/db/tdbEnv.c"
# "src/db/tdbPage.c"
"src/page/tdbPage.c"
"src/page/tdbPageL.c"
)
target_include_directories( target_include_directories(
tdb tdb
@ -17,6 +24,7 @@ target_link_libraries(
PUBLIC util PUBLIC util
) )
# for test
if(${BUILD_TEST}) if(${BUILD_TEST})
add_subdirectory(test) add_subdirectory(test)
endif(${BUILD_TEST}) endif(${BUILD_TEST})

58
source/libs/tdb/inc/tdb.h
View File

@ -22,44 +22,42 @@
extern "C" { extern "C" {
#endif #endif
typedef struct STDb TDB; // typedef struct STDb TDB;
typedef struct STDbEnv TENV; // typedef struct STDbEnv TENV;
typedef struct STDbCurosr TDBC; // typedef struct STDbCurosr TDBC;
typedef int32_t pgsz_t; // typedef int32_t pgsz_t;
typedef int32_t cachesz_t; // typedef int32_t cachesz_t;
typedef int (*TdbKeyCmprFn)(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2); // typedef int (*TdbKeyCmprFn)(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2);
// TEVN // // TEVN
int tdbEnvCreate(TENV **ppEnv, const char *rootDir); // int tdbEnvCreate(TENV **ppEnv, const char *rootDir);
int tdbEnvOpen(TENV *ppEnv); // int tdbEnvOpen(TENV *ppEnv);
int tdbEnvClose(TENV *pEnv); // int tdbEnvClose(TENV *pEnv);
int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize); // int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize);
pgsz_t tdbEnvGetPageSize(TENV *pEnv); // pgsz_t tdbEnvGetPageSize(TENV *pEnv);
cachesz_t tdbEnvGetCacheSize(TENV *pEnv); // cachesz_t tdbEnvGetCacheSize(TENV *pEnv);
int tdbEnvBeginTxn(TENV *pEnv); // int tdbEnvBeginTxn(TENV *pEnv);
int tdbEnvCommit(TENV *pEnv); // int tdbEnvCommit(TENV *pEnv);
// TDB // // TDB
int tdbCreate(TDB **ppDb); // int tdbCreate(TDB **ppDb);
int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv); // int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv);
int tdbClose(TDB *pDb); // int tdbClose(TDB *pDb);
int tdbDrop(TDB *pDb); // int tdbDrop(TDB *pDb);
int tdbSetKeyLen(TDB *pDb, int klen); // int tdbSetKeyLen(TDB *pDb, int klen);
int tdbSetValLen(TDB *pDb, int vlen); // int tdbSetValLen(TDB *pDb, int vlen);
int tdbSetDup(TDB *pDb, int dup); // int tdbSetDup(TDB *pDb, int dup);
int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn); // int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn);
int tdbGetKeyLen(TDB *pDb); // int tdbGetKeyLen(TDB *pDb);
int tdbGetValLen(TDB *pDb); // int tdbGetValLen(TDB *pDb);
int tdbGetDup(TDB *pDb); // int tdbGetDup(TDB *pDb);
int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData); // int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData);
// TDBC
#ifdef __cplusplus #ifdef __cplusplus
} }

205
source/libs/tdb/src/db/tdb.c
View File

@ -1,205 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
struct STDb {
char dbname[TDB_MAX_DBNAME_LEN];
SBTree * pBt; // current access method (may extend)
SPgFile * pPgFile; // backend page file this DB is using
TENV * pEnv; // TENV containing the DB
int klen; // key length if know
int vlen; // value length if know
bool dup; // dup mode
TdbKeyCmprFn cFn; // compare function
};
struct STDbCurosr {
SBtCursor *pBtCur;
};
static int tdbDefaultKeyCmprFn(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2);
int tdbCreate(TDB **ppDb) {
TDB *pDb;
// create the handle
pDb = (TDB *)calloc(1, sizeof(*pDb));
if (pDb == NULL) {
return -1;
}
pDb->klen = TDB_VARIANT_LEN;
pDb->vlen = TDB_VARIANT_LEN;
pDb->dup = false;
pDb->cFn = tdbDefaultKeyCmprFn;
*ppDb = pDb;
return 0;
}
static int tdbDestroy(TDB *pDb) {
if (pDb) {
free(pDb);
}
return 0;
}
int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv) {
int ret;
uint8_t fileid[TDB_FILE_ID_LEN];
SPgFile * pPgFile;
SPgCache *pPgCache;
SBTree * pBt;
bool fileExist;
size_t dbNameLen;
pgno_t dbRootPgno;
char dbfname[128]; // TODO: make this as a macro or malloc on the heap
ASSERT(pDb != NULL);
ASSERT(fname != NULL);
// TODO: Here we simply put an assert here. In the future, make `pEnv`
// can be set as NULL.
ASSERT(pEnv != NULL);
// check the DB name
dbNameLen = 0;
if (dbname) {
dbNameLen = strlen(dbname);
if (dbNameLen >= TDB_MAX_DBNAME_LEN) {
return -1;
}
memcpy(pDb->dbname, dbname, dbNameLen);
}
pDb->dbname[dbNameLen] = '\0';
// get page file from the env, if not opened yet, open it
pPgFile = NULL;
snprintf(dbfname, 128, "%s/%s", tdbEnvGetRootDir(pEnv), fname);
fileExist = taosCheckExistFile(fname);
if (fileExist) {
tdbGnrtFileID(dbfname, fileid, false);
pPgFile = tdbEnvGetPageFile(pEnv, fileid);
}
if (pPgFile == NULL) {
ret = pgFileOpen(&pPgFile, dbfname, pEnv);
if (ret != 0) {
// TODO: handle error
return -1;
}
}
// TODO: get the root page number from the master DB of the page file
// tdbGet(&dbRootPgno);
if (dbRootPgno == 0) {
// DB not exist, create one
ret = pgFileAllocatePage(pPgFile, &dbRootPgno);
if (ret != 0) {
// TODO: handle error
}
// tdbInsert(pPgFile->pMasterDB, dbname, strlen(dbname), &dbRootPgno, sizeof(dbRootPgno));
}
ASSERT(dbRootPgno > 1);
// pDb->pBt->root = dbRootPgno;
// register
pDb->pPgFile = pPgFile;
tdbEnvRgstDB(pEnv, pDb);
pDb->pEnv = pEnv;
return 0;
}
int tdbClose(TDB *pDb) {
if (pDb == NULL) return 0;
return tdbDestroy(pDb);
}
int tdbDrop(TDB *pDb) {
// TODO
return 0;
}
int tdbSetKeyLen(TDB *pDb, int klen) {
// TODO: check `klen`
pDb->klen = klen;
return 0;
}
int tdbSetValLen(TDB *pDb, int vlen) {
// TODO: check `vlen`
pDb->vlen = vlen;
return 0;
}
int tdbSetDup(TDB *pDb, int dup) {
if (dup) {
pDb->dup = true;
} else {
pDb->dup = false;
}
return 0;
}
int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn) {
if (fn == NULL) {
return -1;
} else {
pDb->cFn = fn;
}
return 0;
}
int tdbGetKeyLen(TDB *pDb) { return pDb->klen; }
int tdbGetValLen(TDB *pDb) { return pDb->vlen; }
int tdbGetDup(TDB *pDb) {
if (pDb->dup) {
return 1;
} else {
return 0;
}
}
int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData) {
// TODO
return 0;
}
static int tdbDefaultKeyCmprFn(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2) {
int mlen;
int cret;
ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL);
mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2;
cret = memcmp(pKey1, pKey2, mlen);
if (cret == 0) {
if (keyLen1 < keyLen2) {
cret = -1;
} else if (keyLen1 > keyLen2) {
cret = 1;
} else {
cret = 0;
}
}
return cret;
}

1000
source/libs/tdb/src/db/tdbBtree.c
View File

File diff suppressed because it is too large Load Diff

89
source/libs/tdb/src/db/tdbDb.c Normal file
View File

@ -0,0 +1,89 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
struct STDb {
STEnv *pEnv;
SBTree *pBt;
};
int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb) {
STDb *pDb;
SPager *pPager;
int ret;
char fFullName[TDB_FILENAME_LEN];
SPage *pPage;
SPgno pgno;
*ppDb = NULL;
pDb = (STDb *)calloc(1, sizeof(*pDb));
if (pDb == NULL) {
return -1;
}
// pDb->pEnv
pDb->pEnv = pEnv;
pPager = tdbEnvGetPager(pEnv, fname);
if (pPager == NULL) {
snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname);
ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager);
if (ret < 0) {
return -1;
}
}
ASSERT(pPager != NULL);
// pDb->pBt
ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pDb->pBt));
if (ret < 0) {
return -1;
}
*ppDb = pDb;
return 0;
}
int tdbDbClose(STDb *pDb) {
// TODO
return 0;
}
int tdbDbDrop(STDb *pDb) {
// TODO
return 0;
}
int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) {
SBtCursor btc;
SBtCursor *pCur;
int ret;
pCur = &btc;
ret = tdbBtreeCursor(pCur, pDb->pBt);
if (ret < 0) {
return -1;
}
ret = tdbBtCursorInsert(pCur, pKey, keyLen, pVal, valLen);
if (ret < 0) {
return -1;
}
return 0;
}

167
source/libs/tdb/src/db/tdbEnv.c
View File

@ -15,155 +15,56 @@
#include "tdbInt.h" #include "tdbInt.h"
struct STDbEnv { int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, STEnv **ppEnv) {
char * rootDir; // root directory of the environment STEnv *pEnv;
char * jname; // journal file name int dsize;
TdFilePtr jpFile; // journal file fd int zsize;
pgsz_t pgSize; // page size u8 *pPtr;
cachesz_t cacheSize; // total cache size int ret;
STDbList dbList; // TDB List
SPgFileList pgfList; // SPgFile List
SPgCache * pPgCache; // page cache
struct {
#define TDB_ENV_PGF_HASH_BUCKETS 17
SPgFileList buckets[TDB_ENV_PGF_HASH_BUCKETS];
} pgfht; // page file hash table;
};
#define TDB_ENV_PGF_HASH(fileid) (((uint8_t *)(fileid))[0] + ((uint8_t *)(fileid))[1] + ((uint8_t *)(fileid))[2])
static int tdbEnvDestroy(TENV *pEnv);
int tdbEnvCreate(TENV **ppEnv, const char *rootDir) {
TENV * pEnv;
size_t slen;
size_t jlen;
ASSERT(rootDir != NULL);
*ppEnv = NULL; *ppEnv = NULL;
slen = strlen(rootDir);
jlen = slen + strlen(TDB_JOURNAL_NAME) + 1; dsize = strlen(rootDir);
pEnv = (TENV *)calloc(1, sizeof(*pEnv) + slen + 1 + jlen + 1); zsize = sizeof(*pEnv) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3;
if (pEnv == NULL) {
pPtr = (uint8_t *)calloc(1, zsize);
if (pPtr == NULL) {
return -1; return -1;
} }
pEnv->rootDir = (char *)(&pEnv[1]); pEnv = (STEnv *)pPtr;
pEnv->jname = pEnv->rootDir + slen + 1; pPtr += sizeof(*pEnv);
pEnv->jpFile = NULL; // pEnv->rootDir
pEnv->pgSize = TDB_DEFAULT_PGSIZE; pEnv->rootDir = pPtr;
pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE; memcpy(pEnv->rootDir, rootDir, dsize);
pEnv->rootDir[dsize] = '\0';
pPtr = pPtr + dsize + 1;
// pEnv->jfname
pEnv->jfname = pPtr;
memcpy(pEnv->jfname, rootDir, dsize);
pEnv->jfname[dsize] = '/';
memcpy(pEnv->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME));
pEnv->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0';
memcpy(pEnv->rootDir, rootDir, slen); pEnv->jfd = -1;
pEnv->rootDir[slen] = '\0';
sprintf(pEnv->jname, "%s/%s", rootDir, TDB_JOURNAL_NAME);
TD_DLIST_INIT(&(pEnv->dbList)); ret = tdbPCacheOpen(pageSize, cacheSize, &(pEnv->pCache));
TD_DLIST_INIT(&(pEnv->pgfList)); if (ret < 0) {
return -1;
}
/* TODO */ mkdir(rootDir, 0755);
*ppEnv = pEnv; *ppEnv = pEnv;
return 0; return 0;
} }
int tdbEnvOpen(TENV *pEnv) { int tdbEnvClose(STEnv *pEnv) {
SPgCache *pPgCache;
int ret;
ASSERT(pEnv != NULL);
/* TODO: here we do not need to create the root directory, more
* work should be done here
*/
mkdir(pEnv->rootDir, 0755);
ret = pgCacheOpen(&pPgCache, pEnv);
if (ret != 0) {
goto _err;
}
pEnv->pPgCache = pPgCache;
return 0;
_err:
return -1;
}
int tdbEnvClose(TENV *pEnv) {
if (pEnv == NULL) return 0;
pgCacheClose(pEnv->pPgCache);
tdbEnvDestroy(pEnv);
return 0;
}
int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize) {
if (!TDB_IS_PGSIZE_VLD(pgSize) || cacheSize / pgSize < 10) {
return -1;
}
/* TODO */
pEnv->pgSize = pgSize;
pEnv->cacheSize = cacheSize;
return 0;
}
pgsz_t tdbEnvGetPageSize(TENV *pEnv) { return pEnv->pgSize; }
cachesz_t tdbEnvGetCacheSize(TENV *pEnv) { return pEnv->cacheSize; }
SPgFile *tdbEnvGetPageFile(TENV *pEnv, const uint8_t fileid[]) {
SPgFileList *pBucket;
SPgFile * pPgFile;
pBucket = pEnv->pgfht.buckets + (TDB_ENV_PGF_HASH(fileid) % TDB_ENV_PGF_HASH_BUCKETS); // TODO
for (pPgFile = TD_DLIST_HEAD(pBucket); pPgFile != NULL; pPgFile = TD_DLIST_NODE_NEXT_WITH_FIELD(pPgFile, envHash)) {
if (memcmp(fileid, pPgFile->fileid, TDB_FILE_ID_LEN) == 0) break;
};
return pPgFile;
}
SPgCache *tdbEnvGetPgCache(TENV *pEnv) { return pEnv->pPgCache; }
static int tdbEnvDestroy(TENV *pEnv) {
// TODO // TODO
return 0; return 0;
} }
int tdbEnvBeginTxn(TENV *pEnv) { SPager *tdbEnvGetPager(STEnv *pEnv, const char *fname) {
pEnv->jpFile = taosOpenFile(pEnv->jname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pEnv->jpFile == NULL) {
return -1;
}
return 0;
}
int tdbEnvCommit(TENV *pEnv) {
/* TODO */
taosCloseFile(&pEnv->jpFile);
pEnv->jpFile = NULL;
return 0;
}
const char *tdbEnvGetRootDir(TENV *pEnv) { return pEnv->rootDir; }
int tdbEnvRgstPageFile(TENV *pEnv, SPgFile *pPgFile) {
SPgFileList *pBucket;
TD_DLIST_APPEND_WITH_FIELD(&(pEnv->pgfList), pPgFile, envPgfList);
pBucket = pEnv->pgfht.buckets + (TDB_ENV_PGF_HASH(pPgFile->fileid) % TDB_ENV_PGF_HASH_BUCKETS); // TODO
TD_DLIST_APPEND_WITH_FIELD(pBucket, pPgFile, envHash);
return 0;
}
int tdbEnvRgstDB(TENV *pEnv, TDB *pDb) {
// TODO // TODO
return 0; return NULL;
} }

309
source/libs/tdb/src/db/tdbPCache.c Normal file
View File

@ -0,0 +1,309 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
struct SPCache {
int pageSize;
int cacheSize;
pthread_mutex_t mutex;
int nFree;
SPage *pFree;
int nPage;
int nHash;
SPage **pgHash;
int nRecyclable;
SPage lru;
};
#define PCACHE_PAGE_HASH(pPgid) \
({ \
u32 *t = (u32 *)((pPgid)->fileid); \
t[0] + t[1] + t[2] + t[3] + t[4] + t[5] + (pPgid)->pgno; \
})
#define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL)
// For page ref
#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0)
#if 0
#define TDB_REF_PAGE(pPage) (++(pPage)->nRef)
#define TDB_UNREF_PAGE(pPage) (--(pPage)->nRef)
#define TDB_GET_PAGE_REF(pPage) ((pPage)->nRef)
#else
#define TDB_REF_PAGE(pPage) atomic_add_fetch_32(&((pPage)->nRef), 1)
#define TDB_UNREF_PAGE(pPage) atomic_sub_fetch_32(&((pPage)->nRef), 1)
#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef))
#endif
static int tdbPCacheOpenImpl(SPCache *pCache);
static void tdbPCacheInitLock(SPCache *pCache);
static void tdbPCacheClearLock(SPCache *pCache);
static void tdbPCacheLock(SPCache *pCache);
static void tdbPCacheUnlock(SPCache *pCache);
static bool tdbPCacheLocked(SPCache *pCache);
static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage);
static void tdbPCachePinPage(SPage *pPage);
static void tdbPCacheRemovePageFromHash(SPage *pPage);
static void tdbPCacheAddPageToHash(SPage *pPage);
static void tdbPCacheUnpinPage(SPage *pPage);
static void *tdbOsMalloc(void *arg, size_t size);
static void tdbOsFree(void *arg, void *ptr);
int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
SPCache *pCache;
void *pPtr;
SPage *pPgHdr;
pCache = (SPCache *)calloc(1, sizeof(*pCache));
if (pCache == NULL) {
return -1;
}
pCache->pageSize = pageSize;
pCache->cacheSize = cacheSize;
if (tdbPCacheOpenImpl(pCache) < 0) {
free(pCache);
return -1;
}
*ppCache = pCache;
return 0;
}
int tdbPCacheClose(SPCache *pCache) {
/* TODO */
return 0;
}
SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) {
SPage *pPage;
tdbPCacheLock(pCache);
pPage = tdbPCacheFetchImpl(pCache, pPgid, alcNewPage);
if (pPage) {
TDB_REF_PAGE(pPage);
}
tdbPCacheUnlock(pCache);
return pPage;
}
void tdbPCacheRelease(SPage *pPage) {
i32 nRef;
nRef = TDB_UNREF_PAGE(pPage);
ASSERT(nRef >= 0);
if (nRef == 0) {
if (1 /*TODO: page still clean*/) {
tdbPCacheUnpinPage(pPage);
} else {
// TODO
ASSERT(0);
}
}
}
static void tdbPCacheInitLock(SPCache *pCache) { pthread_mutex_init(&(pCache->mutex), NULL); }
static void tdbPCacheClearLock(SPCache *pCache) { pthread_mutex_destroy(&(pCache->mutex)); }
static void tdbPCacheLock(SPCache *pCache) { pthread_mutex_lock(&(pCache->mutex)); }
static void tdbPCacheUnlock(SPCache *pCache) { pthread_mutex_unlock(&(pCache->mutex)); }
static bool tdbPCacheLocked(SPCache *pCache) {
assert(0);
// TODO
return true;
}
static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) {
SPage *pPage;
// 1. Search the hash table
pPage = pCache->pgHash[PCACHE_PAGE_HASH(pPgid) % pCache->nHash];
while (pPage) {
if (TDB_IS_SAME_PAGE(&(pPage->pgid), pPgid)) break;
pPage = pPage->pHashNext;
}
if (pPage || !alcNewPage) {
if (pPage) {
tdbPCachePinPage(pPage);
}
return pPage;
}
// 2. Try to allocate a new page from the free list
if (pCache->pFree) {
pPage = pCache->pFree;
pCache->pFree = pPage->pFreeNext;
pCache->nFree--;
pPage->pLruNext = NULL;
}
// 3. Try to Recycle a page
if (!pPage && !pCache->lru.pLruPrev->isAnchor) {
pPage = pCache->lru.pLruPrev;
tdbPCacheRemovePageFromHash(pPage);
tdbPCachePinPage(pPage);
}
// 4. Try a stress allocation (TODO)
// 5. Page here are just created from a free list
// or by recycling or allocated streesly,
// need to initialize it
if (pPage) {
memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid));
pPage->pLruNext = NULL;
pPage->pPager = NULL;
tdbPCacheAddPageToHash(pPage);
}
return pPage;
}
static void tdbPCachePinPage(SPage *pPage) {
SPCache *pCache;
pCache = pPage->pCache;
if (!PAGE_IS_PINNED(pPage)) {
pPage->pLruPrev->pLruNext = pPage->pLruNext;
pPage->pLruNext->pLruPrev = pPage->pLruPrev;
pPage->pLruNext = NULL;
pCache->nRecyclable--;
}
}
static void tdbPCacheUnpinPage(SPage *pPage) {
SPCache *pCache;
i32 nRef;
pCache = pPage->pCache;
tdbPCacheLock(pCache);
nRef = TDB_GET_PAGE_REF(pPage);
ASSERT(nRef >= 0);
if (nRef == 0) {
// Add the page to LRU list
ASSERT(pPage->pLruNext == NULL);
pPage->pLruPrev = &(pCache->lru);
pPage->pLruNext = pCache->lru.pLruNext;
pCache->lru.pLruNext->pLruPrev = pPage;
pCache->lru.pLruNext = pPage;
}
pCache->nRecyclable++;
tdbPCacheUnlock(pCache);
}
static void tdbPCacheRemovePageFromHash(SPage *pPage) {
SPCache *pCache;
SPage **ppPage;
int h;
pCache = pPage->pCache;
h = PCACHE_PAGE_HASH(&(pPage->pgid));
for (ppPage = &(pCache->pgHash[h % pCache->nHash]); *ppPage != pPage; ppPage = &((*ppPage)->pHashNext))
;
ASSERT(*ppPage == pPage);
*ppPage = pPage->pHashNext;
pCache->nPage--;
}
static void tdbPCacheAddPageToHash(SPage *pPage) {
SPCache *pCache;
int h;
pCache = pPage->pCache;
h = PCACHE_PAGE_HASH(&(pPage->pgid)) % pCache->nHash;
pPage->pHashNext = pCache->pgHash[h];
pCache->pgHash[h] = pPage;
pCache->nPage++;
}
static int tdbPCacheOpenImpl(SPCache *pCache) {
SPage *pPage;
u8 *pPtr;
int tsize;
int ret;
tdbPCacheInitLock(pCache);
// Open the free list
pCache->nFree = 0;
pCache->pFree = NULL;
for (int i = 0; i < pCache->cacheSize; i++) {
ret = tdbPageCreate(pCache->pageSize, &pPage, tdbOsMalloc, NULL);
if (ret < 0) {
// TODO: handle error
return -1;
}
// pPage->pgid = 0;
pPage->isAnchor = 0;
pPage->isLocalPage = 1;
pPage->pCache = pCache;
TDB_INIT_PAGE_REF(pPage);
pPage->pHashNext = NULL;
pPage->pLruNext = NULL;
pPage->pLruPrev = NULL;
pPage->pDirtyNext = NULL;
pPage->pFreeNext = pCache->pFree;
pCache->pFree = pPage;
pCache->nFree++;
}
// Open the hash table
pCache->nPage = 0;
pCache->nHash = pCache->cacheSize;
pCache->pgHash = (SPage **)calloc(pCache->nHash, sizeof(SPage *));
if (pCache->pgHash == NULL) {
// TODO
return -1;
}
// Open LRU list
pCache->nRecyclable = 0;
pCache->lru.isAnchor = 1;
pCache->lru.pLruNext = &(pCache->lru);
pCache->lru.pLruPrev = &(pCache->lru);
return 0;
}
int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; }
static void *tdbOsMalloc(void *arg, size_t size) {
void *ptr;
ptr = malloc(size);
return ptr;
}
static void tdbOsFree(void *arg, void *ptr) { free(ptr); }

253
source/libs/tdb/src/db/tdbPage.c Normal file
View File

@ -0,0 +1,253 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
typedef struct __attribute__((__packed__)) {
u8 szCell[2];
u8 nxOffset[2];
} SFreeCell;
typedef struct __attribute__((__packed__)) {
u8 szCell[3];
u8 nxOffset[3];
} SFreeCellL;
/* For small page */
#define TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCell *)(PCELL))->szCell)
#define TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCell *)(PCELL))->nxOffset)
#define TDB_SPAGE_FREE_CELL_SIZE(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL))[0]
#define TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL))[0]
#define TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) (TDB_SPAGE_FREE_CELL_SIZE(PCELL) = (SIZE))
#define TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) (TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) = (OFFSET))
/* For large page */
#define TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCellL *)(PCELL))->szCell)
#define TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCellL *)(PCELL))->nxOffset)
#define TDB_LPAGE_FREE_CELL_SIZE(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL))
#define TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL))
#define TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL), SIZE)
#define TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL), OFFSET)
/* For page */
#define TDB_PAGE_FREE_CELL_SIZE_PTR(PPAGE, PCELL) \
(TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) : TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL))
#define TDB_PAGE_FREE_CELL_NXOFFSET_PTR(PPAGE, PCELL) \
(TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL))
#define TDB_PAGE_FREE_CELL_SIZE(PPAGE, PCELL) \
(TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE(PCELL) : TDB_SPAGE_FREE_CELL_SIZE(PCELL))
#define TDB_PAGE_FREE_CELL_NXOFFSET(PPAGE, PCELL) \
(TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL))
#define TDB_PAGE_FREE_CELL_SIZE_SET(PPAGE, PCELL, SIZE) \
do { \
if (TDB_IS_LARGE_PAGE(PPAGE)) { \
TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \
} else { \
TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \
} \
} while (0)
#define TDB_PAGE_FREE_CELL_NXOFFSET_SET(PPAGE, PCELL, OFFSET) \
do { \
if (TDB_IS_LARGE_PAGE(PPAGE)) { \
TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \
} else { \
TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \
} \
} while (0)
static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell);
static int tdbPageDefragment(SPage *pPage);
int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) {
SPage *pPage;
u8 *ptr;
int size;
ASSERT(TDB_IS_PGSIZE_VLD(pageSize));
*ppPage = NULL;
size = pageSize + sizeof(*pPage);
ptr = (u8 *)((*xMalloc)(arg, size));
if (pPage == NULL) {
return -1;
}
memset(ptr, 0, size);
pPage = (SPage *)(ptr + pageSize);
pPage->pData = ptr;
pPage->pageSize = pageSize;
if (pageSize < 65536) {
pPage->szOffset = 2;
pPage->szPageHdr = sizeof(SPageHdr);
pPage->szFreeCell = sizeof(SFreeCell);
} else {
pPage->szOffset = 3;
pPage->szPageHdr = sizeof(SPageHdrL);
pPage->szFreeCell = sizeof(SFreeCellL);
}
TDB_INIT_PAGE_LOCK(pPage);
/* TODO */
*ppPage = pPage;
return 0;
}
int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) {
u8 *ptr;
ptr = pPage->pData;
(*xFree)(arg, ptr);
return 0;
}
int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) {
int ret;
SCell *pTarget;
u8 *pTmp;
int j;
if (pPage->nOverflow || szCell + pPage->szOffset > pPage->nFree) {
// TODO: need to figure out if pCell may be used by outside of this function
j = pPage->nOverflow++;
pPage->apOvfl[j] = pCell;
pPage->aiOvfl[j] = idx;
} else {
ret = tdbPageAllocate(pPage, szCell, &pTarget);
if (ret < 0) {
return -1;
}
memcpy(pTarget, pCell, szCell);
pTmp = pPage->pCellIdx + idx * pPage->szOffset;
memmove(pTmp + pPage->szOffset, pTmp, pPage->pFreeStart - pTmp - pPage->szOffset);
TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData);
TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1);
}
return 0;
}
int tdbPageDropCell(SPage *pPage, int idx) {
// TODO
return 0;
}
static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) {
SCell *pCell;
SFreeCell *pFreeCell;
u8 *pOffset;
int ret;
ASSERT(pPage->nFree > size + pPage->szOffset);
pCell = NULL;
*ppCell = NULL;
// 1. Try to allocate from the free space area
if (pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset) {
pPage->pFreeEnd -= size;
pPage->pFreeStart += pPage->szOffset;
pCell = pPage->pFreeEnd;
}
// 2. Try to allocate from the page free list
if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= pPage->szOffset) && TDB_PAGE_FCELL(pPage)) {
int szCell;
int nxOffset;
pCell = pPage->pData + TDB_PAGE_FCELL(pPage);
pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell
: (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell);
szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell);
nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell);
for (;;) {
// Find a cell
if (szCell >= size) {
if (szCell - size >= pPage->szFreeCell) {
SCell *pTmpCell = pCell + size;
TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size);
TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset);
// TODO: *pOffset = pTmpCell - pPage->pData;
} else {
TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size);
// TODO: *pOffset = nxOffset;
}
break;
}
// Not find a cell yet
if (nxOffset > 0) {
pCell = pPage->pData + nxOffset;
pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell);
szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell);
nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell);
continue;
} else {
pCell = NULL;
break;
}
}
if (pCell) {
pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset;
}
}
// 3. Try to dfragment and allocate again
if (pCell == NULL) {
ret = tdbPageDefragment(pPage);
if (ret < 0) {
return -1;
}
ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset);
ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart);
// Allocate from the free space area again
pPage->pFreeEnd -= size;
pPage->pFreeStart += pPage->szOffset;
pCell = pPage->pFreeEnd;
}
ASSERT(pCell != NULL);
pPage->nFree = pPage->nFree - size - pPage->szOffset;
*ppCell = pCell;
return 0;
}
static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) {
// TODO
return 0;
}
static int tdbPageDefragment(SPage *pPage) {
// TODO
ASSERT(0);
return 0;
}

328
source/libs/tdb/src/db/tdbPager.c Normal file
View File

@ -0,0 +1,328 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
struct SPager {
char *dbFileName;
char *jFileName;
int pageSize;
uint8_t fid[TDB_FILE_ID_LEN];
int fd;
int jfd;
SPCache *pCache;
SPgno dbFileSize;
SPgno dbOrigSize;
int nDirty;
SPage *pDirty;
SPage *pDirtyTail;
u8 inTran;
};
typedef struct __attribute__((__packed__)) {
u8 hdrString[16];
u16 pageSize;
SPgno freePage;
u32 nFreePages;
u8 reserved[102];
} SFileHdr;
TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct");
#define TDB_PAGE_INITIALIZED(pPage) ((pPage)->pPager != NULL)
static int tdbPagerReadPage(SPager *pPager, SPage *pPage);
static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg);
int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
uint8_t *pPtr;
SPager *pPager;
int fsize;
int zsize;
int ret;
*ppPager = NULL;
fsize = strlen(fileName);
zsize = sizeof(*pPager) /* SPager */
+ fsize + 1 /* dbFileName */
+ fsize + 8 + 1; /* jFileName */
pPtr = (uint8_t *)calloc(1, zsize);
if (pPtr == NULL) {
return -1;
}
pPager = (SPager *)pPtr;
pPtr += sizeof(*pPager);
// pPager->dbFileName
pPager->dbFileName = (char *)pPtr;
memcpy(pPager->dbFileName, fileName, fsize);
pPager->dbFileName[fsize] = '\0';
pPtr += fsize + 1;
// pPager->jFileName
pPager->jFileName = (char *)pPtr;
memcpy(pPager->jFileName, fileName, fsize);
memcpy(pPager->jFileName + fsize, "-journal", 8);
pPager->jFileName[fsize + 8] = '\0';
// pPager->pCache
pPager->pCache = pCache;
pPager->fd = open(pPager->dbFileName, O_RDWR | O_CREAT, 0755);
if (pPager->fd < 0) {
return -1;
}
ret = tdbGnrtFileID(pPager->dbFileName, pPager->fid, false);
if (ret < 0) {
return -1;
}
pPager->jfd = -1;
pPager->pageSize = tdbPCacheGetPageSize(pCache);
*ppPager = pPager;
return 0;
}
int tdbPagerClose(SPager *pPager) {
// TODO
return 0;
}
int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate) {
SPgno pgno;
SPage *pPage;
int ret;
{
// TODO: try to search the main DB to get the page number
pgno = 0;
}
// if (pgno == 0 && toCreate) {
// ret = tdbPagerAllocPage(pPager, &pPage, &pgno);
// if (ret < 0) {
// return -1;
// }
// // TODO: Need to zero the page
// ret = tdbPagerWrite(pPager, pPage);
// if (ret < 0) {
// return -1;
// }
// }
*ppgno = pgno;
return 0;
}
int tdbPagerWrite(SPager *pPager, SPage *pPage) {
int ret;
if (pPager->inTran == 0) {
ret = tdbPagerBegin(pPager);
if (ret < 0) {
return -1;
}
}
if (pPage->isDirty == 0) {
pPage->isDirty = 1;
// TODO: add the page to the dirty list
// TODO: write the page to the journal
if (1 /*actually load from the file*/) {
}
}
return 0;
}
int tdbPagerBegin(SPager *pPager) {
if (pPager->inTran) {
return 0;
}
// Open the journal
pPager->jfd = open(pPager->jFileName, O_RDWR | O_CREAT, 0755);
if (pPager->jfd < 0) {
return -1;
}
// TODO: write the size of the file
pPager->inTran = 1;
return 0;
}
int tdbPagerCommit(SPager *pPager) {
// TODO
return 0;
}
static int tdbPagerReadPage(SPager *pPager, SPage *pPage) {
i64 offset;
int ret;
ASSERT(memcmp(pPager->fid, pPage->pgid.fileid, TDB_FILE_ID_LEN) == 0);
offset = (pPage->pgid.pgno - 1) * (i64)(pPager->pageSize);
ret = tdbPRead(pPager->fd, pPage->pData, pPager->pageSize, offset);
if (ret < 0) {
// TODO: handle error
return -1;
}
return 0;
}
int tdbPagerGetPageSize(SPager *pPager) { return pPager->pageSize; }
int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg) {
SPage *pPage;
SPgid pgid;
int ret;
// Fetch a page container from the page cache
memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN);
pgid.pgno = pgno;
pPage = tdbPCacheFetch(pPager->pCache, &pgid, 1);
if (pPage == NULL) {
return -1;
}
// Initialize the page if need
if (!TDB_PAGE_INITIALIZED(pPage)) {
ret = tdbPagerInitPage(pPager, pPage, initPage, arg);
if (ret < 0) {
return -1;
}
}
ASSERT(TDB_PAGE_INITIALIZED(pPage));
ASSERT(pPage->pPager == pPager);
*ppPage = pPage;
return 0;
}
int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg) {
int ret;
SPage *pPage;
SPgid pgid;
// Allocate a page number
ret = tdbPagerAllocPage(pPager, ppgno);
if (ret < 0) {
return -1;
}
ASSERT(*ppgno != 0);
// Fetch a page container from the page cache
memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN);
pgid.pgno = *ppgno;
pPage = tdbPCacheFetch(pPager->pCache, &pgid, 1);
if (pPage == NULL) {
return -1;
}
ASSERT(!TDB_PAGE_INITIALIZED(pPage));
// Initialize the page if need
ret = tdbPagerInitPage(pPager, pPage, initPage, arg);
if (ret < 0) {
return -1;
}
ASSERT(TDB_PAGE_INITIALIZED(pPage));
ASSERT(pPage->pPager == pPager);
*ppPage = pPage;
return 0;
}
static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) {
// TODO: Allocate a page from the free list
return 0;
}
static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) {
*ppgno = ++pPager->dbFileSize;
return 0;
}
static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) {
int ret;
*ppgno = 0;
// Try to allocate from the free list of the pager
ret = tdbPagerAllocFreePage(pPager, ppgno);
if (ret < 0) {
return -1;
}
if (*ppgno != 0) return 0;
// Allocate the page by extending the pager
ret = tdbPagerAllocNewPage(pPager, ppgno);
if (ret < 0) {
return -1;
}
ASSERT(*ppgno != 0);
return 0;
}
static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg) {
int ret;
int lcode;
int nLoops;
lcode = TDB_TRY_LOCK_PAGE(pPage);
if (lcode == P_LOCK_SUCC) {
if (TDB_PAGE_INITIALIZED(pPage)) {
TDB_UNLOCK_PAGE(pPage);
return 0;
}
ret = (*initPage)(pPage, arg);
if (ret < 0) {
TDB_UNLOCK_PAGE(pPage);
return -1;
}
pPage->pPager = pPager;
TDB_UNLOCK_PAGE(pPage);
} else if (lcode == P_LOCK_BUSY) {
nLoops = 0;
for (;;) {
if (TDB_PAGE_INITIALIZED(pPage)) break;
nLoops++;
if (nLoops > 1000) {
sched_yield();
nLoops = 0;
}
}
} else {
return -1;
}
return 0;
}

576
source/libs/tdb/src/db/tdbPgCache.c
View File

@ -1,576 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
typedef TD_DLIST(SPage) SPgList;
struct SPgCache {
TENV * pEnv; // TENV containing this page cache
pgsz_t pgsize;
int32_t npage;
SPage **pages;
SPgList freeList;
SPgList lru;
struct {
int32_t nbucket;
SPgList *buckets;
} pght; // page hash table
};
static void pgCachePinPage(SPage *pPage);
static void pgCacheUnpinPage(SPage *pPage);
int pgCacheOpen(SPgCache **ppPgCache, TENV *pEnv) {
SPgCache *pPgCache;
SPage * pPage;
void * pData;
pgsz_t pgSize;
cachesz_t cacheSize;
int32_t npage;
int32_t nbucket;
size_t msize;
*ppPgCache = NULL;
pgSize = tdbEnvGetPageSize(pEnv);
cacheSize = tdbEnvGetCacheSize(pEnv);
npage = cacheSize / pgSize;
nbucket = npage;
msize = sizeof(*pPgCache) + sizeof(SPage *) * npage + sizeof(SPgList) * nbucket;
// Allocate the handle
pPgCache = (SPgCache *)calloc(1, msize);
if (pPgCache == NULL) {
return -1;
}
// Init the handle
pPgCache->pEnv = pEnv;
pPgCache->pgsize = pgSize;
pPgCache->npage = npage;
pPgCache->pages = (SPage **)(&pPgCache[1]);
pPgCache->pght.nbucket = nbucket;
pPgCache->pght.buckets = (SPgList *)(&(pPgCache->pages[npage]));
TD_DLIST_INIT(&(pPgCache->freeList));
for (int32_t i = 0; i < npage; i++) {
pData = malloc(pgSize + sizeof(SPage));
if (pData == NULL) {
return -1;
// TODO: handle error
}
pPage = POINTER_SHIFT(pData, pgSize);
pPage->pgid = TDB_IVLD_PGID;
pPage->frameid = i;
pPage->pData = pData;
// add current page to the page cache
pPgCache->pages[i] = pPage;
TD_DLIST_APPEND_WITH_FIELD(&(pPgCache->freeList), pPage, freeNode);
}
#if 0
for (int32_t i = 0; i < nbucket; i++) {
TD_DLIST_INIT(pPgCache->pght.buckets + i);
}
#endif
*ppPgCache = pPgCache;
return 0;
}
int pgCacheClose(SPgCache *pPgCache) {
SPage *pPage;
if (pPgCache) {
for (int32_t i = 0; i < pPgCache->npage; i++) {
pPage = pPgCache->pages[i];
tfree(pPage->pData);
}
free(pPgCache);
}
return 0;
}
#define PG_CACHE_HASH(fileid, pgno) (((uint64_t *)(fileid))[0] + ((uint64_t *)(fileid))[1] + ((uint64_t *)(fileid))[2] + (pgno))
SPage *pgCacheFetch(SPgCache *pPgCache, pgid_t pgid) {
SPage * pPage;
SPgFile *pPgFile;
SPgList *pBucket;
// 1. Search the page hash table SPgCache.pght
pBucket = pPgCache->pght.buckets + (PG_CACHE_HASH(pgid.fileid, pgid.pgno) % pPgCache->pght.nbucket);
pPage = TD_DLIST_HEAD(pBucket);
while (pPage && tdbCmprPgId(&(pPage->pgid), &pgid)) {
pPage = TD_DLIST_NODE_NEXT_WITH_FIELD(pPage, pghtNode);
}
if (pPage) {
// Page is found, pin the page and return the page
pgCachePinPage(pPage);
return pPage;
}
// 2. Check the free list
pPage = TD_DLIST_HEAD(&(pPgCache->freeList));
if (pPage) {
TD_DLIST_POP_WITH_FIELD(&(pPgCache->freeList), pPage, freeNode);
pgCachePinPage(pPage);
return pPage;
}
// 3. Try to recycle a page from the LRU list
pPage = TD_DLIST_HEAD(&(pPgCache->lru));
if (pPage) {
TD_DLIST_POP_WITH_FIELD(&(pPgCache->lru), pPage, lruNode);
// TODO: remove from the hash table
pgCachePinPage(pPage);
return pPage;
}
// 4. If a memory allocator is set, try to allocate from the allocator (TODO)
return NULL;
}
int pgCacheRelease(SPage *pPage) {
// TODO
return 0;
}
static void pgCachePinPage(SPage *pPage) {
// TODO
}
static void pgCacheUnpinPage(SPage *pPage) {
// TODO
}
#if 0
// Exposed handle
typedef struct TDB_MPOOL TDB_MPOOL;
typedef struct TDB_MPFILE TDB_MPFILE;
typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t;
typedef struct pg_t {
SRWLatch rwLatch;
frame_id_t frameid;
pgid_t pgid;
uint8_t dirty;
uint8_t rbit;
int32_t pinRef;
pg_free_dlist_node_t free;
pg_hash_dlist_node_t hash;
void * p;
} pg_t;
typedef TD_DLIST(pg_t) pg_list_t;
typedef struct {
SRWLatch latch;
TD_DLIST(TDB_MPFILE);
} mpf_bucket_t;
struct TDB_MPOOL {
int64_t cachesize;
pgsz_t pgsize;
int32_t npages;
pg_t * pages;
pg_list_t freeList;
frame_id_t clockHand;
struct {
int32_t nbucket;
pg_list_t *hashtab;
} pgtab; // page table, hash<pgid_t, pg_t>
struct {
#define MPF_HASH_BUCKETS 16
mpf_bucket_t buckets[MPF_HASH_BUCKETS];
} mpfht; // MPF hash table. MPFs using this MP will be put in this hash table
};
#define MP_PAGE_AT(mp, idx) (mp)->pages[idx]
typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t;
struct TDB_MPFILE {
char * fname; // file name
int fd; // fd
uint8_t fileid[TDB_FILE_ID_LEN]; // file ID
TDB_MPOOL * mp; // underlying memory pool
td_mpf_dlist_node_t node;
};
/*=================================================== Exposed apis ==================================================*/
// TDB_MPOOL
int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize);
int tdbMPoolClose(TDB_MPOOL *mp);
int tdbMPoolSync(TDB_MPOOL *mp);
// TDB_MPFILE
int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp);
int tdbMPoolFileClose(TDB_MPFILE *mpf);
int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr);
int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr);
int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
int tdbMPoolFileSync(TDB_MPFILE *mpf);
static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf);
static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf);
static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid);
static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p);
static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p);
static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp);
int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize) {
TDB_MPOOL *mp = NULL;
size_t tsize;
pg_t * pagep;
// check parameters
if (!TDB_IS_PGSIZE_VLD(pgsize)) {
tdbError("invalid page size");
return -1;
}
// allocate handle
mp = (TDB_MPOOL *)calloc(1, sizeof(*mp));
if (mp == NULL) {
tdbError("failed to malloc memory pool handle");
goto _err;
}
// initialize the handle
mp->cachesize = cachesize;
mp->pgsize = pgsize;
mp->npages = cachesize / pgsize;
mp->clockHand = 0;
TD_DLIST_INIT(&mp->freeList);
mp->pages = (pg_t *)calloc(mp->npages, sizeof(pg_t));
if (mp->pages == NULL) {
tdbError("failed to malloc memory pool pages");
goto _err;
}
for (frame_id_t i = 0; i < mp->npages; i++) {
mp->pages[i].p = malloc(pgsize);
if (mp->pages[i].p == NULL) {
goto _err;
}
taosInitRWLatch(&mp->pages[i].rwLatch);
mp->pages[i].frameid = i;
mp->pages[i].pgid = TDB_IVLD_PGID;
// add new page to the free list
TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), &(mp->pages[i]), free);
}
#define PGTAB_FACTOR 1.0
mp->pgtab.nbucket = mp->npages / PGTAB_FACTOR;
mp->pgtab.hashtab = (pg_list_t *)calloc(mp->pgtab.nbucket, sizeof(pg_list_t));
if (mp->pgtab.hashtab == NULL) {
tdbError("failed to malloc memory pool hash table");
goto _err;
}
// return
*mpp = mp;
return 0;
_err:
tdbMPoolClose(mp);
*mpp = NULL;
return -1;
}
int tdbMPoolClose(TDB_MPOOL *mp) {
if (mp) {
tfree(mp->pgtab.hashtab);
if (mp->pages) {
for (int i = 0; i < mp->npages; i++) {
tfree(mp->pages[i].p);
}
free(mp->pages);
}
free(mp);
}
return 0;
}
int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) {
TDB_MPFILE *mpf;
if ((mpf = (TDB_MPFILE *)calloc(1, sizeof(*mpf))) == NULL) {
return -1;
}
mpf->fd = -1;
if ((mpf->fname = strdup(fname)) == NULL) {
goto _err;
}
if ((mpf->fd = open(fname, O_CREAT | O_RDWR, 0755)) < 0) {
goto _err;
}
if (tdbGnrtFileID(fname, mpf->fileid, false) < 0) {
goto _err;
}
// Register current MPF to MP
tdbMPoolRegFile(mp, mpf);
*mpfp = mpf;
return 0;
_err:
tdbMPoolFileClose(mpf);
*mpfp = NULL;
return -1;
}
int tdbMPoolFileClose(TDB_MPFILE *mpf) {
if (mpf) {
if (mpf->fd > 0) {
close(mpf->fd);
}
tfree(mpf->fname);
free(mpf);
}
return 0;
}
#define MPF_GET_PAGE_BUCKETID(fileid, pgno, nbuckets) \
({ \
uint64_t *tmp = (uint64_t *)fileid; \
(tmp[0] + tmp[1] + tmp[2] + (pgno)) % (nbuckets); \
})
int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr) {
// TODO
return 0;
}
int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr) {
// TODO
return 0;
}
int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) {
pg_t * pagep;
TDB_MPOOL *mp;
pg_list_t *pglist;
mp = mpf->mp;
// check if the page already in pool
pglist = mp->pgtab.hashtab + MPF_GET_PAGE_BUCKETID(mpf->fileid, pgno, mp->pgtab.nbucket);
pagep = TD_DLIST_HEAD(pglist);
while (pagep) {
if (memcmp(mpf->fileid, pagep->pgid.fileid, TDB_FILE_ID_LEN) == 0 && pgno == pagep->pgid.pgno) {
break;
}
pagep = TD_DLIST_NODE_NEXT_WITH_FIELD(pagep, hash);
}
if (pagep) {
// page is found
// todo: pin the page and return
*(void **)addr = pagep->p;
return 0;
}
// page not found
pagep = TD_DLIST_HEAD(&mp->freeList);
if (pagep) {
// has free page
TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free);
} else {
// no free page available
tdbMPoolClockEvictPage(mp, &pagep);
if (pagep) {
if (pagep->dirty) {
// TODO: Handle dirty page eviction
}
}
}
if (pagep == NULL) {
// no available container page
return -1;
}
// load page from the disk if a container page is available
// TODO: load the page from the disk
if (tdbMPoolFileReadPage(mpf, pgno, pagep->p) < 0) {
return -1;
}
memcpy(pagep->pgid.fileid, mpf->fileid, TDB_FILE_ID_LEN);
pagep->pgid.pgno = pgno;
pagep->dirty = 0;
pagep->pinRef = 1;
// add current page to page table
TD_DLIST_APPEND_WITH_FIELD(pglist, pagep, hash);
return 0;
}
int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) {
// TODO
return 0;
}
#define MPF_GET_BUCKETID(fileid) \
({ \
uint64_t *tmp = (uint64_t *)fileid; \
(tmp[0] + tmp[1] + tmp[2]) % MPF_HASH_BUCKETS; \
})
static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) {
mpf_bucket_t *bktp;
bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid);
taosWLockLatch(&(bktp->latch));
TD_DLIST_APPEND_WITH_FIELD(bktp, mpf, node);
taosWUnLockLatch(&(bktp->latch));
mpf->mp = mp;
}
static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid) {
TDB_MPFILE * mpf = NULL;
mpf_bucket_t *bktp;
bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(fileid);
taosRLockLatch(&(bktp->latch));
mpf = TD_DLIST_HEAD(bktp);
while (mpf) {
if (memcmp(fileid, mpf->fileid, TDB_FILE_ID_LEN) == 0) {
break;
}
mpf = TD_DLIST_NODE_NEXT_WITH_FIELD(mpf, node);
}
taosRUnLockLatch(&(bktp->latch));
return mpf;
}
static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) {
mpf_bucket_t *bktp;
TDB_MPFILE * tmpf;
if (mpf->mp == NULL) return;
ASSERT(mpf->mp == mp);
bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid);
taosWLockLatch(&(bktp->latch));
tmpf = TD_DLIST_HEAD(bktp);
while (tmpf) {
if (memcmp(mpf->fileid, tmpf->fileid, TDB_FILE_ID_LEN) == 0) {
TD_DLIST_POP_WITH_FIELD(bktp, tmpf, node);
break;
}
tmpf = TD_DLIST_NODE_NEXT_WITH_FIELD(tmpf, node);
}
taosWUnLockLatch(&(bktp->latch));
ASSERT(tmpf == mpf);
}
static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) {
pgsz_t pgsize;
TDB_MPOOL *mp;
off_t offset;
size_t rsize;
mp = mpf->mp;
pgsize = mp->pgsize;
offset = pgno * pgsize;
// TODO: use loop to read all data
rsize = pread(mpf->fd, p, pgsize, offset);
// TODO: error handle
return 0;
}
static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) {
pgsz_t pgsize;
TDB_MPOOL *mp;
off_t offset;
mp = mpf->mp;
pgsize = mp->pgsize;
offset = pgno * pgsize;
lseek(mpf->fd, offset, SEEK_SET);
// TODO: handle error
write(mpf->fd, p, pgsize);
// TODO: handle error
return 0;
}
static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp) {
pg_t * pagep;
frame_id_t och;
*pagepp = NULL;
och = mp->clockHand;
do {
pagep = mp->pages + mp->clockHand;
mp->clockHand = (mp->clockHand + 1) % mp->npages;
if (pagep->pinRef == 0) {
if (pagep->rbit == 1) {
pagep->rbit = 0;
} else {
break;
}
}
if (mp->clockHand == och) {
return;
}
} while (1);
*pagepp = pagep;
}
#endif

221
source/libs/tdb/src/db/tdbPgFile.c
View File

@ -1,221 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
typedef struct SPage1 {
char magic[64];
pgno_t mdbRootPgno; // master DB root page number
pgno_t freePgno; // free list page number
uint32_t nFree; // number of free pages
} SPage1;
typedef struct SFreePage {
/* TODO */
} SFreePage;
TDB_STATIC_ASSERT(sizeof(SPage1) <= TDB_MIN_PGSIZE, "TDB Page1 definition too large");
static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData);
int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) {
SPgFile * pPgFile;
SPgCache *pPgCache;
size_t fnameLen;
pgno_t fsize;
*ppPgFile = NULL;
// create the handle
fnameLen = strlen(fname);
pPgFile = (SPgFile *)calloc(1, sizeof(*pPgFile) + fnameLen + 1);
if (pPgFile == NULL) {
return -1;
}
ASSERT(pEnv != NULL);
// init the handle
pPgFile->fname = (char *)(&(pPgFile[1]));
memcpy(pPgFile->fname, fname, fnameLen);
pPgFile->fname[fnameLen] = '\0';
pPgFile->pFile = NULL;
pPgFile->pFile = taosOpenFile(fname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pPgFile->pFile == NULL) {
// TODO: handle error
return -1;
}
tdbGnrtFileID(fname, pPgFile->fileid, false);
tdbGetFileSize(fname, tdbEnvGetPageSize(pEnv), &fsize);
pPgFile->fsize = fsize;
pPgFile->lsize = fsize;
if (pPgFile->fsize == 0) {
// A created file
pgno_t pgno;
pgid_t pgid;
pgFileAllocatePage(pPgFile, &pgno);
ASSERT(pgno == 1);
memcpy(pgid.fileid, pPgFile->fileid, TDB_FILE_ID_LEN);
pgid.pgno = pgno;
pgCacheFetch(pPgCache, pgid);
// Need to allocate the first page as a description page
} else {
// An existing file
}
/* TODO: other open operations */
// add the page file to the environment
tdbEnvRgstPageFile(pEnv, pPgFile);
pPgFile->pEnv = pEnv;
*ppPgFile = pPgFile;
return 0;
}
int pgFileClose(SPgFile *pPgFile) {
if (pPgFile) {
if (pPgFile->pFile != NULL) {
taosCloseFile(&pPgFile->pFile);
}
tfree(pPgFile->fname);
free(pPgFile);
}
return 0;
}
SPage *pgFileFetch(SPgFile *pPgFile, pgno_t pgno) {
SPgCache *pPgCache;
SPage * pPage;
pgid_t pgid;
// 1. Fetch from the page cache
// pgCacheFetch(pPgCache, pgid);
// 2. If only get a page frame, no content, maybe
// need to load from the file
if (1 /*page not initialized*/) {
if (pgno < pPgFile->fsize) {
// load the page content from the disk
// ?? How about the freed pages ??
} else {
// zero the page, make the page as a empty
// page with zero records.
}
}
#if 0
pPgCache = pPgFile->pPgCache;
pPage = NULL;
memcpy(pgid.fileid, pPgFile->fileid, TDB_FILE_ID_LEN);
pgid.pgno = pgno;
if (pgno > pPgFile->pgFileSize) {
// TODO
} else {
pPage = pgCacheFetch(pPgCache, pgid);
if (1 /*Page is cached, no need to load from file*/) {
return pPage;
} else {
// TODO: handle error
if (pgFileRead(pPgFile, pgno, (void *)pPage) < 0) {
// todoerr
}
return pPage;
}
}
#endif
return pPage;
}
int pgFileRelease(SPage *pPage) {
pgCacheRelease(pPage);
return 0;
}
int pgFileWrite(SPage *pPage) {
// TODO
return 0;
}
int pgFileAllocatePage(SPgFile *pPgFile, pgno_t *pPgno) {
pgno_t pgno;
SPage1 * pPage1;
SPgCache *pPgCache;
pgid_t pgid;
SPage * pPage;
if (pPgFile->lsize == 0) {
pgno = ++(pPgFile->lsize);
} else {
if (0) {
// TODO: allocate from the free list
pPage = pgCacheFetch(pPgCache, pgid);
if (pPage1->nFree > 0) {
// TODO
} else {
pgno = ++(pPgFile->lsize);
}
} else {
pgno = ++(pPgFile->lsize);
}
}
*pPgno = pgno;
return 0;
}
static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) {
pgsz_t pgSize;
ssize_t rsize;
uint8_t *pTData;
size_t szToRead;
#if 0
// pgSize = ; (TODO)
pTData = pData;
szToRead = pgSize;
for (; szToRead > 0;) {
rsize = pread(pPgFile->pFile, pTData, szToRead, pgno * pgSize);
if (rsize < 0) {
if (errno == EINTR) {
continue;
} else {
return -1;
}
} else if (rsize == 0) {
return -1;
}
szToRead -= rsize;
pTData += rsize;
}
#endif
return 0;
}

28
source/libs/tdb/src/db/tdbUtil.c
View File

@ -51,7 +51,8 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) {
// return access(pathname, flags); // return access(pathname, flags);
// } // }
int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize) { int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize) {
struct stat st;
int ret; int ret;
int64_t file_size = 0; int64_t file_size = 0;
ret = taosStatFile(fname, &file_size, NULL); ret = taosStatFile(fname, &file_size, NULL);
@ -63,4 +64,29 @@ int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize) {
*pSize = file_size / pgSize; *pSize = file_size / pgSize;
return 0; return 0;
}
int tdbPRead(int fd, void *pData, int count, i64 offset) {
void *pBuf;
int nbytes;
i64 ioffset;
int iread;
pBuf = pData;
nbytes = count;
ioffset = offset;
while (nbytes > 0) {
iread = pread(fd, pBuf, nbytes, ioffset);
if (iread < 0) {
/* TODO */
} else if (iread == 0) {
return (count - iread);
}
nbytes = nbytes - iread;
pBuf = (void *)((u8 *)pBuf + iread);
ioffset += iread;
}
return count;
} }

25
source/libs/tdb/src/inc/tdbBtree.h
View File

@ -23,20 +23,21 @@ extern "C" {
typedef struct SBTree SBTree; typedef struct SBTree SBTree;
typedef struct SBtCursor SBtCursor; typedef struct SBtCursor SBtCursor;
// SBTree struct SBtCursor {
int btreeOpen(SBTree **ppBt, SPgFile *pPgFile); SBTree *pBt;
int btreeClose(SBTree *pBt); i8 iPage;
SPage *pPage;
// SBtCursor int idx;
int btreeCursorOpen(SBtCursor *pBtCur, SBTree *pBt); int idxStack[BTREE_MAX_DEPTH + 1];
int btreeCursorClose(SBtCursor *pBtCur); SPage *pgStack[BTREE_MAX_DEPTH + 1];
int btreeCursorMoveTo(SBtCursor *pBtCur, int kLen, const void *pKey); void *pBuf;
int btreeCursorNext(SBtCursor *pBtCur);
struct SBTree {
pgno_t root;
}; };
int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt);
int tdbBtreeClose(SBTree *pBt);
int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt);
int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

34
source/libs/tdb/src/inc/tdbDb.h Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TDB_DB_H_
#define _TD_TDB_DB_H_
#ifdef __cplusplus
extern "C" {
#endif
typedef struct STDb STDb;
int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb);
int tdbDbClose(STDb *pDb);
int tdbDbDrop(STDb *pDb);
int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen);
#ifdef __cplusplus
}
#endif
#endif /*_TD_TDB_DB_H_*/

16
source/libs/tdb/src/inc/tdbEnv.h
View File

@ -20,11 +20,17 @@
extern "C" { extern "C" {
#endif #endif
const char* tdbEnvGetRootDir(TENV* pEnv); typedef struct STEnv {
SPgFile* tdbEnvGetPageFile(TENV* pEnv, const uint8_t fileid[]); char * rootDir;
SPgCache* tdbEnvGetPgCache(TENV* pEnv); char * jfname;
int tdbEnvRgstPageFile(TENV* pEnv, SPgFile* pPgFile); int jfd;
int tdbEnvRgstDB(TENV* pEnv, TDB* pDb); SPCache *pCache;
} STEnv;
int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, STEnv **ppEnv);
int tdbEnvClose(STEnv *pEnv);
SPager *tdbEnvGetPager(STEnv *pEnv, const char *fname);
#ifdef __cplusplus #ifdef __cplusplus
} }

73
source/libs/tdb/src/inc/tdbInt.h
View File

@ -19,16 +19,33 @@
#include "tlist.h" #include "tlist.h"
#include "tlockfree.h" #include "tlockfree.h"
#include "tdb.h" // #include "tdb.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
typedef struct SPgFile SPgFile; typedef int8_t i8;
typedef int16_t i16;
typedef int32_t i32;
typedef int64_t i64;
typedef uint8_t u8;
typedef uint16_t u16;
typedef uint32_t u32;
typedef uint64_t u64;
// pgno_t // p must be u8 *
typedef int32_t pgno_t; #define TDB_GET_U24(p) ((p)[0] * 65536 + *(u16 *)((p) + 1))
#define TDB_PUT_U24(p, v) \
do { \
int tv = (v); \
(p)[2] = tv & 0xff; \
(p)[1] = (tv >> 8) & 0xff; \
(p)[0] = (tv >> 16) & 0xff; \
} while (0)
// SPgno
typedef u32 SPgno;
#define TDB_IVLD_PGNO ((pgno_t)0) #define TDB_IVLD_PGNO ((pgno_t)0)
// fileid // fileid
@ -37,8 +54,8 @@ typedef int32_t pgno_t;
// pgid_t // pgid_t
typedef struct { typedef struct {
uint8_t fileid[TDB_FILE_ID_LEN]; uint8_t fileid[TDB_FILE_ID_LEN];
pgno_t pgno; SPgno pgno;
} pgid_t; } pgid_t, SPgid;
#define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO}; #define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO};
@ -61,18 +78,14 @@ static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) {
} }
} }
// framd_id_t #define TDB_IS_SAME_PAGE(pPgid1, pPgid2) (tdbCmprPgId(pPgid1, pPgid2) == 0)
typedef int32_t frame_id_t;
// pgsz_t // pgsz_t
#define TDB_MIN_PGSIZE 512 #define TDB_MIN_PGSIZE 512 // 512B
#define TDB_MAX_PGSIZE 65536 #define TDB_MAX_PGSIZE 16777216 // 16M
#define TDB_DEFAULT_PGSIZE 4096 #define TDB_DEFAULT_PGSIZE 4096
#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE)) #define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE))
// pgoff_t
typedef pgsz_t pgoff_t;
// cache // cache
#define TDB_DEFAULT_CACHE_SIZE (256 * 4096) // 1M #define TDB_DEFAULT_CACHE_SIZE (256 * 4096) // 1M
@ -100,7 +113,7 @@ typedef TD_DLIST_NODE(SPgFile) SPgFileListNode;
} \ } \
} while (0) } while (0)
#define TDB_VARIANT_LEN (int)-1 #define TDB_VARIANT_LEN ((int)-1)
// page payload format // page payload format
// <keyLen> + <valLen> + [key] + [value] // <keyLen> + <valLen> + [key] + [value]
@ -115,18 +128,40 @@ typedef TD_DLIST_NODE(SPgFile) SPgFileListNode;
/* TODO */ \ /* TODO */ \
} while (0) } while (0)
typedef int (*FKeyComparator)(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
#define TDB_JOURNAL_NAME "tdb.journal" #define TDB_JOURNAL_NAME "tdb.journal"
#define TDB_FILENAME_LEN 128
#define TDB_DEFAULT_FANOUT 6
#define BTREE_MAX_DEPTH 20
#define TDB_FLAG_IS(flags, flag) ((flags) == (flag))
#define TDB_FLAG_HAS(flags, flag) (((flags) & (flag)) != 0)
#define TDB_FLAG_NO(flags, flag) ((flags) & (flag) == 0)
#define TDB_FLAG_ADD(flags, flag) ((flags) |= (flag))
#define TDB_FLAG_REMOVE(flags, flag) ((flags) &= (~(flag)))
typedef struct SPager SPager;
typedef struct SPCache SPCache;
typedef struct SPage SPage;
#include "tdbUtil.h" #include "tdbUtil.h"
#include "tdbPCache.h"
#include "tdbPager.h"
#include "tdbBtree.h" #include "tdbBtree.h"
#include "tdbPgCache.h"
#include "tdbPgFile.h"
#include "tdbEnv.h" #include "tdbEnv.h"
#include "tdbDb.h"
#include "tdbPage.h"
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

37
source/libs/tdb/src/inc/tdbPgCache.h → source/libs/tdb/src/inc/tdbPCache.h
View File

@ -20,26 +20,25 @@
extern "C" { extern "C" {
#endif #endif
typedef struct SPgCache SPgCache; #define TDB_PCACHE_PAGE \
typedef struct SPage SPage; u8 isAnchor; \
u8 isLocalPage; \
u8 isDirty; \
i32 nRef; \
SPCache *pCache; \
SPage *pFreeNext; \
SPage *pHashNext; \
SPage *pLruNext; \
SPage *pLruPrev; \
SPage *pDirtyNext; \
SPager *pPager; \
SPgid pgid;
// SPgCache int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache);
int pgCacheOpen(SPgCache **ppPgCache, TENV *pEnv); int tdbPCacheClose(SPCache *pCache);
int pgCacheClose(SPgCache *pPgCache); SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage);
void tdbPCacheRelease(SPage *pPage);
SPage *pgCacheFetch(SPgCache *pPgCache, pgid_t pgid); int tdbPCacheGetPageSize(SPCache *pCache);
int pgCacheRelease(SPage *pPage);
// SPage
typedef TD_DLIST_NODE(SPage) SPgListNode;
struct SPage {
pgid_t pgid; // page id
frame_id_t frameid; // frame id
uint8_t * pData; // real data
SPgListNode freeNode; // for SPgCache.freeList
SPgListNode pghtNode; // for pght
SPgListNode lruNode; // for LRU
};
#ifdef __cplusplus #ifdef __cplusplus
} }

131
source/libs/tdb/src/inc/tdbPage.h Normal file
View File

@ -0,0 +1,131 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TDB_PAGE_H_
#define _TDB_PAGE_H_
#ifdef __cplusplus
extern "C" {
#endif
typedef u8 SCell;
// PAGE APIS implemented
typedef struct {
int szOffset;
int szPageHdr;
int szFreeCell;
// flags
u16 (*getFlags)(SPage *);
void (*setFlags)(SPage *, u16);
// cell number
int (*getCellNum)(SPage *);
void (*setCellNum)(SPage *, int);
// cell content offset
int (*getCellBody)(SPage *);
void (*setCellBody)(SPage *, int);
// first free cell offset (0 means no free cells)
int (*getCellFree)(SPage *);
void (*setCellFree)(SPage *, int);
// total free bytes
int (*getFreeBytes)(SPage *);
void (*setFreeBytes)(SPage *, int);
// cell offset at idx
int (*getCellOffset)(SPage *, int);
void (*setCellOffset)(SPage *, int, int);
} SPageMethods;
// Page footer
typedef struct __attribute__((__packed__)) {
u8 cksm[4];
} SPageFtr;
struct SPage {
pthread_spinlock_t lock;
u8 *pData;
int pageSize;
SPageMethods *pPageMethods;
// Fields below used by pager and am
u8 szAmHdr;
u8 *pPageHdr;
u8 *pAmHdr;
u8 *pCellIdx;
u8 *pFreeStart;
u8 *pFreeEnd;
SPageFtr *pPageFtr;
int kLen; // key length of the page, -1 for unknown
int vLen; // value length of the page, -1 for unknown
int nFree;
int maxLocal;
int minLocal;
int nOverflow;
SCell *apOvfl[4];
int aiOvfl[4];
// Fields used by SPCache
TDB_PCACHE_PAGE
};
/* For page */
#define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage)
#define TDB_PAGE_NCELLS(pPage) (*(pPage)->pPageMethods->getCellNum)(pPage)
#define TDB_PAGE_CCELLS(pPage) (*(pPage)->pPageMethods->getCellBody)(pPage)
#define TDB_PAGE_FCELL(pPage) (*(pPage)->pPageMethods->getCellFree)(pPage)
#define TDB_PAGE_NFREE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage)
#define TDB_PAGE_CELL_OFFSET_AT(pPage, idx) (*(pPage)->pPageMethods->getCellOffset)(pPage, idx)
#define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS)
#define TDB_PAGE_NCELLS_SET(pPage, NCELLS) (*(pPage)->pPageMethods->setCellNum)(pPage, NCELLS)
#define TDB_PAGE_CCELLS_SET(pPage, CCELLS) (*(pPage)->pPageMethods->setCellBody)(pPage, CCELLS)
#define TDB_PAGE_FCELL_SET(pPage, FCELL) (*(pPage)->pPageMethods->setCellFree)(pPage, FCELL)
#define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE)
#define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET)
#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset)
#define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx))
// For page lock
#define P_LOCK_SUCC 0
#define P_LOCK_BUSY 1
#define P_LOCK_FAIL -1
#define TDB_INIT_PAGE_LOCK(pPage) pthread_spin_init(&((pPage)->lock), 0)
#define TDB_DESTROY_PAGE_LOCK(pPage) pthread_spin_destroy(&((pPage)->lock))
#define TDB_LOCK_PAGE(pPage) pthread_spin_lock(&((pPage)->lock))
#define TDB_UNLOCK_PAGE(pPage) pthread_spin_unlock(&((pPage)->lock))
#define TDB_TRY_LOCK_PAGE(pPage) \
({ \
int ret; \
if (pthread_spin_trylock(&((pPage)->lock)) == 0) { \
ret = P_LOCK_SUCC; \
} else if (errno == EBUSY) { \
ret = P_LOCK_BUSY; \
} else { \
ret = P_LOCK_FAIL; \
} \
ret; \
})
// APIs
int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg);
int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg);
int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell);
int tdbPageDropCell(SPage *pPage, int idx);
#ifdef __cplusplus
}
#endif
#endif /*_TDB_PAGE_H_*/

37
source/libs/tdb/src/inc/tdbPager.h Normal file
View File

@ -0,0 +1,37 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TDB_PAGER_H_
#define _TDB_PAGER_H_
#ifdef __cplusplus
extern "C" {
#endif
int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager);
int tdbPagerClose(SPager *pPager);
int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate);
int tdbPagerWrite(SPager *pPager, SPage *pPage);
int tdbPagerBegin(SPager *pPager);
int tdbPagerCommit(SPager *pPager);
int tdbPagerGetPageSize(SPager *pPager);
int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg);
int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg);
#ifdef __cplusplus
}
#endif
#endif /*_TDB_PAGER_H_*/

61
source/libs/tdb/src/inc/tdbPgFile.h
View File

@ -1,61 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_PAGE_FILE_H_
#define _TD_PAGE_FILE_H_
#ifdef __cplusplus
extern "C" {
#endif
#pragma pack (push,1)
typedef struct {
char hdrInfo[16]; // info string
pgsz_t szPage; // page size of current file
int32_t cno; // commit number counter
pgno_t freePgno; // freelist page number
uint8_t resv[100]; // reserved space
} SPgFileHdr;
#pragma pack(pop)
#define TDB_PG_FILE_HDR_SIZE 128
TDB_STATIC_ASSERT(sizeof(SPgFileHdr) == TDB_PG_FILE_HDR_SIZE, "Page file header size if not 128");
struct SPgFile {
TENV * pEnv; // env containing this page file
char * fname; // backend file name
uint8_t fileid[TDB_FILE_ID_LEN]; // file id
pgno_t lsize; // page file logical size (for count)
pgno_t fsize; // real file size on disk (for rollback)
TdFilePtr pFile;
SPgFileListNode envHash;
SPgFileListNode envPgfList;
};
int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv);
int pgFileClose(SPgFile *pPgFile);
SPage *pgFileFetch(SPgFile *pPgFile, pgno_t pgno);
int pgFileRelease(SPage *pPage);
int pgFileWrite(SPage *pPage);
int pgFileAllocatePage(SPgFile *pPgFile, pgno_t *pPgno);
#ifdef __cplusplus
}
#endif
#endif /*_TD_PAGE_FILE_H_*/

43
source/libs/tdb/src/inc/tdbUtil.h
View File

@ -35,7 +35,48 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique);
// #define TDB_W_OK 0x4 // #define TDB_W_OK 0x4
// int tdbCheckFileAccess(const char *pathname, int mode); // int tdbCheckFileAccess(const char *pathname, int mode);
int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize); int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize);
int tdbPRead(int fd, void *pData, int count, i64 offset);
static inline int tdbPutVarInt(u8 *p, int v) {
int n = 0;
for (;;) {
if (v <= 0x7f) {
p[n++] = v;
break;
}
p[n++] = (v & 0x7f) | 0x80;
v >>= 7;
}
ASSERT(n < 6);
return n;
}
static inline int tdbGetVarInt(const u8 *p, int *v) {
int n = 0;
int tv = 0;
for (;;) {
if (p[n] <= 0x7f) {
tv = (tv << 7) | p[n];
n++;
break;
}
tv = (tv << 7) | (p[n] & 0x7f);
n++;
}
ASSERT(n < 6);
*v = tv;
return n;
}
#ifdef __cplusplus #ifdef __cplusplus
} }

272
source/libs/tdb/src/page/tdbPage.c Normal file
View File

@ -0,0 +1,272 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
extern SPageMethods pageMethods;
extern SPageMethods pageLargeMethods;
typedef struct __attribute__((__packed__)) {
u16 szCell;
u16 nxOffset;
} SFreeCell;
static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell);
static int tdbPageDefragment(SPage *pPage);
int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) {
SPage *pPage;
u8 *ptr;
int size;
ASSERT(TDB_IS_PGSIZE_VLD(pageSize));
*ppPage = NULL;
size = pageSize + sizeof(*pPage);
ptr = (u8 *)((*xMalloc)(arg, size));
if (pPage == NULL) {
return -1;
}
memset(ptr, 0, size);
pPage = (SPage *)(ptr + pageSize);
pPage->pData = ptr;
pPage->pageSize = pageSize;
if (pageSize < 65536) {
pPage->pPageMethods = &pageMethods;
} else {
pPage->pPageMethods = &pageLargeMethods;
}
TDB_INIT_PAGE_LOCK(pPage);
/* TODO */
*ppPage = pPage;
return 0;
}
int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) {
u8 *ptr;
ptr = pPage->pData;
(*xFree)(arg, ptr);
return 0;
}
int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) {
int ret;
SCell *pTarget;
u8 *pTmp;
int j;
if (pPage->nOverflow || szCell + TDB_PAGE_OFFSET_SIZE(pPage) > pPage->nFree) {
// TODO: need to figure out if pCell may be used by outside of this function
j = pPage->nOverflow++;
pPage->apOvfl[j] = pCell;
pPage->aiOvfl[j] = idx;
} else {
ret = tdbPageAllocate(pPage, szCell, &pTarget);
if (ret < 0) {
return -1;
}
memcpy(pTarget, pCell, szCell);
pTmp = pPage->pCellIdx + idx * TDB_PAGE_OFFSET_SIZE(pPage);
memmove(pTmp + TDB_PAGE_OFFSET_SIZE(pPage), pTmp, pPage->pFreeStart - pTmp - TDB_PAGE_OFFSET_SIZE(pPage));
TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData);
TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1);
}
return 0;
}
int tdbPageDropCell(SPage *pPage, int idx) {
// TODO
return 0;
}
static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) {
SCell *pCell;
SFreeCell *pFreeCell;
u8 *pOffset;
int ret;
ASSERT(pPage->nFree > size + TDB_PAGE_OFFSET_SIZE(pPage));
pCell = NULL;
*ppCell = NULL;
// 1. Try to allocate from the free space area
if (pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)) {
pPage->pFreeEnd -= size;
pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage);
pCell = pPage->pFreeEnd;
}
// 2. Try to allocate from the page free list
if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) &&
TDB_PAGE_FCELL(pPage)) {
#if 0
int szCell;
int nxOffset;
pCell = pPage->pData + TDB_PAGE_FCELL(pPage);
pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell
: (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell);
szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell);
nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell);
for (;;) {
// Find a cell
if (szCell >= size) {
if (szCell - size >= pPage->szFreeCell) {
SCell *pTmpCell = pCell + size;
TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size);
TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset);
// TODO: *pOffset = pTmpCell - pPage->pData;
} else {
TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size);
// TODO: *pOffset = nxOffset;
}
break;
}
// Not find a cell yet
if (nxOffset > 0) {
pCell = pPage->pData + nxOffset;
pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell);
szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell);
nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell);
continue;
} else {
pCell = NULL;
break;
}
}
if (pCell) {
pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset;
}
#endif
}
// 3. Try to dfragment and allocate again
if (pCell == NULL) {
ret = tdbPageDefragment(pPage);
if (ret < 0) {
return -1;
}
ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage));
ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart);
// Allocate from the free space area again
pPage->pFreeEnd -= size;
pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage);
pCell = pPage->pFreeEnd;
}
ASSERT(pCell != NULL);
pPage->nFree = pPage->nFree - size - TDB_PAGE_OFFSET_SIZE(pPage);
*ppCell = pCell;
return 0;
}
static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) {
// TODO
return 0;
}
static int tdbPageDefragment(SPage *pPage) {
// TODO
ASSERT(0);
return 0;
}
/* ---------------------------------------------------------------------------------------------------------- */
typedef struct __attribute__((__packed__)) {
u16 flags;
u16 cellNum;
u16 cellBody;
u16 cellFree;
u16 nFree;
} SPageHdr;
// flags
static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].flags; }
static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdr *)(pPage->pPageHdr))[0].flags = flags; }
// cellNum
static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; }
static inline void setPageCellNum(SPage *pPage, int cellNum) {
ASSERT(cellNum < 65536);
((SPageHdr *)(pPage->pPageHdr))[0].cellNum = (u16)cellNum;
}
// cellBody
static inline int getPageCellBody(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellBody; }
static inline void setPageCellBody(SPage *pPage, int cellBody) {
ASSERT(cellBody < 65536);
((SPageHdr *)(pPage->pPageHdr))[0].cellBody = (u16)cellBody;
}
// cellFree
static inline int getPageCellFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellFree; }
static inline void setPageCellFree(SPage *pPage, int cellFree) {
ASSERT(cellFree < 65536);
((SPageHdr *)(pPage->pPageHdr))[0].cellFree = (u16)cellFree;
}
// nFree
static inline int getPageNFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].nFree; }
static inline void setPageNFree(SPage *pPage, int nFree) {
ASSERT(nFree < 65536);
((SPageHdr *)(pPage->pPageHdr))[0].nFree = (u16)nFree;
}
// cell offset
static inline int getPageCellOffset(SPage *pPage, int idx) {
ASSERT(idx >= 0 && idx < getPageCellNum(pPage));
return ((u16 *)pPage->pCellIdx)[idx];
}
static inline void setPageCellOffset(SPage *pPage, int idx, int offset) {
ASSERT(offset < 65536);
((u16 *)pPage->pCellIdx)[idx] = (u16)offset;
}
SPageMethods pageMethods = {
2, // szOffset
sizeof(SPageHdr), // szPageHdr
sizeof(SFreeCell), // szFreeCell
getPageFlags, // getPageFlags
setPageFlags, // setFlagsp
getPageCellNum, // getCellNum
setPageCellNum, // setCellNum
getPageCellBody, // getCellBody
setPageCellBody, // setCellBody
getPageCellFree, // getCellFree
setPageCellFree, // setCellFree
getPageNFree, // getFreeBytes
setPageNFree, // setFreeBytes
getPageCellOffset, // getCellOffset
setPageCellOffset // setCellOffset
};

85
source/libs/tdb/src/page/tdbPageL.c Normal file
View File

@ -0,0 +1,85 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
typedef struct __attribute__((__packed__)) {
u16 flags;
u8 cellNum[3];
u8 cellBody[3];
u8 cellFree[3];
u8 nFree[3];
} SPageHdrL;
typedef struct __attribute__((__packed__)) {
u8 szCell[3];
u8 nxOffset[3];
} SFreeCellL;
// flags
static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdrL *)(pPage->pPageHdr))[0].flags; }
static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdrL *)(pPage->pPageHdr))[0].flags = flags; }
// cellNum
static inline int getPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); }
static inline void setPageCellNum(SPage *pPage, int cellNum) {
TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum, cellNum);
}
// cellBody
static inline int getPageCellBody(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody); }
static inline void setPageCellBody(SPage *pPage, int cellBody) {
TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody, cellBody);
}
// cellFree
static inline int getPageCellFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree); }
static inline void setPageCellFree(SPage *pPage, int cellFree) {
TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree, cellFree);
}
// nFree
static inline int getPageNFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree); }
static inline void setPageNFree(SPage *pPage, int nFree) {
TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree, nFree);
}
// cell offset
static inline int getPageCellOffset(SPage *pPage, int idx) {
ASSERT(idx >= 0 && idx < getPageCellNum(pPage));
return TDB_GET_U24(pPage->pCellIdx + 3 * idx);
}
static inline void setPageCellOffset(SPage *pPage, int idx, int offset) {
TDB_PUT_U24(pPage->pCellIdx + 3 * idx, offset);
}
SPageMethods pageLargeMethods = {
3, // szOffset
sizeof(SPageHdrL), // szPageHdr
sizeof(SFreeCellL), // szFreeCell
getPageFlags, // getPageFlags
setPageFlags, // setFlagsp
getPageCellNum, // getCellNum
setPageCellNum, // setCellNum
getPageCellBody, // getCellBody
setPageCellBody, // setCellBody
getPageCellFree, // getCellFree
setPageCellFree, // setCellFree
getPageNFree, // getFreeBytes
setPageNFree, // setFreeBytes
getPageCellOffset, // getCellOffset
setPageCellOffset // setCellOffset
};

83
source/libs/tdb/test/tdbTest.cpp
View File

@ -1,68 +1,39 @@
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include "tdb.h" #include "tdbInt.h"
TEST(tdb_test, simple_test) { TEST(tdb_test, simple_test) {
TENV * pEnv; int ret;
TDB * pDb1, *pDb2, *pDb3; STEnv *pEnv;
pgsz_t pgSize = 1024; STDb *pDb;
cachesz_t cacheSize = 10240;
// ENV // Open Env
GTEST_ASSERT_EQ(tdbEnvCreate(&pEnv, "./testtdb"), 0); ret = tdbEnvOpen("tdb", 1024, 20, &pEnv);
GTEST_ASSERT_EQ(ret, 0);
GTEST_ASSERT_EQ(tdbEnvSetCache(pEnv, pgSize, cacheSize), 0); // Create a database
ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, NULL, pEnv, &pDb);
GTEST_ASSERT_EQ(ret, 0);
GTEST_ASSERT_EQ(tdbEnvGetCacheSize(pEnv), cacheSize); { // Insert some data
char key[64];
char val[64];
GTEST_ASSERT_EQ(tdbEnvGetPageSize(pEnv), pgSize); for (int i = 1; i <= 1000; i++) {
sprintf(key, "key%d", i);
sprintf(val, "value%d", i);
ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val));
GTEST_ASSERT_EQ(ret, 0);
}
}
GTEST_ASSERT_EQ(tdbEnvOpen(pEnv), 0); ret = tdbDbDrop(pDb);
GTEST_ASSERT_EQ(ret, 0);
#if 1 // Close a database
// DB tdbDbClose(pDb);
GTEST_ASSERT_EQ(tdbCreate(&pDb1), 0);
// GTEST_ASSERT_EQ(tdbSetKeyLen(pDb1, 8), 0); // Close Env
ret = tdbEnvClose(pEnv);
// GTEST_ASSERT_EQ(tdbGetKeyLen(pDb1), 8); GTEST_ASSERT_EQ(ret, 0);
// GTEST_ASSERT_EQ(tdbSetValLen(pDb1, 3), 0);
// GTEST_ASSERT_EQ(tdbGetValLen(pDb1), 3);
// GTEST_ASSERT_EQ(tdbSetDup(pDb1, 1), 0);
// GTEST_ASSERT_EQ(tdbGetDup(pDb1), 1);
// GTEST_ASSERT_EQ(tdbSetCmprFunc(pDb1, NULL), 0);
tdbEnvBeginTxn(pEnv);
GTEST_ASSERT_EQ(tdbOpen(pDb1, "db.db", "db1", pEnv), 0);
// char *key = "key1";
// char *val = "value1";
// tdbInsert(pDb1, (void *)key, strlen(key), (void *)val, strlen(val));
tdbEnvCommit(pEnv);
#if 0
// Insert
// Query
// Delete
// Query
#endif
// GTEST_ASSERT_EQ(tdbOpen(&pDb2, "db.db", "db2", pEnv), 0);
// GTEST_ASSERT_EQ(tdbOpen(&pDb3, "index.db", NULL, pEnv), 0);
// tdbClose(pDb3);
// tdbClose(pDb2);
tdbClose(pDb1);
#endif
tdbEnvClose(pEnv);
} }

3
source/libs/tfs/src/tfs.c
View File

@ -204,7 +204,8 @@ void tfsDirname(const STfsFile *pFile, char *dest) {
void tfsAbsoluteName(STfs *pTfs, SDiskID diskId, const char *rname, char *aname) { void tfsAbsoluteName(STfs *pTfs, SDiskID diskId, const char *rname, char *aname) {
STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId); STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId);
snprintf(aname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname);
snprintf(aname, TSDB_FILENAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname);
} }
int32_t tfsRemoveFile(const STfsFile *pFile) { return taosRemoveFile(pFile->aname); } int32_t tfsRemoveFile(const STfsFile *pFile) { return taosRemoveFile(pFile->aname); }

50
source/libs/transport/inc/transComm.h
View File

@ -14,6 +14,10 @@
*/ */
#ifdef USE_UV #ifdef USE_UV
#ifdef __cplusplus
extern "C" {
#endif
#include <uv.h> #include <uv.h>
#include "lz4.h" #include "lz4.h"
#include "os.h" #include "os.h"
@ -121,24 +125,21 @@ typedef struct {
} SRpcReqContext; } SRpcReqContext;
typedef SRpcMsg STransMsg; typedef SRpcMsg STransMsg;
typedef SRpcCtx STransCtx;
typedef SRpcCtxVal STransCtxVal;
typedef SRpcInfo STrans; typedef SRpcInfo STrans;
typedef SRpcConnInfo STransHandleInfo; typedef SRpcConnInfo STransHandleInfo;
typedef struct { typedef struct {
SEpSet epSet; // ip list provided by app SEpSet epSet; // ip list provided by app
void* ahandle; // handle provided by app void* ahandle; // handle provided by app
tmsg_t msgType; // message type tmsg_t msgType; // message type
uint8_t* pCont; // content provided by app
int32_t contLen; // content length
// int32_t code; // error code
// int16_t numOfTry; // number of try for different servers
// int8_t oldInUse; // server EP inUse passed by app
// int8_t redirect; // flag to indicate redirect
int8_t connType; // connection type cli/srv int8_t connType; // connection type cli/srv
int64_t rid; // refId returned by taosAddRef int64_t rid; // refId returned by taosAddRef
STransMsg* pRsp; // for synchronous API STransCtx appCtx; //
tsem_t* pSem; // for synchronous API STransMsg* pRsp; // for synchronous API
tsem_t* pSem; // for synchronous API
int hThrdIdx; int hThrdIdx;
char* ip; char* ip;
@ -150,11 +151,12 @@ typedef struct {
typedef struct { typedef struct {
char version : 4; // RPC version char version : 4; // RPC version
char comp : 4; // compression algorithm, 0:no compression 1:lz4 char comp : 2; // compression algorithm, 0:no compression 1:lz4
char resflag : 2; // reserved bits char noResp : 2; // noResp bits, 0: resp, 1: resp
char spi : 1; // security parameter index char persist : 2; // persist handle,0: no persit, 1: persist handle
char release : 2;
char secured : 2; char secured : 2;
char encrypt : 3; // encrypt algorithm, 0: no encryption char spi : 2;
uint32_t code; // del later uint32_t code; // del later
uint32_t msgType; uint32_t msgType;
@ -179,6 +181,9 @@ typedef struct {
#pragma pack(pop) #pragma pack(pop)
typedef enum { Normal, Quit, Release, Register } STransMsgType;
typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken } ConnStatus;
#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member)))
#define RPC_RESERVE_SIZE (sizeof(STranConnCtx)) #define RPC_RESERVE_SIZE (sizeof(STranConnCtx))
@ -255,9 +260,10 @@ void transUnrefCliHandle(void* handle);
void transReleaseCliHandle(void* handle); void transReleaseCliHandle(void* handle);
void transReleaseSrvHandle(void* handle); void transReleaseSrvHandle(void* handle);
void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg); void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransCtx* pCtx);
void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransMsg* pRsp); void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransMsg* pRsp);
void transSendResponse(const STransMsg* pMsg); void transSendResponse(const STransMsg* msg);
void transRegisterMsg(const STransMsg* msg);
int transGetConnInfo(void* thandle, STransHandleInfo* pInfo); int transGetConnInfo(void* thandle, STransHandleInfo* pInfo);
void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);
@ -266,4 +272,14 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads,
void transCloseClient(void* arg); void transCloseClient(void* arg);
void transCloseServer(void* arg); void transCloseServer(void* arg);
void transCtxInit(STransCtx* ctx);
void transCtxDestroy(STransCtx* ctx);
void transCtxClear(STransCtx* ctx);
void transCtxMerge(STransCtx* dst, STransCtx* src);
void* transCtxDumpVal(STransCtx* ctx, int32_t key);
#ifdef __cplusplus
}
#endif
#endif #endif

3
source/libs/transport/inc/transportInt.h
View File

@ -63,9 +63,6 @@ typedef struct {
void (*cfp)(void* parent, SRpcMsg*, SEpSet*); void (*cfp)(void* parent, SRpcMsg*, SEpSet*);
int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey);
bool (*pfp)(void* parent, tmsg_t msgType);
void* (*mfp)(void* parent, tmsg_t msgType);
bool (*efp)(void* parent, tmsg_t msgType);
int32_t refCount; int32_t refCount;
void* parent; void* parent;

11
source/libs/transport/src/trans.c
View File

@ -39,9 +39,6 @@ void* rpcOpen(const SRpcInit* pInit) {
// register callback handle // register callback handle
pRpc->cfp = pInit->cfp; pRpc->cfp = pInit->cfp;
pRpc->afp = pInit->afp; pRpc->afp = pInit->afp;
pRpc->pfp = pInit->pfp;
pRpc->mfp = pInit->mfp;
pRpc->efp = pInit->efp;
if (pInit->connType == TAOS_CONN_SERVER) { if (pInit->connType == TAOS_CONN_SERVER) {
pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
@ -121,7 +118,12 @@ void rpcCancelRequest(int64_t rid) { return; }
void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) {
char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn);
uint32_t port = pEpSet->eps[pEpSet->inUse].port; uint32_t port = pEpSet->eps[pEpSet->inUse].port;
transSendRequest(shandle, ip, port, pMsg); transSendRequest(shandle, ip, port, pMsg, NULL);
}
void rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) {
char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn);
uint32_t port = pEpSet->eps[pEpSet->inUse].port;
transSendRequest(shandle, ip, port, pMsg, pCtx);
} }
void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) {
char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn);
@ -142,6 +144,7 @@ void rpcUnrefHandle(void* handle, int8_t type) {
(*taosUnRefHandle[type])(handle); (*taosUnRefHandle[type])(handle);
} }
void rpcRegisterBrokenLinkArg(SRpcMsg* msg) { rpcSendResponse(msg); }
void rpcReleaseHandle(void* handle, int8_t type) { void rpcReleaseHandle(void* handle, int8_t type) {
assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT); assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT);
(*transReleaseHandle[type])(handle); (*transReleaseHandle[type])(handle);

131
source/libs/transport/src/transCli.c
View File

@ -17,11 +17,6 @@
#include "transComm.h" #include "transComm.h"
// Normal(default): send/recv msg
// Quit: quit rpc inst
// Release: release handle to rpc inst
typedef enum { Normal, Quit, Release } SCliMsgType;
typedef struct SCliConn { typedef struct SCliConn {
T_REF_DECLARE() T_REF_DECLARE()
uv_connect_t connReq; uv_connect_t connReq;
@ -35,8 +30,10 @@ typedef struct SCliConn {
uint64_t expireTime; uint64_t expireTime;
int hThrdIdx; int hThrdIdx;
bool broken; // link broken or not bool broken; // link broken or not
STransCtx ctx;
int persist; // ConnStatus status; //
int release; // 1: release
// spi configure // spi configure
char spi; char spi;
char secured; char secured;
@ -55,7 +52,7 @@ typedef struct SCliMsg {
STransMsg msg; STransMsg msg;
queue q; queue q;
uint64_t st; uint64_t st;
SCliMsgType type; STransMsgType type;
} SCliMsg; } SCliMsg;
typedef struct SCliThrdObj { typedef struct SCliThrdObj {
@ -113,10 +110,12 @@ static void cliSend(SCliConn* pConn);
static void cliHandleResp(SCliConn* conn); static void cliHandleResp(SCliConn* conn);
// handle except about conn // handle except about conn
static void cliHandleExcept(SCliConn* conn); static void cliHandleExcept(SCliConn* conn);
// handle req from app // handle req from app
static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd);
static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd); static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd);
static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd); static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd);
static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrdObj* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease};
static void cliSendQuit(SCliThrdObj* thrd); static void cliSendQuit(SCliThrdObj* thrd);
static void destroyUserdata(STransMsg* userdata); static void destroyUserdata(STransMsg* userdata);
@ -133,11 +132,26 @@ static void destroyThrdObj(SCliThrdObj* pThrd);
#define CONN_PERSIST_TIME(para) (para * 1000 * 10) #define CONN_PERSIST_TIME(para) (para * 1000 * 10)
#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) #define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL)
#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) #define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label)
#define CONN_SHOULD_RELEASE(conn, head) \
do { \
if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \
conn->status = ConnRelease; \
transClearBuffer(&conn->readBuf); \
transFreeMsg(transContFromHead((char*)head)); \
tDebug("cli conn %p receive release request", conn); \
if (T_REF_VAL_GET(conn) == 1) { \
SCliThrdObj* thrd = conn->hostThrd; \
addConnToPool(thrd->pool, conn); \
} \
return; \
} \
} while (0)
#define CONN_HANDLE_THREAD_QUIT(conn, thrd) \ #define CONN_HANDLE_THREAD_QUIT(conn, thrd) \
do { \ do { \
if (thrd->quit) { \ if (thrd->quit) { \
cliHandleExcept(conn); \ cliHandleExcept(conn); \
goto _RETURE; \ return; \
} \ } \
} while (0) } while (0)
@ -145,20 +159,22 @@ static void destroyThrdObj(SCliThrdObj* pThrd);
do { \ do { \
if (conn->broken) { \ if (conn->broken) { \
cliHandleExcept(conn); \ cliHandleExcept(conn); \
goto _RETURE; \ return; \
} \ } \
} while (0); } while (0)
#define CONN_SET_PERSIST_BY_APP(conn) \ #define CONN_SET_PERSIST_BY_APP(conn) \
do { \ do { \
if (conn->persist == false) { \ if (conn->status == ConnNormal) { \
conn->persist = true; \ conn->status = ConnAcquire; \
transRefCliHandle(conn); \ transRefCliHandle(conn); \
} \ } \
} while (0) } while (0)
#define CONN_NO_PERSIST_BY_APP(conn) ((conn)->persist == false) #define CONN_NO_PERSIST_BY_APP(conn) ((conn)->status == ConnNormal && T_REF_VAL_GET(conn) == 1)
#define REQUEST_NO_RESP(msg) ((msg)->noResp == 1) #define REQUEST_NO_RESP(msg) ((msg)->noResp == 1)
#define REQUEST_PERSIS_HANDLE(msg) ((msg)->persistHandle == 1)
#define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release)
static void* cliWorkThread(void* arg); static void* cliWorkThread(void* arg);
@ -177,7 +193,6 @@ void cliHandleResp(SCliConn* conn) {
STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf); STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf);
pHead->code = htonl(pHead->code); pHead->code = htonl(pHead->code);
pHead->msgLen = htonl(pHead->msgLen); pHead->msgLen = htonl(pHead->msgLen);
STransMsg transMsg = {0}; STransMsg transMsg = {0};
transMsg.contLen = transContLenFromMsg(pHead->msgLen); transMsg.contLen = transContLenFromMsg(pHead->msgLen);
transMsg.pCont = transContFromHead((char*)pHead); transMsg.pCont = transContFromHead((char*)pHead);
@ -185,6 +200,8 @@ void cliHandleResp(SCliConn* conn) {
transMsg.msgType = pHead->msgType; transMsg.msgType = pHead->msgType;
transMsg.ahandle = NULL; transMsg.ahandle = NULL;
CONN_SHOULD_RELEASE(conn, pHead);
SCliMsg* pMsg = NULL; SCliMsg* pMsg = NULL;
if (taosArrayGetSize(conn->cliMsgs) > 0) { if (taosArrayGetSize(conn->cliMsgs) > 0) {
pMsg = taosArrayGetP(conn->cliMsgs, 0); pMsg = taosArrayGetP(conn->cliMsgs, 0);
@ -193,16 +210,15 @@ void cliHandleResp(SCliConn* conn) {
STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL;
if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) { if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) {
transMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, transMsg.msgType) : NULL; transMsg.ahandle = transCtxDumpVal(&conn->ctx, transMsg.msgType);
} else { } else {
transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; transMsg.ahandle = pCtx ? pCtx->ahandle : NULL;
} }
// buf's mem alread translated to transMsg.pCont // buf's mem alread translated to transMsg.pCont
transClearBuffer(&conn->readBuf); transClearBuffer(&conn->readBuf);
if (pTransInst->pfp != NULL && (*pTransInst->pfp)(pTransInst->parent, transMsg.msgType)) { if (!CONN_NO_PERSIST_BY_APP(conn)) {
transMsg.handle = conn; transMsg.handle = conn;
CONN_SET_PERSIST_BY_APP(conn);
tDebug("%s cli conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); tDebug("%s cli conn %p ref by app", CONN_GET_INST_LABEL(conn), conn);
} }
@ -241,6 +257,8 @@ void cliHandleResp(SCliConn* conn) {
if (!uv_is_active((uv_handle_t*)&pThrd->timer) && pTransInst->idleTime > 0) { if (!uv_is_active((uv_handle_t*)&pThrd->timer) && pTransInst->idleTime > 0) {
// uv_timer_start((uv_timer_t*)&pThrd->timer, cliTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); // uv_timer_start((uv_timer_t*)&pThrd->timer, cliTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0);
} }
_RETURN:
return;
} }
void cliHandleExcept(SCliConn* pConn) { void cliHandleExcept(SCliConn* pConn) {
@ -268,7 +286,7 @@ void cliHandleExcept(SCliConn* pConn) {
transMsg.ahandle = NULL; transMsg.ahandle = NULL;
if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) {
transMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, transMsg.msgType) : NULL; transMsg.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType);
} else { } else {
transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; transMsg.ahandle = pCtx ? pCtx->ahandle : NULL;
} }
@ -359,6 +377,7 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) {
static void addConnToPool(void* pool, SCliConn* conn) { static void addConnToPool(void* pool, SCliConn* conn) {
char key[128] = {0}; char key[128] = {0};
transCtxDestroy(&conn->ctx);
tstrncpy(key, conn->ip, strlen(conn->ip)); tstrncpy(key, conn->ip, strlen(conn->ip));
tstrncpy(key + strlen(key), (char*)(&conn->port), sizeof(conn->port)); tstrncpy(key + strlen(key), (char*)(&conn->port), sizeof(conn->port));
tTrace("cli conn %p added to conn pool, read buf cap: %d", conn, conn->readBuf.cap); tTrace("cli conn %p added to conn pool, read buf cap: %d", conn, conn->readBuf.cap);
@ -367,15 +386,16 @@ static void addConnToPool(void* pool, SCliConn* conn) {
conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime);
SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key));
conn->status = ConnNormal;
// list already create before // list already create before
assert(plist != NULL); assert(plist != NULL);
taosArrayClear(conn->cliMsgs);
QUEUE_PUSH(&plist->conn, &conn->conn); QUEUE_PUSH(&plist->conn, &conn->conn);
assert(!QUEUE_IS_EMPTY(&plist->conn));
} }
static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
SCliConn* conn = handle->data; SCliConn* conn = handle->data;
SConnBuffer* pBuf = &conn->readBuf; SConnBuffer* pBuf = &conn->readBuf;
// avoid conn
QUEUE_REMOVE(&conn->conn);
transAllocBuffer(pBuf, buf); transAllocBuffer(pBuf, buf);
} }
static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
@ -401,6 +421,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
// ref http://docs.libuv.org/en/v1.x/stream.html?highlight=uv_read_start#c.uv_read_cb // ref http://docs.libuv.org/en/v1.x/stream.html?highlight=uv_read_start#c.uv_read_cb
// nread might be 0, which does not indicate an error or EOF. This is equivalent to EAGAIN or EWOULDBLOCK under // nread might be 0, which does not indicate an error or EOF. This is equivalent to EAGAIN or EWOULDBLOCK under
// read(2). // read(2).
tTrace("%s cli conn %p read empty", CONN_GET_INST_LABEL(conn), conn);
return; return;
} }
if (nread < 0) { if (nread < 0) {
@ -420,16 +441,16 @@ static SCliConn* cliCreateConn(SCliThrdObj* pThrd) {
conn->writeReq.data = conn; conn->writeReq.data = conn;
conn->connReq.data = conn; conn->connReq.data = conn;
conn->cliMsgs = taosArrayInit(2, sizeof(void*)); conn->cliMsgs = taosArrayInit(2, sizeof(void*));
QUEUE_INIT(&conn->conn); QUEUE_INIT(&conn->conn);
conn->hostThrd = pThrd; conn->hostThrd = pThrd;
conn->persist = false; conn->status = ConnNormal;
conn->broken = false; conn->broken = 0;
transRefCliHandle(conn); transRefCliHandle(conn);
return conn; return conn;
} }
static void cliDestroyConn(SCliConn* conn, bool clear) { static void cliDestroyConn(SCliConn* conn, bool clear) {
tTrace("%s cli conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn); tTrace("%s cli conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn);
QUEUE_REMOVE(&conn->conn); QUEUE_REMOVE(&conn->conn);
if (clear) { if (clear) {
uv_close((uv_handle_t*)conn->stream, cliDestroy); uv_close((uv_handle_t*)conn->stream, cliDestroy);
@ -439,6 +460,7 @@ static void cliDestroy(uv_handle_t* handle) {
SCliConn* conn = handle->data; SCliConn* conn = handle->data;
free(conn->ip); free(conn->ip);
free(conn->stream); free(conn->stream);
transCtxDestroy(&conn->ctx);
taosArrayDestroy(conn->cliMsgs); taosArrayDestroy(conn->cliMsgs);
tTrace("%s cli conn %p destroy successfully", CONN_GET_INST_LABEL(conn), conn); tTrace("%s cli conn %p destroy successfully", CONN_GET_INST_LABEL(conn), conn);
free(conn); free(conn);
@ -490,7 +512,10 @@ void cliSend(SCliConn* pConn) {
STrans* pTransInst = pThrd->pTransInst; STrans* pTransInst = pThrd->pTransInst;
STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg); STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg);
if (pMsg->pCont == 0) {
pMsg->pCont = (void*)rpcMallocCont(0);
pMsg->contLen = 0;
}
STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); STransMsgHead* pHead = transHeadFromCont(pMsg->pCont);
int msgLen = transMsgLenFromCont(pMsg->contLen); int msgLen = transMsgLenFromCont(pMsg->contLen);
@ -513,20 +538,25 @@ void cliSend(SCliConn* pConn) {
msgLen += sizeof(STransUserMsg); msgLen += sizeof(STransUserMsg);
} }
pHead->resflag = REQUEST_NO_RESP(pMsg) ? 1 : 0; pHead->noResp = REQUEST_NO_RESP(pMsg) ? 1 : 0;
pHead->persist = REQUEST_PERSIS_HANDLE(pMsg) ? 1 : 0;
pHead->msgType = pMsg->msgType; pHead->msgType = pMsg->msgType;
pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); pHead->msgLen = (int32_t)htonl((uint32_t)msgLen);
pHead->release = REQUEST_RELEASE_HANDLE(pCliMsg) ? 1 : 0;
uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); uv_buf_t wb = uv_buf_init((char*)pHead, msgLen);
tDebug("%s cli conn %p %s is send to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn, tDebug("%s cli conn %p %s is send to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn,
TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port),
taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port));
if (pHead->persist == 1) {
CONN_SET_PERSIST_BY_APP(pConn);
}
pConn->writeReq.data = pConn; pConn->writeReq.data = pConn;
uv_write(&pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); uv_write(&pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb);
return;
_RETURE:
return; return;
} }
@ -562,22 +592,14 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) {
} }
static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) { static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) {
SCliConn* conn = pMsg->msg.handle; SCliConn* conn = pMsg->msg.handle;
tDebug("%s cli conn %p release to inst", CONN_GET_INST_LABEL(conn), conn); tDebug("%s cli conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn);
while (taosArrayGetSize(conn->cliMsgs) > 0) { transUnrefCliHandle(conn);
SCliMsg* pMsg = taosArrayGetP(conn->cliMsgs, 0); taosArrayPush(conn->cliMsgs, &pMsg);
destroyCmsg(pMsg); if (taosArrayGetSize(conn->cliMsgs) >= 2) {
taosArrayRemove(conn->cliMsgs, 0); return; // send one by one
}
transDestroyBuffer(&conn->readBuf);
if (conn->persist && T_REF_VAL_GET(conn) >= 2) {
conn->persist = false;
transUnrefCliHandle(conn);
addConnToPool(pThrd->pool, conn);
} else {
transUnrefCliHandle(conn);
} }
cliSend(conn);
} }
SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) {
@ -592,6 +614,8 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) {
conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port);
if (conn != NULL) { if (conn != NULL) {
tTrace("%s cli conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn); tTrace("%s cli conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn);
} else {
tTrace("not found conn in conn pool %p", pThrd->pool);
} }
} }
return conn; return conn;
@ -609,10 +633,12 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
if (conn != NULL) { if (conn != NULL) {
conn->hThrdIdx = pCtx->hThrdIdx; conn->hThrdIdx = pCtx->hThrdIdx;
transCtxMerge(&conn->ctx, &pCtx->appCtx);
if (taosArrayGetSize(conn->cliMsgs) > 0) { if (taosArrayGetSize(conn->cliMsgs) > 0) {
taosArrayPush(conn->cliMsgs, &pMsg); taosArrayPush(conn->cliMsgs, &pMsg);
return; return;
} }
taosArrayPush(conn->cliMsgs, &pMsg); taosArrayPush(conn->cliMsgs, &pMsg);
transDestroyBuffer(&conn->readBuf); transDestroyBuffer(&conn->readBuf);
cliSend(conn); cliSend(conn);
@ -652,14 +678,10 @@ static void cliAsyncCb(uv_async_t* handle) {
QUEUE_REMOVE(h); QUEUE_REMOVE(h);
SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q);
if (pMsg == NULL) {
if (pMsg->type == Normal) { continue;
cliHandleReq(pMsg, pThrd);
} else if (pMsg->type == Quit) {
cliHandleQuit(pMsg, pThrd);
} else if (pMsg->type == Release) {
cliHandleRelease(pMsg, pThrd);
} }
(*cliAsyncHandle[pMsg->type])(pMsg, pThrd);
count++; count++;
} }
if (count >= 2) { if (count >= 2) {
@ -802,13 +824,13 @@ void transReleaseCliHandle(void* handle) {
STransMsg tmsg = {.handle = handle}; STransMsg tmsg = {.handle = handle};
SCliMsg* cmsg = calloc(1, sizeof(SCliMsg)); SCliMsg* cmsg = calloc(1, sizeof(SCliMsg));
cmsg->type = Release;
cmsg->msg = tmsg; cmsg->msg = tmsg;
cmsg->type = Release;
transSendAsync(thrd->asyncPool, &cmsg->q); transSendAsync(thrd->asyncPool, &cmsg->q);
} }
void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg) { void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransCtx* ctx) {
STrans* pTransInst = (STrans*)shandle; STrans* pTransInst = (STrans*)shandle;
int index = CONN_HOST_THREAD_INDEX((SCliConn*)pMsg->handle); int index = CONN_HOST_THREAD_INDEX((SCliConn*)pMsg->handle);
if (index == -1) { if (index == -1) {
@ -818,7 +840,7 @@ void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* p
if (transCompressMsg(pMsg->pCont, pMsg->contLen, &flen)) { if (transCompressMsg(pMsg->pCont, pMsg->contLen, &flen)) {
// imp later // imp later
} }
tDebug("send request at thread:%d %p", index, pMsg); tDebug("send request at thread:%d %p, dst: %s:%d", index, pMsg, ip, port);
STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx));
pCtx->ahandle = pMsg->ahandle; pCtx->ahandle = pMsg->ahandle;
pCtx->msgType = pMsg->msgType; pCtx->msgType = pMsg->msgType;
@ -826,6 +848,10 @@ void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* p
pCtx->port = port; pCtx->port = port;
pCtx->hThrdIdx = index; pCtx->hThrdIdx = index;
if (ctx != NULL) {
pCtx->appCtx = *ctx;
}
assert(pTransInst->connType == TAOS_CONN_CLIENT); assert(pTransInst->connType == TAOS_CONN_CLIENT);
// atomic or not // atomic or not
@ -833,10 +859,12 @@ void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* p
cliMsg->ctx = pCtx; cliMsg->ctx = pCtx;
cliMsg->msg = *pMsg; cliMsg->msg = *pMsg;
cliMsg->st = taosGetTimestampUs(); cliMsg->st = taosGetTimestampUs();
cliMsg->type = Normal;
SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index]; SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index];
transSendAsync(thrd->asyncPool, &(cliMsg->q)); transSendAsync(thrd->asyncPool, &(cliMsg->q));
} }
void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq, STransMsg* pRsp) { void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq, STransMsg* pRsp) {
STrans* pTransInst = (STrans*)shandle; STrans* pTransInst = (STrans*)shandle;
int index = CONN_HOST_THREAD_INDEX(pReq->handle); int index = CONN_HOST_THREAD_INDEX(pReq->handle);
@ -858,6 +886,7 @@ void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq
cliMsg->ctx = pCtx; cliMsg->ctx = pCtx;
cliMsg->msg = *pReq; cliMsg->msg = *pReq;
cliMsg->st = taosGetTimestampUs(); cliMsg->st = taosGetTimestampUs();
cliMsg->type = Normal;
SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index]; SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index];
transSendAsync(thrd->asyncPool, &(cliMsg->q)); transSendAsync(thrd->asyncPool, &(cliMsg->q));

56
source/libs/transport/src/transComm.c
View File

@ -155,9 +155,9 @@ bool transReadComplete(SConnBuffer* connBuf) {
} }
return false; return false;
} }
int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) {return 0;} int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) { return 0; }
int transUnpackMsg(STransMsgHead* msgHead) {return 0;} int transUnpackMsg(STransMsgHead* msgHead) { return 0; }
int transDestroyBuffer(SConnBuffer* buf) { int transDestroyBuffer(SConnBuffer* buf) {
if (buf->cap > 0) { if (buf->cap > 0) {
tfree(buf->buf); tfree(buf->buf);
@ -224,4 +224,56 @@ int transSendAsync(SAsyncPool* pool, queue* q) {
return uv_async_send(async); return uv_async_send(async);
} }
void transCtxInit(STransCtx* ctx) {
// init transCtx
ctx->args = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UINT), true, HASH_NO_LOCK);
}
void transCtxDestroy(STransCtx* ctx) {
if (ctx->args == NULL) {
return;
}
STransCtxVal* iter = taosHashIterate(ctx->args, NULL);
while (iter) {
iter->free(iter->val);
iter = taosHashIterate(ctx->args, iter);
}
taosHashCleanup(ctx->args);
}
void transCtxMerge(STransCtx* dst, STransCtx* src) {
if (dst->args == NULL) {
dst->args = src->args;
src->args = NULL;
return;
}
void* key = NULL;
size_t klen = 0;
void* iter = taosHashIterate(src->args, NULL);
while (iter) {
STransCtxVal* sVal = (STransCtxVal*)iter;
key = taosHashGetKey(sVal, &klen);
STransCtxVal* dVal = taosHashGet(dst->args, key, klen);
if (dVal) {
dVal->free(dVal->val);
}
taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal));
iter = taosHashIterate(src->args, iter);
}
taosHashCleanup(src->args);
}
void* transCtxDumpVal(STransCtx* ctx, int32_t key) {
if (ctx->args == NULL) {
return NULL;
}
STransCtxVal* cVal = taosHashGet(ctx->args, (const void*)&key, sizeof(key));
if (cVal == NULL) {
return NULL;
}
char* ret = calloc(1, cVal->len);
memcpy(ret, (char*)cVal->val, cVal->len);
return (void*)ret;
}
#endif #endif

235
source/libs/transport/src/transSrv.c
View File

@ -17,6 +17,12 @@
#include "transComm.h" #include "transComm.h"
typedef struct {
int notifyCount; //
int init; // init or not
STransMsg msg;
} SSrvRegArg;
typedef struct SSrvConn { typedef struct SSrvConn {
T_REF_DECLARE() T_REF_DECLARE()
uv_tcp_t* pTcp; uv_tcp_t* pTcp;
@ -33,8 +39,10 @@ typedef struct SSrvConn {
void* hostThrd; void* hostThrd;
SArray* srvMsgs; SArray* srvMsgs;
bool broken; // conn broken; SSrvRegArg regArg;
bool broken; // conn broken;
ConnStatus status;
struct sockaddr_in addr; struct sockaddr_in addr;
struct sockaddr_in locaddr; struct sockaddr_in locaddr;
@ -47,18 +55,18 @@ typedef struct SSrvConn {
} SSrvConn; } SSrvConn;
typedef struct SSrvMsg { typedef struct SSrvMsg {
SSrvConn* pConn; SSrvConn* pConn;
STransMsg msg; STransMsg msg;
queue q; queue q;
STransMsgType type;
} SSrvMsg; } SSrvMsg;
typedef struct SWorkThrdObj { typedef struct SWorkThrdObj {
pthread_t thread; pthread_t thread;
uv_pipe_t* pipe; uv_pipe_t* pipe;
uv_os_fd_t fd; uv_os_fd_t fd;
uv_loop_t* loop; uv_loop_t* loop;
SAsyncPool* asyncPool; SAsyncPool* asyncPool;
queue msg; queue msg;
pthread_mutex_t msgMtx; pthread_mutex_t msgMtx;
@ -85,6 +93,27 @@ typedef struct SServerObj {
static const char* notify = "a"; static const char* notify = "a";
#define CONN_SHOULD_RELEASE(conn, head) \
do { \
if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \
conn->status = ConnRelease; \
transClearBuffer(&conn->readBuf); \
transFreeMsg(transContFromHead((char*)head)); \
tTrace("server conn %p received release request", conn); \
\
STransMsg tmsg = {.handle = (void*)conn, .code = 0}; \
SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); \
srvMsg->msg = tmsg; \
srvMsg->type = Release; \
srvMsg->pConn = conn; \
taosArrayPush(conn->srvMsgs, &srvMsg); \
if (taosArrayGetSize(conn->srvMsgs) > 1) { \
return; \
} \
uvStartSendRespInternal(srvMsg); \
return; \
} \
} while (0)
// refactor later // refactor later
static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen); static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen);
@ -113,6 +142,13 @@ static void destroySmsg(SSrvMsg* smsg);
static SSrvConn* createConn(void* hThrd); static SSrvConn* createConn(void* hThrd);
static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/); static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/);
static void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd);
static void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd);
static void uvHandleSendResp(SSrvMsg* msg, SWorkThrdObj* thrd);
static void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd);
static void (*transAsyncHandle[])(SSrvMsg* msg, SWorkThrdObj* thrd) = {uvHandleSendResp, uvHandleQuit, uvHandleRelease,
uvHandleRegister};
static void uvDestroyConn(uv_handle_t* handle); static void uvDestroyConn(uv_handle_t* handle);
// server and worker thread // server and worker thread
@ -217,9 +253,10 @@ static void uvHandleReq(SSrvConn* pConn) {
if (pHead->secured == 1) { if (pHead->secured == 1) {
pHead->msgLen -= sizeof(STransUserMsg); pHead->msgLen -= sizeof(STransUserMsg);
} }
//
} }
CONN_SHOULD_RELEASE(pConn, pHead);
STransMsg transMsg; STransMsg transMsg;
transMsg.contLen = transContLenFromMsg(pHead->msgLen); transMsg.contLen = transContLenFromMsg(pHead->msgLen);
transMsg.pCont = pHead->content; transMsg.pCont = pHead->content;
@ -230,24 +267,32 @@ static void uvHandleReq(SSrvConn* pConn) {
transClearBuffer(&pConn->readBuf); transClearBuffer(&pConn->readBuf);
pConn->inType = pHead->msgType; pConn->inType = pHead->msgType;
if (pConn->status == ConnNormal) {
if (pHead->resflag == 0) { if (pHead->persist == 1) {
pConn->status = ConnAcquire;
transRefSrvHandle(pConn);
}
}
if (pConn->status == ConnNormal && pHead->noResp == 0) {
transRefSrvHandle(pConn); transRefSrvHandle(pConn);
transMsg.handle = pConn;
tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(transMsg.msgType), tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(transMsg.msgType),
taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr),
ntohs(pConn->locaddr.sin_port), transMsg.contLen); ntohs(pConn->locaddr.sin_port), transMsg.contLen);
} else { } else {
tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, no resp ", pConn, tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, resp:%d ", pConn,
TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port),
taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen); taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen, pHead->noResp);
// no ref here
}
if (pHead->noResp == 0) {
transMsg.handle = pConn;
} }
STrans* pTransInst = (STrans*)p->shandle; STrans* pTransInst = (STrans*)p->shandle;
(*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); (*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);
// uv_timer_start(&pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // uv_timer_start(&pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0);
// auth // auth
// validate msg type
} }
void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
@ -272,11 +317,13 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
tError("server conn %p read error: %s", conn, uv_err_name(nread)); tError("server conn %p read error: %s", conn, uv_err_name(nread));
if (nread < 0) { if (nread < 0) {
conn->broken = true; conn->broken = true;
uvNotifyLinkBrokenToApp(conn); if (conn->status == ConnAcquire) {
if (conn->regArg.init) {
// STrans* pTransInst = conn->pTransInst; STrans* pTransInst = conn->pTransInst;
// if (pTransInst->efp != NULL && (pTransInst->efp)(NULL, conn->inType)) { (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL);
//} memset(&conn->regArg, 0, sizeof(conn->regArg));
}
}
transUnrefSrvHandle(conn); transUnrefSrvHandle(conn);
} }
} }
@ -301,8 +348,22 @@ void uvOnSendCb(uv_write_t* req, int status) {
SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0); SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0);
tTrace("server conn %p sending msg size: %d", conn, (int)taosArrayGetSize(conn->srvMsgs)); tTrace("server conn %p sending msg size: %d", conn, (int)taosArrayGetSize(conn->srvMsgs));
taosArrayRemove(conn->srvMsgs, 0); taosArrayRemove(conn->srvMsgs, 0);
if (msg->type == Release && conn->status != ConnNormal) {
conn->status = ConnNormal;
transUnrefSrvHandle(conn);
} else if (msg->type == Register && conn->status == ConnAcquire) {
conn->regArg.notifyCount = 0;
conn->regArg.init = 1;
conn->regArg.msg = msg->msg;
if (conn->broken) {
STrans* pTransInst = conn->pTransInst;
(pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL);
memset(&conn->regArg, 0, sizeof(conn->regArg));
}
free(msg);
return;
}
destroySmsg(msg); destroySmsg(msg);
// send second data, just use for push // send second data, just use for push
if (taosArrayGetSize(conn->srvMsgs) > 0) { if (taosArrayGetSize(conn->srvMsgs) > 0) {
tTrace("resent server conn %p sending msg size: %d", conn, (int)taosArrayGetSize(conn->srvMsgs)); tTrace("resent server conn %p sending msg size: %d", conn, (int)taosArrayGetSize(conn->srvMsgs));
@ -312,7 +373,7 @@ void uvOnSendCb(uv_write_t* req, int status) {
} }
} else { } else {
tError("server conn %p failed to write data, %s", conn, uv_err_name(status)); tError("server conn %p failed to write data, %s", conn, uv_err_name(status));
conn->broken = false; conn->broken = true;
transUnrefSrvHandle(conn); transUnrefSrvHandle(conn);
} }
} }
@ -339,6 +400,7 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) {
pHead->secured = pMsg->code == 0 ? 1 : 0; // pHead->secured = pMsg->code == 0 ? 1 : 0; //
pHead->msgType = smsg->pConn->inType + 1; pHead->msgType = smsg->pConn->inType + 1;
pHead->release = smsg->type == Release ? 1 : 0;
pHead->code = htonl(pMsg->code); pHead->code = htonl(pMsg->code);
// add more info // add more info
char* msg = (char*)pHead; char* msg = (char*)pHead;
@ -368,13 +430,16 @@ static void uvStartSendResp(SSrvMsg* smsg) {
SSrvConn* pConn = smsg->pConn; SSrvConn* pConn = smsg->pConn;
if (pConn->broken == true) { if (pConn->broken == true) {
// persist by
transUnrefSrvHandle(pConn); transUnrefSrvHandle(pConn);
return; return;
} }
transUnrefSrvHandle(pConn); if (pConn->status == ConnNormal) {
transUnrefSrvHandle(pConn);
}
if (taosArrayGetSize(pConn->srvMsgs) > 0) { if (taosArrayGetSize(pConn->srvMsgs) > 0) {
tDebug("server conn %p push data to client %s:%d, local info: %s:%d", pConn, taosInetNtoa(pConn->addr.sin_addr), tDebug("server conn %p send data to client %s:%d, local info: %s:%d", pConn, taosInetNtoa(pConn->addr.sin_addr),
ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port));
taosArrayPush(pConn->srvMsgs, &smsg); taosArrayPush(pConn->srvMsgs, &smsg);
return; return;
@ -384,16 +449,6 @@ static void uvStartSendResp(SSrvMsg* smsg) {
return; return;
} }
static void uvNotifyLinkBrokenToApp(SSrvConn* conn) {
STrans* pTransInst = conn->pTransInst;
if (pTransInst->efp != NULL && (*pTransInst->efp)(NULL, conn->inType) && T_REF_VAL_GET(conn) >= 2) {
STransMsg transMsg = {0};
transMsg.msgType = conn->inType;
transMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
// transRefSrvHandle(conn);
(*pTransInst->cfp)(pTransInst->parent, &transMsg, 0);
}
}
static void destroySmsg(SSrvMsg* smsg) { static void destroySmsg(SSrvMsg* smsg) {
if (smsg == NULL) { if (smsg == NULL) {
return; return;
@ -408,6 +463,9 @@ static void destroyAllConn(SWorkThrdObj* pThrd) {
QUEUE_INIT(h); QUEUE_INIT(h);
SSrvConn* c = QUEUE_DATA(h, SSrvConn, queue); SSrvConn* c = QUEUE_DATA(h, SSrvConn, queue);
while (T_REF_VAL_GET(c) >= 2) {
transUnrefSrvHandle(c);
}
transUnrefSrvHandle(c); transUnrefSrvHandle(c);
} }
} }
@ -431,20 +489,7 @@ void uvWorkerAsyncCb(uv_async_t* handle) {
tError("unexcept occurred, continue"); tError("unexcept occurred, continue");
continue; continue;
} }
if (msg->pConn == NULL) { (*transAsyncHandle[msg->type])(msg, pThrd);
free(msg);
bool noConn = QUEUE_IS_EMPTY(&pThrd->conn);
if (noConn == true) {
uv_loop_close(pThrd->loop);
uv_stop(pThrd->loop);
} else {
destroyAllConn(pThrd);
// uv_loop_close(pThrd->loop);
pThrd->quit = true;
}
} else {
uvStartSendResp(msg);
}
} }
} }
static void uvAcceptAsyncCb(uv_async_t* async) { static void uvAcceptAsyncCb(uv_async_t* async) {
@ -632,7 +677,9 @@ static SSrvConn* createConn(void* hThrd) {
pConn->srvMsgs = taosArrayInit(2, sizeof(void*)); // pConn->srvMsgs = taosArrayInit(2, sizeof(void*)); //
tTrace("conn %p created", pConn); tTrace("conn %p created", pConn);
memset(&pConn->regArg, 0, sizeof(pConn->regArg));
pConn->broken = false; pConn->broken = false;
pConn->status = ConnNormal;
transRefSrvHandle(pConn); transRefSrvHandle(pConn);
return pConn; return pConn;
@ -748,7 +795,57 @@ End:
transCloseServer(srv); transCloseServer(srv);
return NULL; return NULL;
} }
void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) {
if (QUEUE_IS_EMPTY(&thrd->conn)) {
uv_loop_close(thrd->loop);
uv_stop(thrd->loop);
} else {
destroyAllConn(thrd);
thrd->quit = true;
}
free(msg);
}
void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd) {
// release handle to rpc init
SSrvConn* conn = msg->pConn;
if (conn->status == ConnAcquire) {
taosArrayPush(conn->srvMsgs, &msg);
if (taosArrayGetSize(conn->srvMsgs) > 1) {
return;
}
uvStartSendRespInternal(msg);
return;
} else if (conn->status == ConnRelease) {
// already release by server app, do nothing
} else if (conn->status == ConnNormal) {
// no nothing
// user should not call this rpcRelease handle;
}
free(msg);
}
void uvHandleSendResp(SSrvMsg* msg, SWorkThrdObj* thrd) {
// send msg to client
uvStartSendResp(msg);
}
void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd) {
SSrvConn* conn = msg->pConn;
if (conn->status == ConnAcquire) {
if (taosArrayGetSize(conn->srvMsgs) > 0) {
taosArrayPush(conn->srvMsgs, &msg);
return;
}
conn->regArg.notifyCount = 0;
conn->regArg.init = 1;
conn->regArg.msg = msg->msg;
if (conn->broken) {
STrans* pTransInst = conn->pTransInst;
(*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL);
memset(&conn->regArg, 0, sizeof(conn->regArg));
}
free(msg);
}
}
void destroyWorkThrd(SWorkThrdObj* pThrd) { void destroyWorkThrd(SWorkThrdObj* pThrd) {
if (pThrd == NULL) { if (pThrd == NULL) {
return; return;
@ -759,10 +856,10 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) {
free(pThrd); free(pThrd);
} }
void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { void sendQuitToWorkThrd(SWorkThrdObj* pThrd) {
SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); SSrvMsg* msg = calloc(1, sizeof(SSrvMsg));
msg->type = Quit;
tDebug("server send quit msg to work thread"); tDebug("server send quit msg to work thread");
transSendAsync(pThrd->asyncPool, &msg->q);
transSendAsync(pThrd->asyncPool, &srvMsg->q);
} }
void transCloseServer(void* arg) { void transCloseServer(void* arg) {
@ -813,8 +910,21 @@ void transUnrefSrvHandle(void* handle) {
} }
void transReleaseSrvHandle(void* handle) { void transReleaseSrvHandle(void* handle) {
// do nothing currently if (handle == NULL) {
// return;
}
SSrvConn* pConn = handle;
SWorkThrdObj* pThrd = pConn->hostThrd;
STransMsg tmsg = {.handle = handle, .code = 0};
SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg));
srvMsg->msg = tmsg;
srvMsg->type = Release;
srvMsg->pConn = pConn;
tTrace("server conn %p start to release", pConn);
transSendAsync(pThrd->asyncPool, &srvMsg->q);
} }
void transSendResponse(const STransMsg* pMsg) { void transSendResponse(const STransMsg* pMsg) {
if (pMsg->handle == NULL) { if (pMsg->handle == NULL) {
@ -826,6 +936,21 @@ void transSendResponse(const STransMsg* pMsg) {
SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg));
srvMsg->pConn = pConn; srvMsg->pConn = pConn;
srvMsg->msg = *pMsg; srvMsg->msg = *pMsg;
srvMsg->type = Normal;
tTrace("server conn %p start to send resp", pConn);
transSendAsync(pThrd->asyncPool, &srvMsg->q);
}
void transRegisterMsg(const STransMsg* msg) {
if (msg->handle == NULL) {
return;
}
SSrvConn* pConn = msg->handle;
SWorkThrdObj* pThrd = pConn->hostThrd;
SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg));
srvMsg->pConn = pConn;
srvMsg->msg = *msg;
srvMsg->type = Register;
tTrace("server conn %p start to send resp", pConn); tTrace("server conn %p start to send resp", pConn);
transSendAsync(pThrd->asyncPool, &srvMsg->q); transSendAsync(pThrd->asyncPool, &srvMsg->q);
} }

140
source/libs/transport/test/transUT.cc
View File

@ -30,29 +30,8 @@ const char *ckey = "ckey";
class Server; class Server;
int port = 7000; int port = 7000;
// server process // server process
static bool cliPersistHandle(void *parent, tmsg_t msgType) {
// client persist handle
return msgType == 2 || msgType == 4;
}
typedef struct CbArgs {
tmsg_t msgType;
} CbArgs;
static void *ConstructArgForSpecificMsgType(void *parent, tmsg_t msgType) {
if (msgType == 1 || msgType == 2) {
CbArgs *args = (CbArgs *)calloc(1, sizeof(CbArgs));
args->msgType = msgType;
return args;
}
return NULL;
}
// server except // server except
static bool handleExcept(void *parent, tmsg_t msgType) {
//
return msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP;
}
typedef void (*CB)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); typedef void (*CB)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
@ -91,23 +70,6 @@ class Client {
rpcClose(this->transCli); rpcClose(this->transCli);
this->transCli = NULL; this->transCli = NULL;
} }
void SetPersistFP(bool (*pfp)(void *parent, tmsg_t msgType)) {
rpcClose(this->transCli);
rpcInit_.pfp = pfp;
this->transCli = rpcOpen(&rpcInit_);
}
void SetConstructFP(void *(*mfp)(void *parent, tmsg_t msgType)) {
rpcClose(this->transCli);
rpcInit_.mfp = mfp;
this->transCli = rpcOpen(&rpcInit_);
}
void SetPAndMFp(bool (*pfp)(void *parent, tmsg_t msgType), void *(*mfp)(void *parent, tmsg_t msgType)) {
rpcClose(this->transCli);
rpcInit_.pfp = pfp;
rpcInit_.mfp = mfp;
this->transCli = rpcOpen(&rpcInit_);
}
void SendAndRecv(SRpcMsg *req, SRpcMsg *resp) { void SendAndRecv(SRpcMsg *req, SRpcMsg *resp) {
SEpSet epSet = {0}; SEpSet epSet = {0};
@ -126,7 +88,6 @@ class Client {
SendAndRecv(req, resp); SendAndRecv(req, resp);
} }
void SendWithHandle(SRpcMsg *req, SRpcMsg *resp) {}
void SemWait() { tsem_wait(&this->sem); } void SemWait() { tsem_wait(&this->sem); }
void SemPost() { tsem_post(&this->sem); } void SemPost() { tsem_post(&this->sem); }
void Reset() {} void Reset() {}
@ -149,7 +110,6 @@ class Server {
rpcInit_.label = (char *)label; rpcInit_.label = (char *)label;
rpcInit_.numOfThreads = 5; rpcInit_.numOfThreads = 5;
rpcInit_.cfp = processReq; rpcInit_.cfp = processReq;
rpcInit_.efp = NULL;
rpcInit_.user = (char *)user; rpcInit_.user = (char *)user;
rpcInit_.secret = (char *)secret; rpcInit_.secret = (char *)secret;
rpcInit_.ckey = (char *)ckey; rpcInit_.ckey = (char *)ckey;
@ -160,17 +120,17 @@ class Server {
this->transSrv = rpcOpen(&this->rpcInit_); this->transSrv = rpcOpen(&this->rpcInit_);
taosMsleep(1000); taosMsleep(1000);
} }
void SetSrvContinueSend(CB cb) {
this->Stop();
rpcInit_.cfp = cb;
this->Start();
}
void Stop() { void Stop() {
if (this->transSrv == NULL) return; if (this->transSrv == NULL) return;
rpcClose(this->transSrv); rpcClose(this->transSrv);
this->transSrv = NULL; this->transSrv = NULL;
} }
void SetExceptFp(bool (*efp)(void *parent, tmsg_t msgType)) { void SetSrvSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) {
this->Stop();
rpcInit_.efp = efp;
this->Start();
}
void SetSrvContinueSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) {
this->Stop(); this->Stop();
rpcInit_.cfp = cfp; rpcInit_.cfp = cfp;
this->Start(); this->Start();
@ -198,9 +158,6 @@ static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
} }
static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
for (int i = 0; i < 9; i++) {
rpcRefHandle(pMsg->handle, TAOS_CONN_SERVER);
}
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
SRpcMsg rpcMsg = {0}; SRpcMsg rpcMsg = {0};
rpcMsg.pCont = rpcMallocCont(100); rpcMsg.pCont = rpcMallocCont(100);
@ -262,23 +219,7 @@ class TransObj {
// //
srv->Stop(); srv->Stop();
} }
void SetCliPersistFp(bool (*pfp)(void *parent, tmsg_t msgType)) {
// do nothing
cli->SetPersistFP(pfp);
}
void SetCliMFp(void *(*mfp)(void *parent, tmsg_t msgType)) {
// do nothing
cli->SetConstructFP(mfp);
}
void SetCliMAndPFp(bool (*pfp)(void *parent, tmsg_t msgType), void *(*mfp)(void *parent, tmsg_t msgType)) {
// do nothing
cli->SetPAndMFp(pfp, mfp);
}
// call when link broken, and notify query or fetch stop // call when link broken, and notify query or fetch stop
void SetSrvExceptFp(bool (*efp)(void *parent, tmsg_t msgType)) {
////////
srv->SetExceptFp(efp);
}
void SetSrvContinueSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) { void SetSrvContinueSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) {
/////// ///////
srv->SetSrvContinueSend(cfp); srv->SetSrvContinueSend(cfp);
@ -315,7 +256,7 @@ class TransEnv : public ::testing::Test {
}; };
TEST_F(TransEnv, 01sendAndRec) { TEST_F(TransEnv, 01sendAndRec) {
for (int i = 0; i < 1; i++) { for (int i = 0; i < 10; i++) {
SRpcMsg req = {0}, resp = {0}; SRpcMsg req = {0}, resp = {0};
req.msgType = 0; req.msgType = 0;
req.pCont = rpcMallocCont(10); req.pCont = rpcMallocCont(10);
@ -358,51 +299,53 @@ TEST_F(TransEnv, clientUserDefined) {
} }
TEST_F(TransEnv, cliPersistHandle) { TEST_F(TransEnv, cliPersistHandle) {
tr->SetCliPersistFp(cliPersistHandle);
SRpcMsg resp = {0}; SRpcMsg resp = {0};
void * handle = NULL;
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
SRpcMsg req = {.handle = resp.handle, .noResp = 0}; SRpcMsg req = {.handle = resp.handle, .persistHandle = 1};
req.msgType = 1; req.msgType = 1;
req.pCont = rpcMallocCont(10); req.pCont = rpcMallocCont(10);
req.contLen = 10; req.contLen = 10;
tr->cliSendAndRecv(&req, &resp); tr->cliSendAndRecv(&req, &resp);
if (i == 5) {
std::cout << "stop server" << std::endl;
tr->StopSrv();
}
if (i >= 6) {
EXPECT_TRUE(resp.code != 0);
}
}
//////////////////
}
TEST_F(TransEnv, cliReleaseHandle) {
tr->SetCliPersistFp(cliPersistHandle);
SRpcMsg resp = {0};
for (int i = 0; i < 10; i++) {
SRpcMsg req = {.handle = resp.handle};
req.msgType = 1;
req.pCont = rpcMallocCont(10);
req.contLen = 10;
tr->cliSendAndRecvNoHandle(&req, &resp);
// if (i == 5) { // if (i == 5) {
// std::cout << "stop server" << std::endl; // std::cout << "stop server" << std::endl;
// tr->StopSrv(); // tr->StopSrv();
//} //}
// if (i >= 6) { // if (i >= 6) {
// EXPECT_TRUE(resp.code != 0);
//}
handle = resp.handle;
}
rpcReleaseHandle(handle, TAOS_CONN_CLIENT);
for (int i = 0; i < 10; i++) {
SRpcMsg req = {0};
req.msgType = 1;
req.pCont = rpcMallocCont(10);
req.contLen = 10;
tr->cliSendAndRecv(&req, &resp);
}
taosMsleep(1000);
//////////////////
}
TEST_F(TransEnv, cliReleaseHandle) {
SRpcMsg resp = {0};
for (int i = 0; i < 10; i++) {
SRpcMsg req = {.handle = resp.handle, .persistHandle = 1};
req.msgType = 1;
req.pCont = rpcMallocCont(10);
req.contLen = 10;
tr->cliSendAndRecvNoHandle(&req, &resp);
EXPECT_TRUE(resp.code == 0); EXPECT_TRUE(resp.code == 0);
//} //}
} }
////////////////// //////////////////
} }
TEST_F(TransEnv, cliReleaseHandleExcept) { TEST_F(TransEnv, cliReleaseHandleExcept) {
tr->SetCliPersistFp(cliPersistHandle);
SRpcMsg resp = {0}; SRpcMsg resp = {0};
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
SRpcMsg req = {.handle = resp.handle}; SRpcMsg req = {.handle = resp.handle, .persistHandle = 1};
req.msgType = 1; req.msgType = 1;
req.pCont = rpcMallocCont(10); req.pCont = rpcMallocCont(10);
req.contLen = 10; req.contLen = 10;
@ -426,12 +369,12 @@ TEST_F(TransEnv, srvContinueSend) {
req.contLen = 10; req.contLen = 10;
tr->cliSendAndRecv(&req, &resp); tr->cliSendAndRecv(&req, &resp);
} }
taosMsleep(2000); taosMsleep(1000);
} }
TEST_F(TransEnv, srvPersistHandleExcept) { TEST_F(TransEnv, srvPersistHandleExcept) {
tr->SetSrvContinueSend(processContinueSend); tr->SetSrvContinueSend(processContinueSend);
tr->SetCliPersistFp(cliPersistHandle); // tr->SetCliPersistFp(cliPersistHandle);
SRpcMsg resp = {0}; SRpcMsg resp = {0};
for (int i = 0; i < 5; i++) { for (int i = 0; i < 5; i++) {
SRpcMsg req = {.handle = resp.handle}; SRpcMsg req = {.handle = resp.handle};
@ -450,7 +393,6 @@ TEST_F(TransEnv, srvPersistHandleExcept) {
} }
TEST_F(TransEnv, cliPersistHandleExcept) { TEST_F(TransEnv, cliPersistHandleExcept) {
tr->SetSrvContinueSend(processContinueSend); tr->SetSrvContinueSend(processContinueSend);
tr->SetCliPersistFp(cliPersistHandle);
SRpcMsg resp = {0}; SRpcMsg resp = {0};
for (int i = 0; i < 5; i++) { for (int i = 0; i < 5; i++) {
SRpcMsg req = {.handle = resp.handle}; SRpcMsg req = {.handle = resp.handle};
@ -471,11 +413,7 @@ TEST_F(TransEnv, cliPersistHandleExcept) {
TEST_F(TransEnv, multiCliPersistHandleExcept) { TEST_F(TransEnv, multiCliPersistHandleExcept) {
// conn broken // conn broken
} }
TEST_F(TransEnv, queryExcept) { TEST_F(TransEnv, queryExcept) {}
tr->SetSrvExceptFp(handleExcept);
// query and conn is broken
}
TEST_F(TransEnv, noResp) { TEST_F(TransEnv, noResp) {
SRpcMsg resp = {0}; SRpcMsg resp = {0};
for (int i = 0; i < 5; i++) { for (int i = 0; i < 5; i++) {

94
source/libs/transport/test/transportTests.cc
View File

@ -136,4 +136,98 @@ TEST_F(QueueEnv, testIter) {
assert(result.size() == vals.size()); assert(result.size() == vals.size());
} }
class TransCtxEnv : public ::testing::Test {
protected:
virtual void SetUp() {
ctx = (STransCtx *)calloc(1, sizeof(STransCtx));
transCtxInit(ctx);
// TODO
}
virtual void TearDown() {
transCtxDestroy(ctx);
// formate
}
STransCtx *ctx;
};
TEST_F(TransCtxEnv, mergeTest) {
int key = 1;
{
STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx));
transCtxInit(src);
{
STransCtxVal val1 = {.val = NULL, .len = 0, .free = free};
val1.val = malloc(12);
val1.len = 12;
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++;
}
{
STransCtxVal val1 = {.val = NULL, .len = 0, .free = free};
val1.val = malloc(12);
val1.len = 12;
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++;
}
transCtxMerge(ctx, src);
free(src);
}
EXPECT_EQ(2, taosHashGetSize(ctx->args));
{
STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx));
transCtxInit(src);
{
STransCtxVal val1 = {.val = NULL, .len = 0, .free = free};
val1.val = malloc(12);
val1.len = 12;
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++;
}
{
STransCtxVal val1 = {.val = NULL, .len = 0, .free = free};
val1.val = malloc(12);
val1.len = 12;
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++;
}
transCtxMerge(ctx, src);
free(src);
}
std::string val("Hello");
EXPECT_EQ(4, taosHashGetSize(ctx->args));
{
key = 1;
STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx));
transCtxInit(src);
{
STransCtxVal val1 = {.val = NULL, .len = 0, .free = free};
val1.val = calloc(1, 11);
memcpy(val1.val, val.c_str(), val.size());
val1.len = 11;
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++;
}
{
STransCtxVal val1 = {.val = NULL, .len = 0, .free = free};
val1.val = calloc(1, 11);
memcpy(val1.val, val.c_str(), val.size());
val1.len = 11;
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++;
}
transCtxMerge(ctx, src);
free(src);
}
EXPECT_EQ(4, taosHashGetSize(ctx->args));
char *skey = (char *)transCtxDumpVal(ctx, 1);
EXPECT_EQ(0, strcmp(skey, val.c_str()));
free(skey);
skey = (char *)transCtxDumpVal(ctx, 2);
EXPECT_EQ(0, strcmp(skey, val.c_str()));
}
#endif #endif

358
source/os/src/osString.c
View File

@ -15,10 +15,8 @@
#define ALLOW_FORBID_FUNC #define ALLOW_FORBID_FUNC
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include <malloc.h>
#include "os.h" #include "os.h"
// #include "tdef.h"
// #include <wchar.h>
// #include <wctype.h>
#ifndef DISALLOW_NCHAR_WITHOUT_ICONV #ifndef DISALLOW_NCHAR_WITHOUT_ICONV
#include "iconv.h" #include "iconv.h"
@ -32,14 +30,6 @@ int64_t taosStr2int64(const char *str) {
return strtoll(str, &endptr, 10); return strtoll(str, &endptr, 10);
} }
bool taosCheckNcharValid(void) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
return false;
#else
return true;
#endif
}
int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) {
for (int32_t i = 0; i < bytes; i += sizeof(TdUcs4)) { for (int32_t i = 0; i < bytes; i += sizeof(TdUcs4)) {
int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i); int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i);
@ -77,11 +67,13 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) {
TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) { TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) {
memcpy(target_ucs4, source_ucs4, len_ucs4*sizeof(TdUcs4)); assert(malloc_usable_size(target_ucs4)>=len_ucs4*sizeof(TdUcs4));
return memcpy(target_ucs4, source_ucs4, len_ucs4*sizeof(TdUcs4));
} }
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV #ifdef DISALLOW_NCHAR_WITHOUT_ICONV
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n");
return -1; return -1;
#else #else
iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
@ -99,6 +91,7 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) { bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV #ifdef DISALLOW_NCHAR_WITHOUT_ICONV
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n");
return -1; return -1;
#else #else
memset(ucs4, 0, ucs4_max_len); memset(ucs4, 0, ucs4_max_len);
@ -124,7 +117,8 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4
bool taosValidateEncodec(const char *encodec) { bool taosValidateEncodec(const char *encodec) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV #ifdef DISALLOW_NCHAR_WITHOUT_ICONV
return false; printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n");
return true;
#else #else
iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC); iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC);
if (cd == (iconv_t)(-1)) { if (cd == (iconv_t)(-1)) {
@ -164,341 +158,3 @@ int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size) { ret
int32_t taosWcharToMb(char *pStr, TdWchar wchar) { return wctomb(pStr, wchar); } int32_t taosWcharToMb(char *pStr, TdWchar wchar) { return wctomb(pStr, wchar); }
int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size) { return wcstombs(pStrs, pWchars, size); } int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size) { return wcstombs(pStrs, pWchars, size); }
// #ifdef USE_LIBICONV
// #include "iconv.h"
// int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
// iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
// size_t ucs4_input_len = ucs4_max_len;
// size_t outLen = ucs4_max_len;
// if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
// iconv_close(cd);
// return -1;
// }
// iconv_close(cd);
// return (int32_t)(ucs4_max_len - outLen);
// }
// bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
// memset(ucs4, 0, ucs4_max_len);
// iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
// size_t ucs4_input_len = mbsLength;
// size_t outLeft = ucs4_max_len;
// if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) {
// iconv_close(cd);
// return false;
// }
// iconv_close(cd);
// if (len != NULL) {
// *len = (int32_t)(ucs4_max_len - outLeft);
// if (*len < 0) {
// return false;
// }
// }
// return true;
// }
// bool taosValidateEncodec(const char *encodec) {
// iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC);
// if (cd == (iconv_t)(-1)) {
// return false;
// }
// iconv_close(cd);
// return true;
// }
// #else
// int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
// mbstate_t state = {0};
// int32_t len = (int32_t)wcsnrtombs(NULL, (const wchar_t **)&ucs4, ucs4_max_len / 4, 0, &state);
// if (len < 0) {
// return -1;
// }
// memset(&state, 0, sizeof(state));
// len = wcsnrtombs(mbs, (const wchar_t **)&ucs4, ucs4_max_len / 4, (size_t)len, &state);
// if (len < 0) {
// return -1;
// }
// return len;
// }
// bool taosMbsToUcs4(const char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
// memset(ucs4, 0, ucs4_max_len);
// mbstate_t state = {0};
// int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state);
// *len = retlen;
// return retlen >= 0;
// }
// bool taosValidateEncodec(const char *encodec) {
// return true;
// }
// #endif
// #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
// /*
// * windows implementation
// */
// #ifdef HAVE_CONFIG_H
// #include <config.h>
// #endif
// #include <sys/types.h>
// #include <stdio.h>
// #include <assert.h>
// #include <errno.h>
// #include <stdint.h>
// #if STDC_HEADERS
// #include <stdlib.h>
// #else
// char *malloc(), *realloc();
// #endif
// /* Always add at least this many bytes when extending the buffer. */
// #define MIN_CHUNK 64
// /* Read up to (and including) a TERMINATOR from STREAM into *LINEPTR
// + OFFSET (and null-terminate it). *LINEPTR is a pointer returned from
// malloc (or NULL), pointing to *N characters of space. It is realloc'd
// as necessary. Return the number of characters read (not including the
// null terminator), or -1 on error or EOF. On a -1 return, the caller
// should check feof(), if not then errno has been set to indicate
// the error. */
// int32_t getstr(char **lineptr, size_t *n, FILE *stream, char terminator, int32_t offset) {
// int32_t nchars_avail; /* Allocated but unused chars in *LINEPTR. */
// char * read_pos; /* Where we're reading into *LINEPTR. */
// int32_t ret;
// if (!lineptr || !n || !stream) {
// errno = EINVAL;
// return -1;
// }
// if (!*lineptr) {
// *n = MIN_CHUNK;
// *lineptr = malloc(*n);
// if (!*lineptr) {
// errno = ENOMEM;
// return -1;
// }
// }
// nchars_avail = (int32_t)(*n - offset);
// read_pos = *lineptr + offset;
// for (;;) {
// int32_t save_errno;
// register int32_t c = getc(stream);
// save_errno = errno;
// /* We always want at least one char left in the buffer, since we
// always (unless we get an error while reading the first char)
// NUL-terminate the line buffer. */
// assert((*lineptr + *n) == (read_pos + nchars_avail));
// if (nchars_avail < 2) {
// if (*n > MIN_CHUNK)
// *n *= 2;
// else
// *n += MIN_CHUNK;
// nchars_avail = (int32_t)(*n + *lineptr - read_pos);
// char* lineptr1 = realloc(*lineptr, *n);
// if (!lineptr1) {
// errno = ENOMEM;
// return -1;
// }
// *lineptr = lineptr1;
// read_pos = *n - nchars_avail + *lineptr;
// assert((*lineptr + *n) == (read_pos + nchars_avail));
// }
// if (ferror(stream)) {
// /* Might like to return partial line, but there is no
// place for us to store errno. And we don't want to just
// lose errno. */
// errno = save_errno;
// return -1;
// }
// if (c == EOF) {
// /* Return partial line, if any. */
// if (read_pos == *lineptr)
// return -1;
// else
// break;
// }
// *read_pos++ = c;
// nchars_avail--;
// if (c == terminator) /* Return the line. */
// break;
// }
// /* Done - NUL terminate and return the number of chars read. */
// *read_pos = '\0';
// ret = (int32_t)(read_pos - (*lineptr + offset));
// return ret;
// }
// int32_t tgetline(char **lineptr, size_t *n, FILE *stream) { return getstr(lineptr, n, stream, '\n', 0); }
// /*
// * Get next token from string *stringp, where tokens are possibly-empty
// * strings separated by characters from delim.
// *
// * Writes NULs into the string at *stringp to end tokens.
// * delim need not remain constant from call to call.
// * On return, *stringp points past the last NUL written (if there might
// * be further tokens), or is NULL (if there are definitely no moretokens).
// *
// * If *stringp is NULL, strsep returns NULL.
// */
// char *strsep(char **stringp, const char *delim) {
// char * s;
// const char *spanp;
// int32_t c, sc;
// char *tok;
// if ((s = *stringp) == NULL)
// return (NULL);
// for (tok = s;;) {
// c = *s++;
// spanp = delim;
// do {
// if ((sc = *spanp++) == c) {
// if (c == 0)
// s = NULL;
// else
// s[-1] = 0;
// *stringp = s;
// return (tok);
// }
// } while (sc != 0);
// }
// /* NOTREACHED */
// }
// char *getpass(const char *prefix) {
// static char passwd[TSDB_PASSWORD_LEN] = {0};
// memset(passwd, 0, TSDB_PASSWORD_LEN);
// //printf("%s", prefix);
// int32_t index = 0;
// char ch;
// while (index < TSDB_PASSWORD_LEN) {
// ch = getch();
// if (ch == '\n' || ch == '\r') {
// break;
// } else {
// passwd[index++] = ch;
// }
// }
// return passwd;
// }
// int32_t twcslen(const wchar_t *wcs) {
// int32_t *wstr = (int32_t *)wcs;
// if (NULL == wstr) {
// return 0;
// }
// int32_t n = 0;
// while (1) {
// if (0 == *wstr++) {
// break;
// }
// n++;
// }
// return n;
// }
// int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes) {
// for (int32_t i = 0; i < bytes; i += TSDB_NCHAR_SIZE) {
// int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i);
// int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i);
// if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) {
// return f1 - f2;
// } else if (f1 == 0 && f2 == 0) {
// return 0;
// }
// if (f1 != f2) {
// return f1 - f2;
// }
// }
// return 0;
// #if 0
// int32_t ucs4_max_len = bytes + 4;
// char *f1_mbs = calloc(bytes, 1);
// char *f2_mbs = calloc(bytes, 1);
// if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) {
// return -1;
// }
// if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) {
// return -1;
// }
// int32_t ret = strcmp(f1_mbs, f2_mbs);
// free(f1_mbs);
// free(f2_mbs);
// return ret;
// #endif
// }
// /* Copy memory to memory until the specified number of bytes
// has been copied, return pointer to following byte.
// Overlap is NOT handled correctly. */
// void *mempcpy(void *dest, const void *src, size_t len) {
// return (char*)memcpy(dest, src, len) + len;
// }
// /* Copy SRC to DEST, returning the address of the terminating '\0' in DEST. */
// char *stpcpy (char *dest, const char *src) {
// size_t len = strlen (src);
// return (char*)memcpy(dest, src, len + 1) + len;
// }
// /* Copy no more than N characters of SRC to DEST, returning the address of
// the terminating '\0' in DEST, if any, or else DEST + N. */
// char *stpncpy (char *dest, const char *src, size_t n) {
// size_t size = strnlen (src, n);
// memcpy (dest, src, size);
// dest += size;
// if (size == n)
// return dest;
// return memset (dest, '\0', n - size);
// }
// #else
// /*
// * linux and darwin implementation
// */
// int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize) {
// return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / ncharSize);
// }
// #endif

28
source/os/src/osTimezone.c
View File

@ -48,16 +48,11 @@
void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) { void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) {
if (inTimezone == NULL || inTimezone[0] == 0) return; if (inTimezone == NULL || inTimezone[0] == 0) return;
#ifdef WINDOWS #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
char winStr[TD_LOCALE_LEN * 2]; char winStr[TD_LOCALE_LEN * 2];
sprintf(winStr, "TZ=%s", inTimezone); sprintf(winStr, "TZ=%s", inTimezone);
putenv(winStr); putenv(winStr);
#else
setenv("TZ", inTimezone, 1);
#endif
tzset(); tzset();
/*
* get CURRENT time zone. * get CURRENT time zone.
* system current time zone is affected by daylight saving time(DST) * system current time zone is affected by daylight saving time(DST)
* *
@ -75,15 +70,34 @@ void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *ou
int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR);
tz += daylight; tz += daylight;
/* /*
* format: * format:
* (CST, +0800) * (CST, +0800)
* (BST, +0100) * (BST, +0100)
*/ */
sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
*outDaylight = daylight;
#elif defined(_TD_DARWIN_64)
setenv("TZ", inTimezone, 1);
tzset();
int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR);
tz += daylight;
sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
*outDaylight = daylight; *outDaylight = daylight;
#else
setenv("TZ", inTimezone, 1);
tzset();
int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR);
tz += daylight;
sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
*outDaylight = daylight;
#endif
} }
void taosGetSystemTimezone(char *outTimezone) { void taosGetSystemTimezone(char *outTimezone) {

25
source/util/src/tjson.c
View File

@ -202,9 +202,16 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV
return (errno == ERANGE||errno == EINVAL) ? TSDB_CODE_FAILED:TSDB_CODE_SUCCESS; return (errno == ERANGE||errno == EINVAL) ? TSDB_CODE_FAILED:TSDB_CODE_SUCCESS;
} }
int32_t tjsonGetUIntValue(const SJson* pJson, const char* pName, uint32_t* pVal) {
uint64_t val = 0;
int32_t code = tjsonGetUBigIntValue(pJson, pName, &val);
*pVal = val;
return code;
}
int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal) { int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal) {
uint64_t val = 0; uint64_t val = 0;
int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); int32_t code = tjsonGetUBigIntValue(pJson, pName, &val);
*pVal = val; *pVal = val;
return code; return code;
} }
@ -239,6 +246,22 @@ int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, voi
return func(pJsonObj, pObj); return func(pJsonObj, pObj);
} }
int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, void** pObj, int32_t objSize) {
if (objSize <= 0) {
return TSDB_CODE_SUCCESS;
}
SJson* pJsonObj = tjsonGetObjectItem(pJson, pName);
if (NULL == pJsonObj) {
return TSDB_CODE_FAILED;
}
*pObj = calloc(1, objSize);
if (NULL == *pObj) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return func(pJsonObj, *pObj);
}
int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize) { int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize) {
const cJSON* jArray = tjsonGetObjectItem(pJson, pName); const cJSON* jArray = tjsonGetObjectItem(pJson, pName);
int32_t size = (NULL == jArray ? 0 : tjsonGetArraySize(jArray)); int32_t size = (NULL == jArray ? 0 : tjsonGetArraySize(jArray));

5
tests/script/tsim/insert/basic0.sim
View File

@ -26,9 +26,10 @@ endi
print =============== create child table print =============== create child table
sql create table ct1 using stb tags(1000) sql create table ct1 using stb tags(1000)
sql create table ct2 using stb tags(2000) sql create table ct2 using stb tags(2000)
sql create table ct3 using stb tags(3000)
sql show tables sql show tables
if $rows != 2 then if $rows != 3 then
return -1 return -1
endi endi
@ -45,6 +46,8 @@ sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 1
sql_error insert into ct1 values(now+4s, -14, -2.4, -3.4) ct2 values(now+4s, -14, -2.4, -3.4) sql_error insert into ct1 values(now+4s, -14, -2.4, -3.4) ct2 values(now+4s, -14, -2.4, -3.4)
sql_error insert into ct1 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) ct2 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) sql_error insert into ct1 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) ct2 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6)
sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0)
#=================================================================== #===================================================================
#=================================================================== #===================================================================
print =============== query data from child table print =============== query data from child table

305
tests/script/tsim/query/interval-offset.sim Normal file
View File

@ -0,0 +1,305 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 500
sql connect
print =============== create database
sql drop database d0 -x step1
step1:
sql create database d0
sql show databases
if $rows != 1 then
return -1
endi
sql use d0
print =============== create super table and child table
sql create table stb (ts timestamp, tbcol int) tags (t1 int)
sql show stables
print $rows $data00 $data01 $data02
if $rows != 1 then
return -1
endi
sql create table ct1 using stb tags ( 1 )
sql create table ct2 using stb tags ( 2 )
sql show tables
print $rows $data00 $data10 $data20
if $rows != 2 then
return -1
endi
print =============== insert data into child table
sql insert into ct1 values ( '2022-01-01 01:01:01.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:06.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:10.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:16.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:20.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:26.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:30.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:36.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:40.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:46.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:50.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:56.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:00.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:06.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:10.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:16.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:20.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:26.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:30.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:36.000', 1 )
sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s)
print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s)
print ===> $rows $data00 $data01 $data02 $data03 $data04
if $rows != 10 then
return -1
endi
if $data00 != 2 then
return -1
endi
if $data04 != 2 then
return -1
endi
sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-01 12:00:01.000', 2 )
sql insert into ct2 values ( '2022-01-01 23:00:01.000', 3 )
sql insert into ct2 values ( '2022-01-02 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-03 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-04 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-05 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-06 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-07 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-08 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-09 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-10 10:00:01.000', 1 )
sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h)
print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2w)
print ===> rows0: $data00 $data01 $data02 $data03 $data04
print ===> rows1: $data10 $data11 $data12 $data13 $data14
print ===> rows2: $data20 $data21 $data22 $data23 $data24
if $rows != 11 then
return -1
endi
if $data00 != 1 then
return -1
endi
if $data10 != 2 then
return -1
endi
return
sql select count(*) from car interval(1n, 10d) order by ts desc
# tdSql.checkData(0, 1, 1)
# tdSql.checkData(1, 1, 2)
# tdSql.checkData(2, 1, 3)
# tdSql.checkData(3, 1, 3)
# tdSql.checkData(4, 1, 6)
# tdSql.checkData(5, 1, 1)
# tdSql.checkData(6, 1, 1)
#
sql select count(*) from car interval(2n, 5d)
# tdSql.checkData(0, 1, 1)
# tdSql.checkData(1, 1, 1)
# tdSql.checkData(2, 1, 6)
# tdSql.checkData(3, 1, 6)
# tdSql.checkData(4, 1, 3)
sql select count(*) from car interval(2n) order by ts desc
# tdSql.checkData(0, 1, 3)
# tdSql.checkData(1, 1, 6)
# tdSql.checkData(2, 1, 6)
# tdSql.checkData(3, 1, 1)
# tdSql.checkData(4, 1, 1)
#
sql select count(*) from car interval(1y, 1n)
# tdSql.checkData(0, 1, 1)
# tdSql.checkData(1, 1, 8)
# tdSql.checkData(2, 1, 8)
#
sql select count(*) from car interval(1y, 2n)
# tdSql.checkData(0, 1, 1)
# tdSql.checkData(1, 1, 11)
# tdSql.checkData(2, 1, 5)
sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d)
# tdSql.checkData(0, 1, 6)
# tdSql.checkData(1, 1, 9)
sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int)
print ====== start create child tables and insert data
$i = 0
while $i < $tbNum
$tb = $tbPrefix . $i
sql create table $tb using $mt tags( $i )
$x = 0
while $x < $rowNum
$cc = $x * 60000
$ms = 1601481600000 + $cc
sql insert into $tb values ($ms , $x )
$x = $x + 1
endw
$i = $i + 1
endw
print =============== step2
$i = 1
$tb = $tbPrefix . $i
sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m)
print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m)
print ===> $rows $data01 $data05
if $rows != $rowNum then
return -1
endi
if $data00 != 1 then
return -1
endi
if $data04 != 1 then
return -1
endi
#print =============== step3
#$cc = 4 * 60000
#$ms = 1601481600000 + $cc
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m)
#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m)
#print ===> $rows $data01 $data05
#if $rows != 5 then
# return -1
#endi
#if $data00 != 1 then
# return -1
#endi
#if $data04 != 1 then
# return -1
#endi
#print =============== step4
#$cc = 40 * 60000
#$ms = 1601481600000 + $cc
#$cc = 1 * 60000
#$ms2 = 1601481600000 - $cc
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m)
#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m)
#print ===> $rows $data01 $data05
#if $rows != 20 then
# return -1
#endi
#if $data00 != 1 then
# return -1
#endi
#if $data04 != 1 then
# return -1
#endi
#print =============== step5
#$cc = 40 * 60000
#$ms = 1601481600000 + $cc
#$cc = 1 * 60000
#$ms2 = 1601481600000 - $cc
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0)
#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0)
#print ===> $rows $data21 $data25
#if $rows != 42 then
# return -1
#endi
#if $data20 != 1 then
# return -1
#endi
#if $data24 != 1 then
# return -1
#endi
#print =============== step6
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m)
#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m)
#print ===> $rows $data11
#if $rows != 20 then
# return -1
#endi
#if $data11 != 10 then
# return -1
#endi
#print =============== step7
#$cc = 4 * 60000
#$ms = 1601481600000 + $cc
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m)
#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m)
#print ===> $rows $data11
#if $rows != 5 then
# return -1
#endi
#if $data11 != 10 then
# return -1
#endi
#print =============== step8
#$cc = 40 * 60000
#$ms1 = 1601481600000 + $cc
#
#$cc = 1 * 60000
#$ms2 = 1601481600000 - $cc
#
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m)
#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m)
#print ===> $rows $data11
#if $rows != 20 then
# return -1
#endi
#if $data11 != 10 then
# return -1
#endi
#
#print =============== step9
#$cc = 40 * 60000
#$ms1 = 1601481600000 + $cc
#
#$cc = 1 * 60000
#$ms2 = 1601481600000 - $cc
#
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0)
#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0)
#print ===> $rows $data11
#if $rows != 42 then
# return -1
#endi
#if $data11 != 10 then
# return -1
#endi
print =============== clear
#sql drop database $db
#sql show databases
#if $rows != 0 then
# return -1
#endi
#system sh/exec.sh -n dnode1 -s stop -x SIGINT

2
tests/script/tsim/query/interval.sim
View File

@ -3,7 +3,7 @@ system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c wal -v 1 system sh/cfg.sh -n dnode1 -c wal -v 1
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 2000 sleep 500
sql connect sql connect
$dbPrefix = m_in_db $dbPrefix = m_in_db