xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/shm

This commit is contained in:
Shengliang Guan 2022-03-20 00:30:24 +08:00
parent fd7f61a069 a4ea6c7191
commit 916e569229
93 changed files with 7494 additions and 4296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
include/common/tmsg.h
View File

@ -115,11 +115,12 @@ typedef enum _mgmt_table {
#define TSDB_ALTER_TABLE_DROP_TAG 2
#define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3
#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4
#define TSDB_ALTER_TABLE_ADD_COLUMN 5
#define TSDB_ALTER_TABLE_DROP_COLUMN 6
#define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES 7
#define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8
#define TSDB_ALTER_TABLE_UPDATE_OPTIONS 9
#define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME 10
#define TSDB_FILL_NONE 0
#define TSDB_FILL_NULL 1
@ -2053,27 +2054,19 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) {
buf = taosDecodeFixedI64(buf, &pSma->sliding);
if (pSma->exprLen > 0) {
pSma->expr = (char*)calloc(pSma->exprLen, 1);
if (pSma->expr != NULL) {
buf = taosDecodeStringTo(buf, pSma->expr);
} else {
if ((buf = taosDecodeString(buf, &pSma->expr)) == NULL) {
tdDestroyTSma(pSma);
return NULL;
}
} else {
pSma->expr = NULL;
}
if (pSma->tagsFilterLen > 0) {
pSma->tagsFilter = (char*)calloc(pSma->tagsFilterLen, 1);
if (pSma->tagsFilter != NULL) {
buf = taosDecodeStringTo(buf, pSma->tagsFilter);
} else {
if ((buf = taosDecodeString(buf, &pSma->tagsFilter)) == NULL) {
tdDestroyTSma(pSma);
return NULL;
}
} else {
pSma->tagsFilter = NULL;
}

263
include/common/ttokendef.h
View File

@ -34,128 +34,147 @@
#define TK_NK_REM 16
#define TK_NK_CONCAT 17
#define TK_CREATE 18
#define TK_USER 19
#define TK_PASS 20
#define TK_NK_STRING 21
#define TK_ALTER 22
#define TK_PRIVILEGE 23
#define TK_DROP 24
#define TK_SHOW 25
#define TK_USERS 26
#define TK_DNODE 27
#define TK_PORT 28
#define TK_NK_INTEGER 29
#define TK_DNODES 30
#define TK_NK_ID 31
#define TK_NK_IPTOKEN 32
#define TK_QNODE 33
#define TK_ON 34
#define TK_QNODES 35
#define TK_DATABASE 36
#define TK_DATABASES 37
#define TK_USE 38
#define TK_IF 39
#define TK_NOT 40
#define TK_EXISTS 41
#define TK_BLOCKS 42
#define TK_CACHE 43
#define TK_CACHELAST 44
#define TK_COMP 45
#define TK_DAYS 46
#define TK_FSYNC 47
#define TK_MAXROWS 48
#define TK_MINROWS 49
#define TK_KEEP 50
#define TK_PRECISION 51
#define TK_QUORUM 52
#define TK_REPLICA 53
#define TK_TTL 54
#define TK_WAL 55
#define TK_VGROUPS 56
#define TK_SINGLE_STABLE 57
#define TK_STREAM_MODE 58
#define TK_TABLE 59
#define TK_NK_LP 60
#define TK_NK_RP 61
#define TK_STABLE 62
#define TK_TABLES 63
#define TK_STABLES 64
#define TK_USING 65
#define TK_TAGS 66
#define TK_NK_DOT 67
#define TK_NK_COMMA 68
#define TK_COMMENT 69
#define TK_BOOL 70
#define TK_TINYINT 71
#define TK_SMALLINT 72
#define TK_INT 73
#define TK_INTEGER 74
#define TK_BIGINT 75
#define TK_FLOAT 76
#define TK_DOUBLE 77
#define TK_BINARY 78
#define TK_TIMESTAMP 79
#define TK_NCHAR 80
#define TK_UNSIGNED 81
#define TK_JSON 82
#define TK_VARCHAR 83
#define TK_MEDIUMBLOB 84
#define TK_BLOB 85
#define TK_VARBINARY 86
#define TK_DECIMAL 87
#define TK_SMA 88
#define TK_INDEX 89
#define TK_FULLTEXT 90
#define TK_FUNCTION 91
#define TK_INTERVAL 92
#define TK_TOPIC 93
#define TK_AS 94
#define TK_MNODES 95
#define TK_NK_FLOAT 96
#define TK_NK_BOOL 97
#define TK_NK_VARIABLE 98
#define TK_BETWEEN 99
#define TK_IS 100
#define TK_NULL 101
#define TK_NK_LT 102
#define TK_NK_GT 103
#define TK_NK_LE 104
#define TK_NK_GE 105
#define TK_NK_NE 106
#define TK_NK_EQ 107
#define TK_LIKE 108
#define TK_MATCH 109
#define TK_NMATCH 110
#define TK_IN 111
#define TK_FROM 112
#define TK_JOIN 113
#define TK_INNER 114
#define TK_SELECT 115
#define TK_DISTINCT 116
#define TK_WHERE 117
#define TK_PARTITION 118
#define TK_BY 119
#define TK_SESSION 120
#define TK_STATE_WINDOW 121
#define TK_SLIDING 122
#define TK_FILL 123
#define TK_VALUE 124
#define TK_NONE 125
#define TK_PREV 126
#define TK_LINEAR 127
#define TK_NEXT 128
#define TK_GROUP 129
#define TK_HAVING 130
#define TK_ORDER 131
#define TK_SLIMIT 132
#define TK_SOFFSET 133
#define TK_LIMIT 134
#define TK_OFFSET 135
#define TK_ASC 136
#define TK_DESC 137
#define TK_NULLS 138
#define TK_FIRST 139
#define TK_LAST 140
#define TK_ACCOUNT 19
#define TK_NK_ID 20
#define TK_PASS 21
#define TK_NK_STRING 22
#define TK_ALTER 23
#define TK_PPS 24
#define TK_TSERIES 25
#define TK_STORAGE 26
#define TK_STREAMS 27
#define TK_QTIME 28
#define TK_DBS 29
#define TK_USERS 30
#define TK_CONNS 31
#define TK_STATE 32
#define TK_USER 33
#define TK_PRIVILEGE 34
#define TK_DROP 35
#define TK_SHOW 36
#define TK_DNODE 37
#define TK_PORT 38
#define TK_NK_INTEGER 39
#define TK_DNODES 40
#define TK_NK_IPTOKEN 41
#define TK_LOCAL 42
#define TK_QNODE 43
#define TK_ON 44
#define TK_QNODES 45
#define TK_DATABASE 46
#define TK_DATABASES 47
#define TK_USE 48
#define TK_IF 49
#define TK_NOT 50
#define TK_EXISTS 51
#define TK_BLOCKS 52
#define TK_CACHE 53
#define TK_CACHELAST 54
#define TK_COMP 55
#define TK_DAYS 56
#define TK_FSYNC 57
#define TK_MAXROWS 58
#define TK_MINROWS 59
#define TK_KEEP 60
#define TK_PRECISION 61
#define TK_QUORUM 62
#define TK_REPLICA 63
#define TK_TTL 64
#define TK_WAL 65
#define TK_VGROUPS 66
#define TK_SINGLE_STABLE 67
#define TK_STREAM_MODE 68
#define TK_RETENTIONS 69
#define TK_FILE_FACTOR 70
#define TK_NK_FLOAT 71
#define TK_TABLE 72
#define TK_NK_LP 73
#define TK_NK_RP 74
#define TK_STABLE 75
#define TK_TABLES 76
#define TK_STABLES 77
#define TK_ADD 78
#define TK_COLUMN 79
#define TK_MODIFY 80
#define TK_RENAME 81
#define TK_TAG 82
#define TK_SET 83
#define TK_NK_EQ 84
#define TK_USING 85
#define TK_TAGS 86
#define TK_NK_DOT 87
#define TK_NK_COMMA 88
#define TK_COMMENT 89
#define TK_BOOL 90
#define TK_TINYINT 91
#define TK_SMALLINT 92
#define TK_INT 93
#define TK_INTEGER 94
#define TK_BIGINT 95
#define TK_FLOAT 96
#define TK_DOUBLE 97
#define TK_BINARY 98
#define TK_TIMESTAMP 99
#define TK_NCHAR 100
#define TK_UNSIGNED 101
#define TK_JSON 102
#define TK_VARCHAR 103
#define TK_MEDIUMBLOB 104
#define TK_BLOB 105
#define TK_VARBINARY 106
#define TK_DECIMAL 107
#define TK_SMA 108
#define TK_ROLLUP 109
#define TK_INDEX 110
#define TK_FULLTEXT 111
#define TK_FUNCTION 112
#define TK_INTERVAL 113
#define TK_TOPIC 114
#define TK_AS 115
#define TK_MNODES 116
#define TK_NK_BOOL 117
#define TK_NK_VARIABLE 118
#define TK_BETWEEN 119
#define TK_IS 120
#define TK_NULL 121
#define TK_NK_LT 122
#define TK_NK_GT 123
#define TK_NK_LE 124
#define TK_NK_GE 125
#define TK_NK_NE 126
#define TK_LIKE 127
#define TK_MATCH 128
#define TK_NMATCH 129
#define TK_IN 130
#define TK_FROM 131
#define TK_JOIN 132
#define TK_INNER 133
#define TK_SELECT 134
#define TK_DISTINCT 135
#define TK_WHERE 136
#define TK_PARTITION 137
#define TK_BY 138
#define TK_SESSION 139
#define TK_STATE_WINDOW 140
#define TK_SLIDING 141
#define TK_FILL 142
#define TK_VALUE 143
#define TK_NONE 144
#define TK_PREV 145
#define TK_LINEAR 146
#define TK_NEXT 147
#define TK_GROUP 148
#define TK_HAVING 149
#define TK_ORDER 150
#define TK_SLIMIT 151
#define TK_SOFFSET 152
#define TK_LIMIT 153
#define TK_OFFSET 154
#define TK_ASC 155
#define TK_DESC 156
#define TK_NULLS 157
#define TK_FIRST 158
#define TK_LAST 159
#define TK_NK_SPACE 300
#define TK_NK_COMMENT 301

2
include/common/ttypes.h
View File

@ -27,7 +27,7 @@ extern "C" {
typedef int32_t VarDataOffsetT;
typedef uint32_t TDRowLenT;
typedef uint8_t TDRowValT;
typedef uint16_t col_id_t;
typedef int16_t col_id_t;
typedef int8_t col_type_t;
#pragma pack(push, 1)

25
include/libs/nodes/cmdnodes.h
View File

@ -127,6 +127,18 @@ typedef struct SDropSuperTableStmt {
bool ignoreNotExists;
} SDropSuperTableStmt;
typedef struct SAlterTableStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN];
int8_t alterType;
char colName[TSDB_COL_NAME_LEN];
char newColName[TSDB_COL_NAME_LEN];
STableOptions* pOptions;
SDataType dataType;
SValueNode* pVal;
} SAlterTableStmt;
typedef struct SCreateUserStmt {
ENodeType type;
char useName[TSDB_USER_LEN];
@ -158,6 +170,13 @@ typedef struct SDropDnodeStmt {
int32_t port;
} SDropDnodeStmt;
typedef struct SAlterDnodeStmt {
ENodeType type;
int32_t dnodeId;
char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_DNODE_VALUE_LEN];
} SAlterDnodeStmt;
typedef struct SShowStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
@ -215,6 +234,12 @@ typedef struct SDropTopicStmt {
bool ignoreNotExists;
} SDropTopicStmt;
typedef struct SAlterLocalStmt {
ENodeType type;
char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_DNODE_VALUE_LEN];
} SAlterLocalStmt;
#ifdef __cplusplus
}
#endif

4
include/libs/nodes/nodes.h
View File

@ -85,6 +85,7 @@ typedef enum ENodeType {
QUERY_NODE_DROP_TABLE_CLAUSE,
QUERY_NODE_DROP_TABLE_STMT,
QUERY_NODE_DROP_SUPER_TABLE_STMT,
QUERY_NODE_ALTER_TABLE_STMT,
QUERY_NODE_SHOW_TABLES_STMT, // temp
QUERY_NODE_SHOW_STABLES_STMT,
QUERY_NODE_CREATE_USER_STMT,
@ -94,6 +95,7 @@ typedef enum ENodeType {
QUERY_NODE_USE_DATABASE_STMT,
QUERY_NODE_CREATE_DNODE_STMT,
QUERY_NODE_DROP_DNODE_STMT,
QUERY_NODE_ALTER_DNODE_STMT,
QUERY_NODE_SHOW_DNODES_STMT,
QUERY_NODE_SHOW_VGROUPS_STMT,
QUERY_NODE_SHOW_MNODES_STMT,
@ -104,6 +106,7 @@ typedef enum ENodeType {
QUERY_NODE_DROP_QNODE_STMT,
QUERY_NODE_CREATE_TOPIC_STMT,
QUERY_NODE_DROP_TOPIC_STMT,
QUERY_NODE_ALTER_LOCAL_STMT,
// logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN,
@ -163,6 +166,7 @@ SNodeList* nodesMakeList();
int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode);
int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode);
int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
SNodeptr nodesListGetNode(SNodeList* pList, int32_t index);
void nodesDestroyList(SNodeList* pList);

3
include/libs/nodes/plannodes.h
View File

@ -26,7 +26,6 @@ extern "C" {
typedef struct SLogicNode {
ENodeType type;
int32_t id;
SNodeList* pTargets; // SColumnNode
SNode* pConditions;
SNodeList* pChildren;
@ -37,6 +36,7 @@ typedef enum EScanType {
SCAN_TYPE_TAG,
SCAN_TYPE_TABLE,
SCAN_TYPE_STABLE,
SCAN_TYPE_TOPIC,
SCAN_TYPE_STREAM
} EScanType;
@ -167,6 +167,7 @@ typedef struct SScanPhysiNode {
typedef SScanPhysiNode SSystemTableScanPhysiNode;
typedef SScanPhysiNode STagScanPhysiNode;
typedef SScanPhysiNode SStreamScanPhysiNode;
typedef struct STableScanPhysiNode {
SScanPhysiNode scan;

4
include/libs/nodes/querynodes.h
View File

@ -23,6 +23,10 @@ extern "C" {
#include "nodes.h"
#include "tmsg.h"
#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags)
#define TABLE_META_SIZE(pMeta) (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema)))
#define VGROUPS_INFO_SIZE(pInfo) (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo)))
typedef struct SRawExprNode {
ENodeType nodeType;
char* p;

1
include/libs/parser/parser.h
View File

@ -26,6 +26,7 @@ typedef struct SParseContext {
uint64_t requestId;
int32_t acctId;
const char *db;
bool topicQuery;
void *pTransporter;
SEpSet mgmtEpSet;
const char *pSql; // sql string

1
include/libs/planner/planner.h
View File

@ -26,6 +26,7 @@ typedef struct SPlanContext {
uint64_t queryId;
int32_t acctId;
SNode* pAstRoot;
bool topicQuery;
bool streamQuery;
} SPlanContext;

30
include/libs/transport/trpc.h
View File

@ -38,13 +38,13 @@ typedef struct SRpcConnInfo {
typedef struct SRpcMsg {
tmsg_t msgType;
tmsg_t expectMsgType;
void * pCont;
int contLen;
int32_t code;
void * handle; // rpc handle returned to app
void * ahandle; // app handle set by client
int noResp; // has response or not(default 0 indicate resp);
int noResp; // has response or not(default 0, 0: resp, 1: no resp);
int persistHandle; // persist handle or not
} SRpcMsg;
@ -77,18 +77,19 @@ typedef struct SRpcInit {
// call back to retrieve the client auth info, for server app only
int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
// call back to keep conn or not
bool (*pfp)(void *parent, tmsg_t msgType);
// to support Send messages multiple times on a link
void *(*mfp)(void *parent, tmsg_t msgType);
// call back to handle except when query/fetch in progress
bool (*efp)(void *parent, tmsg_t msgType);
void *parent;
} SRpcInit;
typedef struct {
void * val;
int32_t len;
void (*free)(void *arg);
} SRpcCtxVal;
typedef struct {
SHashObj *args;
} SRpcCtx;
int32_t rpcInit();
void rpcCleanup();
void * rpcOpen(const SRpcInit *pRpc);
@ -97,16 +98,17 @@ void * rpcMallocCont(int contLen);
void rpcFreeCont(void *pCont);
void * rpcReallocCont(void *ptr, int contLen);
void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
void rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx);
void rpcSendResponse(const SRpcMsg *pMsg);
void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet);
int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
int rpcReportProgress(void *pConn, char *pCont, int contLen);
void rpcCancelRequest(int64_t rid);
void rpcRegisterBrokenLinkArg(SRpcMsg *msg);
// just release client conn to rpc instance, no close sock
void rpcReleaseHandle(void *handle, int8_t type);
void rpcReleaseHandle(void *handle, int8_t type); //
void rpcRefHandle(void *handle, int8_t type);
void rpcUnrefHandle(void *handle, int8_t type);

1
include/util/taoserror.h
View File

@ -463,6 +463,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_PASSWD_EMPTY TAOS_DEF_ERROR_CODE(0, 0x2611)
#define TSDB_CODE_PAR_INVALID_PORT TAOS_DEF_ERROR_CODE(0, 0x2612)
#define TSDB_CODE_PAR_INVALID_ENDPOINT TAOS_DEF_ERROR_CODE(0, 0x2613)
#define TSDB_CODE_PAR_EXPRIE_STATEMENT TAOS_DEF_ERROR_CODE(0, 0x2614)
#ifdef __cplusplus
}

10
include/util/tjson.h
View File

@ -22,6 +22,14 @@
extern "C" {
#endif
#define tjsonGetNumberValue(pJson, pName, val) \
({ \
uint64_t _tmp = 0; \
int32_t _code = tjsonGetUBigIntValue(pJson, pName, &_tmp); \
val = _tmp; \
_code; \
})
typedef void SJson;
SJson* tjsonCreateObject();
@ -44,6 +52,7 @@ int32_t tjsonGetIntValue(const SJson* pJson, const char* pName, int32_t* pVal);
int32_t tjsonGetSmallIntValue(const SJson* pJson, const char* pName, int16_t* pVal);
int32_t tjsonGetTinyIntValue(const SJson* pJson, const char* pName, int8_t* pVal);
int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pVal);
int32_t tjsonGetUIntValue(const SJson* pJson, const char* pName, uint32_t* pVal);
int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal);
int32_t tjsonGetBoolValue(const SJson* pJson, const char* pName, bool* pVal);
int32_t tjsonGetDoubleValue(const SJson* pJson, const char* pName, double* pVal);
@ -60,6 +69,7 @@ int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void*
typedef int32_t (*FToObject)(const SJson* pJson, void* pObj);
int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj);
int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, void** pObj, int32_t objSize);
int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize);
char* tjsonToString(const SJson* pJson);

3
source/client/inc/clientInt.h
View File

@ -179,6 +179,7 @@ typedef struct SRequestObj {
uint64_t requestId;
int32_t type; // request type
STscObj* pTscObj;
char* pDb;
char* sqlstr; // sql string
int32_t sqlLen;
int64_t self;
@ -229,7 +230,7 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t
int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery);
int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery);
int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);
// --- heartbeat

3
source/client/src/clientEnv.c
View File

@ -90,7 +90,6 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) {
rpcInit.label = "TSC";
rpcInit.numOfThreads = numOfThread;
rpcInit.cfp = processMsgFromServer;
rpcInit.pfp = persistConnForSpecificMsg;
rpcInit.sessions = tsMaxConnections;
rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.user = (char *)user;
@ -150,6 +149,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty
return NULL;
}
pRequest->pDb = getDbOfConnection(pObj);
pRequest->requestId = generateRequestId();
pRequest->metric.start = taosGetTimestampMs();
@ -180,6 +180,7 @@ static void doDestroyRequest(void *p) {
tfree(pRequest->msgBuf);
tfree(pRequest->sqlstr);
tfree(pRequest->pInfo);
tfree(pRequest->pDb);
doFreeReqResultInfo(&pRequest->body.resInfo);
qDestroyQueryPlan(pRequest->body.pDag);

9
source/client/src/clientImpl.c
View File

@ -137,13 +137,14 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj*
return TSDB_CODE_SUCCESS;
}
int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) {
int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) {
STscObj* pTscObj = pRequest->pTscObj;
SParseContext cxt = {
.requestId = pRequest->requestId,
.acctId = pTscObj->acctId,
.db = getDbOfConnection(pTscObj),
.db = pRequest->pDb,
.topicQuery = topicQuery,
.pSql = pRequest->sqlstr,
.sqlLen = pRequest->sqlLen,
.pMsg = pRequest->msgBuf,
@ -154,7 +155,6 @@ int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) {
cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
if (code != TSDB_CODE_SUCCESS) {
tfree(cxt.db);
return code;
}
@ -163,7 +163,6 @@ int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) {
setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols);
}
tfree(cxt.db);
return code;
}
@ -249,7 +248,7 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) {
terrno = TSDB_CODE_SUCCESS;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return);
CHECK_CODE_GOTO(parseSql(pRequest, false, &pQuery), _return);
if (pQuery->directRpc) {
CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return);

30
source/client/src/tmq.c
View File

@ -482,38 +482,24 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
}
tscDebug("start to create topic, %s", topicName);
#if 0
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return);
pQueryNode->streamQuery = true;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, true, &pQueryNode), _return);
// todo check for invalid sql statement and return with error code
SSchema* schema = NULL;
int32_t numOfCols = 0;
CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, NULL), _return);
pStr = qQueryPlanToString(pRequest->body.pDag);
if (pStr == NULL) {
goto _return;
}
CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &pStr, NULL), _return);
/*printf("%s\n", pStr);*/
// The topic should be related to a database that the queried table is belonged to.
SName name = {0};
char dbName[TSDB_DB_FNAME_LEN] = {0};
// tNameGetFullDbName(&((SQueryStmtInfo*)pQueryNode)->pTableMetaInfo[0]->name, dbName);
tNameFromString(&name, dbName, T_NAME_ACCT | T_NAME_DB);
tNameFromString(&name, topicName, T_NAME_TABLE);
SName name = { .acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T };
strcpy(name.dbname, pRequest->pDb);
strcpy(name.tname, topicName);
SCMCreateTopicReq req = {
.igExists = 1,
.physicalPlan = (char*)pStr,
.ast = (char*)pStr,
.sql = (char*)sql,
.logicalPlan = (char*)"no logic plan",
};
tNameExtractFullName(&name, req.name);
@ -536,7 +522,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
tsem_wait(&pRequest->body.rspSem);
#endif
_return:
qDestroyQuery(pQueryNode);
/*if (sendInfo != NULL) {*/

10
source/common/src/tdatablock.c
View File

@ -1072,6 +1072,10 @@ void blockDataClearup(SSDataBlock* pDataBlock) {
}
int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) {
if (0 == numOfRows) {
return TSDB_CODE_SUCCESS;
}
if (IS_VAR_DATA_TYPE(pColumn->info.type)) {
char* tmp = realloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows);
if (tmp == NULL) {
@ -1092,7 +1096,7 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo
pColumn->nullbitmap = tmp;
memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows));
assert(pColumn->info.bytes);
tmp = realloc(pColumn->pData, numOfRows * pColumn->info.bytes);
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -1137,7 +1141,7 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
taosArrayDestroy(pBlock->pDataBlock);
tfree(pBlock->pBlockAgg);
tfree(pBlock);
// tfree(pBlock);
return NULL;
}
@ -1190,7 +1194,7 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
}
int32_t len = colDataGetLength(pColData, rows);
taosEncodeFixedI32(buf, len);
tlen += taosEncodeFixedI32(buf, len);
tlen += taosEncodeBinary(buf, pColData->pData, len);
}

2
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -246,7 +246,7 @@ static int32_t mndGetPlanString(SCMCreateTopicReq *pCreate, char **pStr) {
SQueryPlan* pPlan = NULL;
if (TSDB_CODE_SUCCESS == code) {
SPlanContext cxt = { .pAstRoot = pAst, .streamQuery = true };
SPlanContext cxt = { .pAstRoot = pAst, .topicQuery = true };
code = qCreateQueryPlan(&cxt, &pPlan, NULL);
}

1
source/dnode/vnode/src/inc/tsdbDBDef.h
View File

@ -26,6 +26,7 @@ typedef struct SDBFile SDBFile;
typedef DB_ENV* TDBEnv;
struct SDBFile {
int32_t fid;
DB* pDB;
char* path;
};

1
source/dnode/vnode/src/inc/tsdbDef.h
View File

@ -63,6 +63,7 @@ struct STsdb {
#define REPO_ID(r) ((r)->vgId)
#define REPO_CFG(r) (&(r)->config)
#define REPO_FS(r) (r)->fs
#define REPO_TFS(r) (r)->pTfs
#define IS_REPO_LOCKED(r) (r)->repoLocked
#define REPO_SMA_ENV(r, t) ((TSDB_SMA_TYPE_ROLLUP == (t)) ? (r)->pRSmaEnv : (r)->pTSmaEnv)

24
source/dnode/vnode/src/inc/tsdbFS.h
View File

@ -42,23 +42,23 @@ typedef struct {
typedef struct {
STsdbFSMeta meta; // FS meta
SArray * df; // data file array
SArray * sf; // sma data file array v2(t|r)1900.index_name_1
SArray * sf; // sma data file array v2f1900.index_name_1
} SFSStatus;
/**
* @brief Directory structure of .tsma data files.
*
* /vnode2/tsdb $ tree .sma/
* .sma/
* ├── v2t100.index_name_1
* ├── v2t101.index_name_1
* ├── v2t102.index_name_1
* ├── v2t1900.index_name_3
* ├── v2t1901.index_name_3
* ├── v2t1902.index_name_3
* ├── v2t200.index_name_2
* ├── v2t201.index_name_2
* └── v2t202.index_name_2
* /vnode2/tsdb $ tree tsma/
* tsma/
* ├── v2f100.index_name_1
* ├── v2f101.index_name_1
* ├── v2f102.index_name_1
* ├── v2f1900.index_name_3
* ├── v2f1901.index_name_3
* ├── v2f1902.index_name_3
* ├── v2f200.index_name_2
* ├── v2f201.index_name_2
* └── v2f202.index_name_2
*
* 0 directories, 9 files
*/

6
source/dnode/vnode/src/inc/tsdbSma.h
View File

@ -21,12 +21,14 @@ typedef struct SSmaEnv SSmaEnv;
struct SSmaEnv {
pthread_rwlock_t lock;
TDBEnv dbEnv;
char * path;
SDiskID did;
TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level?
char * path; // relative path
SSmaStat * pStat;
};
#define SMA_ENV_LOCK(env) ((env)->lock)
#define SMA_ENV_DID(env) ((env)->did)
#define SMA_ENV_ENV(env) ((env)->dbEnv)
#define SMA_ENV_PATH(env) ((env)->path)
#define SMA_ENV_STAT(env) ((env)->pStat)

2
source/dnode/vnode/src/meta/metaBDBImpl.c
View File

@ -884,7 +884,7 @@ const char *metaSmaCursorNext(SMSmaCursor *pCur) {
STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
STSmaWrapper *pSW = NULL;
pSW = calloc(sizeof(*pSW), 1);
pSW = calloc(1, sizeof(*pSW));
if (pSW == NULL) {
return NULL;
}

8
source/dnode/vnode/src/tq/tqRead.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdatablock.h"
#include "vnode.h"
STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
@ -128,10 +129,13 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
int j = 0;
for (int32_t i = 0; i < colNumNeed; i++) {
int32_t colId = *(int32_t*)taosArrayGet(pHandle->pColIdList, i);
int16_t colId = *(int16_t*)taosArrayGet(pHandle->pColIdList, i);
while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) {
j++;
}
if (j >= pSchemaWrapper->nCols) {
continue;
}
SSchema* pColSchema = &pSchemaWrapper->pSchema[j];
SColumnInfoData colInfo = {0};
int sz = numOfRows * pColSchema->bytes;
@ -145,6 +149,8 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
taosArrayDestroy(pArray);
return NULL;
}
blockDataEnsureColumnCapacity(&colInfo, numOfRows);
taosArrayPush(pArray, &colInfo);
}

138
source/dnode/vnode/src/tsdb/tsdbSma.c
View File

@ -67,7 +67,7 @@ typedef struct {
*/
int8_t state; // ETsdbSmaStat
SHashObj *expiredWindows; // key: skey of time window, value: N/A
STSma * pSma;
STSma * pSma; // cache schema
} SSmaStatItem;
struct SSmaStat {
@ -81,8 +81,8 @@ struct SSmaStat {
static int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg);
static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat);
static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat);
static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path);
static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv);
static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did);
static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv);
static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey);
static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat);
static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat);
@ -102,8 +102,8 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p
static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen);
static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision);
static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel);
static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid);
static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey);
static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int64_t indexUid, int32_t fid);
static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey);
static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey);
static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]);
static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg);
@ -111,10 +111,11 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg);
// implementation
static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) {
snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s", vgId, TSDB_SMA_DNAME[smaType]);
snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%stsdb%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TD_DIRSEP,
TSDB_SMA_DNAME[smaType]);
}
static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) {
static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) {
SSmaEnv *pEnv = NULL;
pEnv = (SSmaEnv *)calloc(1, sizeof(SSmaEnv));
@ -137,12 +138,16 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) {
return NULL;
}
pEnv->did = did;
if (tsdbInitSmaStat(&pEnv->pStat) != TSDB_CODE_SUCCESS) {
tsdbFreeSmaEnv(pEnv);
return NULL;
}
if (tsdbOpenBDBEnv(&pEnv->dbEnv, pEnv->path) != TSDB_CODE_SUCCESS) {
char aname[TSDB_FILENAME_LEN] = {0};
tfsAbsoluteName(pTsdb->pTfs, did, path, aname);
if (tsdbOpenBDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) {
tsdbFreeSmaEnv(pEnv);
return NULL;
}
@ -150,18 +155,14 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) {
return pEnv;
}
static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv) {
static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv) {
if (!pEnv) {
terrno = TSDB_CODE_INVALID_PTR;
return TSDB_CODE_FAILED;
}
if (*pEnv) {
return TSDB_CODE_SUCCESS;
}
if (*pEnv == NULL) {
if ((*pEnv = tsdbNewSmaEnv(pTsdb, path)) == NULL) {
if ((*pEnv = tsdbNewSmaEnv(pTsdb, path, did)) == NULL) {
return TSDB_CODE_FAILED;
}
}
@ -260,10 +261,15 @@ static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) {
int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) {
if (pSmaStat) {
// TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
SSmaStatItem *item = taosHashIterate(pSmaStat->smaStatItems, NULL);
void *item = taosHashIterate(pSmaStat->smaStatItems, NULL);
while (item != NULL) {
tfree(item->pSma);
taosHashCleanup(item->expiredWindows);
SSmaStatItem *pItem = *(SSmaStatItem **)item;
if (pItem != NULL) {
tdDestroyTSma(pItem->pSma);
tfree(pItem->pSma);
taosHashCleanup(pItem->expiredWindows);
tfree(pItem);
}
item = taosHashIterate(pSmaStat->smaStatItems, item);
}
taosHashCleanup(pSmaStat->smaStatItems);
@ -292,9 +298,9 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) {
// init sma env
tsdbLockRepo(pTsdb);
if (pTsdb->pTSmaEnv == NULL) {
pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&pTsdb->pTSmaEnv) : atomic_load_ptr(&pTsdb->pRSmaEnv);
if (pEnv == NULL) {
char rname[TSDB_FILENAME_LEN] = {0};
char aname[TSDB_FILENAME_LEN * 2 + 32] = {0}; // TODO: make TMPNAME_LEN public as TSDB_FILENAME_LEN?
SDiskID did = {0};
tfsAllocDisk(pTsdb->pTfs, TFS_PRIMARY_LEVEL, &did);
@ -303,23 +309,19 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) {
return TSDB_CODE_FAILED;
}
tsdbGetSmaDir(REPO_ID(pTsdb), smaType, rname);
tfsAbsoluteName(pTsdb->pTfs, did, rname, aname);
if (tfsMkdirRecurAt(pTsdb->pTfs, rname, did) != TSDB_CODE_SUCCESS) {
tsdbUnlockRepo(pTsdb);
return TSDB_CODE_FAILED;
}
if (tsdbInitSmaEnv(pTsdb, aname, &pEnv) != TSDB_CODE_SUCCESS) {
if (tsdbInitSmaEnv(pTsdb, rname, did, &pEnv) != TSDB_CODE_SUCCESS) {
tsdbUnlockRepo(pTsdb);
return TSDB_CODE_FAILED;
}
if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
atomic_store_ptr(&pTsdb->pTSmaEnv, pEnv);
} else {
atomic_store_ptr(&pTsdb->pRSmaEnv, pEnv);
}
(smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&pTsdb->pTSmaEnv, pEnv)
: atomic_store_ptr(&pTsdb->pRSmaEnv, pEnv);
}
tsdbUnlockRepo(pTsdb);
@ -340,11 +342,6 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg) {
return TSDB_CODE_FAILED;
}
if (tsdbCheckAndInitSmaEnv(pTsdb, smaType) != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TDB_INIT_FAILED;
return TSDB_CODE_FAILED;
}
// TODO: decode the msg from Stream Computing module => start
int64_t indexUid = SMA_TEST_INDEX_UID;
const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10;
@ -355,12 +352,19 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg) {
}
// TODO: decode the msg <= end
if (tsdbCheckAndInitSmaEnv(pTsdb, smaType) != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TDB_INIT_FAILED;
return TSDB_CODE_FAILED;
}
SSmaEnv * pEnv = REPO_SMA_ENV(pTsdb, smaType);
SSmaStat *pStat = SMA_ENV_STAT(pEnv);
SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv);
TASSERT(pEnv != NULL && pStat != NULL && pItemsHash != NULL);
tsdbRefSmaStat(pTsdb, pStat);
SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pItemsHash, &indexUid, sizeof(indexUid));
SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid));
if (pItem == NULL) {
pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED); // TODO use the real state
if (pItem == NULL) {
@ -421,9 +425,9 @@ static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t ind
tsdbRefSmaStat(pTsdb, pStat);
if (pStat && pStat->smaStatItems) {
pItem = *(SSmaStatItem **)taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid));
pItem = taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid));
}
if (pItem != NULL) {
if ((pItem != NULL) && ((pItem = *(SSmaStatItem **)pItem) != NULL)) {
// pItem resides in hash buffer all the time unless drop sma index
// TODO: multithread protect
if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) {
@ -494,7 +498,7 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
* @brief Insert TSma data blocks to DB File build by B+Tree
*
* @param pSmaH
* @param smaKey
* @param smaKey tableUid-colId-skeyOfWindow(8-2-8)
* @param keyLen
* @param pData
* @param dataLen
@ -502,12 +506,11 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
*/
static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) {
SDBFile *pDBFile = &pSmaH->dFile;
// TODO: insert sma data blocks into B+Tree
tsdbDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d",
REPO_ID(pSmaH->pTsdb), pDBFile->path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8),
*(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen);
// TODO: insert sma data blocks into B+Tree(TDB)
if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) {
return TSDB_CODE_FAILED;
}
@ -564,34 +567,34 @@ static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit
return interval / 1e3;
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second
return interval / 1e6;
} else {
} else { // ms
return interval;
}
break;
case TSDB_TIME_PRECISION_MICRO:
if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
return interval;
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns
return interval / 1e3;
} else {
} else { // ms
return interval * 1e3;
}
break;
case TSDB_TIME_PRECISION_NANO:
if (TIME_UNIT_MICROSECOND == intervalUnit) {
if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
return interval * 1e3;
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns
return interval;
} else {
} else { // ms
return interval * 1e6;
}
break;
default: // ms
if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
return interval / 1e3;
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns
return interval / 1e6;
} else {
} else { // ms
return interval;
}
break;
@ -660,12 +663,15 @@ static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH) {
}
}
static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid) {
static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int64_t indexUid, int32_t fid) {
STsdb *pTsdb = pSmaH->pTsdb;
ASSERT(pSmaH->dFile.path == NULL && pSmaH->dFile.pDB == NULL);
pSmaH->dFile.fid = fid;
char tSmaFile[TSDB_FILENAME_LEN] = {0};
snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pTsdb), fid);
snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid);
pSmaH->dFile.path = strdup(tSmaFile);
return TSDB_CODE_SUCCESS;
}
@ -704,8 +710,9 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe
static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
STsdbCfg * pCfg = REPO_CFG(pTsdb);
STSmaDataWrapper *pData = (STSmaDataWrapper *)msg;
SSmaEnv * pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv);
if (!pTsdb->pTSmaEnv) {
if (pEnv == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb));
return terrno;
@ -723,6 +730,17 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
return TSDB_CODE_FAILED;
}
int64_t indexUid = SMA_TEST_INDEX_UID;
char rPath[TSDB_FILENAME_LEN] = {0};
char aPath[TSDB_FILENAME_LEN] = {0};
snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid);
tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath);
if (!taosCheckExistFile(aPath)) {
if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED;
}
}
// Step 1: Judge the storage level and days
int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit);
int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel);
@ -731,7 +749,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
// Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file
// - Set and open the DFile or the B+Tree file
// TODO: tsdbStartTSmaCommit();
tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid);
tsdbSetTSmaDataFile(&tSmaH, pData, indexUid, fid);
if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) {
tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb),
tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno));
@ -818,13 +836,16 @@ static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interv
* @brief Init of tSma FS
*
* @param pReadH
* @param indexUid
* @param skey
* @return int32_t
*/
static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey) {
int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pSmaH->pTsdb)->precision));
static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) {
STsdb *pTsdb = pSmaH->pTsdb;
int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pTsdb)->precision));
char tSmaFile[TSDB_FILENAME_LEN] = {0};
snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pSmaH->pTsdb), fid);
snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, REPO_ID(pTsdb), fid);
pSmaH->dFile.path = strdup(tSmaFile);
pSmaH->smaFsIter.iter = 0;
pSmaH->smaFsIter.fid = fid;
@ -883,15 +904,17 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) {
static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval,
int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey,
int32_t nMaxResult) {
if (!pTsdb->pTSmaEnv) {
SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv);
if (!pEnv) {
terrno = TSDB_CODE_INVALID_PTR;
tsdbWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", REPO_ID(pTsdb));
return TSDB_CODE_FAILED;
}
tsdbRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv));
SSmaStatItem *pItem = *(SSmaStatItem **)taosHashGet(SMA_ENV_STAT_ITEMS(pTsdb->pTSmaEnv), &indexUid, sizeof(indexUid));
if (pItem == NULL) {
tsdbRefSmaStat(pTsdb, SMA_ENV_STAT(pEnv));
SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid));
if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL)) {
// Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if
// it's NULL.
tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv));
@ -922,11 +945,12 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_
tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv));
#endif
STSmaReadH tReadH = {0};
tsdbInitTSmaReadH(&tReadH, pTsdb, interval, intervalUnit);
tsdbCloseDBF(&tReadH.dFile);
tsdbInitTSmaFile(&tReadH, querySKey);
tsdbInitTSmaFile(&tReadH, indexUid, querySKey);
if (tsdbOpenDBF(SMA_ENV_ENV(pTsdb->pTSmaEnv), &tReadH.dFile) != 0) {
tsdbWarn("vgId:%d open DBF %s failed since %s", REPO_ID(pTsdb), tReadH.dFile.path, tstrerror(terrno));
return TSDB_CODE_FAILED;

3
source/dnode/vnode/src/vnd/vnodeWrite.c
View File

@ -146,6 +146,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
return -1;
}
// record current timezone of server side
tstrncpy(vCreateSmaReq.tSma.timezone, tsTimezone, TD_TIMEZONE_LEN);
if (metaCreateTSma(pVnode->pMeta, &vCreateSmaReq) < 0) {
// TODO: handle error
tdDestroyTSma(&vCreateSmaReq.tSma);

48
source/dnode/vnode/test/tsdbSmaTest.cpp
View File

@ -33,6 +33,7 @@ int main(int argc, char **argv) {
return RUN_ALL_TESTS();
}
#if 1
TEST(testCase, tSma_Meta_Encode_Decode_Test) {
// encode
STSma tSma = {0};
@ -49,7 +50,7 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) {
STSmaWrapper tSmaWrapper = {.number = 1, .tSma = &tSma};
uint32_t bufLen = tEncodeTSmaWrapper(NULL, &tSmaWrapper);
void *buf = calloc(bufLen, 1);
void *buf = calloc(1, bufLen);
ASSERT_NE(buf, nullptr);
STSmaWrapper *pSW = (STSmaWrapper *)buf;
@ -84,9 +85,11 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) {
}
// resource release
tfree(pSW);
tdDestroyTSma(&tSma);
tdDestroyTSmaWrapper(&dstTSmaWrapper);
}
#endif
#if 1
TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
@ -113,7 +116,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
tSma.tableUid = tbUid;
tSma.exprLen = strlen(expr);
tSma.expr = (char *)calloc(tSma.exprLen + 1, 1);
tSma.expr = (char *)calloc(1, tSma.exprLen + 1);
ASSERT_NE(tSma.expr, nullptr);
tstrncpy(tSma.expr, expr, tSma.exprLen + 1);
@ -251,12 +254,12 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
tSma.tableUid = tbUid;
tSma.exprLen = strlen(expr);
tSma.expr = (char *)calloc(tSma.exprLen + 1, 1);
tSma.expr = (char *)calloc(1, tSma.exprLen + 1);
ASSERT_NE(tSma.expr, nullptr);
tstrncpy(tSma.expr, expr, tSma.exprLen + 1);
tSma.tagsFilterLen = strlen(tagsFilter);
tSma.tagsFilter = (char *)calloc(tSma.tagsFilterLen + 1, 1);
tSma.tagsFilter = (char *)calloc(1, tSma.tagsFilterLen + 1);
ASSERT_NE(tSma.tagsFilter, nullptr);
tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1);
@ -273,20 +276,20 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
// step 2: insert data
STSmaDataWrapper *pSmaData = NULL;
STsdb tsdb = {0};
STsdbCfg * pCfg = &tsdb.config;
STsdb * pTsdb = (STsdb *)calloc(1, sizeof(STsdb));
STsdbCfg * pCfg = &pTsdb->config;
tsdb.pMeta = pMeta;
tsdb.vgId = 2;
tsdb.config.daysPerFile = 10; // default days is 10
tsdb.config.keep1 = 30;
tsdb.config.keep2 = 90;
tsdb.config.keep = 365;
tsdb.config.precision = TSDB_TIME_PRECISION_MILLI;
tsdb.config.update = TD_ROW_OVERWRITE_UPDATE;
tsdb.config.compression = TWO_STAGE_COMP;
pTsdb->pMeta = pMeta;
pTsdb->vgId = 2;
pTsdb->config.daysPerFile = 10; // default days is 10
pTsdb->config.keep1 = 30;
pTsdb->config.keep2 = 90;
pTsdb->config.keep = 365;
pTsdb->config.precision = TSDB_TIME_PRECISION_MILLI;
pTsdb->config.update = TD_ROW_OVERWRITE_UPDATE;
pTsdb->config.compression = TWO_STAGE_COMP;
switch (tsdb.config.precision) {
switch (pTsdb->config.precision) {
case TSDB_TIME_PRECISION_MILLI:
skey1 *= 1e3;
break;
@ -304,12 +307,12 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
SDiskCfg pDisks = {.level = 0, .primary = 1};
strncpy(pDisks.dir, "/var/lib/taos", TSDB_FILENAME_LEN);
int32_t numOfDisks = 1;
tsdb.pTfs = tfsOpen(&pDisks, numOfDisks);
ASSERT_NE(tsdb.pTfs, nullptr);
pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks);
ASSERT_NE(pTsdb->pTfs, nullptr);
char *msg = (char *)calloc(1, 100);
ASSERT_NE(msg, nullptr);
ASSERT_EQ(tsdbUpdateSmaWindow(&tsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0);
ASSERT_EQ(tsdbUpdateSmaWindow(pTsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0);
// init
int32_t allocCnt = 0;
@ -367,13 +370,13 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
ASSERT_GE(bufSize, pSmaData->dataLen);
// execute
ASSERT_EQ(tsdbInsertTSmaData(&tsdb, (char *)pSmaData), TSDB_CODE_SUCCESS);
ASSERT_EQ(tsdbInsertTSmaData(pTsdb, (char *)pSmaData), TSDB_CODE_SUCCESS);
// step 3: query
uint32_t checkDataCnt = 0;
for (int32_t t = 0; t < numOfTables; ++t) {
for (col_id_t c = 0; c < numOfCols; ++c) {
ASSERT_EQ(tsdbGetTSmaData(&tsdb, NULL, indexUid1, interval1, intervalUnit1, tbUid + t,
ASSERT_EQ(tsdbGetTSmaData(pTsdb, NULL, indexUid1, interval1, intervalUnit1, tbUid + t,
c + PRIMARYKEY_TIMESTAMP_COL_ID, skey1, 1),
TSDB_CODE_SUCCESS);
++checkDataCnt;
@ -383,9 +386,12 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
printf("%s:%d The sma data check count for insert and query is %" PRIu32 "\n", __FILE__, __LINE__, checkDataCnt);
// release data
tfree(msg);
taosTZfree(buf);
// release meta
tdDestroyTSma(&tSma);
tfsClose(pTsdb->pTfs);
tsdbClose(pTsdb);
metaClose(pMeta);
}
#endif

2
source/libs/CMakeLists.txt
View File

@ -1,6 +1,6 @@
add_subdirectory(transport)
add_subdirectory(sync)
add_subdirectory(tdb)
# add_subdirectory(tdb)
add_subdirectory(index)
add_subdirectory(wal)
add_subdirectory(parser)

5
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -190,7 +190,6 @@ static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) {
}
static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) {
COPY_SCALAR_FIELD(id);
CLONE_NODE_LIST_FIELD(pTargets);
CLONE_NODE_FIELD(pConditions);
CLONE_NODE_LIST_FIELD(pChildren);
@ -198,7 +197,7 @@ static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) {
}
static STableMeta* tableMetaClone(const STableMeta* pSrc) {
int32_t len = sizeof(STableMeta) + (pSrc->tableInfo.numOfTags + pSrc->tableInfo.numOfColumns) * sizeof(SSchema);
int32_t len = TABLE_META_SIZE(pSrc);
STableMeta* pDst = malloc(len);
if (NULL == pDst) {
return NULL;
@ -208,7 +207,7 @@ static STableMeta* tableMetaClone(const STableMeta* pSrc) {
}
static SVgroupsInfo* vgroupsInfoClone(const SVgroupsInfo* pSrc) {
int32_t len = sizeof(SVgroupsInfo) + pSrc->numOfVgroups * sizeof(SVgroupInfo);
int32_t len = VGROUPS_INFO_SIZE(pSrc);
SVgroupsInfo* pDst = malloc(len);
if (NULL == pDst) {
return NULL;

500
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "cmdnodes.h"
#include "nodesUtil.h"
#include "plannodes.h"
#include "querynodes.h"
@ -85,6 +86,8 @@ const char* nodesNodeName(ENodeType type) {
return "ShowDatabaseStmt";
case QUERY_NODE_SHOW_TABLES_STMT:
return "ShowTablesStmt";
case QUERY_NODE_CREATE_TOPIC_STMT:
return "CreateTopicStmt";
case QUERY_NODE_LOGIC_PLAN_SCAN:
return "LogicScan";
case QUERY_NODE_LOGIC_PLAN_JOIN:
@ -179,21 +182,151 @@ static int32_t jsonToNodeList(const SJson* pJson, const char* pName, SNodeList**
return jsonToNodeListImpl(tjsonGetObjectItem(pJson, pName), pList);
}
static const char* jkTableMetaUid = "TableMetaUid";
static const char* jkTableMetaSuid = "TableMetaSuid";
static const char* jkTableComInfoNumOfTags = "NumOfTags";
static const char* jkTableComInfoPrecision = "Precision";
static const char* jkTableComInfoNumOfColumns = "NumOfColumns";
static const char* jkTableComInfoRowSize = "RowSize";
static int32_t tableMetaToJson(const void* pObj, SJson* pJson) {
const STableMeta* pNode = (const STableMeta*)pObj;
static int32_t tableComInfoToJson(const void* pObj, SJson* pJson) {
const STableComInfo* pNode = (const STableComInfo*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkTableMetaUid, pNode->uid);
int32_t code = tjsonAddIntegerToObject(pJson, jkTableComInfoNumOfTags, pNode->numOfTags);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableMetaSuid, pNode->suid);
code = tjsonAddIntegerToObject(pJson, jkTableComInfoPrecision, pNode->precision);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableComInfoRowSize, pNode->rowSize);
}
return code;
}
static int32_t jsonToTableComInfo(const SJson* pJson, void* pObj) {
STableComInfo* pNode = (STableComInfo*)pObj;
int32_t code = tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize);
}
return code;
}
static const char* jkSchemaType = "Type";
static const char* jkSchemaColId = "ColId";
static const char* jkSchemaBytes = "bytes";
static const char* jkSchemaName = "Name";
static int32_t schemaToJson(const void* pObj, SJson* pJson) {
const SSchema* pNode = (const SSchema*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkSchemaType, pNode->type);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkSchemaColId, pNode->colId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkSchemaBytes, pNode->bytes);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkSchemaName, pNode->name);
}
return code;
}
static int32_t jsonToSchema(const SJson* pJson, void* pObj) {
SSchema* pNode = (SSchema*)pObj;
int32_t code = tjsonGetNumberValue(pJson, jkSchemaType, pNode->type);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkSchemaName, pNode->name);
}
return code;
}
static const char* jkTableMetaVgId = "VgId";
static const char* jkTableMetaTableType = "TableType";
static const char* jkTableMetaUid = "Uid";
static const char* jkTableMetaSuid = "Suid";
static const char* jkTableMetaSversion = "Sversion";
static const char* jkTableMetaTversion = "Tversion";
static const char* jkTableMetaComInfo = "ComInfo";
static const char* jkTableMetaColSchemas = "ColSchemas";
static int32_t tableMetaToJson(const void* pObj, SJson* pJson) {
const STableMeta* pNode = (const STableMeta*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkTableMetaVgId, pNode->vgId);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableMetaTableType, pNode->tableType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableMetaUid, pNode->uid);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableMetaSuid, pNode->suid);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableMetaSversion, pNode->sversion);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTableMetaTversion, pNode->tversion);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkTableMetaComInfo, tableComInfoToJson, &pNode->tableInfo);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddArray(pJson, jkTableMetaColSchemas, schemaToJson, pNode->schema, sizeof(SSchema), TABLE_TOTAL_COL_NUM(pNode));
}
return code;
}
static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) {
STableMeta* pNode = (STableMeta*)pObj;
int32_t code = tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, jkTableMetaComInfo, jsonToTableComInfo, &pNode->tableInfo);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToArray(pJson, jkTableMetaColSchemas, jsonToSchema, pNode->schema, sizeof(SSchema));
}
return code;
}
static const char* jkLogicPlanId = "Id";
static const char* jkLogicPlanTargets = "Targets";
static const char* jkLogicPlanConditions = "Conditions";
static const char* jkLogicPlanChildren = "Children";
@ -201,10 +334,7 @@ static const char* jkLogicPlanChildren = "Children";
static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) {
const SLogicNode* pNode = (const SLogicNode*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkLogicPlanId, pNode->id);
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkLogicPlanTargets, pNode->pTargets);
}
int32_t code = nodeListToJson(pJson, jkLogicPlanTargets, pNode->pTargets);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkLogicPlanConditions, nodeToJson, pNode->pConditions);
}
@ -215,7 +345,22 @@ static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) {
return code;
}
static int32_t jsonToLogicPlanNode(const SJson* pJson, void* pObj) {
SLogicNode* pNode = (SLogicNode*)pObj;
int32_t code = jsonToNodeList(pJson, jkLogicPlanTargets, &pNode->pTargets);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkLogicPlanConditions, &pNode->pConditions);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkLogicPlanChildren, &pNode->pChildren);
}
return code;
}
static const char* jkScanLogicPlanScanCols = "ScanCols";
static const char* jkScanLogicPlanTableMetaSize = "TableMetaSize";
static const char* jkScanLogicPlanTableMeta = "TableMeta";
static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
@ -225,6 +370,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkScanLogicPlanScanCols, pNode->pScanCols);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, TABLE_META_SIZE(pNode->pMeta));
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkScanLogicPlanTableMeta, tableMetaToJson, pNode->pMeta);
}
@ -232,6 +380,24 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
return code;
}
static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
SScanLogicNode* pNode = (SScanLogicNode*)pObj;
int32_t objSize = 0;
int32_t code = jsonToLogicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkScanLogicPlanScanCols, &pNode->pScanCols);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkScanLogicPlanTableMetaSize, &objSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonMakeObject(pJson, jkScanLogicPlanTableMeta, jsonToTableMeta, (void**)&pNode->pMeta, objSize);
}
return code;
}
static const char* jkProjectLogicPlanProjections = "Projections";
static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
@ -245,6 +411,17 @@ static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
return code;
}
static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) {
SProjectLogicNode* pNode = (SProjectLogicNode*)pObj;
int32_t code = jsonToLogicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkProjectLogicPlanProjections, &pNode->pProjections);
}
return code;
}
static const char* jkJoinLogicPlanJoinType = "JoinType";
static const char* jkJoinLogicPlanOnConditions = "OnConditions";
@ -445,6 +622,14 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
return code;
}
static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) {
return physiScanNodeToJson(pObj, pJson);
}
static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) {
return jsonToPhysiScanNode(pJson, pObj);
}
static const char* jkProjectPhysiPlanProjections = "Projections";
static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
@ -1273,6 +1458,193 @@ static int32_t jsonToFunctionNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkTableDbName = "DbName";
static const char* jkTableTableName = "tableName";
static const char* jkTableTableAlias = "tableAlias";
static int32_t tableNodeToJson(const void* pObj, SJson* pJson) {
const STableNode* pNode = (const STableNode*)pObj;
int32_t code = exprNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkTableDbName, pNode->dbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkTableTableName, pNode->tableName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkTableTableAlias, pNode->tableAlias);
}
return code;
}
static int32_t jsonToTableNode(const SJson* pJson, void* pObj) {
STableNode* pNode = (STableNode*)pObj;
int32_t code = jsonToExprNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkTableDbName, pNode->dbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkTableTableName, pNode->tableName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkTableTableAlias, pNode->tableAlias);
}
return code;
}
static const char* jkEpSetInUse = "InUse";
static const char* jkEpSetNumOfEps = "NumOfEps";
static const char* jkEpSetEps = "Eps";
static int32_t epSetToJson(const void* pObj, SJson* pJson) {
const SEpSet* pNode = (const SEpSet*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkEpSetInUse, pNode->inUse);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkEpSetNumOfEps, pNode->numOfEps);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddArray(pJson, jkEpSetEps, epToJson, pNode->eps, sizeof(SEp), pNode->numOfEps);
}
return code;
}
static int32_t jsonToEpSet(const SJson* pJson, void* pObj) {
SEpSet* pNode = (SEpSet*)pObj;
int32_t code = tjsonGetTinyIntValue(pJson, jkEpSetInUse, &pNode->inUse);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkEpSetNumOfEps, &pNode->numOfEps);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToArray(pJson, jkEpSetEps, jsonToEp, pNode->eps, sizeof(SEp));
}
return code;
}
static const char* jkVgroupInfoVgId = "VgId";
static const char* jkVgroupInfoHashBegin = "HashBegin";
static const char* jkVgroupInfoHashEnd = "HashEnd";
static const char* jkVgroupInfoEpSet = "EpSet";
static const char* jkVgroupInfoNumOfTable = "NumOfTable";
static int32_t vgroupInfoToJson(const void* pObj, SJson* pJson) {
const SVgroupInfo* pNode = (const SVgroupInfo*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupInfoVgId, pNode->vgId);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashBegin, pNode->hashBegin);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkVgroupInfoHashEnd, pNode->hashEnd);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkVgroupInfoEpSet, epSetToJson, &pNode->epSet);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkVgroupInfoNumOfTable, pNode->numOfTable);
}
return code;
}
static int32_t jsonToVgroupInfo(const SJson* pJson, void* pObj) {
SVgroupInfo* pNode = (SVgroupInfo*)pObj;
int32_t code = tjsonGetIntValue(pJson, jkVgroupInfoVgId, &pNode->vgId);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetUIntValue(pJson, jkVgroupInfoHashBegin, &pNode->hashBegin);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetUIntValue(pJson, jkVgroupInfoHashEnd, &pNode->hashEnd);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, jkVgroupInfoEpSet, jsonToEpSet, &pNode->epSet);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkVgroupInfoNumOfTable, &pNode->numOfTable);
}
return code;
}
static const char* jkVgroupsInfoNum = "Num";
static const char* jkVgroupsInfoVgroups = "Vgroups";
static int32_t vgroupsInfoToJson(const void* pObj, SJson* pJson) {
const SVgroupsInfo* pNode = (const SVgroupsInfo*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupsInfoNum, pNode->numOfVgroups);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddArray(pJson, jkVgroupsInfoVgroups, vgroupInfoToJson, pNode->vgroups, sizeof(SVgroupInfo), pNode->numOfVgroups);
}
return code;
}
static int32_t jsonToVgroupsInfo(const SJson* pJson, void* pObj) {
SVgroupsInfo* pNode = (SVgroupsInfo*)pObj;
int32_t code = tjsonGetIntValue(pJson, jkVgroupsInfoNum, &pNode->numOfVgroups);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToArray(pJson, jkVgroupsInfoVgroups, jsonToVgroupInfo, pNode->vgroups, sizeof(SVgroupInfo));
}
return code;
}
static const char* jkRealTableMetaSize = "MetaSize";
static const char* jkRealTableMeta = "Meta";
static const char* jkRealTableVgroupsInfoSize = "VgroupsInfoSize";
static const char* jkRealTableVgroupsInfo = "VgroupsInfo";
static int32_t realTableNodeToJson(const void* pObj, SJson* pJson) {
const SRealTableNode* pNode = (const SRealTableNode*)pObj;
int32_t code = tableNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkRealTableMetaSize, TABLE_META_SIZE(pNode->pMeta));
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkRealTableMeta, tableMetaToJson, pNode->pMeta);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkRealTableVgroupsInfoSize, VGROUPS_INFO_SIZE(pNode->pVgroupList));
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkRealTableVgroupsInfo, vgroupsInfoToJson, pNode->pVgroupList);
}
return code;
}
static int32_t jsonToRealTableNode(const SJson* pJson, void* pObj) {
SRealTableNode* pNode = (SRealTableNode*)pObj;
int32_t objSize = 0;
int32_t code = jsonToTableNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkRealTableMetaSize, &objSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonMakeObject(pJson, jkRealTableMeta, jsonToTableMeta, (void**)&pNode->pMeta, objSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkRealTableVgroupsInfoSize, &objSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonMakeObject(pJson, jkRealTableVgroupsInfo, jsonToVgroupsInfo, (void**)&pNode->pVgroupList, objSize);
}
return code;
}
static const char* jkGroupingSetType = "GroupingSetType";
static const char* jkGroupingSetParameter = "Parameters";
@ -1464,7 +1836,7 @@ static const char* jkSelectStmtSlimit = "Slimit";
static int32_t selectStmtTojson(const void* pObj, SJson* pJson) {
const SSelectStmt* pNode = (const SSelectStmt*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct);
int32_t code = tjsonAddBoolToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct);
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkSelectStmtProjections, pNode->pProjectionList);
}
@ -1499,6 +1871,83 @@ static int32_t selectStmtTojson(const void* pObj, SJson* pJson) {
return code;
}
static int32_t jsonToSelectStmt(const SJson* pJson, void* pObj) {
SSelectStmt* pNode = (SSelectStmt*)pObj;
int32_t code = tjsonGetBoolValue(pJson, jkSelectStmtDistinct, &pNode->isDistinct);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSelectStmtProjections, &pNode->pProjectionList);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtFrom, &pNode->pFromTable);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtWhere, &pNode->pWhere);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSelectStmtPartitionBy, &pNode->pPartitionByList);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtWindow, &pNode->pWindow);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSelectStmtGroupBy, &pNode->pGroupByList);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtHaving, &pNode->pHaving);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSelectStmtOrderBy, &pNode->pOrderByList);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtLimit, &pNode->pLimit);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSelectStmtSlimit, &pNode->pSlimit);
}
return code;
}
static const char* jkCreateTopicStmtTopicName = "TopicName";
static const char* jkCreateTopicStmtSubscribeDbName = "SubscribeDbName";
static const char* jkCreateTopicStmtIgnoreExists = "IgnoreExists";
static const char* jkCreateTopicStmtQuery = "Query";
static int32_t createTopicStmtToJson(const void* pObj, SJson* pJson) {
const SCreateTopicStmt* pNode = (const SCreateTopicStmt*)pObj;
int32_t code = tjsonAddStringToObject(pJson, jkCreateTopicStmtTopicName, pNode->topicName);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subscribeDbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkCreateTopicStmtIgnoreExists, pNode->ignoreExists);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkCreateTopicStmtQuery, nodeToJson, pNode->pQuery);
}
return code;
}
static int32_t jsonToCreateTopicStmt(const SJson* pJson, void* pObj) {
SCreateTopicStmt* pNode = (SCreateTopicStmt*)pObj;
int32_t code = tjsonGetStringValue(pJson, jkCreateTopicStmtTopicName, pNode->topicName);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subscribeDbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkCreateTopicStmtIgnoreExists, &pNode->ignoreExists);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkCreateTopicStmtQuery, &pNode->pQuery);
}
return code;
}
static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
switch (nodeType(pObj)) {
case QUERY_NODE_COLUMN:
@ -1512,6 +1961,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_FUNCTION:
return functionNodeToJson(pObj, pJson);
case QUERY_NODE_REAL_TABLE:
return realTableNodeToJson(pObj, pJson);
case QUERY_NODE_TEMP_TABLE:
case QUERY_NODE_JOIN_TABLE:
break;
@ -1549,6 +1999,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_SHOW_DATABASES_STMT:
case QUERY_NODE_SHOW_TABLES_STMT:
break;
case QUERY_NODE_CREATE_TOPIC_STMT:
return createTopicStmtToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_SCAN:
return logicScanNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_JOIN:
@ -1565,9 +2017,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return physiTagScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
return physiTableScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
break;
return physiStreamScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return physiProjectNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_JOIN:
@ -1589,6 +2040,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN:
return planToJson(pObj, pJson);
default:
assert(0);
break;
}
nodesWarn("specificNodeToJson unknown node = %s", nodesNodeName(nodeType(pObj)));
@ -1607,7 +2059,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToLogicConditionNode(pJson, pObj);
case QUERY_NODE_FUNCTION:
return jsonToFunctionNode(pJson, pObj);
// case QUERY_NODE_REAL_TABLE:
case QUERY_NODE_REAL_TABLE:
return jsonToRealTableNode(pJson, pObj);
// case QUERY_NODE_TEMP_TABLE:
// case QUERY_NODE_JOIN_TABLE:
// break;
@ -1633,20 +2086,24 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToDownstreamSourceNode(pJson, pObj);
// case QUERY_NODE_SET_OPERATOR:
// break;
// case QUERY_NODE_SELECT_STMT:
// return jsonToSelectStmt(pJson, pObj);
// case QUERY_NODE_LOGIC_PLAN_SCAN:
// return jsonToLogicScanNode(pJson, pObj);
case QUERY_NODE_SELECT_STMT:
return jsonToSelectStmt(pJson, pObj);
case QUERY_NODE_CREATE_TOPIC_STMT:
return jsonToCreateTopicStmt(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_SCAN:
return jsonToLogicScanNode(pJson, pObj);
// case QUERY_NODE_LOGIC_PLAN_JOIN:
// return jsonToLogicJoinNode(pJson, pObj);
// case QUERY_NODE_LOGIC_PLAN_AGG:
// return jsonToLogicAggNode(pJson, pObj);
// case QUERY_NODE_LOGIC_PLAN_PROJECT:
// return jsonToLogicProjectNode(pJson, pObj);
case QUERY_NODE_LOGIC_PLAN_PROJECT:
return jsonToLogicProjectNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
return jsonToPhysiTagScanNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
return jsonToPhysiTableScanNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
return jsonToPhysiStreamScanNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return jsonToPhysiProjectNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_JOIN:
@ -1768,6 +2225,7 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode) {
int32_t code = makeNodeByJson(pJson, pNode);
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode(*pNode);
*pNode = NULL;
terrno = code;
return code;
}

17
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -125,6 +125,8 @@ SNodeptr nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SCreateDnodeStmt));
case QUERY_NODE_DROP_DNODE_STMT:
return makeNode(type, sizeof(SDropDnodeStmt));
case QUERY_NODE_ALTER_DNODE_STMT:
return makeNode(type, sizeof(SAlterDnodeStmt));
case QUERY_NODE_SHOW_DNODES_STMT:
return makeNode(type, sizeof(SShowStmt));
case QUERY_NODE_SHOW_VGROUPS_STMT:
@ -168,7 +170,7 @@ SNodeptr nodesMakeNode(ENodeType type) {
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
return makeNode(type, sizeof(STableSeqScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
return makeNode(type, sizeof(SNode));
return makeNode(type, sizeof(SStreamScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return makeNode(type, sizeof(SProjectPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_JOIN:
@ -357,6 +359,17 @@ int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) {
return TSDB_CODE_SUCCESS;
}
int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc) {
if (NULL == pSrc) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = nodesListAppendList(pTarget, pSrc);
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pSrc);
}
return code;
}
SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) {
if (NULL == pCell->pPrev) {
pList->pHead = pCell->pNext;
@ -571,7 +584,7 @@ typedef struct SCollectFuncsCxt {
static EDealRes collectFuncs(SNode* pNode, void* pContext) {
SCollectFuncsCxt* pCxt = (SCollectFuncsCxt*)pContext;
if (QUERY_NODE_FUNCTION == nodeType(pNode) && pCxt->classifier(((SFunctionNode*)pNode)->funcId)) {
pCxt->errCode = nodesListAppend(pCxt->pFuncs, pNode);
pCxt->errCode = nodesListStrictAppend(pCxt->pFuncs, nodesCloneNode(pNode));
return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR);
}
return DEAL_RES_CONTINUE;

21
source/libs/parser/inc/parAst.h
View File

@ -50,8 +50,10 @@ typedef enum EDatabaseOptionType {
DB_OPTION_TTL,
DB_OPTION_WAL,
DB_OPTION_VGROUPS,
DB_OPTION_SINGLESTABLE,
DB_OPTION_STREAMMODE,
DB_OPTION_SINGLE_STABLE,
DB_OPTION_STREAM_MODE,
DB_OPTION_RETENTIONS,
DB_OPTION_FILE_FACTOR,
DB_OPTION_MAX
} EDatabaseOptionType;
@ -65,6 +67,11 @@ typedef enum ETableOptionType {
TABLE_OPTION_MAX
} ETableOptionType;
typedef struct SAlterOption {
int32_t type;
SToken val;
} SAlterOption;
extern SToken nil_token;
void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt);
@ -110,13 +117,16 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr
SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight);
SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt);
SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt);
SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal);
SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pDbName, SNode* pOptions);
SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pDbName);
SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName, SNode* pOptions);
SNode* createDefaultTableOptions(SAstCreateContext* pCxt);
SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt);
SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal);
SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pSma);
SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs);
SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment);
SDataType createDataType(uint8_t type);
SDataType createVarLenDataType(uint8_t type, const SToken* pLen);
@ -126,6 +136,11 @@ SNode* createCreateMultiTableStmt(SAstCreateContext* pCxt, SNodeList* pSubTables
SNode* createDropTableClause(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable);
SNode* createDropTableStmt(SAstCreateContext* pCxt, SNodeList* pTables);
SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable);
SNode* createAlterTableOption(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions);
SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName, SDataType dataType);
SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName);
SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pOldColName, const SToken* pNewColName);
SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const SToken* pTagName, SNode* pVal);
SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName);
SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDbName);
SNode* createCreateUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, const SToken* pPassword);
@ -133,6 +148,7 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, const SToken* pUserName, int
SNode* createDropUserStmt(SAstCreateContext* pCxt, const SToken* pUserName);
SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort);
SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode);
SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue);
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions);
SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding);
SNode* createDropIndexStmt(SAstCreateContext* pCxt, const SToken* pIndexName, const SToken* pTableName);
@ -140,6 +156,7 @@ SNode* createCreateQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId);
SNode* createDropQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId);
SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName);
SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName);
SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue);
#ifdef __cplusplus
}

105
source/libs/parser/inc/sql.y
View File

@ -22,6 +22,7 @@
}
%syntax_error {
if (pCxt->valid) {
if(TOKEN.z) {
generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z);
} else {
@ -29,6 +30,7 @@
}
pCxt->valid = false;
}
}
%left OR.
%left AND.
@ -41,6 +43,41 @@
%left NK_CONCAT.
//%right NK_BITNOT.
/************************************************ create/alter account *****************************************/
cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); }
cmd ::= ALTER ACCOUNT NK_ID alter_account_options. { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); }
%type account_options { int32_t }
%destructor account_options { }
account_options ::= . { }
account_options ::= account_options PPS literal. { }
account_options ::= account_options TSERIES literal. { }
account_options ::= account_options STORAGE literal. { }
account_options ::= account_options STREAMS literal. { }
account_options ::= account_options QTIME literal. { }
account_options ::= account_options DBS literal. { }
account_options ::= account_options USERS literal. { }
account_options ::= account_options CONNS literal. { }
account_options ::= account_options STATE literal. { }
%type alter_account_options { int32_t }
%destructor alter_account_options { }
alter_account_options ::= alter_account_option. { }
alter_account_options ::= alter_account_options alter_account_option. { }
%type alter_account_option { int32_t }
%destructor alter_account_option { }
alter_account_option ::= PASS literal. { }
alter_account_option ::= PPS literal. { }
alter_account_option ::= TSERIES literal. { }
alter_account_option ::= STORAGE literal. { }
alter_account_option ::= STREAMS literal. { }
alter_account_option ::= QTIME literal. { }
alter_account_option ::= DBS literal. { }
alter_account_option ::= USERS literal. { }
alter_account_option ::= CONNS literal. { }
alter_account_option ::= STATE literal. { }
/************************************************ create/alter/drop/show user *****************************************/
cmd ::= CREATE USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B); }
cmd ::= ALTER USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B); }
@ -48,12 +85,16 @@ cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B).
cmd ::= DROP USER user_name(A). { pCxt->pRootNode = createDropUserStmt(pCxt, &A); }
cmd ::= SHOW USERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT, NULL); }
/************************************************ create/drop/show dnode **********************************************/
/************************************************ create/drop/alter/show dnode ****************************************/
cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); }
cmd ::= CREATE DNODE dnode_host_name(A) PORT NK_INTEGER(B). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, &B); }
cmd ::= DROP DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); }
cmd ::= DROP DNODE dnode_endpoint(A). { pCxt->pRootNode = createDropDnodeStmt(pCxt, &A); }
cmd ::= SHOW DNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL); }
cmd ::= ALTER DNODE NK_INTEGER(A) NK_STRING(B). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &A, &B, NULL); }
cmd ::= ALTER DNODE NK_INTEGER(A) NK_STRING(B) NK_STRING(C). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &A, &B, &C); }
cmd ::= ALTER ALL DNODES NK_STRING(A). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &A, NULL); }
cmd ::= ALTER ALL DNODES NK_STRING(A) NK_STRING(B). { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &A, &B); }
%type dnode_endpoint { SToken }
%destructor dnode_endpoint { }
@ -64,6 +105,10 @@ dnode_endpoint(A) ::= NK_STRING(B).
dnode_host_name(A) ::= NK_ID(B). { A = B; }
dnode_host_name(A) ::= NK_IPTOKEN(B). { A = B; }
/************************************************ alter local *********************************************************/
cmd ::= ALTER LOCAL NK_STRING(A). { pCxt->pRootNode = createAlterLocalStmt(pCxt, &A, NULL); }
cmd ::= ALTER LOCAL NK_STRING(A) NK_STRING(B). { pCxt->pRootNode = createAlterLocalStmt(pCxt, &A, &B); }
/************************************************ create/drop qnode ***************************************************/
cmd ::= CREATE QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createCreateQnodeStmt(pCxt, &A); }
cmd ::= DROP QNODE ON DNODE NK_INTEGER(A). { pCxt->pRootNode = createDropQnodeStmt(pCxt, &A); }
@ -74,7 +119,7 @@ cmd ::= CREATE DATABASE not_exists_opt(A) db_name(B) db_options(C).
cmd ::= DROP DATABASE exists_opt(A) db_name(B). { pCxt->pRootNode = createDropDatabaseStmt(pCxt, A, &B); }
cmd ::= SHOW DATABASES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL); }
cmd ::= USE db_name(A). { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &A); }
cmd ::= ALTER DATABASE db_name(A) db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); }
cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); }
%type not_exists_opt { bool }
%destructor not_exists_opt { }
@ -102,8 +147,22 @@ db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C).
db_options(A) ::= db_options(B) TTL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TTL, &C); }
db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL, &C); }
db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); }
db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLESTABLE, &C); }
db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAMMODE, &C); }
db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLE_STABLE, &C); }
db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAM_MODE, &C); }
db_options(A) ::= db_options(B) RETENTIONS NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_RETENTIONS, &C); }
db_options(A) ::= db_options(B) FILE_FACTOR NK_FLOAT(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_FILE_FACTOR, &C); }
alter_db_options(A) ::= alter_db_option(B). { A = createDefaultAlterDatabaseOptions(pCxt); A = setDatabaseOption(pCxt, A, B.type, &B.val); }
alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setDatabaseOption(pCxt, B, C.type, &C.val); }
%type alter_db_option { SAlterOption }
%destructor alter_db_option { }
alter_db_option(A) ::= BLOCKS NK_INTEGER(B). { A.type = DB_OPTION_BLOCKS; A.val = B; }
alter_db_option(A) ::= FSYNC NK_INTEGER(B). { A.type = DB_OPTION_FSYNC; A.val = B; }
alter_db_option(A) ::= KEEP NK_INTEGER(B). { A.type = DB_OPTION_KEEP; A.val = B; }
alter_db_option(A) ::= WAL NK_INTEGER(B). { A.type = DB_OPTION_WAL; A.val = B; }
alter_db_option(A) ::= QUORUM NK_INTEGER(B). { A.type = DB_OPTION_QUORUM; A.val = B; }
alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.val = B; }
/************************************************ create/drop/show table/stable ***************************************/
cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B)
@ -116,6 +175,27 @@ cmd ::= DROP STABLE exists_opt(A) full_table_name(B).
cmd ::= SHOW TABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, NULL); }
cmd ::= SHOW STABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, NULL); }
cmd ::= ALTER TABLE alter_table_clause(A). { pCxt->pRootNode = A; }
cmd ::= ALTER STABLE alter_table_clause(A). { pCxt->pRootNode = A; }
alter_table_clause(A) ::= full_table_name(B) alter_table_options(C). { A = createAlterTableOption(pCxt, B, C); }
alter_table_clause(A) ::=
full_table_name(B) ADD COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_ADD_COLUMN, &C, D); }
alter_table_clause(A) ::= full_table_name(B) DROP COLUMN column_name(C). { A = createAlterTableDropCol(pCxt, B, TSDB_ALTER_TABLE_DROP_COLUMN, &C); }
alter_table_clause(A) ::=
full_table_name(B) MODIFY COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &C, D); }
alter_table_clause(A) ::=
full_table_name(B) RENAME COLUMN column_name(C) column_name(D). { A = createAlterTableRenameCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &C, &D); }
alter_table_clause(A) ::=
full_table_name(B) ADD TAG column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_ADD_TAG, &C, D); }
alter_table_clause(A) ::= full_table_name(B) DROP TAG column_name(C). { A = createAlterTableDropCol(pCxt, B, TSDB_ALTER_TABLE_DROP_TAG, &C); }
alter_table_clause(A) ::=
full_table_name(B) MODIFY TAG column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &C, D); }
alter_table_clause(A) ::=
full_table_name(B) RENAME TAG column_name(C) column_name(D). { A = createAlterTableRenameCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &C, &D); }
alter_table_clause(A) ::=
full_table_name(B) SET TAG column_name(C) NK_EQ literal(D). { A = createAlterTableSetTag(pCxt, B, &C, D); }
%type multi_create_clause { SNodeList* }
%destructor multi_create_clause { nodesDestroyList($$); }
multi_create_clause(A) ::= create_subtable_clause(B). { A = createNodeList(pCxt, B); }
@ -188,6 +268,16 @@ table_options(A) ::= table_options(B) COMMENT NK_STRING(C).
table_options(A) ::= table_options(B) KEEP NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_KEEP, &C); }
table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); }
table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableSmaOption(pCxt, B, C); }
table_options(A) ::= table_options(B) ROLLUP NK_LP func_name_list(C) NK_RP. { A = setTableRollupOption(pCxt, B, C); }
alter_table_options(A) ::= alter_table_option(B). { A = createDefaultAlterTableOptions(pCxt); A = setTableOption(pCxt, A, B.type, &B.val); }
alter_table_options(A) ::= alter_table_options(B) alter_table_option(C). { A = setTableOption(pCxt, B, C.type, &C.val); }
%type alter_table_option { SAlterOption }
%destructor alter_table_option { }
alter_table_option(A) ::= COMMENT NK_STRING(B). { A.type = TABLE_OPTION_COMMENT; A.val = B; }
alter_table_option(A) ::= KEEP NK_INTEGER(B). { A.type = TABLE_OPTION_KEEP; A.val = B; }
alter_table_option(A) ::= TTL NK_INTEGER(B). { A.type = TABLE_OPTION_TTL; A.val = B; }
%type col_name_list { SNodeList* }
%destructor col_name_list { nodesDestroyList($$); }
@ -196,6 +286,13 @@ col_name_list(A) ::= col_name_list(B) NK_COMMA col_name(C).
col_name(A) ::= column_name(B). { A = createColumnNode(pCxt, NULL, &B); }
%type func_name_list { SNodeList* }
%destructor func_name_list { nodesDestroyList($$); }
func_name_list(A) ::= func_name(B). { A = createNodeList(pCxt, B); }
func_name_list(A) ::= func_name_list(B) NK_COMMA col_name(C). { A = addNodeToList(pCxt, B, C); }
func_name(A) ::= function_name(B). { A = createFunctionNode(pCxt, &B, NULL); }
/************************************************ create index ********************************************************/
cmd ::= CREATE SMA INDEX index_name(A) ON table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, &A, &B, NULL, C); }
cmd ::= CREATE FULLTEXT INDEX

116
source/libs/parser/src/parAstCreater.c
View File

@ -246,6 +246,16 @@ static SDatabaseOptions* setDbStreamMode(SAstCreateContext* pCxt, SDatabaseOptio
return pOptions;
}
static SDatabaseOptions* setDbRetentions(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
// todo
return pOptions;
}
static SDatabaseOptions* setDbFileFactor(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
// todo
return pOptions;
}
static void initSetDatabaseOptionFp() {
setDbOptionFuncs[DB_OPTION_BLOCKS] = setDbBlocks;
setDbOptionFuncs[DB_OPTION_CACHE] = setDbCache;
@ -262,8 +272,10 @@ static void initSetDatabaseOptionFp() {
setDbOptionFuncs[DB_OPTION_TTL] = setDbTtl;
setDbOptionFuncs[DB_OPTION_WAL] = setDbWal;
setDbOptionFuncs[DB_OPTION_VGROUPS] = setDbVgroups;
setDbOptionFuncs[DB_OPTION_SINGLESTABLE] = setDbSingleStable;
setDbOptionFuncs[DB_OPTION_STREAMMODE] = setDbStreamMode;
setDbOptionFuncs[DB_OPTION_SINGLE_STABLE] = setDbSingleStable;
setDbOptionFuncs[DB_OPTION_STREAM_MODE] = setDbStreamMode;
setDbOptionFuncs[DB_OPTION_RETENTIONS] = setDbRetentions;
setDbOptionFuncs[DB_OPTION_FILE_FACTOR] = setDbFileFactor;
}
static STableOptions* setTableKeep(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) {
@ -772,6 +784,29 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
return (SNode*)pOptions;
}
SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt) {
SDatabaseOptions* pOptions = nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions);
pOptions->numOfBlocks = -1;
pOptions->cacheBlockSize = -1;
pOptions->cachelast = -1;
pOptions->compressionLevel = -1;
pOptions->daysPerFile = -1;
pOptions->fsyncPeriod = -1;
pOptions->maxRowsPerBlock = -1;
pOptions->minRowsPerBlock = -1;
pOptions->keep = -1;
pOptions->precision = -1;
pOptions->quorum = -1;
pOptions->replica = -1;
pOptions->ttl = -1;
pOptions->walLevel = -1;
pOptions->numOfVgroups = -1;
pOptions->singleStable = -1;
pOptions->streamMode = -1;
return (SNode*)pOptions;
}
SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal) {
return (SNode*)setDbOptionFuncs[type](pCxt, (SDatabaseOptions*)pOptions, pVal);
}
@ -818,6 +853,14 @@ SNode* createDefaultTableOptions(SAstCreateContext* pCxt) {
return (SNode*)pOptions;
}
SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt) {
STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions);
pOptions->keep = -1;
pOptions->ttl = -1;
return (SNode*)pOptions;
}
SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal) {
return (SNode*)setTableOptionFuncs[type](pCxt, (STableOptions*)pOptions, pVal);
}
@ -827,6 +870,11 @@ SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pS
return pOptions;
}
SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs) {
// todo
return pOptions;
}
SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment) {
SColumnDefNode* pCol = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF);
CHECK_OUT_OF_MEM(pCol);
@ -912,6 +960,49 @@ SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, S
return (SNode*)pStmt;
}
SNode* createAlterTableOption(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions) {
SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = TSDB_ALTER_TABLE_UPDATE_OPTIONS;
pStmt->pOptions = (STableOptions*)pOptions;
return (SNode*)pStmt;
}
SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName, SDataType dataType) {
SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = alterType;
strncpy(pStmt->colName, pColName->z, pColName->n);
pStmt->dataType = dataType;
return (SNode*)pStmt;
}
SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName) {
SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = alterType;
strncpy(pStmt->colName, pColName->z, pColName->n);
return (SNode*)pStmt;
}
SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pOldColName, const SToken* pNewColName) {
SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = alterType;
strncpy(pStmt->colName, pOldColName->z, pOldColName->n);
strncpy(pStmt->newColName, pNewColName->z, pNewColName->n);
return (SNode*)pStmt;
}
SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const SToken* pTagName, SNode* pVal) {
SAlterTableStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->alterType = TSDB_ALTER_TABLE_UPDATE_TAG_VAL;
strncpy(pStmt->colName, pTagName->z, pTagName->n);
pStmt->pVal = (SValueNode*)pVal;
return (SNode*)pStmt;
}
SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName) {
SUseDatabaseStmt* pStmt = (SUseDatabaseStmt*)nodesMakeNode(QUERY_NODE_USE_DATABASE_STMT);
CHECK_OUT_OF_MEM(pStmt);
@ -1009,6 +1100,17 @@ SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode) {
return (SNode*)pStmt;
}
SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue) {
SAlterDnodeStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_DNODE_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->dnodeId = strtol(pDnode->z, NULL, 10);
trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config));
if (NULL != pValue) {
trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value));
}
return (SNode*)pStmt;
}
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, const SToken* pIndexName, const SToken* pTableName, SNodeList* pCols, SNode* pOptions) {
if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) {
return NULL;
@ -1077,3 +1179,13 @@ SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const
pStmt->ignoreNotExists = ignoreNotExists;
return (SNode*)pStmt;
}
SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue) {
SAlterLocalStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_LOCAL_STMT);
CHECK_OUT_OF_MEM(pStmt);
trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config));
if (NULL != pValue) {
trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value));
}
return (SNode*)pStmt;
}

2
source/libs/parser/src/parInsert.c
View File

@ -391,7 +391,7 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha
}
// Remove quotation marks
if (TSDB_DATA_TYPE_BINARY == type) {
if (TK_NK_STRING == pToken->type) {
if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) {
return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z);
}

2
source/libs/parser/src/parTokenizer.c
View File

@ -28,6 +28,7 @@ typedef struct SKeyword {
// keywords in sql string
static SKeyword keywordTable[] = {
{"ACCOUNT", TK_ACCOUNT},
{"ALL", TK_ALL},
{"ALTER", TK_ALTER},
{"AND", TK_AND},
@ -168,7 +169,6 @@ static SKeyword keywordTable[] = {
// {"SCORES", TK_SCORES},
// {"GRANTS", TK_GRANTS},
// {"DOT", TK_DOT},
// {"ACCOUNT", TK_ACCOUNT},
// {"DESCRIBE", TK_DESCRIBE},
// {"SYNCDB", TK_SYNCDB},
// {"LOCAL", TK_LOCAL},

118
source/libs/parser/src/parTranslater.c
View File

@ -499,6 +499,10 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect)
}
static int32_t setTableVgroupList(SParseContext* pCxt, SName* name, SRealTableNode* pRealTable) {
if (pCxt->topicQuery) {
return TSDB_CODE_SUCCESS;
}
if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) {
SArray* vgroupList = NULL;
int32_t code = catalogGetTableDistVgInfo(pCxt->pCatalog, pCxt->pTransporter, &pCxt->mgmtEpSet, name, &vgroupList);
@ -962,6 +966,73 @@ static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableS
return doTranslateDropSuperTable(pCxt, &tableName, pStmt->ignoreNotExists);
}
static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAltertbReq* pAlterReq) {
pAlterReq->pFields = taosArrayInit(2, sizeof(TAOS_FIELD));
if (NULL == pAlterReq->pFields) {
return TSDB_CODE_OUT_OF_MEMORY;
}
switch (pStmt->alterType) {
case TSDB_ALTER_TABLE_ADD_TAG:
case TSDB_ALTER_TABLE_DROP_TAG:
case TSDB_ALTER_TABLE_ADD_COLUMN:
case TSDB_ALTER_TABLE_DROP_COLUMN:
case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: {
TAOS_FIELD field = { .type = pStmt->dataType.type, .bytes = pStmt->dataType.bytes };
strcpy(field.name, pStmt->colName);
taosArrayPush(pAlterReq->pFields, &field);
break;
}
case TSDB_ALTER_TABLE_UPDATE_TAG_NAME:
case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: {
TAOS_FIELD oldField = {0};
strcpy(oldField.name, pStmt->colName);
taosArrayPush(pAlterReq->pFields, &oldField);
TAOS_FIELD newField = {0};
strcpy(oldField.name, pStmt->newColName);
taosArrayPush(pAlterReq->pFields, &newField);
break;
}
default:
break;
}
return TSDB_CODE_SUCCESS;
}
static int32_t translateAlterTable(STranslateContext* pCxt, SAlterTableStmt* pStmt) {
SMAltertbReq alterReq = {0};
SName tableName = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId };
strcpy(tableName.dbname, pStmt->dbName);
strcpy(tableName.tname, pStmt->tableName);
tNameExtractFullName(&tableName, alterReq.name);
alterReq.alterType = pStmt->alterType;
alterReq.numOfFields = 1;
if (TSDB_ALTER_TABLE_UPDATE_OPTIONS == pStmt->alterType) {
// todo
} else {
if (TSDB_CODE_SUCCESS != setAlterTableField(pStmt, &alterReq)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo));
if (NULL == pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_STB;
pCxt->pCmdMsg->msgLen = tSerializeSMAlterStbReq(NULL, 0, &alterReq);
pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tSerializeSMAlterStbReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &alterReq);
return TSDB_CODE_SUCCESS;
}
static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* pStmt) {
SName name = {0};
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
@ -1099,6 +1170,28 @@ static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt
return TSDB_CODE_SUCCESS;
}
static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pStmt) {
SMCfgDnodeReq cfgReq = {0};
cfgReq.dnodeId = pStmt->dnodeId;
strcpy(cfgReq.config, pStmt->config);
strcpy(cfgReq.value, pStmt->value);
pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo));
if (NULL == pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_MND_CONFIG_DNODE;
pCxt->pCmdMsg->msgLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq);
pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tSerializeSMCfgDnodeReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &cfgReq);
return TSDB_CODE_SUCCESS;
}
static int32_t nodeTypeToShowType(ENodeType nt) {
switch (nt) {
case QUERY_NODE_SHOW_DATABASES_STMT:
@ -1300,6 +1393,7 @@ static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* p
SCMCreateTopicReq createReq = {0};
if (NULL != pStmt->pQuery) {
pCxt->pParseCxt->topicQuery = true;
int32_t code = translateQuery(pCxt, pStmt->pQuery);
if (TSDB_CODE_SUCCESS == code) {
code = nodesNodeToString(pStmt->pQuery, false, &createReq.ast, NULL);
@ -1364,6 +1458,11 @@ static int32_t translateDropTopic(STranslateContext* pCxt, SDropTopicStmt* pStmt
return TSDB_CODE_SUCCESS;
}
static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pStmt) {
// todo
return TSDB_CODE_SUCCESS;
}
static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pNode)) {
@ -1388,6 +1487,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_DROP_SUPER_TABLE_STMT:
code = translateDropSuperTable(pCxt, (SDropSuperTableStmt*)pNode);
break;
case QUERY_NODE_ALTER_TABLE_STMT:
code = translateAlterTable(pCxt, (SAlterTableStmt*)pNode);
break;
case QUERY_NODE_CREATE_USER_STMT:
code = translateCreateUser(pCxt, (SCreateUserStmt*)pNode);
break;
@ -1406,6 +1508,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_DROP_DNODE_STMT:
code = translateDropDnode(pCxt, (SDropDnodeStmt*)pNode);
break;
case QUERY_NODE_ALTER_DNODE_STMT:
code = translateAlterDnode(pCxt, (SAlterDnodeStmt*)pNode);
break;
case QUERY_NODE_SHOW_DATABASES_STMT:
case QUERY_NODE_SHOW_STABLES_STMT:
case QUERY_NODE_SHOW_USERS_STMT:
@ -1436,6 +1541,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_DROP_TOPIC_STMT:
code = translateDropTopic(pCxt, (SDropTopicStmt*)pNode);
break;
case QUERY_NODE_ALTER_LOCAL_STMT:
code = translateAlterLocal(pCxt, (SAlterLocalStmt*)pNode);
break;
default:
break;
}
@ -1855,6 +1963,11 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery)
return rewriteToVnodeModifOpStmt(pQuery, pBufArray);
}
static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) {
// todo
return TSDB_CODE_SUCCESS;
}
static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pQuery->pRoot)) {
@ -1866,6 +1979,11 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_CREATE_MULTI_TABLE_STMT:
code = rewriteCreateMultiTable(pCxt, pQuery);
break;
case QUERY_NODE_ALTER_TABLE_STMT:
if (TSDB_ALTER_TABLE_UPDATE_TAG_VAL == ((SAlterTableStmt*)pQuery->pRoot)->alterType) {
code = rewriteAlterTable(pCxt, pQuery);
}
break;
default:
break;
}

14
source/libs/parser/src/parUtil.c
View File

@ -46,17 +46,19 @@ static char* getSyntaxErrFormat(int32_t errCode) {
case TSDB_CODE_PAR_NOT_SINGLE_GROUP:
return "Not a single-group group function";
case TSDB_CODE_PAR_TAGS_NOT_MATCHED:
return "tags number not matched";
return "Tags number not matched";
case TSDB_CODE_PAR_INVALID_TAG_NAME:
return "invalid tag name : %s";
return "Invalid tag name : %s";
case TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG:
return "name or password too long";
return "Name or password too long";
case TSDB_CODE_PAR_PASSWD_EMPTY:
return "password can not be empty";
return "Password can not be empty";
case TSDB_CODE_PAR_INVALID_PORT:
return "port should be an integer that is less than 65535 and greater than 0";
return "Port should be an integer that is less than 65535 and greater than 0";
case TSDB_CODE_PAR_INVALID_ENDPOINT:
return "endpoint should be in the format of 'fqdn:port'";
return "Endpoint should be in the format of 'fqdn:port'";
case TSDB_CODE_PAR_EXPRIE_STATEMENT:
return "This statement is no longer supported";
case TSDB_CODE_OUT_OF_MEMORY:
return "Out of memory";
default:

3748
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

17
source/libs/parser/test/parserAstTest.cpp
View File

@ -302,6 +302,13 @@ TEST_F(ParserTest, createUser) {
ASSERT_TRUE(run());
}
TEST_F(ParserTest, alterAccount) {
setDatabase("root", "test");
bind("alter account ac_wxy pass '123456'");
ASSERT_TRUE(run(TSDB_CODE_PAR_EXPRIE_STATEMENT));
}
TEST_F(ParserTest, createDnode) {
setDatabase("root", "test");
@ -312,6 +319,16 @@ TEST_F(ParserTest, createDnode) {
ASSERT_TRUE(run());
}
TEST_F(ParserTest, alterDnode) {
setDatabase("root", "test");
bind("alter dnode 1 'resetLog'");
ASSERT_TRUE(run());
bind("alter dnode 1 'debugFlag' '134'");
ASSERT_TRUE(run());
}
TEST_F(ParserTest, createDatabase) {
setDatabase("root", "test");

468
source/libs/planner/src/planLogicCreater.c
View File

@ -18,13 +18,13 @@
#include "functionMgt.h"
typedef struct SLogicPlanContext {
int32_t errCode;
int32_t planNodeId;
int32_t acctId;
SPlanContext* pPlanCxt;
} SLogicPlanContext;
static SLogicNode* createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt);
static SLogicNode* createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable);
typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, SSelectStmt*, SLogicNode**);
static int32_t doCreateLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode);
static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogicNode** pLogicNode);
typedef struct SRewriteExprCxt {
int32_t errCode;
@ -66,7 +66,6 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) {
}
typedef struct SNameExprCxt {
int32_t planNodeId;
int32_t rewriteId;
} SNameExprCxt;
@ -76,7 +75,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) {
case QUERY_NODE_LOGIC_CONDITION:
case QUERY_NODE_FUNCTION: {
SNameExprCxt* pCxt = (SNameExprCxt*)pContext;
sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d_%d", pCxt->planNodeId, pCxt->rewriteId++);
sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d", pCxt->rewriteId++);
return DEAL_RES_IGNORE_CHILD;
}
default:
@ -86,132 +85,195 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
static int32_t rewriteExpr(int32_t planNodeId, int32_t rewriteId, SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) {
SNameExprCxt nameCxt = { .planNodeId = planNodeId, .rewriteId = rewriteId };
static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) {
static int32_t rewriteId = 1;
SNameExprCxt nameCxt = { .rewriteId = rewriteId };
nodesWalkList(pExprs, doNameExpr, &nameCxt);
SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs };
nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt);
return cxt.errCode;
}
static SLogicNode* pushLogicNode(SLogicPlanContext* pCxt, SLogicNode* pRoot, SLogicNode* pNode) {
if (TSDB_CODE_SUCCESS != pCxt->errCode) {
goto error;
static int32_t pushLogicNode(SLogicPlanContext* pCxt, SLogicNode** pOldRoot, SLogicNode* pNewRoot) {
if (NULL == pNewRoot->pChildren) {
pNewRoot->pChildren = nodesMakeList();
if (NULL == pNewRoot->pChildren) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS != nodesListAppend(pNewRoot->pChildren, (SNode*)*pOldRoot)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (NULL == pRoot) {
return pNode;
(*pOldRoot)->pParent = pNewRoot;
*pOldRoot = pNewRoot;
return TSDB_CODE_SUCCESS;
}
if (NULL == pNode) {
return pRoot;
static int32_t createChildLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, FCreateLogicNode func, SLogicNode** pRoot) {
SLogicNode* pNode = NULL;
int32_t code = func(pCxt, pSelect, &pNode);
if (TSDB_CODE_SUCCESS == code && NULL != pNode) {
code = pushLogicNode(pCxt, pRoot, pNode);
}
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode(pNode);
}
return code;
}
if (NULL == pNode->pChildren) {
pNode->pChildren = nodesMakeList();
if (NULL == pNode->pChildren) {
goto error;
}
}
if (TSDB_CODE_SUCCESS != nodesListAppend(pNode->pChildren, (SNode*)pRoot)) {
goto error;
}
pRoot->pParent = pNode;
return pNode;
error:
nodesDestroyNode((SNode*)pNode);
return pRoot;
}
static SLogicNode* createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable) {
static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, SLogicNode** pLogicNode) {
SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN);
CHECK_ALLOC(pScan, NULL);
pScan->node.id = pCxt->planNodeId++;
if (NULL == pScan) {
return TSDB_CODE_OUT_OF_MEMORY;
}
TSWAP(pScan->pMeta, pRealTable->pMeta, STableMeta*);
TSWAP(pScan->pVgroupList, pRealTable->pVgroupList, SVgroupsInfo*);
// set columns to scan
SNodeList* pCols = NULL;
CHECK_CODE(nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols), (SLogicNode*)pScan);
if (NULL != pCols) {
pScan->pScanCols = nodesCloneList(pCols);
CHECK_ALLOC(pScan->pScanCols, (SLogicNode*)pScan);
}
// set output
if (NULL != pCols) {
pScan->node.pTargets = nodesCloneList(pCols);
CHECK_ALLOC(pScan->node.pTargets, (SLogicNode*)pScan);
}
pScan->scanType = SCAN_TYPE_TABLE;
pScan->scanType = pCxt->pPlanCxt->topicQuery ? SCAN_TYPE_TOPIC : SCAN_TYPE_TABLE;
pScan->scanFlag = MAIN_SCAN;
pScan->scanRange = TSWINDOW_INITIALIZER;
pScan->tableName.type = TSDB_TABLE_NAME_T;
pScan->tableName.acctId = pCxt->acctId;
pScan->tableName.acctId = pCxt->pPlanCxt->acctId;
strcpy(pScan->tableName.dbname, pRealTable->table.dbName);
strcpy(pScan->tableName.tname, pRealTable->table.tableName);
return (SLogicNode*)pScan;
// set columns to scan
SNodeList* pCols = NULL;
int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols);
if (TSDB_CODE_SUCCESS == code && NULL != pCols) {
pScan->pScanCols = nodesCloneList(pCols);
if (NULL == pScan) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
static SLogicNode* createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable) {
SLogicNode* pRoot = createQueryLogicNode(pCxt, pTable->pSubquery);
CHECK_ALLOC(pRoot, NULL);
// set output
if (TSDB_CODE_SUCCESS == code && NULL != pCols) {
pScan->node.pTargets = nodesCloneList(pCols);
if (NULL == pScan) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pScan;
} else {
nodesDestroyNode(pScan);
}
return code;
}
static int32_t createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable, SLogicNode** pLogicNode) {
int32_t code = createQueryLogicNode(pCxt, pTable->pSubquery, pLogicNode);
if (TSDB_CODE_SUCCESS == code) {
SNode* pNode;
FOREACH(pNode, pRoot->pTargets) {
FOREACH(pNode, (*pLogicNode)->pTargets) {
strcpy(((SColumnNode*)pNode)->tableAlias, pTable->table.tableAlias);
}
return pRoot;
}
return code;
}
static SLogicNode* createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SJoinTableNode* pJoinTable) {
static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SJoinTableNode* pJoinTable, SLogicNode** pLogicNode) {
SJoinLogicNode* pJoin = (SJoinLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_JOIN);
CHECK_ALLOC(pJoin, NULL);
pJoin->node.id = pCxt->planNodeId++;
if (NULL == pJoin) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pJoin->joinType = pJoinTable->joinType;
int32_t code = TSDB_CODE_SUCCESS;
// set left and right node
pJoin->node.pChildren = nodesMakeList();
CHECK_ALLOC(pJoin->node.pChildren, (SLogicNode*)pJoin);
SLogicNode* pLeft = createLogicNodeByTable(pCxt, pSelect, pJoinTable->pLeft);
CHECK_ALLOC(pLeft, (SLogicNode*)pJoin);
CHECK_CODE(nodesListAppend(pJoin->node.pChildren, (SNode*)pLeft), (SLogicNode*)pJoin);
SLogicNode* pRight = createLogicNodeByTable(pCxt, pSelect, pJoinTable->pRight);
CHECK_ALLOC(pRight, (SLogicNode*)pJoin);
CHECK_CODE(nodesListAppend(pJoin->node.pChildren, (SNode*)pRight), (SLogicNode*)pJoin);
if (NULL == pJoin->node.pChildren) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
SLogicNode* pLeft = NULL;
if (TSDB_CODE_SUCCESS == code) {
code = doCreateLogicNodeByTable(pCxt, pSelect, pJoinTable->pLeft, &pLeft);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppend(pJoin->node.pChildren, (SNode*)pLeft);
}
}
SLogicNode* pRight = NULL;
if (TSDB_CODE_SUCCESS == code) {
code = doCreateLogicNodeByTable(pCxt, pSelect, pJoinTable->pRight, &pRight);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppend(pJoin->node.pChildren, (SNode*)pRight);
}
}
// set on conditions
if (NULL != pJoinTable->pOnCond) {
if (TSDB_CODE_SUCCESS == code && NULL != pJoinTable->pOnCond) {
pJoin->pOnConditions = nodesCloneNode(pJoinTable->pOnCond);
CHECK_ALLOC(pJoin->pOnConditions, (SLogicNode*)pJoin);
if (NULL == pJoin->pOnConditions) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
// set the output
if (TSDB_CODE_SUCCESS == code) {
pJoin->node.pTargets = nodesCloneList(pLeft->pTargets);
CHECK_ALLOC(pJoin->node.pTargets, (SLogicNode*)pJoin);
SNodeList* pTargets = nodesCloneList(pRight->pTargets);
CHECK_ALLOC(pTargets, (SLogicNode*)pJoin);
nodesListAppendList(pJoin->node.pTargets, pTargets);
return (SLogicNode*)pJoin;
if (NULL == pJoin->pOnConditions) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppendList(pJoin->node.pTargets, nodesCloneList(pRight->pTargets));
}
}
static SLogicNode* createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable) {
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pJoin;
} else {
nodesDestroyNode(pJoin);
}
return code;
}
static int32_t doCreateLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode) {
switch (nodeType(pTable)) {
case QUERY_NODE_REAL_TABLE:
return createScanLogicNode(pCxt, pSelect, (SRealTableNode*)pTable);
return createScanLogicNode(pCxt, pSelect, (SRealTableNode*)pTable, pLogicNode);
case QUERY_NODE_TEMP_TABLE:
return createSubqueryLogicNode(pCxt, pSelect, (STempTableNode*)pTable);
return createSubqueryLogicNode(pCxt, pSelect, (STempTableNode*)pTable, pLogicNode);
case QUERY_NODE_JOIN_TABLE:
return createJoinLogicNode(pCxt, pSelect, (SJoinTableNode*)pTable);
return createJoinLogicNode(pCxt, pSelect, (SJoinTableNode*)pTable, pLogicNode);
default:
break;
}
return TSDB_CODE_FAILED;
}
static int32_t createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode) {
SLogicNode* pNode = NULL;
int32_t code = doCreateLogicNodeByTable(pCxt, pSelect, pTable, &pNode);
if (TSDB_CODE_SUCCESS == code) {
pNode->pConditions = nodesCloneNode(pSelect->pWhere);
if (NULL != pSelect->pWhere && NULL == pNode->pConditions) {
nodesDestroyNode(pNode);
return TSDB_CODE_OUT_OF_MEMORY;
}
*pLogicNode = pNode;
}
return code;
}
static SColumnNode* createColumnByExpr(SExprNode* pExpr) {
SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
return NULL;
}
pCol->node.resType = pExpr->resType;
strcpy(pCol->colName, pExpr->aliasName);
return pCol;
}
typedef struct SCreateColumnCxt {
int32_t errCode;
@ -245,197 +307,241 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
static SNodeList* createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pExprs) {
SCreateColumnCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pList = nodesMakeList() };
CHECK_ALLOC(cxt.pList, NULL);
static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pList) {
SCreateColumnCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pList = (NULL == *pList ? nodesMakeList() : *pList) };
if (NULL == cxt.pList) {
return TSDB_CODE_OUT_OF_MEMORY;
}
nodesWalkList(pExprs, doCreateColumn, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyList(cxt.pList);
return NULL;
return cxt.errCode;
}
return cxt.pList;
if (NULL == *pList) {
*pList = cxt.pList;
}
return cxt.errCode;
}
static SLogicNode* createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) {
static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
SNodeList* pAggFuncs = NULL;
CHECK_CODE(nodesCollectFuncs(pSelect, fmIsAggFunc, &pAggFuncs), NULL);
int32_t code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pAggFuncs);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
if (NULL == pAggFuncs && NULL == pSelect->pGroupByList) {
return NULL;
return TSDB_CODE_SUCCESS;
}
SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG);
CHECK_ALLOC(pAgg, NULL);
pAgg->node.id = pCxt->planNodeId++;
if (NULL == pAgg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
// set grouyp keys, agg funcs and having conditions
if (NULL != pSelect->pGroupByList) {
pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList);
CHECK_ALLOC(pAgg->pGroupKeys, (SLogicNode*)pAgg);
if (NULL == pAgg->pGroupKeys) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
if (NULL != pAggFuncs) {
}
if (TSDB_CODE_SUCCESS == code && NULL != pAggFuncs) {
pAgg->pAggFuncs = nodesCloneList(pAggFuncs);
CHECK_ALLOC(pAgg->pAggFuncs, (SLogicNode*)pAgg);
if (NULL == pAgg->pAggFuncs) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
// rewrite the expression in subsequent clauses
CHECK_CODE(rewriteExpr(pAgg->node.id, 1, pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg);
CHECK_CODE(rewriteExpr(pAgg->node.id, 1 + LIST_LENGTH(pAgg->pGroupKeys), pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg);
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExpr(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY);
}
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExpr(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY);
}
if (NULL != pSelect->pHaving) {
if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) {
pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving);
CHECK_ALLOC(pAgg->node.pConditions, (SLogicNode*)pAgg);
if (NULL == pAgg->node.pConditions) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
// set the output
pAgg->node.pTargets = nodesMakeList();
CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg);
if (NULL != pAgg->pGroupKeys) {
SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys);
CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg);
nodesListAppendList(pAgg->node.pTargets, pTargets);
if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pGroupKeys) {
code = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys, &pAgg->node.pTargets);
}
if (NULL != pAgg->pAggFuncs) {
SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs);
CHECK_ALLOC(pTargets, (SLogicNode*)pAgg);
nodesListAppendList(pAgg->node.pTargets, pTargets);
if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pAggFuncs) {
code = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs, &pAgg->node.pTargets);
}
return (SLogicNode*)pAgg;
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pAgg;
} else {
nodesDestroyNode(pAgg);
}
static SLogicNode* createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SIntervalWindowNode* pInterval, SSelectStmt* pSelect) {
return code;
}
static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SIntervalWindowNode* pInterval, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW);
CHECK_ALLOC(pWindow, NULL);
pWindow->node.id = pCxt->planNodeId++;
if (NULL == pWindow) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pWindow->winType = WINDOW_TYPE_INTERVAL;
//SValueNode* pIntervalNode = (SValueNode*)((SRawExprNode*)(pInterval->pInterval))->pNode;
SValueNode* pIntervalNode = (SValueNode*)(pInterval->pInterval);
pWindow->interval = pIntervalNode->datum.i;
pWindow->interval = ((SValueNode*)pInterval->pInterval)->datum.i;
pWindow->intervalUnit = ((SValueNode*)pInterval->pInterval)->unit;
pWindow->offset = (NULL != pInterval->pOffset ? ((SValueNode*)pInterval->pOffset)->datum.i : 0);
pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval);
pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit);
int32_t code = TSDB_CODE_SUCCESS;
if (NULL != pInterval->pFill) {
pWindow->pFill = nodesCloneNode(pInterval->pFill);
CHECK_ALLOC(pWindow->pFill, (SLogicNode*)pWindow);
if (NULL == pWindow->pFill) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
SNodeList* pFuncs = NULL;
CHECK_CODE(nodesCollectFuncs(pSelect, fmIsAggFunc, &pFuncs), NULL);
if (NULL != pFuncs) {
pWindow->pFuncs = nodesCloneList(pFuncs);
CHECK_ALLOC(pWindow->pFuncs, (SLogicNode*)pWindow);
if (TSDB_CODE_SUCCESS == code) {
code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pWindow->pFuncs);
}
CHECK_CODE(rewriteExpr(pWindow->node.id, 1, pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW), (SLogicNode*)pWindow);
pWindow->node.pTargets = createColumnByRewriteExps(pCxt, pWindow->pFuncs);
CHECK_ALLOC(pWindow->node.pTargets, (SLogicNode*)pWindow);
return (SLogicNode*)pWindow;
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW);
}
static SLogicNode* createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) {
if (TSDB_CODE_SUCCESS == code) {
code = createColumnByRewriteExps(pCxt, pWindow->pFuncs, &pWindow->node.pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pWindow;
} else {
nodesDestroyNode(pWindow);
}
return code;
}
static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
if (NULL == pSelect->pWindow) {
return NULL;
return TSDB_CODE_SUCCESS;
}
switch (nodeType(pSelect->pWindow)) {
case QUERY_NODE_INTERVAL_WINDOW:
return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect);
return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect, pLogicNode);
default:
break;
}
return NULL;
return TSDB_CODE_FAILED;
}
static SNodeList* createColumnByProjections(SLogicPlanContext* pCxt, SNodeList* pExprs) {
static int32_t createColumnByProjections(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pCols) {
SNodeList* pList = nodesMakeList();
CHECK_ALLOC(pList, NULL);
if (NULL == pList) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SNode* pNode;
FOREACH(pNode, pExprs) {
SExprNode* pExpr = (SExprNode*)pNode;
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
goto error;
}
pCol->node.resType = pExpr->resType;
strcpy(pCol->colName, pExpr->aliasName);
if (TSDB_CODE_SUCCESS != nodesListAppend(pList, (SNode*)pCol)) {
goto error;
}
}
return pList;
error:
if (TSDB_CODE_SUCCESS != nodesListAppend(pList, createColumnByExpr((SExprNode*)pNode))) {
nodesDestroyList(pList);
return NULL;
return TSDB_CODE_OUT_OF_MEMORY;
}
}
static SLogicNode* createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) {
*pCols = pList;
return TSDB_CODE_SUCCESS;
}
static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
SProjectLogicNode* pProject = (SProjectLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PROJECT);
CHECK_ALLOC(pProject, NULL);
pProject->node.id = pCxt->planNodeId++;
if (NULL == pProject) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = TSDB_CODE_SUCCESS;
pProject->pProjections = nodesCloneList(pSelect->pProjectionList);
pProject->node.pTargets = createColumnByProjections(pCxt,pSelect->pProjectionList);
CHECK_ALLOC(pProject->node.pTargets, (SLogicNode*)pProject);
return (SLogicNode*)pProject;
if (NULL == pProject->pProjections) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
static SLogicNode* createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect) {
SLogicNode* pRoot = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable);
if (TSDB_CODE_SUCCESS == pCxt->errCode && NULL != pSelect->pWhere) {
pRoot->pConditions = nodesCloneNode(pSelect->pWhere);
CHECK_ALLOC(pRoot->pConditions, pRoot);
if (TSDB_CODE_SUCCESS == code) {
code = createColumnByProjections(pCxt,pSelect->pProjectionList, &pProject->node.pTargets);
}
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pRoot = pushLogicNode(pCxt, pRoot, createWindowLogicNode(pCxt, pSelect));
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pProject;
} else {
nodesDestroyNode(pProject);
}
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pRoot = pushLogicNode(pCxt, pRoot, createAggLogicNode(pCxt, pSelect));
return code;
}
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
pRoot = pushLogicNode(pCxt, pRoot, createProjectLogicNode(pCxt, pSelect));
static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
SLogicNode* pRoot = NULL;
int32_t code = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable, &pRoot);
if (TSDB_CODE_SUCCESS == code) {
code = createChildLogicNode(pCxt, pSelect, createWindowLogicNode, &pRoot);
}
return pRoot;
if (TSDB_CODE_SUCCESS == code) {
code = createChildLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot);
}
if (TSDB_CODE_SUCCESS == code) {
code = createChildLogicNode(pCxt, pSelect, createProjectLogicNode, &pRoot);
}
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = pRoot;
} else {
nodesDestroyNode(pRoot);
}
return code;
}
static int32_t getMsgType(ENodeType sqlType) {
return (QUERY_NODE_CREATE_TABLE_STMT == sqlType || QUERY_NODE_CREATE_MULTI_TABLE_STMT == sqlType) ? TDMT_VND_CREATE_TABLE : TDMT_VND_SUBMIT;
}
static SLogicNode* createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt) {
SVnodeModifLogicNode* pModif = (SVnodeModifLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIF);
CHECK_ALLOC(pModif, NULL);
static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt, SLogicNode** pLogicNode) {
SVnodeModifLogicNode* pModif = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIF);
if (NULL == pModif) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pModif->pDataBlocks = pStmt->pDataBlocks;
pModif->msgType = getMsgType(pStmt->sqlNodeType);
return (SLogicNode*)pModif;
*pLogicNode = (SLogicNode*)pModif;
return TSDB_CODE_SUCCESS;
}
static SLogicNode* createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt) {
static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogicNode** pLogicNode) {
switch (nodeType(pStmt)) {
case QUERY_NODE_SELECT_STMT:
return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt);
return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt, pLogicNode);
case QUERY_NODE_VNODE_MODIF_STMT:
return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt);
return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt, pLogicNode);
default:
break;
}
return NULL; // to avoid compiler error
return TSDB_CODE_FAILED;
}
int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode) {
SLogicPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .planNodeId = 1, .acctId = pCxt->acctId };
SLogicNode* pRoot = createQueryLogicNode(&cxt, pCxt->pAstRoot);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyNode((SNode*)pRoot);
return cxt.errCode;
SLogicPlanContext cxt = { .pPlanCxt = pCxt };
int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, pLogicNode);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
*pLogicNode = pRoot;
return TSDB_CODE_SUCCESS;
}

22
source/libs/planner/src/planPhysiCreater.c
View File

@ -259,15 +259,22 @@ static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* p
return (SPhysiNode*)pTableScan;
}
static SPhysiNode* createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) {
SStreamScanPhysiNode* pTableScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
CHECK_ALLOC(pTableScan, NULL);
CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan);
return (SPhysiNode*)pTableScan;
}
static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) {
switch (pScanLogicNode->scanType) {
case SCAN_TYPE_TAG:
return createTagScanPhysiNode(pCxt, pScanLogicNode);
case SCAN_TYPE_TABLE:
return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode);
case SCAN_TYPE_STABLE:
case SCAN_TYPE_TOPIC:
case SCAN_TYPE_STREAM:
break;
return createStreamScanPhysiNode(pCxt, pSubplan, pScanLogicNode);
default:
break;
}
@ -466,12 +473,21 @@ static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC
}
static SPhysiNode* createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode) {
if (pCxt->pPlanCxt->streamQuery) {
SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
CHECK_ALLOC(pScan, NULL);
pScan->pScanCols = nodesCloneList(pExchangeLogicNode->node.pTargets);
CHECK_ALLOC(pScan->pScanCols, (SPhysiNode*)pScan);
CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pScan->node.pOutputDataBlockDesc), (SPhysiNode*)pScan);
return (SPhysiNode*)pScan;
} else {
SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE);
CHECK_ALLOC(pExchange, NULL);
CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pExchange->node.pOutputDataBlockDesc), (SPhysiNode*)pExchange);
pExchange->srcGroupId = pExchangeLogicNode->srcGroupId;
return (SPhysiNode*)pExchange;
}
}
static SPhysiNode* createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode) {
SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_INTERVAL);
@ -608,7 +624,9 @@ static SSubplan* createPhysiSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLog
taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode);
} else {
pSubplan->pNode = createPhysiNode(pCxt, pSubplan, pLogicSubplan->pNode);
if (!pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) {
pSubplan->pDataSink = createDataDispatcher(pCxt, pSubplan->pNode);
}
pSubplan->msgType = TDMT_VND_QUERY;
}
return pSubplan;

3
source/libs/planner/src/planSpliter.c
View File

@ -44,7 +44,8 @@ typedef struct SStsInfo {
} SStsInfo;
static SLogicNode* stsMatchByNode(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->pMeta->tableType) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->pMeta->tableType &&
SCAN_TYPE_TOPIC != ((SScanLogicNode*)pNode)->scanType) {
return pNode;
}
SNode* pChild;

20
source/libs/planner/test/plannerTest.cpp
View File

@ -17,6 +17,7 @@
#include <gtest/gtest.h>
#include "cmdnodes.h"
#include "parser.h"
#include "planInt.h"
@ -56,7 +57,8 @@ protected:
const string syntaxTreeStr = toString(query_->pRoot, false);
SLogicNode* pLogicPlan = nullptr;
SPlanContext cxt = { .queryId = 1, .acctId = 0, .pAstRoot = query_->pRoot };
SPlanContext cxt = { .queryId = 1, .acctId = 0 };
setPlanContext(query_, &cxt);
code = createLogicPlan(&cxt, &pLogicPlan);
if (code != TSDB_CODE_SUCCESS) {
cout << "sql:[" << cxt_.pSql << "] logic plan code:" << code << ", strerror:" << tstrerror(code) << endl;
@ -94,6 +96,15 @@ protected:
private:
static const int max_err_len = 1024;
void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) {
if (QUERY_NODE_CREATE_TOPIC_STMT == nodeType(pQuery->pRoot)) {
pCxt->pAstRoot = ((SCreateTopicStmt*)pQuery->pRoot)->pQuery;
pCxt->topicQuery = true;
} else {
pCxt->pAstRoot = pQuery->pRoot;
}
}
void reset() {
memset(&cxt_, 0, sizeof(cxt_));
memset(errMagBuf_, 0, max_err_len);
@ -173,3 +184,10 @@ TEST_F(PlannerTest, interval) {
bind("SELECT count(*) FROM t1 interval(10s)");
ASSERT_TRUE(run());
}
TEST_F(PlannerTest, createTopic) {
setDatabase("root", "test");
bind("create topic tp as SELECT * FROM st1");
ASSERT_TRUE(run());
}

4
source/libs/qcom/src/queryUtil.c
View File

@ -155,6 +155,10 @@ int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransp
.ahandle = (void*)pInfo,
.handle = pInfo->msgInfo.handle,
.code = 0};
if (pInfo->msgType == TDMT_VND_QUERY || pInfo->msgType == TDMT_VND_FETCH ||
pInfo->msgType == TDMT_VND_QUERY_CONTINUE) {
rpcMsg.persistHandle = 1;
}
assert(pInfo->fp != NULL);

5
source/libs/sync/src/syncAppendEntries.c
View File

@ -87,7 +87,10 @@
//
int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
int32_t ret = 0;
syncAppendEntriesLog2("==syncNodeOnAppendEntriesCb==", pMsg);
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesCb== term:%lu", ths->pRaftStore->currentTerm);
syncAppendEntriesLog2(logBuf, pMsg);
if (pMsg->term > ths->pRaftStore->currentTerm) {
syncNodeUpdateTerm(ths, pMsg->term);

5
source/libs/sync/src/syncAppendEntriesReply.c
View File

@ -37,7 +37,10 @@
//
int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) {
int32_t ret = 0;
syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplyCb==", pMsg);
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesReplyCb== term:%lu", ths->pRaftStore->currentTerm);
syncAppendEntriesReplyLog2(logBuf, pMsg);
if (pMsg->term < ths->pRaftStore->currentTerm) {
sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term,

29
source/libs/sync/src/syncCommit.c
View File

@ -18,6 +18,7 @@
#include "syncInt.h"
#include "syncRaftLog.h"
#include "syncRaftStore.h"
#include "syncUtil.h"
// \* Leader i advances its commitIndex.
// \* This is done as a separate step from handling AppendEntries responses,
@ -49,8 +50,11 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
// update commit index
SyncIndex newCommitIndex = pSyncNode->commitIndex;
for (SyncIndex index = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore); index > pSyncNode->commitIndex;
++index) {
if (syncAgree(pSyncNode, index)) {
--index) {
bool agree = syncAgree(pSyncNode, index);
sTrace("syncMaybeAdvanceCommitIndex syncAgree:%d, index:%ld, pSyncNode->commitIndex:%ld", agree, index,
pSyncNode->commitIndex);
if (agree) {
// term
SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index);
assert(pEntry != NULL);
@ -68,6 +72,8 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
SyncIndex beginIndex = pSyncNode->commitIndex + 1;
SyncIndex endIndex = newCommitIndex;
sTrace("syncMaybeAdvanceCommitIndex sync commit %ld", newCommitIndex);
// update commit index
pSyncNode->commitIndex = newCommitIndex;
@ -97,14 +103,19 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
}
bool syncAgreeIndex(SSyncNode* pSyncNode, SRaftId* pRaftId, SyncIndex index) {
SyncIndex matchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, pRaftId);
// b for debug
bool b = false;
if (matchIndex >= index) {
b = true;
// I am leader, I agree
if (syncUtilSameId(pRaftId, &(pSyncNode->myRaftId)) && pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
return true;
}
return b;
// follower agree
SyncIndex matchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, pRaftId);
if (matchIndex >= index) {
return true;
}
// not agree
return false;
}
bool syncAgree(SSyncNode* pSyncNode, SyncIndex index) {

24
source/libs/sync/src/syncMain.c
View File

@ -60,11 +60,14 @@ int64_t syncStart(const SSyncInfo* pSyncInfo) {
int32_t ret = 0;
SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo);
assert(pSyncNode != NULL);
// todo : return ref id
return ret;
}
void syncStop(int64_t rid) {
SSyncNode* pSyncNode = NULL; // get pointer from rid
// todo : get pointer from rid
SSyncNode* pSyncNode = NULL;
syncNodeClose(pSyncNode);
}
@ -75,7 +78,9 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) {
int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) {
int32_t ret = 0;
SSyncNode* pSyncNode = NULL; // get pointer from rid
// todo : get pointer from rid
SSyncNode* pSyncNode = NULL;
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, 0, isWeak);
SRpcMsg rpcMsg;
@ -86,13 +91,14 @@ int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) {
} else {
sTrace("syncForwardToPeer not leader, %s", syncUtilState2String(pSyncNode->state));
ret = -1; // need define err code !!
ret = -1; // todo : need define err code !!
}
return ret;
}
ESyncState syncGetMyRole(int64_t rid) {
SSyncNode* pSyncNode = NULL; // get pointer from rid
// todo : get pointer from rid
SSyncNode* pSyncNode = NULL;
return pSyncNode->state;
}
@ -195,7 +201,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
// init TLA+ log vars
pSyncNode->pLogStore = logStoreCreate(pSyncNode);
assert(pSyncNode->pLogStore != NULL);
pSyncNode->commitIndex = 0;
pSyncNode->commitIndex = SYNC_INDEX_INVALID;
// init ping timer
pSyncNode->pPingTimer = NULL;
@ -774,9 +780,6 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg
if (ths->state == TAOS_SYNC_STATE_LEADER) {
ths->pLogStore->appendEntry(ths->pLogStore, pEntry);
// only myself, maybe commit
syncMaybeAdvanceCommitIndex(ths);
// start replicate right now!
syncNodeReplicate(ths);
@ -791,6 +794,9 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg
}
rpcFreeCont(rpcMsg.pCont);
// only myself, maybe commit
syncMaybeAdvanceCommitIndex(ths);
} else {
// pre commit
SRpcMsg rpcMsg;
@ -798,7 +804,7 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg
if (ths->pFsm != NULL) {
if (ths->pFsm->FpPreCommitCb != NULL) {
ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, -1);
ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, -2);
}
}
rpcFreeCont(rpcMsg.pCont);

5
source/libs/sync/src/syncRequestVote.c
View File

@ -43,7 +43,10 @@
//
int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) {
int32_t ret = 0;
syncRequestVoteLog2("==syncNodeOnRequestVoteCb==", pMsg);
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteCb== term:%lu", ths->pRaftStore->currentTerm);
syncRequestVoteLog2(logBuf, pMsg);
if (pMsg->term > ths->pRaftStore->currentTerm) {
syncNodeUpdateTerm(ths, pMsg->term);

5
source/libs/sync/src/syncRequestVoteReply.c
View File

@ -38,7 +38,10 @@
//
int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) {
int32_t ret = 0;
syncRequestVoteReplyLog2("==syncNodeOnRequestVoteReplyCb==", pMsg);
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplyCb== term:%lu", ths->pRaftStore->currentTerm);
syncRequestVoteReplyLog2(logBuf, pMsg);
if (pMsg->term < ths->pRaftStore->currentTerm) {
sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term,

14
source/libs/sync/test/CMakeLists.txt
View File

@ -30,6 +30,7 @@ add_executable(syncPingSelfTest "")
add_executable(syncElectTest "")
add_executable(syncEncodeTest "")
add_executable(syncWriteTest "")
add_executable(syncReplicateTest "")
target_sources(syncTest
@ -160,6 +161,10 @@ target_sources(syncWriteTest
PRIVATE
"syncWriteTest.cpp"
)
target_sources(syncReplicateTest
PRIVATE
"syncReplicateTest.cpp"
)
target_include_directories(syncTest
@ -327,6 +332,11 @@ target_include_directories(syncWriteTest
"${CMAKE_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories(syncReplicateTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries(syncTest
@ -457,6 +467,10 @@ target_link_libraries(syncWriteTest
sync
gtest_main
)
target_link_libraries(syncReplicateTest
sync
gtest_main
)
enable_testing()

5
source/libs/sync/test/syncElectTest.cpp
View File

@ -116,10 +116,9 @@ int main(int argc, char** argv) {
//---------------------------
while (1) {
sTrace("while 1 sleep, state: %d, %s, electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d",
gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->electTimerLogicClock,
sTrace("elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d",
gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, gSyncNode->electTimerLogicClock,
gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS);
taosMsleep(1000);
}
return 0;

188
source/libs/sync/test/syncReplicateTest.cpp Normal file
View File

@ -0,0 +1,188 @@
#include <gtest/gtest.h>
#include <stdio.h>
#include "syncEnv.h"
#include "syncIO.h"
#include "syncInt.h"
#include "syncMessage.h"
#include "syncRaftEntry.h"
#include "syncRaftLog.h"
#include "syncRaftStore.h"
#include "syncUtil.h"
void logTest() {
sTrace("--- sync log test: trace");
sDebug("--- sync log test: debug");
sInfo("--- sync log test: info");
sWarn("--- sync log test: warn");
sError("--- sync log test: error");
sFatal("--- sync log test: fatal");
}
uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
int32_t replicaNum = 3;
int32_t myIndex = 0;
SRaftId ids[TSDB_MAX_REPLICA];
SSyncInfo syncInfo;
SSyncFSM * pFsm;
SWal * pWal;
SSyncNode *gSyncNode;
void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) {
printf("==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code);
syncRpcMsgPrint2((char *)"==CommitCb==", (SRpcMsg *)pBuf);
}
void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) {
printf("==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code);
syncRpcMsgPrint2((char *)"==PreCommitCb==", (SRpcMsg *)pBuf);
}
void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) {
printf("==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code);
syncRpcMsgPrint2((char *)"==RollBackCb==", (SRpcMsg *)pBuf);
}
void initFsm() {
pFsm = (SSyncFSM *)malloc(sizeof(SSyncFSM));
pFsm->FpCommitCb = CommitCb;
pFsm->FpPreCommitCb = PreCommitCb;
pFsm->FpRollBackCb = RollBackCb;
}
SSyncNode *syncNodeInit() {
syncInfo.vgId = 1234;
syncInfo.rpcClient = gSyncIO->clientRpc;
syncInfo.FpSendMsg = syncIOSendMsg;
syncInfo.queue = gSyncIO->pMsgQ;
syncInfo.FpEqMsg = syncIOEqMsg;
syncInfo.pFsm = pFsm;
snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate_test_%d", myIndex);
int code = walInit();
assert(code == 0);
SWalCfg walCfg;
memset(&walCfg, 0, sizeof(SWalCfg));
walCfg.vgId = syncInfo.vgId;
walCfg.fsyncPeriod = 1000;
walCfg.retentionPeriod = 1000;
walCfg.rollPeriod = 1000;
walCfg.retentionSize = 1000;
walCfg.segSize = 1000;
walCfg.level = TAOS_WAL_FSYNC;
char tmpdir[128];
snprintf(tmpdir, sizeof(tmpdir), "./replicate_test_wal_%d", myIndex);
pWal = walOpen(tmpdir, &walCfg);
assert(pWal != NULL);
syncInfo.pWal = pWal;
SSyncCfg *pCfg = &syncInfo.syncCfg;
pCfg->myIndex = myIndex;
pCfg->replicaNum = replicaNum;
for (int i = 0; i < replicaNum; ++i) {
pCfg->nodeInfo[i].nodePort = ports[i];
snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
// taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
}
SSyncNode *pSyncNode = syncNodeOpen(&syncInfo);
assert(pSyncNode != NULL);
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout;
gSyncIO->pSyncNode = pSyncNode;
return pSyncNode;
}
SSyncNode *syncInitTest() { return syncNodeInit(); }
void initRaftId(SSyncNode *pSyncNode) {
for (int i = 0; i < replicaNum; ++i) {
ids[i] = pSyncNode->replicasId[i];
char *s = syncUtilRaftId2Str(&ids[i]);
printf("raftId[%d] : %s\n", i, s);
free(s);
}
}
SRpcMsg *step0(int i) {
SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg));
memset(pMsg, 0, sizeof(SRpcMsg));
pMsg->msgType = 9999;
pMsg->contLen = 128;
pMsg->pCont = malloc(pMsg->contLen);
snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i);
return pMsg;
}
SyncClientRequest *step1(const SRpcMsg *pMsg) {
SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true);
return pRetMsg;
}
int main(int argc, char **argv) {
// taosInitLog((char *)"syncTest.log", 100000, 10);
tsAsyncLog = 0;
sDebugFlag = 143 + 64;
void logTest();
myIndex = 0;
if (argc >= 2) {
myIndex = atoi(argv[1]);
}
int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]);
assert(ret == 0);
ret = syncEnvStart();
assert(ret == 0);
taosRemoveDir("./wal_test");
initFsm();
gSyncNode = syncInitTest();
assert(gSyncNode != NULL);
syncNodePrint2((char *)"", gSyncNode);
initRaftId(gSyncNode);
for (int i = 0; i < 30; ++i) {
// step0
SRpcMsg *pMsg0 = step0(i);
syncRpcMsgPrint2((char *)"==step0==", pMsg0);
// step1
SyncClientRequest *pMsg1 = step1(pMsg0);
syncClientRequestPrint2((char *)"==step1==", pMsg1);
SyncClientRequest *pSyncClientRequest = pMsg1;
SRpcMsg rpcMsg;
syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg);
gSyncNode->FpEqMsg(gSyncNode->queue, &rpcMsg);
taosMsleep(1000);
sTrace("replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d",
gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, gSyncNode->electTimerLogicClock,
gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS);
}
while (1) {
sTrace("replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d",
gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, gSyncNode->electTimerLogicClock,
gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS);
taosMsleep(1000);
}
return 0;
}

22
source/libs/tdb/CMakeLists.txt
View File

@ -1,10 +1,17 @@
set(TDB_SUBDIRS "db")
foreach(TDB_SUBDIR ${TDB_SUBDIRS})
aux_source_directory("src/${TDB_SUBDIR}" TDB_SRC)
endforeach()
add_library(tdb STATIC ${TDB_SRC})
# tdb
add_library(tdb "")
target_sources(tdb
PRIVATE
"src/db/tdbPCache.c"
"src/db/tdbPager.c"
"src/db/tdbUtil.c"
"src/db/tdbBtree.c"
"src/db/tdbDb.c"
"src/db/tdbEnv.c"
# "src/db/tdbPage.c"
"src/page/tdbPage.c"
"src/page/tdbPageL.c"
)
target_include_directories(
tdb
@ -17,6 +24,7 @@ target_link_libraries(
PUBLIC util
)
# for test
if(${BUILD_TEST})
add_subdirectory(test)
endif(${BUILD_TEST})

58
source/libs/tdb/inc/tdb.h
View File

@ -22,44 +22,42 @@
extern "C" {
#endif
typedef struct STDb TDB;
typedef struct STDbEnv TENV;
typedef struct STDbCurosr TDBC;
// typedef struct STDb TDB;
// typedef struct STDbEnv TENV;
// typedef struct STDbCurosr TDBC;
typedef int32_t pgsz_t;
typedef int32_t cachesz_t;
// typedef int32_t pgsz_t;
// typedef int32_t cachesz_t;
typedef int (*TdbKeyCmprFn)(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2);
// typedef int (*TdbKeyCmprFn)(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2);
// TEVN
int tdbEnvCreate(TENV **ppEnv, const char *rootDir);
int tdbEnvOpen(TENV *ppEnv);
int tdbEnvClose(TENV *pEnv);
// // TEVN
// int tdbEnvCreate(TENV **ppEnv, const char *rootDir);
// int tdbEnvOpen(TENV *ppEnv);
// int tdbEnvClose(TENV *pEnv);
int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize);
pgsz_t tdbEnvGetPageSize(TENV *pEnv);
cachesz_t tdbEnvGetCacheSize(TENV *pEnv);
// int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize);
// pgsz_t tdbEnvGetPageSize(TENV *pEnv);
// cachesz_t tdbEnvGetCacheSize(TENV *pEnv);
int tdbEnvBeginTxn(TENV *pEnv);
int tdbEnvCommit(TENV *pEnv);
// int tdbEnvBeginTxn(TENV *pEnv);
// int tdbEnvCommit(TENV *pEnv);
// TDB
int tdbCreate(TDB **ppDb);
int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv);
int tdbClose(TDB *pDb);
int tdbDrop(TDB *pDb);
// // TDB
// int tdbCreate(TDB **ppDb);
// int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv);
// int tdbClose(TDB *pDb);
// int tdbDrop(TDB *pDb);
int tdbSetKeyLen(TDB *pDb, int klen);
int tdbSetValLen(TDB *pDb, int vlen);
int tdbSetDup(TDB *pDb, int dup);
int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn);
int tdbGetKeyLen(TDB *pDb);
int tdbGetValLen(TDB *pDb);
int tdbGetDup(TDB *pDb);
// int tdbSetKeyLen(TDB *pDb, int klen);
// int tdbSetValLen(TDB *pDb, int vlen);
// int tdbSetDup(TDB *pDb, int dup);
// int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn);
// int tdbGetKeyLen(TDB *pDb);
// int tdbGetValLen(TDB *pDb);
// int tdbGetDup(TDB *pDb);
int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData);
// TDBC
// int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData);
#ifdef __cplusplus
}

205
source/libs/tdb/src/db/tdb.c
View File

@ -1,205 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
struct STDb {
char dbname[TDB_MAX_DBNAME_LEN];
SBTree * pBt; // current access method (may extend)
SPgFile * pPgFile; // backend page file this DB is using
TENV * pEnv; // TENV containing the DB
int klen; // key length if know
int vlen; // value length if know
bool dup; // dup mode
TdbKeyCmprFn cFn; // compare function
};
struct STDbCurosr {
SBtCursor *pBtCur;
};
static int tdbDefaultKeyCmprFn(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2);
int tdbCreate(TDB **ppDb) {
TDB *pDb;
// create the handle
pDb = (TDB *)calloc(1, sizeof(*pDb));
if (pDb == NULL) {
return -1;
}
pDb->klen = TDB_VARIANT_LEN;
pDb->vlen = TDB_VARIANT_LEN;
pDb->dup = false;
pDb->cFn = tdbDefaultKeyCmprFn;
*ppDb = pDb;
return 0;
}
static int tdbDestroy(TDB *pDb) {
if (pDb) {
free(pDb);
}
return 0;
}
int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv) {
int ret;
uint8_t fileid[TDB_FILE_ID_LEN];
SPgFile * pPgFile;
SPgCache *pPgCache;
SBTree * pBt;
bool fileExist;
size_t dbNameLen;
pgno_t dbRootPgno;
char dbfname[128]; // TODO: make this as a macro or malloc on the heap
ASSERT(pDb != NULL);
ASSERT(fname != NULL);
// TODO: Here we simply put an assert here. In the future, make `pEnv`
// can be set as NULL.
ASSERT(pEnv != NULL);
// check the DB name
dbNameLen = 0;
if (dbname) {
dbNameLen = strlen(dbname);
if (dbNameLen >= TDB_MAX_DBNAME_LEN) {
return -1;
}
memcpy(pDb->dbname, dbname, dbNameLen);
}
pDb->dbname[dbNameLen] = '\0';
// get page file from the env, if not opened yet, open it
pPgFile = NULL;
snprintf(dbfname, 128, "%s/%s", tdbEnvGetRootDir(pEnv), fname);
fileExist = taosCheckExistFile(fname);
if (fileExist) {
tdbGnrtFileID(dbfname, fileid, false);
pPgFile = tdbEnvGetPageFile(pEnv, fileid);
}
if (pPgFile == NULL) {
ret = pgFileOpen(&pPgFile, dbfname, pEnv);
if (ret != 0) {
// TODO: handle error
return -1;
}
}
// TODO: get the root page number from the master DB of the page file
// tdbGet(&dbRootPgno);
if (dbRootPgno == 0) {
// DB not exist, create one
ret = pgFileAllocatePage(pPgFile, &dbRootPgno);
if (ret != 0) {
// TODO: handle error
}
// tdbInsert(pPgFile->pMasterDB, dbname, strlen(dbname), &dbRootPgno, sizeof(dbRootPgno));
}
ASSERT(dbRootPgno > 1);
// pDb->pBt->root = dbRootPgno;
// register
pDb->pPgFile = pPgFile;
tdbEnvRgstDB(pEnv, pDb);
pDb->pEnv = pEnv;
return 0;
}
int tdbClose(TDB *pDb) {
if (pDb == NULL) return 0;
return tdbDestroy(pDb);
}
int tdbDrop(TDB *pDb) {
// TODO
return 0;
}
int tdbSetKeyLen(TDB *pDb, int klen) {
// TODO: check `klen`
pDb->klen = klen;
return 0;
}
int tdbSetValLen(TDB *pDb, int vlen) {
// TODO: check `vlen`
pDb->vlen = vlen;
return 0;
}
int tdbSetDup(TDB *pDb, int dup) {
if (dup) {
pDb->dup = true;
} else {
pDb->dup = false;
}
return 0;
}
int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn) {
if (fn == NULL) {
return -1;
} else {
pDb->cFn = fn;
}
return 0;
}
int tdbGetKeyLen(TDB *pDb) { return pDb->klen; }
int tdbGetValLen(TDB *pDb) { return pDb->vlen; }
int tdbGetDup(TDB *pDb) {
if (pDb->dup) {
return 1;
} else {
return 0;
}
}
int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData) {
// TODO
return 0;
}
static int tdbDefaultKeyCmprFn(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2) {
int mlen;
int cret;
ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL);
mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2;
cret = memcmp(pKey1, pKey2, mlen);
if (cret == 0) {
if (keyLen1 < keyLen2) {
cret = -1;
} else if (keyLen1 > keyLen2) {
cret = 1;
} else {
cret = 0;
}
}
return cret;
}

970
source/libs/tdb/src/db/tdbBtree.c
View File

File diff suppressed because it is too large Load Diff

89
source/libs/tdb/src/db/tdbDb.c Normal file
View File

@ -0,0 +1,89 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
struct STDb {
STEnv *pEnv;
SBTree *pBt;
};
int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb) {
STDb *pDb;
SPager *pPager;
int ret;
char fFullName[TDB_FILENAME_LEN];
SPage *pPage;
SPgno pgno;
*ppDb = NULL;
pDb = (STDb *)calloc(1, sizeof(*pDb));
if (pDb == NULL) {
return -1;
}
// pDb->pEnv
pDb->pEnv = pEnv;
pPager = tdbEnvGetPager(pEnv, fname);
if (pPager == NULL) {
snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname);
ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager);
if (ret < 0) {
return -1;
}
}
ASSERT(pPager != NULL);
// pDb->pBt
ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pDb->pBt));
if (ret < 0) {
return -1;
}
*ppDb = pDb;
return 0;
}
int tdbDbClose(STDb *pDb) {
// TODO
return 0;
}
int tdbDbDrop(STDb *pDb) {
// TODO
return 0;
}
int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) {
SBtCursor btc;
SBtCursor *pCur;
int ret;
pCur = &btc;
ret = tdbBtreeCursor(pCur, pDb->pBt);
if (ret < 0) {
return -1;
}
ret = tdbBtCursorInsert(pCur, pKey, keyLen, pVal, valLen);
if (ret < 0) {
return -1;
}
return 0;
}

167
source/libs/tdb/src/db/tdbEnv.c
View File

@ -15,155 +15,56 @@
#include "tdbInt.h"
struct STDbEnv {
char * rootDir; // root directory of the environment
char * jname; // journal file name
TdFilePtr jpFile; // journal file fd
pgsz_t pgSize; // page size
cachesz_t cacheSize; // total cache size
STDbList dbList; // TDB List
SPgFileList pgfList; // SPgFile List
SPgCache * pPgCache; // page cache
struct {
#define TDB_ENV_PGF_HASH_BUCKETS 17
SPgFileList buckets[TDB_ENV_PGF_HASH_BUCKETS];
} pgfht; // page file hash table;
};
#define TDB_ENV_PGF_HASH(fileid) (((uint8_t *)(fileid))[0] + ((uint8_t *)(fileid))[1] + ((uint8_t *)(fileid))[2])
static int tdbEnvDestroy(TENV *pEnv);
int tdbEnvCreate(TENV **ppEnv, const char *rootDir) {
TENV * pEnv;
size_t slen;
size_t jlen;
ASSERT(rootDir != NULL);
int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, STEnv **ppEnv) {
STEnv *pEnv;
int dsize;
int zsize;
u8 *pPtr;
int ret;
*ppEnv = NULL;
slen = strlen(rootDir);
jlen = slen + strlen(TDB_JOURNAL_NAME) + 1;
pEnv = (TENV *)calloc(1, sizeof(*pEnv) + slen + 1 + jlen + 1);
if (pEnv == NULL) {
dsize = strlen(rootDir);
zsize = sizeof(*pEnv) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3;
pPtr = (uint8_t *)calloc(1, zsize);
if (pPtr == NULL) {
return -1;
}
pEnv->rootDir = (char *)(&pEnv[1]);
pEnv->jname = pEnv->rootDir + slen + 1;
pEnv->jpFile = NULL;
pEnv->pgSize = TDB_DEFAULT_PGSIZE;
pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE;
pEnv = (STEnv *)pPtr;
pPtr += sizeof(*pEnv);
// pEnv->rootDir
pEnv->rootDir = pPtr;
memcpy(pEnv->rootDir, rootDir, dsize);
pEnv->rootDir[dsize] = '\0';
pPtr = pPtr + dsize + 1;
// pEnv->jfname
pEnv->jfname = pPtr;
memcpy(pEnv->jfname, rootDir, dsize);
pEnv->jfname[dsize] = '/';
memcpy(pEnv->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME));
pEnv->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0';
memcpy(pEnv->rootDir, rootDir, slen);
pEnv->rootDir[slen] = '\0';
sprintf(pEnv->jname, "%s/%s", rootDir, TDB_JOURNAL_NAME);
pEnv->jfd = -1;
TD_DLIST_INIT(&(pEnv->dbList));
TD_DLIST_INIT(&(pEnv->pgfList));
ret = tdbPCacheOpen(pageSize, cacheSize, &(pEnv->pCache));
if (ret < 0) {
return -1;
}
/* TODO */
mkdir(rootDir, 0755);
*ppEnv = pEnv;
return 0;
}
int tdbEnvOpen(TENV *pEnv) {
SPgCache *pPgCache;
int ret;
ASSERT(pEnv != NULL);
/* TODO: here we do not need to create the root directory, more
* work should be done here
*/
mkdir(pEnv->rootDir, 0755);
ret = pgCacheOpen(&pPgCache, pEnv);
if (ret != 0) {
goto _err;
}
pEnv->pPgCache = pPgCache;
return 0;
_err:
return -1;
}
int tdbEnvClose(TENV *pEnv) {
if (pEnv == NULL) return 0;
pgCacheClose(pEnv->pPgCache);
tdbEnvDestroy(pEnv);
return 0;
}
int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize) {
if (!TDB_IS_PGSIZE_VLD(pgSize) || cacheSize / pgSize < 10) {
return -1;
}
/* TODO */
pEnv->pgSize = pgSize;
pEnv->cacheSize = cacheSize;
return 0;
}
pgsz_t tdbEnvGetPageSize(TENV *pEnv) { return pEnv->pgSize; }
cachesz_t tdbEnvGetCacheSize(TENV *pEnv) { return pEnv->cacheSize; }
SPgFile *tdbEnvGetPageFile(TENV *pEnv, const uint8_t fileid[]) {
SPgFileList *pBucket;
SPgFile * pPgFile;
pBucket = pEnv->pgfht.buckets + (TDB_ENV_PGF_HASH(fileid) % TDB_ENV_PGF_HASH_BUCKETS); // TODO
for (pPgFile = TD_DLIST_HEAD(pBucket); pPgFile != NULL; pPgFile = TD_DLIST_NODE_NEXT_WITH_FIELD(pPgFile, envHash)) {
if (memcmp(fileid, pPgFile->fileid, TDB_FILE_ID_LEN) == 0) break;
};
return pPgFile;
}
SPgCache *tdbEnvGetPgCache(TENV *pEnv) { return pEnv->pPgCache; }
static int tdbEnvDestroy(TENV *pEnv) {
int tdbEnvClose(STEnv *pEnv) {
// TODO
return 0;
}
int tdbEnvBeginTxn(TENV *pEnv) {
pEnv->jpFile = taosOpenFile(pEnv->jname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pEnv->jpFile == NULL) {
return -1;
}
return 0;
}
int tdbEnvCommit(TENV *pEnv) {
/* TODO */
taosCloseFile(&pEnv->jpFile);
pEnv->jpFile = NULL;
return 0;
}
const char *tdbEnvGetRootDir(TENV *pEnv) { return pEnv->rootDir; }
int tdbEnvRgstPageFile(TENV *pEnv, SPgFile *pPgFile) {
SPgFileList *pBucket;
TD_DLIST_APPEND_WITH_FIELD(&(pEnv->pgfList), pPgFile, envPgfList);
pBucket = pEnv->pgfht.buckets + (TDB_ENV_PGF_HASH(pPgFile->fileid) % TDB_ENV_PGF_HASH_BUCKETS); // TODO
TD_DLIST_APPEND_WITH_FIELD(pBucket, pPgFile, envHash);
return 0;
}
int tdbEnvRgstDB(TENV *pEnv, TDB *pDb) {
SPager *tdbEnvGetPager(STEnv *pEnv, const char *fname) {
// TODO
return 0;
return NULL;
}

309
source/libs/tdb/src/db/tdbPCache.c Normal file
View File

@ -0,0 +1,309 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
struct SPCache {
int pageSize;
int cacheSize;
pthread_mutex_t mutex;
int nFree;
SPage *pFree;
int nPage;
int nHash;
SPage **pgHash;
int nRecyclable;
SPage lru;
};
#define PCACHE_PAGE_HASH(pPgid) \
({ \
u32 *t = (u32 *)((pPgid)->fileid); \
t[0] + t[1] + t[2] + t[3] + t[4] + t[5] + (pPgid)->pgno; \
})
#define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL)
// For page ref
#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0)
#if 0
#define TDB_REF_PAGE(pPage) (++(pPage)->nRef)
#define TDB_UNREF_PAGE(pPage) (--(pPage)->nRef)
#define TDB_GET_PAGE_REF(pPage) ((pPage)->nRef)
#else
#define TDB_REF_PAGE(pPage) atomic_add_fetch_32(&((pPage)->nRef), 1)
#define TDB_UNREF_PAGE(pPage) atomic_sub_fetch_32(&((pPage)->nRef), 1)
#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef))
#endif
static int tdbPCacheOpenImpl(SPCache *pCache);
static void tdbPCacheInitLock(SPCache *pCache);
static void tdbPCacheClearLock(SPCache *pCache);
static void tdbPCacheLock(SPCache *pCache);
static void tdbPCacheUnlock(SPCache *pCache);
static bool tdbPCacheLocked(SPCache *pCache);
static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage);
static void tdbPCachePinPage(SPage *pPage);
static void tdbPCacheRemovePageFromHash(SPage *pPage);
static void tdbPCacheAddPageToHash(SPage *pPage);
static void tdbPCacheUnpinPage(SPage *pPage);
static void *tdbOsMalloc(void *arg, size_t size);
static void tdbOsFree(void *arg, void *ptr);
int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
SPCache *pCache;
void *pPtr;
SPage *pPgHdr;
pCache = (SPCache *)calloc(1, sizeof(*pCache));
if (pCache == NULL) {
return -1;
}
pCache->pageSize = pageSize;
pCache->cacheSize = cacheSize;
if (tdbPCacheOpenImpl(pCache) < 0) {
free(pCache);
return -1;
}
*ppCache = pCache;
return 0;
}
int tdbPCacheClose(SPCache *pCache) {
/* TODO */
return 0;
}
SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) {
SPage *pPage;
tdbPCacheLock(pCache);
pPage = tdbPCacheFetchImpl(pCache, pPgid, alcNewPage);
if (pPage) {
TDB_REF_PAGE(pPage);
}
tdbPCacheUnlock(pCache);
return pPage;
}
void tdbPCacheRelease(SPage *pPage) {
i32 nRef;
nRef = TDB_UNREF_PAGE(pPage);
ASSERT(nRef >= 0);
if (nRef == 0) {
if (1 /*TODO: page still clean*/) {
tdbPCacheUnpinPage(pPage);
} else {
// TODO
ASSERT(0);
}
}
}
static void tdbPCacheInitLock(SPCache *pCache) { pthread_mutex_init(&(pCache->mutex), NULL); }
static void tdbPCacheClearLock(SPCache *pCache) { pthread_mutex_destroy(&(pCache->mutex)); }
static void tdbPCacheLock(SPCache *pCache) { pthread_mutex_lock(&(pCache->mutex)); }
static void tdbPCacheUnlock(SPCache *pCache) { pthread_mutex_unlock(&(pCache->mutex)); }
static bool tdbPCacheLocked(SPCache *pCache) {
assert(0);
// TODO
return true;
}
static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) {
SPage *pPage;
// 1. Search the hash table
pPage = pCache->pgHash[PCACHE_PAGE_HASH(pPgid) % pCache->nHash];
while (pPage) {
if (TDB_IS_SAME_PAGE(&(pPage->pgid), pPgid)) break;
pPage = pPage->pHashNext;
}
if (pPage || !alcNewPage) {
if (pPage) {
tdbPCachePinPage(pPage);
}
return pPage;
}
// 2. Try to allocate a new page from the free list
if (pCache->pFree) {
pPage = pCache->pFree;
pCache->pFree = pPage->pFreeNext;
pCache->nFree--;
pPage->pLruNext = NULL;
}
// 3. Try to Recycle a page
if (!pPage && !pCache->lru.pLruPrev->isAnchor) {
pPage = pCache->lru.pLruPrev;
tdbPCacheRemovePageFromHash(pPage);
tdbPCachePinPage(pPage);
}
// 4. Try a stress allocation (TODO)
// 5. Page here are just created from a free list
// or by recycling or allocated streesly,
// need to initialize it
if (pPage) {
memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid));
pPage->pLruNext = NULL;
pPage->pPager = NULL;
tdbPCacheAddPageToHash(pPage);
}
return pPage;
}
static void tdbPCachePinPage(SPage *pPage) {
SPCache *pCache;
pCache = pPage->pCache;
if (!PAGE_IS_PINNED(pPage)) {
pPage->pLruPrev->pLruNext = pPage->pLruNext;
pPage->pLruNext->pLruPrev = pPage->pLruPrev;
pPage->pLruNext = NULL;
pCache->nRecyclable--;
}
}
static void tdbPCacheUnpinPage(SPage *pPage) {
SPCache *pCache;
i32 nRef;
pCache = pPage->pCache;
tdbPCacheLock(pCache);
nRef = TDB_GET_PAGE_REF(pPage);
ASSERT(nRef >= 0);
if (nRef == 0) {
// Add the page to LRU list
ASSERT(pPage->pLruNext == NULL);
pPage->pLruPrev = &(pCache->lru);
pPage->pLruNext = pCache->lru.pLruNext;
pCache->lru.pLruNext->pLruPrev = pPage;
pCache->lru.pLruNext = pPage;
}
pCache->nRecyclable++;
tdbPCacheUnlock(pCache);
}
static void tdbPCacheRemovePageFromHash(SPage *pPage) {
SPCache *pCache;
SPage **ppPage;
int h;
pCache = pPage->pCache;
h = PCACHE_PAGE_HASH(&(pPage->pgid));
for (ppPage = &(pCache->pgHash[h % pCache->nHash]); *ppPage != pPage; ppPage = &((*ppPage)->pHashNext))
;
ASSERT(*ppPage == pPage);
*ppPage = pPage->pHashNext;
pCache->nPage--;
}
static void tdbPCacheAddPageToHash(SPage *pPage) {
SPCache *pCache;
int h;
pCache = pPage->pCache;
h = PCACHE_PAGE_HASH(&(pPage->pgid)) % pCache->nHash;
pPage->pHashNext = pCache->pgHash[h];
pCache->pgHash[h] = pPage;
pCache->nPage++;
}
static int tdbPCacheOpenImpl(SPCache *pCache) {
SPage *pPage;
u8 *pPtr;
int tsize;
int ret;
tdbPCacheInitLock(pCache);
// Open the free list
pCache->nFree = 0;
pCache->pFree = NULL;
for (int i = 0; i < pCache->cacheSize; i++) {
ret = tdbPageCreate(pCache->pageSize, &pPage, tdbOsMalloc, NULL);
if (ret < 0) {
// TODO: handle error
return -1;
}
// pPage->pgid = 0;
pPage->isAnchor = 0;
pPage->isLocalPage = 1;
pPage->pCache = pCache;
TDB_INIT_PAGE_REF(pPage);
pPage->pHashNext = NULL;
pPage->pLruNext = NULL;
pPage->pLruPrev = NULL;
pPage->pDirtyNext = NULL;
pPage->pFreeNext = pCache->pFree;
pCache->pFree = pPage;
pCache->nFree++;
}
// Open the hash table
pCache->nPage = 0;
pCache->nHash = pCache->cacheSize;
pCache->pgHash = (SPage **)calloc(pCache->nHash, sizeof(SPage *));
if (pCache->pgHash == NULL) {
// TODO
return -1;
}
// Open LRU list
pCache->nRecyclable = 0;
pCache->lru.isAnchor = 1;
pCache->lru.pLruNext = &(pCache->lru);
pCache->lru.pLruPrev = &(pCache->lru);
return 0;
}
int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; }
static void *tdbOsMalloc(void *arg, size_t size) {
void *ptr;
ptr = malloc(size);
return ptr;
}
static void tdbOsFree(void *arg, void *ptr) { free(ptr); }

253
source/libs/tdb/src/db/tdbPage.c Normal file
View File

@ -0,0 +1,253 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
typedef struct __attribute__((__packed__)) {
u8 szCell[2];
u8 nxOffset[2];
} SFreeCell;
typedef struct __attribute__((__packed__)) {
u8 szCell[3];
u8 nxOffset[3];
} SFreeCellL;
/* For small page */
#define TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCell *)(PCELL))->szCell)
#define TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCell *)(PCELL))->nxOffset)
#define TDB_SPAGE_FREE_CELL_SIZE(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL))[0]
#define TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL))[0]
#define TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) (TDB_SPAGE_FREE_CELL_SIZE(PCELL) = (SIZE))
#define TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) (TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) = (OFFSET))
/* For large page */
#define TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCellL *)(PCELL))->szCell)
#define TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCellL *)(PCELL))->nxOffset)
#define TDB_LPAGE_FREE_CELL_SIZE(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL))
#define TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL))
#define TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL), SIZE)
#define TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL), OFFSET)
/* For page */
#define TDB_PAGE_FREE_CELL_SIZE_PTR(PPAGE, PCELL) \
(TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) : TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL))
#define TDB_PAGE_FREE_CELL_NXOFFSET_PTR(PPAGE, PCELL) \
(TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL))
#define TDB_PAGE_FREE_CELL_SIZE(PPAGE, PCELL) \
(TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE(PCELL) : TDB_SPAGE_FREE_CELL_SIZE(PCELL))
#define TDB_PAGE_FREE_CELL_NXOFFSET(PPAGE, PCELL) \
(TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL))
#define TDB_PAGE_FREE_CELL_SIZE_SET(PPAGE, PCELL, SIZE) \
do { \
if (TDB_IS_LARGE_PAGE(PPAGE)) { \
TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \
} else { \
TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \
} \
} while (0)
#define TDB_PAGE_FREE_CELL_NXOFFSET_SET(PPAGE, PCELL, OFFSET) \
do { \
if (TDB_IS_LARGE_PAGE(PPAGE)) { \
TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \
} else { \
TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \
} \
} while (0)
static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell);
static int tdbPageDefragment(SPage *pPage);
int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) {
SPage *pPage;
u8 *ptr;
int size;
ASSERT(TDB_IS_PGSIZE_VLD(pageSize));
*ppPage = NULL;
size = pageSize + sizeof(*pPage);
ptr = (u8 *)((*xMalloc)(arg, size));
if (pPage == NULL) {
return -1;
}
memset(ptr, 0, size);
pPage = (SPage *)(ptr + pageSize);
pPage->pData = ptr;
pPage->pageSize = pageSize;
if (pageSize < 65536) {
pPage->szOffset = 2;
pPage->szPageHdr = sizeof(SPageHdr);
pPage->szFreeCell = sizeof(SFreeCell);
} else {
pPage->szOffset = 3;
pPage->szPageHdr = sizeof(SPageHdrL);
pPage->szFreeCell = sizeof(SFreeCellL);
}
TDB_INIT_PAGE_LOCK(pPage);
/* TODO */
*ppPage = pPage;
return 0;
}
int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) {
u8 *ptr;
ptr = pPage->pData;
(*xFree)(arg, ptr);
return 0;
}
int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) {
int ret;
SCell *pTarget;
u8 *pTmp;
int j;
if (pPage->nOverflow || szCell + pPage->szOffset > pPage->nFree) {
// TODO: need to figure out if pCell may be used by outside of this function
j = pPage->nOverflow++;
pPage->apOvfl[j] = pCell;
pPage->aiOvfl[j] = idx;
} else {
ret = tdbPageAllocate(pPage, szCell, &pTarget);
if (ret < 0) {
return -1;
}
memcpy(pTarget, pCell, szCell);
pTmp = pPage->pCellIdx + idx * pPage->szOffset;
memmove(pTmp + pPage->szOffset, pTmp, pPage->pFreeStart - pTmp - pPage->szOffset);
TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData);
TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1);
}
return 0;
}
int tdbPageDropCell(SPage *pPage, int idx) {
// TODO
return 0;
}
static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) {
SCell *pCell;
SFreeCell *pFreeCell;
u8 *pOffset;
int ret;
ASSERT(pPage->nFree > size + pPage->szOffset);
pCell = NULL;
*ppCell = NULL;
// 1. Try to allocate from the free space area
if (pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset) {
pPage->pFreeEnd -= size;
pPage->pFreeStart += pPage->szOffset;
pCell = pPage->pFreeEnd;
}
// 2. Try to allocate from the page free list
if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= pPage->szOffset) && TDB_PAGE_FCELL(pPage)) {
int szCell;
int nxOffset;
pCell = pPage->pData + TDB_PAGE_FCELL(pPage);
pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell
: (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell);
szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell);
nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell);
for (;;) {
// Find a cell
if (szCell >= size) {
if (szCell - size >= pPage->szFreeCell) {
SCell *pTmpCell = pCell + size;
TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size);
TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset);
// TODO: *pOffset = pTmpCell - pPage->pData;
} else {
TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size);
// TODO: *pOffset = nxOffset;
}
break;
}
// Not find a cell yet
if (nxOffset > 0) {
pCell = pPage->pData + nxOffset;
pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell);
szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell);
nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell);
continue;
} else {
pCell = NULL;
break;
}
}
if (pCell) {
pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset;
}
}
// 3. Try to dfragment and allocate again
if (pCell == NULL) {
ret = tdbPageDefragment(pPage);
if (ret < 0) {
return -1;
}
ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset);
ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart);
// Allocate from the free space area again
pPage->pFreeEnd -= size;
pPage->pFreeStart += pPage->szOffset;
pCell = pPage->pFreeEnd;
}
ASSERT(pCell != NULL);
pPage->nFree = pPage->nFree - size - pPage->szOffset;
*ppCell = pCell;
return 0;
}
static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) {
// TODO
return 0;
}
static int tdbPageDefragment(SPage *pPage) {
// TODO
ASSERT(0);
return 0;
}

328
source/libs/tdb/src/db/tdbPager.c Normal file
View File

@ -0,0 +1,328 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
struct SPager {
char *dbFileName;
char *jFileName;
int pageSize;
uint8_t fid[TDB_FILE_ID_LEN];
int fd;
int jfd;
SPCache *pCache;
SPgno dbFileSize;
SPgno dbOrigSize;
int nDirty;
SPage *pDirty;
SPage *pDirtyTail;
u8 inTran;
};
typedef struct __attribute__((__packed__)) {
u8 hdrString[16];
u16 pageSize;
SPgno freePage;
u32 nFreePages;
u8 reserved[102];
} SFileHdr;
TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct");
#define TDB_PAGE_INITIALIZED(pPage) ((pPage)->pPager != NULL)
static int tdbPagerReadPage(SPager *pPager, SPage *pPage);
static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg);
int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
uint8_t *pPtr;
SPager *pPager;
int fsize;
int zsize;
int ret;
*ppPager = NULL;
fsize = strlen(fileName);
zsize = sizeof(*pPager) /* SPager */
+ fsize + 1 /* dbFileName */
+ fsize + 8 + 1; /* jFileName */
pPtr = (uint8_t *)calloc(1, zsize);
if (pPtr == NULL) {
return -1;
}
pPager = (SPager *)pPtr;
pPtr += sizeof(*pPager);
// pPager->dbFileName
pPager->dbFileName = (char *)pPtr;
memcpy(pPager->dbFileName, fileName, fsize);
pPager->dbFileName[fsize] = '\0';
pPtr += fsize + 1;
// pPager->jFileName
pPager->jFileName = (char *)pPtr;
memcpy(pPager->jFileName, fileName, fsize);
memcpy(pPager->jFileName + fsize, "-journal", 8);
pPager->jFileName[fsize + 8] = '\0';
// pPager->pCache
pPager->pCache = pCache;
pPager->fd = open(pPager->dbFileName, O_RDWR | O_CREAT, 0755);
if (pPager->fd < 0) {
return -1;
}
ret = tdbGnrtFileID(pPager->dbFileName, pPager->fid, false);
if (ret < 0) {
return -1;
}
pPager->jfd = -1;
pPager->pageSize = tdbPCacheGetPageSize(pCache);
*ppPager = pPager;
return 0;
}
int tdbPagerClose(SPager *pPager) {
// TODO
return 0;
}
int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate) {
SPgno pgno;
SPage *pPage;
int ret;
{
// TODO: try to search the main DB to get the page number
pgno = 0;
}
// if (pgno == 0 && toCreate) {
// ret = tdbPagerAllocPage(pPager, &pPage, &pgno);
// if (ret < 0) {
// return -1;
// }
// // TODO: Need to zero the page
// ret = tdbPagerWrite(pPager, pPage);
// if (ret < 0) {
// return -1;
// }
// }
*ppgno = pgno;
return 0;
}
int tdbPagerWrite(SPager *pPager, SPage *pPage) {
int ret;
if (pPager->inTran == 0) {
ret = tdbPagerBegin(pPager);
if (ret < 0) {
return -1;
}
}
if (pPage->isDirty == 0) {
pPage->isDirty = 1;
// TODO: add the page to the dirty list
// TODO: write the page to the journal
if (1 /*actually load from the file*/) {
}
}
return 0;
}
int tdbPagerBegin(SPager *pPager) {
if (pPager->inTran) {
return 0;
}
// Open the journal
pPager->jfd = open(pPager->jFileName, O_RDWR | O_CREAT, 0755);
if (pPager->jfd < 0) {
return -1;
}
// TODO: write the size of the file
pPager->inTran = 1;
return 0;
}
int tdbPagerCommit(SPager *pPager) {
// TODO
return 0;
}
static int tdbPagerReadPage(SPager *pPager, SPage *pPage) {
i64 offset;
int ret;
ASSERT(memcmp(pPager->fid, pPage->pgid.fileid, TDB_FILE_ID_LEN) == 0);
offset = (pPage->pgid.pgno - 1) * (i64)(pPager->pageSize);
ret = tdbPRead(pPager->fd, pPage->pData, pPager->pageSize, offset);
if (ret < 0) {
// TODO: handle error
return -1;
}
return 0;
}
int tdbPagerGetPageSize(SPager *pPager) { return pPager->pageSize; }
int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg) {
SPage *pPage;
SPgid pgid;
int ret;
// Fetch a page container from the page cache
memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN);
pgid.pgno = pgno;
pPage = tdbPCacheFetch(pPager->pCache, &pgid, 1);
if (pPage == NULL) {
return -1;
}
// Initialize the page if need
if (!TDB_PAGE_INITIALIZED(pPage)) {
ret = tdbPagerInitPage(pPager, pPage, initPage, arg);
if (ret < 0) {
return -1;
}
}
ASSERT(TDB_PAGE_INITIALIZED(pPage));
ASSERT(pPage->pPager == pPager);
*ppPage = pPage;
return 0;
}
int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg) {
int ret;
SPage *pPage;
SPgid pgid;
// Allocate a page number
ret = tdbPagerAllocPage(pPager, ppgno);
if (ret < 0) {
return -1;
}
ASSERT(*ppgno != 0);
// Fetch a page container from the page cache
memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN);
pgid.pgno = *ppgno;
pPage = tdbPCacheFetch(pPager->pCache, &pgid, 1);
if (pPage == NULL) {
return -1;
}
ASSERT(!TDB_PAGE_INITIALIZED(pPage));
// Initialize the page if need
ret = tdbPagerInitPage(pPager, pPage, initPage, arg);
if (ret < 0) {
return -1;
}
ASSERT(TDB_PAGE_INITIALIZED(pPage));
ASSERT(pPage->pPager == pPager);
*ppPage = pPage;
return 0;
}
static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) {
// TODO: Allocate a page from the free list
return 0;
}
static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) {
*ppgno = ++pPager->dbFileSize;
return 0;
}
static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) {
int ret;
*ppgno = 0;
// Try to allocate from the free list of the pager
ret = tdbPagerAllocFreePage(pPager, ppgno);
if (ret < 0) {
return -1;
}
if (*ppgno != 0) return 0;
// Allocate the page by extending the pager
ret = tdbPagerAllocNewPage(pPager, ppgno);
if (ret < 0) {
return -1;
}
ASSERT(*ppgno != 0);
return 0;
}
static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg) {
int ret;
int lcode;
int nLoops;
lcode = TDB_TRY_LOCK_PAGE(pPage);
if (lcode == P_LOCK_SUCC) {
if (TDB_PAGE_INITIALIZED(pPage)) {
TDB_UNLOCK_PAGE(pPage);
return 0;
}
ret = (*initPage)(pPage, arg);
if (ret < 0) {
TDB_UNLOCK_PAGE(pPage);
return -1;
}
pPage->pPager = pPager;
TDB_UNLOCK_PAGE(pPage);
} else if (lcode == P_LOCK_BUSY) {
nLoops = 0;
for (;;) {
if (TDB_PAGE_INITIALIZED(pPage)) break;
nLoops++;
if (nLoops > 1000) {
sched_yield();
nLoops = 0;
}
}
} else {
return -1;
}
return 0;
}

576
source/libs/tdb/src/db/tdbPgCache.c
View File

@ -1,576 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
typedef TD_DLIST(SPage) SPgList;
struct SPgCache {
TENV * pEnv; // TENV containing this page cache
pgsz_t pgsize;
int32_t npage;
SPage **pages;
SPgList freeList;
SPgList lru;
struct {
int32_t nbucket;
SPgList *buckets;
} pght; // page hash table
};
static void pgCachePinPage(SPage *pPage);
static void pgCacheUnpinPage(SPage *pPage);
int pgCacheOpen(SPgCache **ppPgCache, TENV *pEnv) {
SPgCache *pPgCache;
SPage * pPage;
void * pData;
pgsz_t pgSize;
cachesz_t cacheSize;
int32_t npage;
int32_t nbucket;
size_t msize;
*ppPgCache = NULL;
pgSize = tdbEnvGetPageSize(pEnv);
cacheSize = tdbEnvGetCacheSize(pEnv);
npage = cacheSize / pgSize;
nbucket = npage;
msize = sizeof(*pPgCache) + sizeof(SPage *) * npage + sizeof(SPgList) * nbucket;
// Allocate the handle
pPgCache = (SPgCache *)calloc(1, msize);
if (pPgCache == NULL) {
return -1;
}
// Init the handle
pPgCache->pEnv = pEnv;
pPgCache->pgsize = pgSize;
pPgCache->npage = npage;
pPgCache->pages = (SPage **)(&pPgCache[1]);
pPgCache->pght.nbucket = nbucket;
pPgCache->pght.buckets = (SPgList *)(&(pPgCache->pages[npage]));
TD_DLIST_INIT(&(pPgCache->freeList));
for (int32_t i = 0; i < npage; i++) {
pData = malloc(pgSize + sizeof(SPage));
if (pData == NULL) {
return -1;
// TODO: handle error
}
pPage = POINTER_SHIFT(pData, pgSize);
pPage->pgid = TDB_IVLD_PGID;
pPage->frameid = i;
pPage->pData = pData;
// add current page to the page cache
pPgCache->pages[i] = pPage;
TD_DLIST_APPEND_WITH_FIELD(&(pPgCache->freeList), pPage, freeNode);
}
#if 0
for (int32_t i = 0; i < nbucket; i++) {
TD_DLIST_INIT(pPgCache->pght.buckets + i);
}
#endif
*ppPgCache = pPgCache;
return 0;
}
int pgCacheClose(SPgCache *pPgCache) {
SPage *pPage;
if (pPgCache) {
for (int32_t i = 0; i < pPgCache->npage; i++) {
pPage = pPgCache->pages[i];
tfree(pPage->pData);
}
free(pPgCache);
}
return 0;
}
#define PG_CACHE_HASH(fileid, pgno) (((uint64_t *)(fileid))[0] + ((uint64_t *)(fileid))[1] + ((uint64_t *)(fileid))[2] + (pgno))
SPage *pgCacheFetch(SPgCache *pPgCache, pgid_t pgid) {
SPage * pPage;
SPgFile *pPgFile;
SPgList *pBucket;
// 1. Search the page hash table SPgCache.pght
pBucket = pPgCache->pght.buckets + (PG_CACHE_HASH(pgid.fileid, pgid.pgno) % pPgCache->pght.nbucket);
pPage = TD_DLIST_HEAD(pBucket);
while (pPage && tdbCmprPgId(&(pPage->pgid), &pgid)) {
pPage = TD_DLIST_NODE_NEXT_WITH_FIELD(pPage, pghtNode);
}
if (pPage) {
// Page is found, pin the page and return the page
pgCachePinPage(pPage);
return pPage;
}
// 2. Check the free list
pPage = TD_DLIST_HEAD(&(pPgCache->freeList));
if (pPage) {
TD_DLIST_POP_WITH_FIELD(&(pPgCache->freeList), pPage, freeNode);
pgCachePinPage(pPage);
return pPage;
}
// 3. Try to recycle a page from the LRU list
pPage = TD_DLIST_HEAD(&(pPgCache->lru));
if (pPage) {
TD_DLIST_POP_WITH_FIELD(&(pPgCache->lru), pPage, lruNode);
// TODO: remove from the hash table
pgCachePinPage(pPage);
return pPage;
}
// 4. If a memory allocator is set, try to allocate from the allocator (TODO)
return NULL;
}
int pgCacheRelease(SPage *pPage) {
// TODO
return 0;
}
static void pgCachePinPage(SPage *pPage) {
// TODO
}
static void pgCacheUnpinPage(SPage *pPage) {
// TODO
}
#if 0
// Exposed handle
typedef struct TDB_MPOOL TDB_MPOOL;
typedef struct TDB_MPFILE TDB_MPFILE;
typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t;
typedef struct pg_t {
SRWLatch rwLatch;
frame_id_t frameid;
pgid_t pgid;
uint8_t dirty;
uint8_t rbit;
int32_t pinRef;
pg_free_dlist_node_t free;
pg_hash_dlist_node_t hash;
void * p;
} pg_t;
typedef TD_DLIST(pg_t) pg_list_t;
typedef struct {
SRWLatch latch;
TD_DLIST(TDB_MPFILE);
} mpf_bucket_t;
struct TDB_MPOOL {
int64_t cachesize;
pgsz_t pgsize;
int32_t npages;
pg_t * pages;
pg_list_t freeList;
frame_id_t clockHand;
struct {
int32_t nbucket;
pg_list_t *hashtab;
} pgtab; // page table, hash<pgid_t, pg_t>
struct {
#define MPF_HASH_BUCKETS 16
mpf_bucket_t buckets[MPF_HASH_BUCKETS];
} mpfht; // MPF hash table. MPFs using this MP will be put in this hash table
};
#define MP_PAGE_AT(mp, idx) (mp)->pages[idx]
typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t;
struct TDB_MPFILE {
char * fname; // file name
int fd; // fd
uint8_t fileid[TDB_FILE_ID_LEN]; // file ID
TDB_MPOOL * mp; // underlying memory pool
td_mpf_dlist_node_t node;
};
/*=================================================== Exposed apis ==================================================*/
// TDB_MPOOL
int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize);
int tdbMPoolClose(TDB_MPOOL *mp);
int tdbMPoolSync(TDB_MPOOL *mp);
// TDB_MPFILE
int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp);
int tdbMPoolFileClose(TDB_MPFILE *mpf);
int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr);
int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr);
int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
int tdbMPoolFileSync(TDB_MPFILE *mpf);
static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf);
static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf);
static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid);
static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p);
static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p);
static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp);
int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize) {
TDB_MPOOL *mp = NULL;
size_t tsize;
pg_t * pagep;
// check parameters
if (!TDB_IS_PGSIZE_VLD(pgsize)) {
tdbError("invalid page size");
return -1;
}
// allocate handle
mp = (TDB_MPOOL *)calloc(1, sizeof(*mp));
if (mp == NULL) {
tdbError("failed to malloc memory pool handle");
goto _err;
}
// initialize the handle
mp->cachesize = cachesize;
mp->pgsize = pgsize;
mp->npages = cachesize / pgsize;
mp->clockHand = 0;
TD_DLIST_INIT(&mp->freeList);
mp->pages = (pg_t *)calloc(mp->npages, sizeof(pg_t));
if (mp->pages == NULL) {
tdbError("failed to malloc memory pool pages");
goto _err;
}
for (frame_id_t i = 0; i < mp->npages; i++) {
mp->pages[i].p = malloc(pgsize);
if (mp->pages[i].p == NULL) {
goto _err;
}
taosInitRWLatch(&mp->pages[i].rwLatch);
mp->pages[i].frameid = i;
mp->pages[i].pgid = TDB_IVLD_PGID;
// add new page to the free list
TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), &(mp->pages[i]), free);
}
#define PGTAB_FACTOR 1.0
mp->pgtab.nbucket = mp->npages / PGTAB_FACTOR;
mp->pgtab.hashtab = (pg_list_t *)calloc(mp->pgtab.nbucket, sizeof(pg_list_t));
if (mp->pgtab.hashtab == NULL) {
tdbError("failed to malloc memory pool hash table");
goto _err;
}
// return
*mpp = mp;
return 0;
_err:
tdbMPoolClose(mp);
*mpp = NULL;
return -1;
}
int tdbMPoolClose(TDB_MPOOL *mp) {
if (mp) {
tfree(mp->pgtab.hashtab);
if (mp->pages) {
for (int i = 0; i < mp->npages; i++) {
tfree(mp->pages[i].p);
}
free(mp->pages);
}
free(mp);
}
return 0;
}
int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) {
TDB_MPFILE *mpf;
if ((mpf = (TDB_MPFILE *)calloc(1, sizeof(*mpf))) == NULL) {
return -1;
}
mpf->fd = -1;
if ((mpf->fname = strdup(fname)) == NULL) {
goto _err;
}
if ((mpf->fd = open(fname, O_CREAT | O_RDWR, 0755)) < 0) {
goto _err;
}
if (tdbGnrtFileID(fname, mpf->fileid, false) < 0) {
goto _err;
}
// Register current MPF to MP
tdbMPoolRegFile(mp, mpf);
*mpfp = mpf;
return 0;
_err:
tdbMPoolFileClose(mpf);
*mpfp = NULL;
return -1;
}
int tdbMPoolFileClose(TDB_MPFILE *mpf) {
if (mpf) {
if (mpf->fd > 0) {
close(mpf->fd);
}
tfree(mpf->fname);
free(mpf);
}
return 0;
}
#define MPF_GET_PAGE_BUCKETID(fileid, pgno, nbuckets) \
({ \
uint64_t *tmp = (uint64_t *)fileid; \
(tmp[0] + tmp[1] + tmp[2] + (pgno)) % (nbuckets); \
})
int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr) {
// TODO
return 0;
}
int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr) {
// TODO
return 0;
}
int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) {
pg_t * pagep;
TDB_MPOOL *mp;
pg_list_t *pglist;
mp = mpf->mp;
// check if the page already in pool
pglist = mp->pgtab.hashtab + MPF_GET_PAGE_BUCKETID(mpf->fileid, pgno, mp->pgtab.nbucket);
pagep = TD_DLIST_HEAD(pglist);
while (pagep) {
if (memcmp(mpf->fileid, pagep->pgid.fileid, TDB_FILE_ID_LEN) == 0 && pgno == pagep->pgid.pgno) {
break;
}
pagep = TD_DLIST_NODE_NEXT_WITH_FIELD(pagep, hash);
}
if (pagep) {
// page is found
// todo: pin the page and return
*(void **)addr = pagep->p;
return 0;
}
// page not found
pagep = TD_DLIST_HEAD(&mp->freeList);
if (pagep) {
// has free page
TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free);
} else {
// no free page available
tdbMPoolClockEvictPage(mp, &pagep);
if (pagep) {
if (pagep->dirty) {
// TODO: Handle dirty page eviction
}
}
}
if (pagep == NULL) {
// no available container page
return -1;
}
// load page from the disk if a container page is available
// TODO: load the page from the disk
if (tdbMPoolFileReadPage(mpf, pgno, pagep->p) < 0) {
return -1;
}
memcpy(pagep->pgid.fileid, mpf->fileid, TDB_FILE_ID_LEN);
pagep->pgid.pgno = pgno;
pagep->dirty = 0;
pagep->pinRef = 1;
// add current page to page table
TD_DLIST_APPEND_WITH_FIELD(pglist, pagep, hash);
return 0;
}
int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) {
// TODO
return 0;
}
#define MPF_GET_BUCKETID(fileid) \
({ \
uint64_t *tmp = (uint64_t *)fileid; \
(tmp[0] + tmp[1] + tmp[2]) % MPF_HASH_BUCKETS; \
})
static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) {
mpf_bucket_t *bktp;
bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid);
taosWLockLatch(&(bktp->latch));
TD_DLIST_APPEND_WITH_FIELD(bktp, mpf, node);
taosWUnLockLatch(&(bktp->latch));
mpf->mp = mp;
}
static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid) {
TDB_MPFILE * mpf = NULL;
mpf_bucket_t *bktp;
bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(fileid);
taosRLockLatch(&(bktp->latch));
mpf = TD_DLIST_HEAD(bktp);
while (mpf) {
if (memcmp(fileid, mpf->fileid, TDB_FILE_ID_LEN) == 0) {
break;
}
mpf = TD_DLIST_NODE_NEXT_WITH_FIELD(mpf, node);
}
taosRUnLockLatch(&(bktp->latch));
return mpf;
}
static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) {
mpf_bucket_t *bktp;
TDB_MPFILE * tmpf;
if (mpf->mp == NULL) return;
ASSERT(mpf->mp == mp);
bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid);
taosWLockLatch(&(bktp->latch));
tmpf = TD_DLIST_HEAD(bktp);
while (tmpf) {
if (memcmp(mpf->fileid, tmpf->fileid, TDB_FILE_ID_LEN) == 0) {
TD_DLIST_POP_WITH_FIELD(bktp, tmpf, node);
break;
}
tmpf = TD_DLIST_NODE_NEXT_WITH_FIELD(tmpf, node);
}
taosWUnLockLatch(&(bktp->latch));
ASSERT(tmpf == mpf);
}
static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) {
pgsz_t pgsize;
TDB_MPOOL *mp;
off_t offset;
size_t rsize;
mp = mpf->mp;
pgsize = mp->pgsize;
offset = pgno * pgsize;
// TODO: use loop to read all data
rsize = pread(mpf->fd, p, pgsize, offset);
// TODO: error handle
return 0;
}
static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) {
pgsz_t pgsize;
TDB_MPOOL *mp;
off_t offset;
mp = mpf->mp;
pgsize = mp->pgsize;
offset = pgno * pgsize;
lseek(mpf->fd, offset, SEEK_SET);
// TODO: handle error
write(mpf->fd, p, pgsize);
// TODO: handle error
return 0;
}
static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp) {
pg_t * pagep;
frame_id_t och;
*pagepp = NULL;
och = mp->clockHand;
do {
pagep = mp->pages + mp->clockHand;
mp->clockHand = (mp->clockHand + 1) % mp->npages;
if (pagep->pinRef == 0) {
if (pagep->rbit == 1) {
pagep->rbit = 0;
} else {
break;
}
}
if (mp->clockHand == och) {
return;
}
} while (1);
*pagepp = pagep;
}
#endif

221
source/libs/tdb/src/db/tdbPgFile.c
View File

@ -1,221 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
typedef struct SPage1 {
char magic[64];
pgno_t mdbRootPgno; // master DB root page number
pgno_t freePgno; // free list page number
uint32_t nFree; // number of free pages
} SPage1;
typedef struct SFreePage {
/* TODO */
} SFreePage;
TDB_STATIC_ASSERT(sizeof(SPage1) <= TDB_MIN_PGSIZE, "TDB Page1 definition too large");
static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData);
int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) {
SPgFile * pPgFile;
SPgCache *pPgCache;
size_t fnameLen;
pgno_t fsize;
*ppPgFile = NULL;
// create the handle
fnameLen = strlen(fname);
pPgFile = (SPgFile *)calloc(1, sizeof(*pPgFile) + fnameLen + 1);
if (pPgFile == NULL) {
return -1;
}
ASSERT(pEnv != NULL);
// init the handle
pPgFile->fname = (char *)(&(pPgFile[1]));
memcpy(pPgFile->fname, fname, fnameLen);
pPgFile->fname[fnameLen] = '\0';
pPgFile->pFile = NULL;
pPgFile->pFile = taosOpenFile(fname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pPgFile->pFile == NULL) {
// TODO: handle error
return -1;
}
tdbGnrtFileID(fname, pPgFile->fileid, false);
tdbGetFileSize(fname, tdbEnvGetPageSize(pEnv), &fsize);
pPgFile->fsize = fsize;
pPgFile->lsize = fsize;
if (pPgFile->fsize == 0) {
// A created file
pgno_t pgno;
pgid_t pgid;
pgFileAllocatePage(pPgFile, &pgno);
ASSERT(pgno == 1);
memcpy(pgid.fileid, pPgFile->fileid, TDB_FILE_ID_LEN);
pgid.pgno = pgno;
pgCacheFetch(pPgCache, pgid);
// Need to allocate the first page as a description page
} else {
// An existing file
}
/* TODO: other open operations */
// add the page file to the environment
tdbEnvRgstPageFile(pEnv, pPgFile);
pPgFile->pEnv = pEnv;
*ppPgFile = pPgFile;
return 0;
}
int pgFileClose(SPgFile *pPgFile) {
if (pPgFile) {
if (pPgFile->pFile != NULL) {
taosCloseFile(&pPgFile->pFile);
}
tfree(pPgFile->fname);
free(pPgFile);
}
return 0;
}
SPage *pgFileFetch(SPgFile *pPgFile, pgno_t pgno) {
SPgCache *pPgCache;
SPage * pPage;
pgid_t pgid;
// 1. Fetch from the page cache
// pgCacheFetch(pPgCache, pgid);
// 2. If only get a page frame, no content, maybe
// need to load from the file
if (1 /*page not initialized*/) {
if (pgno < pPgFile->fsize) {
// load the page content from the disk
// ?? How about the freed pages ??
} else {
// zero the page, make the page as a empty
// page with zero records.
}
}
#if 0
pPgCache = pPgFile->pPgCache;
pPage = NULL;
memcpy(pgid.fileid, pPgFile->fileid, TDB_FILE_ID_LEN);
pgid.pgno = pgno;
if (pgno > pPgFile->pgFileSize) {
// TODO
} else {
pPage = pgCacheFetch(pPgCache, pgid);
if (1 /*Page is cached, no need to load from file*/) {
return pPage;
} else {
// TODO: handle error
if (pgFileRead(pPgFile, pgno, (void *)pPage) < 0) {
// todoerr
}
return pPage;
}
}
#endif
return pPage;
}
int pgFileRelease(SPage *pPage) {
pgCacheRelease(pPage);
return 0;
}
int pgFileWrite(SPage *pPage) {
// TODO
return 0;
}
int pgFileAllocatePage(SPgFile *pPgFile, pgno_t *pPgno) {
pgno_t pgno;
SPage1 * pPage1;
SPgCache *pPgCache;
pgid_t pgid;
SPage * pPage;
if (pPgFile->lsize == 0) {
pgno = ++(pPgFile->lsize);
} else {
if (0) {
// TODO: allocate from the free list
pPage = pgCacheFetch(pPgCache, pgid);
if (pPage1->nFree > 0) {
// TODO
} else {
pgno = ++(pPgFile->lsize);
}
} else {
pgno = ++(pPgFile->lsize);
}
}
*pPgno = pgno;
return 0;
}
static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) {
pgsz_t pgSize;
ssize_t rsize;
uint8_t *pTData;
size_t szToRead;
#if 0
// pgSize = ; (TODO)
pTData = pData;
szToRead = pgSize;
for (; szToRead > 0;) {
rsize = pread(pPgFile->pFile, pTData, szToRead, pgno * pgSize);
if (rsize < 0) {
if (errno == EINTR) {
continue;
} else {
return -1;
}
} else if (rsize == 0) {
return -1;
}
szToRead -= rsize;
pTData += rsize;
}
#endif
return 0;
}

28
source/libs/tdb/src/db/tdbUtil.c
View File

@ -51,7 +51,8 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) {
// return access(pathname, flags);
// }
int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize) {
int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize) {
struct stat st;
int ret;
int64_t file_size = 0;
ret = taosStatFile(fname, &file_size, NULL);
@ -64,3 +65,28 @@ int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize) {
*pSize = file_size / pgSize;
return 0;
}
int tdbPRead(int fd, void *pData, int count, i64 offset) {
void *pBuf;
int nbytes;
i64 ioffset;
int iread;
pBuf = pData;
nbytes = count;
ioffset = offset;
while (nbytes > 0) {
iread = pread(fd, pBuf, nbytes, ioffset);
if (iread < 0) {
/* TODO */
} else if (iread == 0) {
return (count - iread);
}
nbytes = nbytes - iread;
pBuf = (void *)((u8 *)pBuf + iread);
ioffset += iread;
}
return count;
}

25
source/libs/tdb/src/inc/tdbBtree.h
View File

@ -23,20 +23,21 @@ extern "C" {
typedef struct SBTree SBTree;
typedef struct SBtCursor SBtCursor;
// SBTree
int btreeOpen(SBTree **ppBt, SPgFile *pPgFile);
int btreeClose(SBTree *pBt);
// SBtCursor
int btreeCursorOpen(SBtCursor *pBtCur, SBTree *pBt);
int btreeCursorClose(SBtCursor *pBtCur);
int btreeCursorMoveTo(SBtCursor *pBtCur, int kLen, const void *pKey);
int btreeCursorNext(SBtCursor *pBtCur);
struct SBTree {
pgno_t root;
struct SBtCursor {
SBTree *pBt;
i8 iPage;
SPage *pPage;
int idx;
int idxStack[BTREE_MAX_DEPTH + 1];
SPage *pgStack[BTREE_MAX_DEPTH + 1];
void *pBuf;
};
int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt);
int tdbBtreeClose(SBTree *pBt);
int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt);
int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen);
#ifdef __cplusplus
}
#endif

34
source/libs/tdb/src/inc/tdbDb.h Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TDB_DB_H_
#define _TD_TDB_DB_H_
#ifdef __cplusplus
extern "C" {
#endif
typedef struct STDb STDb;
int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb);
int tdbDbClose(STDb *pDb);
int tdbDbDrop(STDb *pDb);
int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen);
#ifdef __cplusplus
}
#endif
#endif /*_TD_TDB_DB_H_*/

16
source/libs/tdb/src/inc/tdbEnv.h
View File

@ -20,11 +20,17 @@
extern "C" {
#endif
const char* tdbEnvGetRootDir(TENV* pEnv);
SPgFile* tdbEnvGetPageFile(TENV* pEnv, const uint8_t fileid[]);
SPgCache* tdbEnvGetPgCache(TENV* pEnv);
int tdbEnvRgstPageFile(TENV* pEnv, SPgFile* pPgFile);
int tdbEnvRgstDB(TENV* pEnv, TDB* pDb);
typedef struct STEnv {
char * rootDir;
char * jfname;
int jfd;
SPCache *pCache;
} STEnv;
int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, STEnv **ppEnv);
int tdbEnvClose(STEnv *pEnv);
SPager *tdbEnvGetPager(STEnv *pEnv, const char *fname);
#ifdef __cplusplus
}

71
source/libs/tdb/src/inc/tdbInt.h
View File

@ -19,16 +19,33 @@
#include "tlist.h"
#include "tlockfree.h"
#include "tdb.h"
// #include "tdb.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SPgFile SPgFile;
typedef int8_t i8;
typedef int16_t i16;
typedef int32_t i32;
typedef int64_t i64;
typedef uint8_t u8;
typedef uint16_t u16;
typedef uint32_t u32;
typedef uint64_t u64;
// pgno_t
typedef int32_t pgno_t;
// p must be u8 *
#define TDB_GET_U24(p) ((p)[0] * 65536 + *(u16 *)((p) + 1))
#define TDB_PUT_U24(p, v) \
do { \
int tv = (v); \
(p)[2] = tv & 0xff; \
(p)[1] = (tv >> 8) & 0xff; \
(p)[0] = (tv >> 16) & 0xff; \
} while (0)
// SPgno
typedef u32 SPgno;
#define TDB_IVLD_PGNO ((pgno_t)0)
// fileid
@ -37,8 +54,8 @@ typedef int32_t pgno_t;
// pgid_t
typedef struct {
uint8_t fileid[TDB_FILE_ID_LEN];
pgno_t pgno;
} pgid_t;
SPgno pgno;
} pgid_t, SPgid;
#define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO};
@ -61,18 +78,14 @@ static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) {
}
}
// framd_id_t
typedef int32_t frame_id_t;
#define TDB_IS_SAME_PAGE(pPgid1, pPgid2) (tdbCmprPgId(pPgid1, pPgid2) == 0)
// pgsz_t
#define TDB_MIN_PGSIZE 512
#define TDB_MAX_PGSIZE 65536
#define TDB_MIN_PGSIZE 512 // 512B
#define TDB_MAX_PGSIZE 16777216 // 16M
#define TDB_DEFAULT_PGSIZE 4096
#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE))
// pgoff_t
typedef pgsz_t pgoff_t;
// cache
#define TDB_DEFAULT_CACHE_SIZE (256 * 4096) // 1M
@ -100,7 +113,7 @@ typedef TD_DLIST_NODE(SPgFile) SPgFileListNode;
} \
} while (0)
#define TDB_VARIANT_LEN (int)-1
#define TDB_VARIANT_LEN ((int)-1)
// page payload format
// <keyLen> + <valLen> + [key] + [value]
@ -115,18 +128,40 @@ typedef TD_DLIST_NODE(SPgFile) SPgFileListNode;
/* TODO */ \
} while (0)
typedef int (*FKeyComparator)(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
#define TDB_JOURNAL_NAME "tdb.journal"
#define TDB_FILENAME_LEN 128
#define TDB_DEFAULT_FANOUT 6
#define BTREE_MAX_DEPTH 20
#define TDB_FLAG_IS(flags, flag) ((flags) == (flag))
#define TDB_FLAG_HAS(flags, flag) (((flags) & (flag)) != 0)
#define TDB_FLAG_NO(flags, flag) ((flags) & (flag) == 0)
#define TDB_FLAG_ADD(flags, flag) ((flags) |= (flag))
#define TDB_FLAG_REMOVE(flags, flag) ((flags) &= (~(flag)))
typedef struct SPager SPager;
typedef struct SPCache SPCache;
typedef struct SPage SPage;
#include "tdbUtil.h"
#include "tdbPCache.h"
#include "tdbPager.h"
#include "tdbBtree.h"
#include "tdbPgCache.h"
#include "tdbPgFile.h"
#include "tdbEnv.h"
#include "tdbDb.h"
#include "tdbPage.h"
#ifdef __cplusplus
}
#endif

37
source/libs/tdb/src/inc/tdbPgCache.h → source/libs/tdb/src/inc/tdbPCache.h
View File

@ -20,26 +20,25 @@
extern "C" {
#endif
typedef struct SPgCache SPgCache;
typedef struct SPage SPage;
#define TDB_PCACHE_PAGE \
u8 isAnchor; \
u8 isLocalPage; \
u8 isDirty; \
i32 nRef; \
SPCache *pCache; \
SPage *pFreeNext; \
SPage *pHashNext; \
SPage *pLruNext; \
SPage *pLruPrev; \
SPage *pDirtyNext; \
SPager *pPager; \
SPgid pgid;
// SPgCache
int pgCacheOpen(SPgCache **ppPgCache, TENV *pEnv);
int pgCacheClose(SPgCache *pPgCache);
SPage *pgCacheFetch(SPgCache *pPgCache, pgid_t pgid);
int pgCacheRelease(SPage *pPage);
// SPage
typedef TD_DLIST_NODE(SPage) SPgListNode;
struct SPage {
pgid_t pgid; // page id
frame_id_t frameid; // frame id
uint8_t * pData; // real data
SPgListNode freeNode; // for SPgCache.freeList
SPgListNode pghtNode; // for pght
SPgListNode lruNode; // for LRU
};
int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache);
int tdbPCacheClose(SPCache *pCache);
SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage);
void tdbPCacheRelease(SPage *pPage);
int tdbPCacheGetPageSize(SPCache *pCache);
#ifdef __cplusplus
}

131
source/libs/tdb/src/inc/tdbPage.h Normal file
View File

@ -0,0 +1,131 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TDB_PAGE_H_
#define _TDB_PAGE_H_
#ifdef __cplusplus
extern "C" {
#endif
typedef u8 SCell;
// PAGE APIS implemented
typedef struct {
int szOffset;
int szPageHdr;
int szFreeCell;
// flags
u16 (*getFlags)(SPage *);
void (*setFlags)(SPage *, u16);
// cell number
int (*getCellNum)(SPage *);
void (*setCellNum)(SPage *, int);
// cell content offset
int (*getCellBody)(SPage *);
void (*setCellBody)(SPage *, int);
// first free cell offset (0 means no free cells)
int (*getCellFree)(SPage *);
void (*setCellFree)(SPage *, int);
// total free bytes
int (*getFreeBytes)(SPage *);
void (*setFreeBytes)(SPage *, int);
// cell offset at idx
int (*getCellOffset)(SPage *, int);
void (*setCellOffset)(SPage *, int, int);
} SPageMethods;
// Page footer
typedef struct __attribute__((__packed__)) {
u8 cksm[4];
} SPageFtr;
struct SPage {
pthread_spinlock_t lock;
u8 *pData;
int pageSize;
SPageMethods *pPageMethods;
// Fields below used by pager and am
u8 szAmHdr;
u8 *pPageHdr;
u8 *pAmHdr;
u8 *pCellIdx;
u8 *pFreeStart;
u8 *pFreeEnd;
SPageFtr *pPageFtr;
int kLen; // key length of the page, -1 for unknown
int vLen; // value length of the page, -1 for unknown
int nFree;
int maxLocal;
int minLocal;
int nOverflow;
SCell *apOvfl[4];
int aiOvfl[4];
// Fields used by SPCache
TDB_PCACHE_PAGE
};
/* For page */
#define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage)
#define TDB_PAGE_NCELLS(pPage) (*(pPage)->pPageMethods->getCellNum)(pPage)
#define TDB_PAGE_CCELLS(pPage) (*(pPage)->pPageMethods->getCellBody)(pPage)
#define TDB_PAGE_FCELL(pPage) (*(pPage)->pPageMethods->getCellFree)(pPage)
#define TDB_PAGE_NFREE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage)
#define TDB_PAGE_CELL_OFFSET_AT(pPage, idx) (*(pPage)->pPageMethods->getCellOffset)(pPage, idx)
#define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS)
#define TDB_PAGE_NCELLS_SET(pPage, NCELLS) (*(pPage)->pPageMethods->setCellNum)(pPage, NCELLS)
#define TDB_PAGE_CCELLS_SET(pPage, CCELLS) (*(pPage)->pPageMethods->setCellBody)(pPage, CCELLS)
#define TDB_PAGE_FCELL_SET(pPage, FCELL) (*(pPage)->pPageMethods->setCellFree)(pPage, FCELL)
#define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE)
#define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET)
#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset)
#define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx))
// For page lock
#define P_LOCK_SUCC 0
#define P_LOCK_BUSY 1
#define P_LOCK_FAIL -1
#define TDB_INIT_PAGE_LOCK(pPage) pthread_spin_init(&((pPage)->lock), 0)
#define TDB_DESTROY_PAGE_LOCK(pPage) pthread_spin_destroy(&((pPage)->lock))
#define TDB_LOCK_PAGE(pPage) pthread_spin_lock(&((pPage)->lock))
#define TDB_UNLOCK_PAGE(pPage) pthread_spin_unlock(&((pPage)->lock))
#define TDB_TRY_LOCK_PAGE(pPage) \
({ \
int ret; \
if (pthread_spin_trylock(&((pPage)->lock)) == 0) { \
ret = P_LOCK_SUCC; \
} else if (errno == EBUSY) { \
ret = P_LOCK_BUSY; \
} else { \
ret = P_LOCK_FAIL; \
} \
ret; \
})
// APIs
int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg);
int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg);
int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell);
int tdbPageDropCell(SPage *pPage, int idx);
#ifdef __cplusplus
}
#endif
#endif /*_TDB_PAGE_H_*/

37
source/libs/tdb/src/inc/tdbPager.h Normal file
View File

@ -0,0 +1,37 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TDB_PAGER_H_
#define _TDB_PAGER_H_
#ifdef __cplusplus
extern "C" {
#endif
int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager);
int tdbPagerClose(SPager *pPager);
int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate);
int tdbPagerWrite(SPager *pPager, SPage *pPage);
int tdbPagerBegin(SPager *pPager);
int tdbPagerCommit(SPager *pPager);
int tdbPagerGetPageSize(SPager *pPager);
int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg);
int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg);
#ifdef __cplusplus
}
#endif
#endif /*_TDB_PAGER_H_*/

61
source/libs/tdb/src/inc/tdbPgFile.h
View File

@ -1,61 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_PAGE_FILE_H_
#define _TD_PAGE_FILE_H_
#ifdef __cplusplus
extern "C" {
#endif
#pragma pack (push,1)
typedef struct {
char hdrInfo[16]; // info string
pgsz_t szPage; // page size of current file
int32_t cno; // commit number counter
pgno_t freePgno; // freelist page number
uint8_t resv[100]; // reserved space
} SPgFileHdr;
#pragma pack(pop)
#define TDB_PG_FILE_HDR_SIZE 128
TDB_STATIC_ASSERT(sizeof(SPgFileHdr) == TDB_PG_FILE_HDR_SIZE, "Page file header size if not 128");
struct SPgFile {
TENV * pEnv; // env containing this page file
char * fname; // backend file name
uint8_t fileid[TDB_FILE_ID_LEN]; // file id
pgno_t lsize; // page file logical size (for count)
pgno_t fsize; // real file size on disk (for rollback)
TdFilePtr pFile;
SPgFileListNode envHash;
SPgFileListNode envPgfList;
};
int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv);
int pgFileClose(SPgFile *pPgFile);
SPage *pgFileFetch(SPgFile *pPgFile, pgno_t pgno);
int pgFileRelease(SPage *pPage);
int pgFileWrite(SPage *pPage);
int pgFileAllocatePage(SPgFile *pPgFile, pgno_t *pPgno);
#ifdef __cplusplus
}
#endif
#endif /*_TD_PAGE_FILE_H_*/

43
source/libs/tdb/src/inc/tdbUtil.h
View File

@ -35,7 +35,48 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique);
// #define TDB_W_OK 0x4
// int tdbCheckFileAccess(const char *pathname, int mode);
int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize);
int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize);
int tdbPRead(int fd, void *pData, int count, i64 offset);
static inline int tdbPutVarInt(u8 *p, int v) {
int n = 0;
for (;;) {
if (v <= 0x7f) {
p[n++] = v;
break;
}
p[n++] = (v & 0x7f) | 0x80;
v >>= 7;
}
ASSERT(n < 6);
return n;
}
static inline int tdbGetVarInt(const u8 *p, int *v) {
int n = 0;
int tv = 0;
for (;;) {
if (p[n] <= 0x7f) {
tv = (tv << 7) | p[n];
n++;
break;
}
tv = (tv << 7) | (p[n] & 0x7f);
n++;
}
ASSERT(n < 6);
*v = tv;
return n;
}
#ifdef __cplusplus
}

272
source/libs/tdb/src/page/tdbPage.c Normal file
View File

@ -0,0 +1,272 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
extern SPageMethods pageMethods;
extern SPageMethods pageLargeMethods;
typedef struct __attribute__((__packed__)) {
u16 szCell;
u16 nxOffset;
} SFreeCell;
static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell);
static int tdbPageDefragment(SPage *pPage);
int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) {
SPage *pPage;
u8 *ptr;
int size;
ASSERT(TDB_IS_PGSIZE_VLD(pageSize));
*ppPage = NULL;
size = pageSize + sizeof(*pPage);
ptr = (u8 *)((*xMalloc)(arg, size));
if (pPage == NULL) {
return -1;
}
memset(ptr, 0, size);
pPage = (SPage *)(ptr + pageSize);
pPage->pData = ptr;
pPage->pageSize = pageSize;
if (pageSize < 65536) {
pPage->pPageMethods = &pageMethods;
} else {
pPage->pPageMethods = &pageLargeMethods;
}
TDB_INIT_PAGE_LOCK(pPage);
/* TODO */
*ppPage = pPage;
return 0;
}
int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) {
u8 *ptr;
ptr = pPage->pData;
(*xFree)(arg, ptr);
return 0;
}
int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) {
int ret;
SCell *pTarget;
u8 *pTmp;
int j;
if (pPage->nOverflow || szCell + TDB_PAGE_OFFSET_SIZE(pPage) > pPage->nFree) {
// TODO: need to figure out if pCell may be used by outside of this function
j = pPage->nOverflow++;
pPage->apOvfl[j] = pCell;
pPage->aiOvfl[j] = idx;
} else {
ret = tdbPageAllocate(pPage, szCell, &pTarget);
if (ret < 0) {
return -1;
}
memcpy(pTarget, pCell, szCell);
pTmp = pPage->pCellIdx + idx * TDB_PAGE_OFFSET_SIZE(pPage);
memmove(pTmp + TDB_PAGE_OFFSET_SIZE(pPage), pTmp, pPage->pFreeStart - pTmp - TDB_PAGE_OFFSET_SIZE(pPage));
TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData);
TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1);
}
return 0;
}
int tdbPageDropCell(SPage *pPage, int idx) {
// TODO
return 0;
}
static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) {
SCell *pCell;
SFreeCell *pFreeCell;
u8 *pOffset;
int ret;
ASSERT(pPage->nFree > size + TDB_PAGE_OFFSET_SIZE(pPage));
pCell = NULL;
*ppCell = NULL;
// 1. Try to allocate from the free space area
if (pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)) {
pPage->pFreeEnd -= size;
pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage);
pCell = pPage->pFreeEnd;
}
// 2. Try to allocate from the page free list
if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) &&
TDB_PAGE_FCELL(pPage)) {
#if 0
int szCell;
int nxOffset;
pCell = pPage->pData + TDB_PAGE_FCELL(pPage);
pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell
: (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell);
szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell);
nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell);
for (;;) {
// Find a cell
if (szCell >= size) {
if (szCell - size >= pPage->szFreeCell) {
SCell *pTmpCell = pCell + size;
TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size);
TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset);
// TODO: *pOffset = pTmpCell - pPage->pData;
} else {
TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size);
// TODO: *pOffset = nxOffset;
}
break;
}
// Not find a cell yet
if (nxOffset > 0) {
pCell = pPage->pData + nxOffset;
pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell);
szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell);
nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell);
continue;
} else {
pCell = NULL;
break;
}
}
if (pCell) {
pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset;
}
#endif
}
// 3. Try to dfragment and allocate again
if (pCell == NULL) {
ret = tdbPageDefragment(pPage);
if (ret < 0) {
return -1;
}
ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage));
ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart);
// Allocate from the free space area again
pPage->pFreeEnd -= size;
pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage);
pCell = pPage->pFreeEnd;
}
ASSERT(pCell != NULL);
pPage->nFree = pPage->nFree - size - TDB_PAGE_OFFSET_SIZE(pPage);
*ppCell = pCell;
return 0;
}
static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) {
// TODO
return 0;
}
static int tdbPageDefragment(SPage *pPage) {
// TODO
ASSERT(0);
return 0;
}
/* ---------------------------------------------------------------------------------------------------------- */
typedef struct __attribute__((__packed__)) {
u16 flags;
u16 cellNum;
u16 cellBody;
u16 cellFree;
u16 nFree;
} SPageHdr;
// flags
static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].flags; }
static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdr *)(pPage->pPageHdr))[0].flags = flags; }
// cellNum
static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; }
static inline void setPageCellNum(SPage *pPage, int cellNum) {
ASSERT(cellNum < 65536);
((SPageHdr *)(pPage->pPageHdr))[0].cellNum = (u16)cellNum;
}
// cellBody
static inline int getPageCellBody(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellBody; }
static inline void setPageCellBody(SPage *pPage, int cellBody) {
ASSERT(cellBody < 65536);
((SPageHdr *)(pPage->pPageHdr))[0].cellBody = (u16)cellBody;
}
// cellFree
static inline int getPageCellFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellFree; }
static inline void setPageCellFree(SPage *pPage, int cellFree) {
ASSERT(cellFree < 65536);
((SPageHdr *)(pPage->pPageHdr))[0].cellFree = (u16)cellFree;
}
// nFree
static inline int getPageNFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].nFree; }
static inline void setPageNFree(SPage *pPage, int nFree) {
ASSERT(nFree < 65536);
((SPageHdr *)(pPage->pPageHdr))[0].nFree = (u16)nFree;
}
// cell offset
static inline int getPageCellOffset(SPage *pPage, int idx) {
ASSERT(idx >= 0 && idx < getPageCellNum(pPage));
return ((u16 *)pPage->pCellIdx)[idx];
}
static inline void setPageCellOffset(SPage *pPage, int idx, int offset) {
ASSERT(offset < 65536);
((u16 *)pPage->pCellIdx)[idx] = (u16)offset;
}
SPageMethods pageMethods = {
2, // szOffset
sizeof(SPageHdr), // szPageHdr
sizeof(SFreeCell), // szFreeCell
getPageFlags, // getPageFlags
setPageFlags, // setFlagsp
getPageCellNum, // getCellNum
setPageCellNum, // setCellNum
getPageCellBody, // getCellBody
setPageCellBody, // setCellBody
getPageCellFree, // getCellFree
setPageCellFree, // setCellFree
getPageNFree, // getFreeBytes
setPageNFree, // setFreeBytes
getPageCellOffset, // getCellOffset
setPageCellOffset // setCellOffset
};

85
source/libs/tdb/src/page/tdbPageL.c Normal file
View File

@ -0,0 +1,85 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
typedef struct __attribute__((__packed__)) {
u16 flags;
u8 cellNum[3];
u8 cellBody[3];
u8 cellFree[3];
u8 nFree[3];
} SPageHdrL;
typedef struct __attribute__((__packed__)) {
u8 szCell[3];
u8 nxOffset[3];
} SFreeCellL;
// flags
static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdrL *)(pPage->pPageHdr))[0].flags; }
static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdrL *)(pPage->pPageHdr))[0].flags = flags; }
// cellNum
static inline int getPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); }
static inline void setPageCellNum(SPage *pPage, int cellNum) {
TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum, cellNum);
}
// cellBody
static inline int getPageCellBody(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody); }
static inline void setPageCellBody(SPage *pPage, int cellBody) {
TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody, cellBody);
}
// cellFree
static inline int getPageCellFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree); }
static inline void setPageCellFree(SPage *pPage, int cellFree) {
TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree, cellFree);
}
// nFree
static inline int getPageNFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree); }
static inline void setPageNFree(SPage *pPage, int nFree) {
TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree, nFree);
}
// cell offset
static inline int getPageCellOffset(SPage *pPage, int idx) {
ASSERT(idx >= 0 && idx < getPageCellNum(pPage));
return TDB_GET_U24(pPage->pCellIdx + 3 * idx);
}
static inline void setPageCellOffset(SPage *pPage, int idx, int offset) {
TDB_PUT_U24(pPage->pCellIdx + 3 * idx, offset);
}
SPageMethods pageLargeMethods = {
3, // szOffset
sizeof(SPageHdrL), // szPageHdr
sizeof(SFreeCellL), // szFreeCell
getPageFlags, // getPageFlags
setPageFlags, // setFlagsp
getPageCellNum, // getCellNum
setPageCellNum, // setCellNum
getPageCellBody, // getCellBody
setPageCellBody, // setCellBody
getPageCellFree, // getCellFree
setPageCellFree, // setCellFree
getPageNFree, // getFreeBytes
setPageNFree, // setFreeBytes
getPageCellOffset, // getCellOffset
setPageCellOffset // setCellOffset
};

89
source/libs/tdb/test/tdbTest.cpp
View File

@ -1,68 +1,39 @@
#include <gtest/gtest.h>
#include "tdb.h"
#include "tdbInt.h"
TEST(tdb_test, simple_test) {
TENV * pEnv;
TDB * pDb1, *pDb2, *pDb3;
pgsz_t pgSize = 1024;
cachesz_t cacheSize = 10240;
int ret;
STEnv *pEnv;
STDb *pDb;
// ENV
GTEST_ASSERT_EQ(tdbEnvCreate(&pEnv, "./testtdb"), 0);
// Open Env
ret = tdbEnvOpen("tdb", 1024, 20, &pEnv);
GTEST_ASSERT_EQ(ret, 0);
GTEST_ASSERT_EQ(tdbEnvSetCache(pEnv, pgSize, cacheSize), 0);
// Create a database
ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, NULL, pEnv, &pDb);
GTEST_ASSERT_EQ(ret, 0);
GTEST_ASSERT_EQ(tdbEnvGetCacheSize(pEnv), cacheSize);
{ // Insert some data
char key[64];
char val[64];
GTEST_ASSERT_EQ(tdbEnvGetPageSize(pEnv), pgSize);
GTEST_ASSERT_EQ(tdbEnvOpen(pEnv), 0);
#if 1
// DB
GTEST_ASSERT_EQ(tdbCreate(&pDb1), 0);
// GTEST_ASSERT_EQ(tdbSetKeyLen(pDb1, 8), 0);
// GTEST_ASSERT_EQ(tdbGetKeyLen(pDb1), 8);
// GTEST_ASSERT_EQ(tdbSetValLen(pDb1, 3), 0);
// GTEST_ASSERT_EQ(tdbGetValLen(pDb1), 3);
// GTEST_ASSERT_EQ(tdbSetDup(pDb1, 1), 0);
// GTEST_ASSERT_EQ(tdbGetDup(pDb1), 1);
// GTEST_ASSERT_EQ(tdbSetCmprFunc(pDb1, NULL), 0);
tdbEnvBeginTxn(pEnv);
GTEST_ASSERT_EQ(tdbOpen(pDb1, "db.db", "db1", pEnv), 0);
// char *key = "key1";
// char *val = "value1";
// tdbInsert(pDb1, (void *)key, strlen(key), (void *)val, strlen(val));
tdbEnvCommit(pEnv);
#if 0
// Insert
// Query
// Delete
// Query
#endif
// GTEST_ASSERT_EQ(tdbOpen(&pDb2, "db.db", "db2", pEnv), 0);
// GTEST_ASSERT_EQ(tdbOpen(&pDb3, "index.db", NULL, pEnv), 0);
// tdbClose(pDb3);
// tdbClose(pDb2);
tdbClose(pDb1);
#endif
tdbEnvClose(pEnv);
for (int i = 1; i <= 1000; i++) {
sprintf(key, "key%d", i);
sprintf(val, "value%d", i);
ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val));
GTEST_ASSERT_EQ(ret, 0);
}
}
ret = tdbDbDrop(pDb);
GTEST_ASSERT_EQ(ret, 0);
// Close a database
tdbDbClose(pDb);
// Close Env
ret = tdbEnvClose(pEnv);
GTEST_ASSERT_EQ(ret, 0);
}

3
source/libs/tfs/src/tfs.c
View File

@ -204,7 +204,8 @@ void tfsDirname(const STfsFile *pFile, char *dest) {
void tfsAbsoluteName(STfs *pTfs, SDiskID diskId, const char *rname, char *aname) {
STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId);
snprintf(aname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname);
snprintf(aname, TSDB_FILENAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname);
}
int32_t tfsRemoveFile(const STfsFile *pFile) { return taosRemoveFile(pFile->aname); }

40
source/libs/transport/inc/transComm.h
View File

@ -14,6 +14,10 @@
*/
#ifdef USE_UV
#ifdef __cplusplus
extern "C" {
#endif
#include <uv.h>
#include "lz4.h"
#include "os.h"
@ -121,6 +125,8 @@ typedef struct {
} SRpcReqContext;
typedef SRpcMsg STransMsg;
typedef SRpcCtx STransCtx;
typedef SRpcCtxVal STransCtxVal;
typedef SRpcInfo STrans;
typedef SRpcConnInfo STransHandleInfo;
@ -128,15 +134,10 @@ typedef struct {
SEpSet epSet; // ip list provided by app
void* ahandle; // handle provided by app
tmsg_t msgType; // message type
uint8_t* pCont; // content provided by app
int32_t contLen; // content length
// int32_t code; // error code
// int16_t numOfTry; // number of try for different servers
// int8_t oldInUse; // server EP inUse passed by app
// int8_t redirect; // flag to indicate redirect
int8_t connType; // connection type cli/srv
int64_t rid; // refId returned by taosAddRef
STransCtx appCtx; //
STransMsg* pRsp; // for synchronous API
tsem_t* pSem; // for synchronous API
@ -150,11 +151,12 @@ typedef struct {
typedef struct {
char version : 4; // RPC version
char comp : 4; // compression algorithm, 0:no compression 1:lz4
char resflag : 2; // reserved bits
char spi : 1; // security parameter index
char comp : 2; // compression algorithm, 0:no compression 1:lz4
char noResp : 2; // noResp bits, 0: resp, 1: resp
char persist : 2; // persist handle,0: no persit, 1: persist handle
char release : 2;
char secured : 2;
char encrypt : 3; // encrypt algorithm, 0: no encryption
char spi : 2;
uint32_t code; // del later
uint32_t msgType;
@ -179,6 +181,9 @@ typedef struct {
#pragma pack(pop)
typedef enum { Normal, Quit, Release, Register } STransMsgType;
typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken } ConnStatus;
#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member)))
#define RPC_RESERVE_SIZE (sizeof(STranConnCtx))
@ -255,9 +260,10 @@ void transUnrefCliHandle(void* handle);
void transReleaseCliHandle(void* handle);
void transReleaseSrvHandle(void* handle);
void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg);
void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransCtx* pCtx);
void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransMsg* pRsp);
void transSendResponse(const STransMsg* pMsg);
void transSendResponse(const STransMsg* msg);
void transRegisterMsg(const STransMsg* msg);
int transGetConnInfo(void* thandle, STransHandleInfo* pInfo);
void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);
@ -266,4 +272,14 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads,
void transCloseClient(void* arg);
void transCloseServer(void* arg);
void transCtxInit(STransCtx* ctx);
void transCtxDestroy(STransCtx* ctx);
void transCtxClear(STransCtx* ctx);
void transCtxMerge(STransCtx* dst, STransCtx* src);
void* transCtxDumpVal(STransCtx* ctx, int32_t key);
#ifdef __cplusplus
}
#endif
#endif

3
source/libs/transport/inc/transportInt.h
View File

@ -63,9 +63,6 @@ typedef struct {
void (*cfp)(void* parent, SRpcMsg*, SEpSet*);
int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey);
bool (*pfp)(void* parent, tmsg_t msgType);
void* (*mfp)(void* parent, tmsg_t msgType);
bool (*efp)(void* parent, tmsg_t msgType);
int32_t refCount;
void* parent;

11
source/libs/transport/src/trans.c
View File

@ -39,9 +39,6 @@ void* rpcOpen(const SRpcInit* pInit) {
// register callback handle
pRpc->cfp = pInit->cfp;
pRpc->afp = pInit->afp;
pRpc->pfp = pInit->pfp;
pRpc->mfp = pInit->mfp;
pRpc->efp = pInit->efp;
if (pInit->connType == TAOS_CONN_SERVER) {
pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
@ -121,7 +118,12 @@ void rpcCancelRequest(int64_t rid) { return; }
void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) {
char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn);
uint32_t port = pEpSet->eps[pEpSet->inUse].port;
transSendRequest(shandle, ip, port, pMsg);
transSendRequest(shandle, ip, port, pMsg, NULL);
}
void rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) {
char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn);
uint32_t port = pEpSet->eps[pEpSet->inUse].port;
transSendRequest(shandle, ip, port, pMsg, pCtx);
}
void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) {
char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn);
@ -142,6 +144,7 @@ void rpcUnrefHandle(void* handle, int8_t type) {
(*taosUnRefHandle[type])(handle);
}
void rpcRegisterBrokenLinkArg(SRpcMsg* msg) { rpcSendResponse(msg); }
void rpcReleaseHandle(void* handle, int8_t type) {
assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT);
(*transReleaseHandle[type])(handle);

129
source/libs/transport/src/transCli.c
View File

@ -17,11 +17,6 @@
#include "transComm.h"
// Normal(default): send/recv msg
// Quit: quit rpc inst
// Release: release handle to rpc inst
typedef enum { Normal, Quit, Release } SCliMsgType;
typedef struct SCliConn {
T_REF_DECLARE()
uv_connect_t connReq;
@ -35,8 +30,10 @@ typedef struct SCliConn {
uint64_t expireTime;
int hThrdIdx;
bool broken; // link broken or not
STransCtx ctx;
int persist; //
ConnStatus status; //
int release; // 1: release
// spi configure
char spi;
char secured;
@ -55,7 +52,7 @@ typedef struct SCliMsg {
STransMsg msg;
queue q;
uint64_t st;
SCliMsgType type;
STransMsgType type;
} SCliMsg;
typedef struct SCliThrdObj {
@ -113,10 +110,12 @@ static void cliSend(SCliConn* pConn);
static void cliHandleResp(SCliConn* conn);
// handle except about conn
static void cliHandleExcept(SCliConn* conn);
// handle req from app
static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd);
static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd);
static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd);
static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrdObj* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease};
static void cliSendQuit(SCliThrdObj* thrd);
static void destroyUserdata(STransMsg* userdata);
@ -133,11 +132,26 @@ static void destroyThrdObj(SCliThrdObj* pThrd);
#define CONN_PERSIST_TIME(para) (para * 1000 * 10)
#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL)
#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label)
#define CONN_SHOULD_RELEASE(conn, head) \
do { \
if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \
conn->status = ConnRelease; \
transClearBuffer(&conn->readBuf); \
transFreeMsg(transContFromHead((char*)head)); \
tDebug("cli conn %p receive release request", conn); \
if (T_REF_VAL_GET(conn) == 1) { \
SCliThrdObj* thrd = conn->hostThrd; \
addConnToPool(thrd->pool, conn); \
} \
return; \
} \
} while (0)
#define CONN_HANDLE_THREAD_QUIT(conn, thrd) \
do { \
if (thrd->quit) { \
cliHandleExcept(conn); \
goto _RETURE; \
return; \
} \
} while (0)
@ -145,20 +159,22 @@ static void destroyThrdObj(SCliThrdObj* pThrd);
do { \
if (conn->broken) { \
cliHandleExcept(conn); \
goto _RETURE; \
return; \
} \
} while (0);
} while (0)
#define CONN_SET_PERSIST_BY_APP(conn) \
do { \
if (conn->persist == false) { \
conn->persist = true; \
if (conn->status == ConnNormal) { \
conn->status = ConnAcquire; \
transRefCliHandle(conn); \
} \
} while (0)
#define CONN_NO_PERSIST_BY_APP(conn) ((conn)->persist == false)
#define CONN_NO_PERSIST_BY_APP(conn) ((conn)->status == ConnNormal && T_REF_VAL_GET(conn) == 1)
#define REQUEST_NO_RESP(msg) ((msg)->noResp == 1)
#define REQUEST_PERSIS_HANDLE(msg) ((msg)->persistHandle == 1)
#define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release)
static void* cliWorkThread(void* arg);
@ -177,7 +193,6 @@ void cliHandleResp(SCliConn* conn) {
STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf);
pHead->code = htonl(pHead->code);
pHead->msgLen = htonl(pHead->msgLen);
STransMsg transMsg = {0};
transMsg.contLen = transContLenFromMsg(pHead->msgLen);
transMsg.pCont = transContFromHead((char*)pHead);
@ -185,6 +200,8 @@ void cliHandleResp(SCliConn* conn) {
transMsg.msgType = pHead->msgType;
transMsg.ahandle = NULL;
CONN_SHOULD_RELEASE(conn, pHead);
SCliMsg* pMsg = NULL;
if (taosArrayGetSize(conn->cliMsgs) > 0) {
pMsg = taosArrayGetP(conn->cliMsgs, 0);
@ -193,16 +210,15 @@ void cliHandleResp(SCliConn* conn) {
STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL;
if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) {
transMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, transMsg.msgType) : NULL;
transMsg.ahandle = transCtxDumpVal(&conn->ctx, transMsg.msgType);
} else {
transMsg.ahandle = pCtx ? pCtx->ahandle : NULL;
}
// buf's mem alread translated to transMsg.pCont
transClearBuffer(&conn->readBuf);
if (pTransInst->pfp != NULL && (*pTransInst->pfp)(pTransInst->parent, transMsg.msgType)) {
if (!CONN_NO_PERSIST_BY_APP(conn)) {
transMsg.handle = conn;
CONN_SET_PERSIST_BY_APP(conn);
tDebug("%s cli conn %p ref by app", CONN_GET_INST_LABEL(conn), conn);
}
@ -241,6 +257,8 @@ void cliHandleResp(SCliConn* conn) {
if (!uv_is_active((uv_handle_t*)&pThrd->timer) && pTransInst->idleTime > 0) {
// uv_timer_start((uv_timer_t*)&pThrd->timer, cliTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0);
}
_RETURN:
return;
}
void cliHandleExcept(SCliConn* pConn) {
@ -268,7 +286,7 @@ void cliHandleExcept(SCliConn* pConn) {
transMsg.ahandle = NULL;
if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) {
transMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, transMsg.msgType) : NULL;
transMsg.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType);
} else {
transMsg.ahandle = pCtx ? pCtx->ahandle : NULL;
}
@ -359,6 +377,7 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) {
static void addConnToPool(void* pool, SCliConn* conn) {
char key[128] = {0};
transCtxDestroy(&conn->ctx);
tstrncpy(key, conn->ip, strlen(conn->ip));
tstrncpy(key + strlen(key), (char*)(&conn->port), sizeof(conn->port));
tTrace("cli conn %p added to conn pool, read buf cap: %d", conn, conn->readBuf.cap);
@ -367,15 +386,16 @@ static void addConnToPool(void* pool, SCliConn* conn) {
conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime);
SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key));
conn->status = ConnNormal;
// list already create before
assert(plist != NULL);
taosArrayClear(conn->cliMsgs);
QUEUE_PUSH(&plist->conn, &conn->conn);
assert(!QUEUE_IS_EMPTY(&plist->conn));
}
static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
SCliConn* conn = handle->data;
SConnBuffer* pBuf = &conn->readBuf;
// avoid conn
QUEUE_REMOVE(&conn->conn);
transAllocBuffer(pBuf, buf);
}
static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
@ -401,6 +421,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
// ref http://docs.libuv.org/en/v1.x/stream.html?highlight=uv_read_start#c.uv_read_cb
// nread might be 0, which does not indicate an error or EOF. This is equivalent to EAGAIN or EWOULDBLOCK under
// read(2).
tTrace("%s cli conn %p read empty", CONN_GET_INST_LABEL(conn), conn);
return;
}
if (nread < 0) {
@ -420,16 +441,16 @@ static SCliConn* cliCreateConn(SCliThrdObj* pThrd) {
conn->writeReq.data = conn;
conn->connReq.data = conn;
conn->cliMsgs = taosArrayInit(2, sizeof(void*));
QUEUE_INIT(&conn->conn);
conn->hostThrd = pThrd;
conn->persist = false;
conn->broken = false;
conn->status = ConnNormal;
conn->broken = 0;
transRefCliHandle(conn);
return conn;
}
static void cliDestroyConn(SCliConn* conn, bool clear) {
tTrace("%s cli conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn);
QUEUE_REMOVE(&conn->conn);
if (clear) {
uv_close((uv_handle_t*)conn->stream, cliDestroy);
@ -439,6 +460,7 @@ static void cliDestroy(uv_handle_t* handle) {
SCliConn* conn = handle->data;
free(conn->ip);
free(conn->stream);
transCtxDestroy(&conn->ctx);
taosArrayDestroy(conn->cliMsgs);
tTrace("%s cli conn %p destroy successfully", CONN_GET_INST_LABEL(conn), conn);
free(conn);
@ -490,7 +512,10 @@ void cliSend(SCliConn* pConn) {
STrans* pTransInst = pThrd->pTransInst;
STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg);
if (pMsg->pCont == 0) {
pMsg->pCont = (void*)rpcMallocCont(0);
pMsg->contLen = 0;
}
STransMsgHead* pHead = transHeadFromCont(pMsg->pCont);
int msgLen = transMsgLenFromCont(pMsg->contLen);
@ -513,20 +538,25 @@ void cliSend(SCliConn* pConn) {
msgLen += sizeof(STransUserMsg);
}
pHead->resflag = REQUEST_NO_RESP(pMsg) ? 1 : 0;
pHead->noResp = REQUEST_NO_RESP(pMsg) ? 1 : 0;
pHead->persist = REQUEST_PERSIS_HANDLE(pMsg) ? 1 : 0;
pHead->msgType = pMsg->msgType;
pHead->msgLen = (int32_t)htonl((uint32_t)msgLen);
pHead->release = REQUEST_RELEASE_HANDLE(pCliMsg) ? 1 : 0;
uv_buf_t wb = uv_buf_init((char*)pHead, msgLen);
tDebug("%s cli conn %p %s is send to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn,
TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port),
taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port));
if (pHead->persist == 1) {
CONN_SET_PERSIST_BY_APP(pConn);
}
pConn->writeReq.data = pConn;
uv_write(&pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb);
return;
_RETURE:
return;
}
@ -562,22 +592,14 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) {
}
static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) {
SCliConn* conn = pMsg->msg.handle;
tDebug("%s cli conn %p release to inst", CONN_GET_INST_LABEL(conn), conn);
tDebug("%s cli conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn);
while (taosArrayGetSize(conn->cliMsgs) > 0) {
SCliMsg* pMsg = taosArrayGetP(conn->cliMsgs, 0);
destroyCmsg(pMsg);
taosArrayRemove(conn->cliMsgs, 0);
}
transDestroyBuffer(&conn->readBuf);
if (conn->persist && T_REF_VAL_GET(conn) >= 2) {
conn->persist = false;
transUnrefCliHandle(conn);
addConnToPool(pThrd->pool, conn);
} else {
transUnrefCliHandle(conn);
taosArrayPush(conn->cliMsgs, &pMsg);
if (taosArrayGetSize(conn->cliMsgs) >= 2) {
return; // send one by one
}
cliSend(conn);
}
SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) {
@ -592,6 +614,8 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) {
conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port);
if (conn != NULL) {
tTrace("%s cli conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn);
} else {
tTrace("not found conn in conn pool %p", pThrd->pool);
}
}
return conn;
@ -609,10 +633,12 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
if (conn != NULL) {
conn->hThrdIdx = pCtx->hThrdIdx;
transCtxMerge(&conn->ctx, &pCtx->appCtx);
if (taosArrayGetSize(conn->cliMsgs) > 0) {
taosArrayPush(conn->cliMsgs, &pMsg);
return;
}
taosArrayPush(conn->cliMsgs, &pMsg);
transDestroyBuffer(&conn->readBuf);
cliSend(conn);
@ -652,14 +678,10 @@ static void cliAsyncCb(uv_async_t* handle) {
QUEUE_REMOVE(h);
SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q);
if (pMsg->type == Normal) {
cliHandleReq(pMsg, pThrd);
} else if (pMsg->type == Quit) {
cliHandleQuit(pMsg, pThrd);
} else if (pMsg->type == Release) {
cliHandleRelease(pMsg, pThrd);
if (pMsg == NULL) {
continue;
}
(*cliAsyncHandle[pMsg->type])(pMsg, pThrd);
count++;
}
if (count >= 2) {
@ -802,13 +824,13 @@ void transReleaseCliHandle(void* handle) {
STransMsg tmsg = {.handle = handle};
SCliMsg* cmsg = calloc(1, sizeof(SCliMsg));
cmsg->type = Release;
cmsg->msg = tmsg;
cmsg->type = Release;
transSendAsync(thrd->asyncPool, &cmsg->q);
}
void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg) {
void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransCtx* ctx) {
STrans* pTransInst = (STrans*)shandle;
int index = CONN_HOST_THREAD_INDEX((SCliConn*)pMsg->handle);
if (index == -1) {
@ -818,7 +840,7 @@ void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* p
if (transCompressMsg(pMsg->pCont, pMsg->contLen, &flen)) {
// imp later
}
tDebug("send request at thread:%d %p", index, pMsg);
tDebug("send request at thread:%d %p, dst: %s:%d", index, pMsg, ip, port);
STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx));
pCtx->ahandle = pMsg->ahandle;
pCtx->msgType = pMsg->msgType;
@ -826,6 +848,10 @@ void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* p
pCtx->port = port;
pCtx->hThrdIdx = index;
if (ctx != NULL) {
pCtx->appCtx = *ctx;
}
assert(pTransInst->connType == TAOS_CONN_CLIENT);
// atomic or not
@ -833,10 +859,12 @@ void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* p
cliMsg->ctx = pCtx;
cliMsg->msg = *pMsg;
cliMsg->st = taosGetTimestampUs();
cliMsg->type = Normal;
SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index];
transSendAsync(thrd->asyncPool, &(cliMsg->q));
}
void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq, STransMsg* pRsp) {
STrans* pTransInst = (STrans*)shandle;
int index = CONN_HOST_THREAD_INDEX(pReq->handle);
@ -858,6 +886,7 @@ void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq
cliMsg->ctx = pCtx;
cliMsg->msg = *pReq;
cliMsg->st = taosGetTimestampUs();
cliMsg->type = Normal;
SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index];
transSendAsync(thrd->asyncPool, &(cliMsg->q));

52
source/libs/transport/src/transComm.c
View File

@ -224,4 +224,56 @@ int transSendAsync(SAsyncPool* pool, queue* q) {
return uv_async_send(async);
}
void transCtxInit(STransCtx* ctx) {
// init transCtx
ctx->args = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UINT), true, HASH_NO_LOCK);
}
void transCtxDestroy(STransCtx* ctx) {
if (ctx->args == NULL) {
return;
}
STransCtxVal* iter = taosHashIterate(ctx->args, NULL);
while (iter) {
iter->free(iter->val);
iter = taosHashIterate(ctx->args, iter);
}
taosHashCleanup(ctx->args);
}
void transCtxMerge(STransCtx* dst, STransCtx* src) {
if (dst->args == NULL) {
dst->args = src->args;
src->args = NULL;
return;
}
void* key = NULL;
size_t klen = 0;
void* iter = taosHashIterate(src->args, NULL);
while (iter) {
STransCtxVal* sVal = (STransCtxVal*)iter;
key = taosHashGetKey(sVal, &klen);
STransCtxVal* dVal = taosHashGet(dst->args, key, klen);
if (dVal) {
dVal->free(dVal->val);
}
taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal));
iter = taosHashIterate(src->args, iter);
}
taosHashCleanup(src->args);
}
void* transCtxDumpVal(STransCtx* ctx, int32_t key) {
if (ctx->args == NULL) {
return NULL;
}
STransCtxVal* cVal = taosHashGet(ctx->args, (const void*)&key, sizeof(key));
if (cVal == NULL) {
return NULL;
}
char* ret = calloc(1, cVal->len);
memcpy(ret, (char*)cVal->val, cVal->len);
return (void*)ret;
}
#endif

215
source/libs/transport/src/transSrv.c
View File

@ -17,6 +17,12 @@
#include "transComm.h"
typedef struct {
int notifyCount; //
int init; // init or not
STransMsg msg;
} SSrvRegArg;
typedef struct SSrvConn {
T_REF_DECLARE()
uv_tcp_t* pTcp;
@ -33,8 +39,10 @@ typedef struct SSrvConn {
void* hostThrd;
SArray* srvMsgs;
SSrvRegArg regArg;
bool broken; // conn broken;
ConnStatus status;
struct sockaddr_in addr;
struct sockaddr_in locaddr;
@ -50,6 +58,7 @@ typedef struct SSrvMsg {
SSrvConn* pConn;
STransMsg msg;
queue q;
STransMsgType type;
} SSrvMsg;
typedef struct SWorkThrdObj {
@ -58,7 +67,6 @@ typedef struct SWorkThrdObj {
uv_os_fd_t fd;
uv_loop_t* loop;
SAsyncPool* asyncPool;
queue msg;
pthread_mutex_t msgMtx;
@ -85,6 +93,27 @@ typedef struct SServerObj {
static const char* notify = "a";
#define CONN_SHOULD_RELEASE(conn, head) \
do { \
if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \
conn->status = ConnRelease; \
transClearBuffer(&conn->readBuf); \
transFreeMsg(transContFromHead((char*)head)); \
tTrace("server conn %p received release request", conn); \
\
STransMsg tmsg = {.handle = (void*)conn, .code = 0}; \
SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); \
srvMsg->msg = tmsg; \
srvMsg->type = Release; \
srvMsg->pConn = conn; \
taosArrayPush(conn->srvMsgs, &srvMsg); \
if (taosArrayGetSize(conn->srvMsgs) > 1) { \
return; \
} \
uvStartSendRespInternal(srvMsg); \
return; \
} \
} while (0)
// refactor later
static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen);
@ -113,6 +142,13 @@ static void destroySmsg(SSrvMsg* smsg);
static SSrvConn* createConn(void* hThrd);
static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/);
static void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd);
static void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd);
static void uvHandleSendResp(SSrvMsg* msg, SWorkThrdObj* thrd);
static void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd);
static void (*transAsyncHandle[])(SSrvMsg* msg, SWorkThrdObj* thrd) = {uvHandleSendResp, uvHandleQuit, uvHandleRelease,
uvHandleRegister};
static void uvDestroyConn(uv_handle_t* handle);
// server and worker thread
@ -217,9 +253,10 @@ static void uvHandleReq(SSrvConn* pConn) {
if (pHead->secured == 1) {
pHead->msgLen -= sizeof(STransUserMsg);
}
//
}
CONN_SHOULD_RELEASE(pConn, pHead);
STransMsg transMsg;
transMsg.contLen = transContLenFromMsg(pHead->msgLen);
transMsg.pCont = pHead->content;
@ -230,24 +267,32 @@ static void uvHandleReq(SSrvConn* pConn) {
transClearBuffer(&pConn->readBuf);
pConn->inType = pHead->msgType;
if (pHead->resflag == 0) {
if (pConn->status == ConnNormal) {
if (pHead->persist == 1) {
pConn->status = ConnAcquire;
transRefSrvHandle(pConn);
}
}
if (pConn->status == ConnNormal && pHead->noResp == 0) {
transRefSrvHandle(pConn);
transMsg.handle = pConn;
tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(transMsg.msgType),
taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr),
ntohs(pConn->locaddr.sin_port), transMsg.contLen);
} else {
tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, no resp ", pConn,
tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, resp:%d ", pConn,
TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port),
taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen);
taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen, pHead->noResp);
// no ref here
}
if (pHead->noResp == 0) {
transMsg.handle = pConn;
}
STrans* pTransInst = (STrans*)p->shandle;
(*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);
// uv_timer_start(&pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0);
// auth
// validate msg type
}
void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
@ -272,11 +317,13 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
tError("server conn %p read error: %s", conn, uv_err_name(nread));
if (nread < 0) {
conn->broken = true;
uvNotifyLinkBrokenToApp(conn);
// STrans* pTransInst = conn->pTransInst;
// if (pTransInst->efp != NULL && (pTransInst->efp)(NULL, conn->inType)) {
//}
if (conn->status == ConnAcquire) {
if (conn->regArg.init) {
STrans* pTransInst = conn->pTransInst;
(*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL);
memset(&conn->regArg, 0, sizeof(conn->regArg));
}
}
transUnrefSrvHandle(conn);
}
}
@ -301,8 +348,22 @@ void uvOnSendCb(uv_write_t* req, int status) {
SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0);
tTrace("server conn %p sending msg size: %d", conn, (int)taosArrayGetSize(conn->srvMsgs));
taosArrayRemove(conn->srvMsgs, 0);
if (msg->type == Release && conn->status != ConnNormal) {
conn->status = ConnNormal;
transUnrefSrvHandle(conn);
} else if (msg->type == Register && conn->status == ConnAcquire) {
conn->regArg.notifyCount = 0;
conn->regArg.init = 1;
conn->regArg.msg = msg->msg;
if (conn->broken) {
STrans* pTransInst = conn->pTransInst;
(pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL);
memset(&conn->regArg, 0, sizeof(conn->regArg));
}
free(msg);
return;
}
destroySmsg(msg);
// send second data, just use for push
if (taosArrayGetSize(conn->srvMsgs) > 0) {
tTrace("resent server conn %p sending msg size: %d", conn, (int)taosArrayGetSize(conn->srvMsgs));
@ -312,7 +373,7 @@ void uvOnSendCb(uv_write_t* req, int status) {
}
} else {
tError("server conn %p failed to write data, %s", conn, uv_err_name(status));
conn->broken = false;
conn->broken = true;
transUnrefSrvHandle(conn);
}
}
@ -339,6 +400,7 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) {
pHead->secured = pMsg->code == 0 ? 1 : 0; //
pHead->msgType = smsg->pConn->inType + 1;
pHead->release = smsg->type == Release ? 1 : 0;
pHead->code = htonl(pMsg->code);
// add more info
char* msg = (char*)pHead;
@ -368,13 +430,16 @@ static void uvStartSendResp(SSrvMsg* smsg) {
SSrvConn* pConn = smsg->pConn;
if (pConn->broken == true) {
// persist by
transUnrefSrvHandle(pConn);
return;
}
if (pConn->status == ConnNormal) {
transUnrefSrvHandle(pConn);
}
if (taosArrayGetSize(pConn->srvMsgs) > 0) {
tDebug("server conn %p push data to client %s:%d, local info: %s:%d", pConn, taosInetNtoa(pConn->addr.sin_addr),
tDebug("server conn %p send data to client %s:%d, local info: %s:%d", pConn, taosInetNtoa(pConn->addr.sin_addr),
ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port));
taosArrayPush(pConn->srvMsgs, &smsg);
return;
@ -384,16 +449,6 @@ static void uvStartSendResp(SSrvMsg* smsg) {
return;
}
static void uvNotifyLinkBrokenToApp(SSrvConn* conn) {
STrans* pTransInst = conn->pTransInst;
if (pTransInst->efp != NULL && (*pTransInst->efp)(NULL, conn->inType) && T_REF_VAL_GET(conn) >= 2) {
STransMsg transMsg = {0};
transMsg.msgType = conn->inType;
transMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
// transRefSrvHandle(conn);
(*pTransInst->cfp)(pTransInst->parent, &transMsg, 0);
}
}
static void destroySmsg(SSrvMsg* smsg) {
if (smsg == NULL) {
return;
@ -408,6 +463,9 @@ static void destroyAllConn(SWorkThrdObj* pThrd) {
QUEUE_INIT(h);
SSrvConn* c = QUEUE_DATA(h, SSrvConn, queue);
while (T_REF_VAL_GET(c) >= 2) {
transUnrefSrvHandle(c);
}
transUnrefSrvHandle(c);
}
}
@ -431,20 +489,7 @@ void uvWorkerAsyncCb(uv_async_t* handle) {
tError("unexcept occurred, continue");
continue;
}
if (msg->pConn == NULL) {
free(msg);
bool noConn = QUEUE_IS_EMPTY(&pThrd->conn);
if (noConn == true) {
uv_loop_close(pThrd->loop);
uv_stop(pThrd->loop);
} else {
destroyAllConn(pThrd);
// uv_loop_close(pThrd->loop);
pThrd->quit = true;
}
} else {
uvStartSendResp(msg);
}
(*transAsyncHandle[msg->type])(msg, pThrd);
}
}
static void uvAcceptAsyncCb(uv_async_t* async) {
@ -632,7 +677,9 @@ static SSrvConn* createConn(void* hThrd) {
pConn->srvMsgs = taosArrayInit(2, sizeof(void*)); //
tTrace("conn %p created", pConn);
memset(&pConn->regArg, 0, sizeof(pConn->regArg));
pConn->broken = false;
pConn->status = ConnNormal;
transRefSrvHandle(pConn);
return pConn;
@ -748,7 +795,57 @@ End:
transCloseServer(srv);
return NULL;
}
void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) {
if (QUEUE_IS_EMPTY(&thrd->conn)) {
uv_loop_close(thrd->loop);
uv_stop(thrd->loop);
} else {
destroyAllConn(thrd);
thrd->quit = true;
}
free(msg);
}
void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd) {
// release handle to rpc init
SSrvConn* conn = msg->pConn;
if (conn->status == ConnAcquire) {
taosArrayPush(conn->srvMsgs, &msg);
if (taosArrayGetSize(conn->srvMsgs) > 1) {
return;
}
uvStartSendRespInternal(msg);
return;
} else if (conn->status == ConnRelease) {
// already release by server app, do nothing
} else if (conn->status == ConnNormal) {
// no nothing
// user should not call this rpcRelease handle;
}
free(msg);
}
void uvHandleSendResp(SSrvMsg* msg, SWorkThrdObj* thrd) {
// send msg to client
uvStartSendResp(msg);
}
void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd) {
SSrvConn* conn = msg->pConn;
if (conn->status == ConnAcquire) {
if (taosArrayGetSize(conn->srvMsgs) > 0) {
taosArrayPush(conn->srvMsgs, &msg);
return;
}
conn->regArg.notifyCount = 0;
conn->regArg.init = 1;
conn->regArg.msg = msg->msg;
if (conn->broken) {
STrans* pTransInst = conn->pTransInst;
(*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL);
memset(&conn->regArg, 0, sizeof(conn->regArg));
}
free(msg);
}
}
void destroyWorkThrd(SWorkThrdObj* pThrd) {
if (pThrd == NULL) {
return;
@ -759,10 +856,10 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) {
free(pThrd);
}
void sendQuitToWorkThrd(SWorkThrdObj* pThrd) {
SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg));
SSrvMsg* msg = calloc(1, sizeof(SSrvMsg));
msg->type = Quit;
tDebug("server send quit msg to work thread");
transSendAsync(pThrd->asyncPool, &srvMsg->q);
transSendAsync(pThrd->asyncPool, &msg->q);
}
void transCloseServer(void* arg) {
@ -813,8 +910,21 @@ void transUnrefSrvHandle(void* handle) {
}
void transReleaseSrvHandle(void* handle) {
// do nothing currently
//
if (handle == NULL) {
return;
}
SSrvConn* pConn = handle;
SWorkThrdObj* pThrd = pConn->hostThrd;
STransMsg tmsg = {.handle = handle, .code = 0};
SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg));
srvMsg->msg = tmsg;
srvMsg->type = Release;
srvMsg->pConn = pConn;
tTrace("server conn %p start to release", pConn);
transSendAsync(pThrd->asyncPool, &srvMsg->q);
}
void transSendResponse(const STransMsg* pMsg) {
if (pMsg->handle == NULL) {
@ -826,6 +936,21 @@ void transSendResponse(const STransMsg* pMsg) {
SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg));
srvMsg->pConn = pConn;
srvMsg->msg = *pMsg;
srvMsg->type = Normal;
tTrace("server conn %p start to send resp", pConn);
transSendAsync(pThrd->asyncPool, &srvMsg->q);
}
void transRegisterMsg(const STransMsg* msg) {
if (msg->handle == NULL) {
return;
}
SSrvConn* pConn = msg->handle;
SWorkThrdObj* pThrd = pConn->hostThrd;
SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg));
srvMsg->pConn = pConn;
srvMsg->msg = *msg;
srvMsg->type = Register;
tTrace("server conn %p start to send resp", pConn);
transSendAsync(pThrd->asyncPool, &srvMsg->q);
}

140
source/libs/transport/test/transUT.cc
View File

@ -30,29 +30,8 @@ const char *ckey = "ckey";
class Server;
int port = 7000;
// server process
static bool cliPersistHandle(void *parent, tmsg_t msgType) {
// client persist handle
return msgType == 2 || msgType == 4;
}
typedef struct CbArgs {
tmsg_t msgType;
} CbArgs;
static void *ConstructArgForSpecificMsgType(void *parent, tmsg_t msgType) {
if (msgType == 1 || msgType == 2) {
CbArgs *args = (CbArgs *)calloc(1, sizeof(CbArgs));
args->msgType = msgType;
return args;
}
return NULL;
}
// server except
static bool handleExcept(void *parent, tmsg_t msgType) {
//
return msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP;
}
typedef void (*CB)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
@ -91,23 +70,6 @@ class Client {
rpcClose(this->transCli);
this->transCli = NULL;
}
void SetPersistFP(bool (*pfp)(void *parent, tmsg_t msgType)) {
rpcClose(this->transCli);
rpcInit_.pfp = pfp;
this->transCli = rpcOpen(&rpcInit_);
}
void SetConstructFP(void *(*mfp)(void *parent, tmsg_t msgType)) {
rpcClose(this->transCli);
rpcInit_.mfp = mfp;
this->transCli = rpcOpen(&rpcInit_);
}
void SetPAndMFp(bool (*pfp)(void *parent, tmsg_t msgType), void *(*mfp)(void *parent, tmsg_t msgType)) {
rpcClose(this->transCli);
rpcInit_.pfp = pfp;
rpcInit_.mfp = mfp;
this->transCli = rpcOpen(&rpcInit_);
}
void SendAndRecv(SRpcMsg *req, SRpcMsg *resp) {
SEpSet epSet = {0};
@ -126,7 +88,6 @@ class Client {
SendAndRecv(req, resp);
}
void SendWithHandle(SRpcMsg *req, SRpcMsg *resp) {}
void SemWait() { tsem_wait(&this->sem); }
void SemPost() { tsem_post(&this->sem); }
void Reset() {}
@ -149,7 +110,6 @@ class Server {
rpcInit_.label = (char *)label;
rpcInit_.numOfThreads = 5;
rpcInit_.cfp = processReq;
rpcInit_.efp = NULL;
rpcInit_.user = (char *)user;
rpcInit_.secret = (char *)secret;
rpcInit_.ckey = (char *)ckey;
@ -160,17 +120,17 @@ class Server {
this->transSrv = rpcOpen(&this->rpcInit_);
taosMsleep(1000);
}
void SetSrvContinueSend(CB cb) {
this->Stop();
rpcInit_.cfp = cb;
this->Start();
}
void Stop() {
if (this->transSrv == NULL) return;
rpcClose(this->transSrv);
this->transSrv = NULL;
}
void SetExceptFp(bool (*efp)(void *parent, tmsg_t msgType)) {
this->Stop();
rpcInit_.efp = efp;
this->Start();
}
void SetSrvContinueSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) {
void SetSrvSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) {
this->Stop();
rpcInit_.cfp = cfp;
this->Start();
@ -198,9 +158,6 @@ static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
}
static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
for (int i = 0; i < 9; i++) {
rpcRefHandle(pMsg->handle, TAOS_CONN_SERVER);
}
for (int i = 0; i < 10; i++) {
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = rpcMallocCont(100);
@ -262,23 +219,7 @@ class TransObj {
//
srv->Stop();
}
void SetCliPersistFp(bool (*pfp)(void *parent, tmsg_t msgType)) {
// do nothing
cli->SetPersistFP(pfp);
}
void SetCliMFp(void *(*mfp)(void *parent, tmsg_t msgType)) {
// do nothing
cli->SetConstructFP(mfp);
}
void SetCliMAndPFp(bool (*pfp)(void *parent, tmsg_t msgType), void *(*mfp)(void *parent, tmsg_t msgType)) {
// do nothing
cli->SetPAndMFp(pfp, mfp);
}
// call when link broken, and notify query or fetch stop
void SetSrvExceptFp(bool (*efp)(void *parent, tmsg_t msgType)) {
////////
srv->SetExceptFp(efp);
}
void SetSrvContinueSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) {
///////
srv->SetSrvContinueSend(cfp);
@ -315,7 +256,7 @@ class TransEnv : public ::testing::Test {
};
TEST_F(TransEnv, 01sendAndRec) {
for (int i = 0; i < 1; i++) {
for (int i = 0; i < 10; i++) {
SRpcMsg req = {0}, resp = {0};
req.msgType = 0;
req.pCont = rpcMallocCont(10);
@ -358,51 +299,53 @@ TEST_F(TransEnv, clientUserDefined) {
}
TEST_F(TransEnv, cliPersistHandle) {
tr->SetCliPersistFp(cliPersistHandle);
SRpcMsg resp = {0};
void * handle = NULL;
for (int i = 0; i < 10; i++) {
SRpcMsg req = {.handle = resp.handle, .noResp = 0};
SRpcMsg req = {.handle = resp.handle, .persistHandle = 1};
req.msgType = 1;
req.pCont = rpcMallocCont(10);
req.contLen = 10;
tr->cliSendAndRecv(&req, &resp);
if (i == 5) {
std::cout << "stop server" << std::endl;
tr->StopSrv();
}
if (i >= 6) {
EXPECT_TRUE(resp.code != 0);
}
}
//////////////////
}
TEST_F(TransEnv, cliReleaseHandle) {
tr->SetCliPersistFp(cliPersistHandle);
SRpcMsg resp = {0};
for (int i = 0; i < 10; i++) {
SRpcMsg req = {.handle = resp.handle};
req.msgType = 1;
req.pCont = rpcMallocCont(10);
req.contLen = 10;
tr->cliSendAndRecvNoHandle(&req, &resp);
// if (i == 5) {
// std::cout << "stop server" << std::endl;
// tr->StopSrv();
//}
// if (i >= 6) {
// EXPECT_TRUE(resp.code != 0);
//}
handle = resp.handle;
}
rpcReleaseHandle(handle, TAOS_CONN_CLIENT);
for (int i = 0; i < 10; i++) {
SRpcMsg req = {0};
req.msgType = 1;
req.pCont = rpcMallocCont(10);
req.contLen = 10;
tr->cliSendAndRecv(&req, &resp);
}
taosMsleep(1000);
//////////////////
}
TEST_F(TransEnv, cliReleaseHandle) {
SRpcMsg resp = {0};
for (int i = 0; i < 10; i++) {
SRpcMsg req = {.handle = resp.handle, .persistHandle = 1};
req.msgType = 1;
req.pCont = rpcMallocCont(10);
req.contLen = 10;
tr->cliSendAndRecvNoHandle(&req, &resp);
EXPECT_TRUE(resp.code == 0);
//}
}
//////////////////
}
TEST_F(TransEnv, cliReleaseHandleExcept) {
tr->SetCliPersistFp(cliPersistHandle);
SRpcMsg resp = {0};
for (int i = 0; i < 10; i++) {
SRpcMsg req = {.handle = resp.handle};
SRpcMsg req = {.handle = resp.handle, .persistHandle = 1};
req.msgType = 1;
req.pCont = rpcMallocCont(10);
req.contLen = 10;
@ -426,12 +369,12 @@ TEST_F(TransEnv, srvContinueSend) {
req.contLen = 10;
tr->cliSendAndRecv(&req, &resp);
}
taosMsleep(2000);
taosMsleep(1000);
}
TEST_F(TransEnv, srvPersistHandleExcept) {
tr->SetSrvContinueSend(processContinueSend);
tr->SetCliPersistFp(cliPersistHandle);
// tr->SetCliPersistFp(cliPersistHandle);
SRpcMsg resp = {0};
for (int i = 0; i < 5; i++) {
SRpcMsg req = {.handle = resp.handle};
@ -450,7 +393,6 @@ TEST_F(TransEnv, srvPersistHandleExcept) {
}
TEST_F(TransEnv, cliPersistHandleExcept) {
tr->SetSrvContinueSend(processContinueSend);
tr->SetCliPersistFp(cliPersistHandle);
SRpcMsg resp = {0};
for (int i = 0; i < 5; i++) {
SRpcMsg req = {.handle = resp.handle};
@ -471,11 +413,7 @@ TEST_F(TransEnv, cliPersistHandleExcept) {
TEST_F(TransEnv, multiCliPersistHandleExcept) {
// conn broken
}
TEST_F(TransEnv, queryExcept) {
tr->SetSrvExceptFp(handleExcept);
// query and conn is broken
}
TEST_F(TransEnv, queryExcept) {}
TEST_F(TransEnv, noResp) {
SRpcMsg resp = {0};
for (int i = 0; i < 5; i++) {

94
source/libs/transport/test/transportTests.cc
View File

@ -136,4 +136,98 @@ TEST_F(QueueEnv, testIter) {
assert(result.size() == vals.size());
}
class TransCtxEnv : public ::testing::Test {
protected:
virtual void SetUp() {
ctx = (STransCtx *)calloc(1, sizeof(STransCtx));
transCtxInit(ctx);
// TODO
}
virtual void TearDown() {
transCtxDestroy(ctx);
// formate
}
STransCtx *ctx;
};
TEST_F(TransCtxEnv, mergeTest) {
int key = 1;
{
STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx));
transCtxInit(src);
{
STransCtxVal val1 = {.val = NULL, .len = 0, .free = free};
val1.val = malloc(12);
val1.len = 12;
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++;
}
{
STransCtxVal val1 = {.val = NULL, .len = 0, .free = free};
val1.val = malloc(12);
val1.len = 12;
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++;
}
transCtxMerge(ctx, src);
free(src);
}
EXPECT_EQ(2, taosHashGetSize(ctx->args));
{
STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx));
transCtxInit(src);
{
STransCtxVal val1 = {.val = NULL, .len = 0, .free = free};
val1.val = malloc(12);
val1.len = 12;
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++;
}
{
STransCtxVal val1 = {.val = NULL, .len = 0, .free = free};
val1.val = malloc(12);
val1.len = 12;
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++;
}
transCtxMerge(ctx, src);
free(src);
}
std::string val("Hello");
EXPECT_EQ(4, taosHashGetSize(ctx->args));
{
key = 1;
STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx));
transCtxInit(src);
{
STransCtxVal val1 = {.val = NULL, .len = 0, .free = free};
val1.val = calloc(1, 11);
memcpy(val1.val, val.c_str(), val.size());
val1.len = 11;
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++;
}
{
STransCtxVal val1 = {.val = NULL, .len = 0, .free = free};
val1.val = calloc(1, 11);
memcpy(val1.val, val.c_str(), val.size());
val1.len = 11;
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++;
}
transCtxMerge(ctx, src);
free(src);
}
EXPECT_EQ(4, taosHashGetSize(ctx->args));
char *skey = (char *)transCtxDumpVal(ctx, 1);
EXPECT_EQ(0, strcmp(skey, val.c_str()));
free(skey);
skey = (char *)transCtxDumpVal(ctx, 2);
EXPECT_EQ(0, strcmp(skey, val.c_str()));
}
#endif

358
source/os/src/osString.c
View File

@ -15,10 +15,8 @@
#define ALLOW_FORBID_FUNC
#define _DEFAULT_SOURCE
#include <malloc.h>
#include "os.h"
// #include "tdef.h"
// #include <wchar.h>
// #include <wctype.h>
#ifndef DISALLOW_NCHAR_WITHOUT_ICONV
#include "iconv.h"
@ -32,14 +30,6 @@ int64_t taosStr2int64(const char *str) {
return strtoll(str, &endptr, 10);
}
bool taosCheckNcharValid(void) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
return false;
#else
return true;
#endif
}
int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) {
for (int32_t i = 0; i < bytes; i += sizeof(TdUcs4)) {
int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i);
@ -77,11 +67,13 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) {
TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) {
memcpy(target_ucs4, source_ucs4, len_ucs4*sizeof(TdUcs4));
assert(malloc_usable_size(target_ucs4)>=len_ucs4*sizeof(TdUcs4));
return memcpy(target_ucs4, source_ucs4, len_ucs4*sizeof(TdUcs4));
}
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n");
return -1;
#else
iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
@ -99,6 +91,7 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n");
return -1;
#else
memset(ucs4, 0, ucs4_max_len);
@ -124,7 +117,8 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4
bool taosValidateEncodec(const char *encodec) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
return false;
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n");
return true;
#else
iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC);
if (cd == (iconv_t)(-1)) {
@ -164,341 +158,3 @@ int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size) { ret
int32_t taosWcharToMb(char *pStr, TdWchar wchar) { return wctomb(pStr, wchar); }
int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size) { return wcstombs(pStrs, pWchars, size); }
// #ifdef USE_LIBICONV
// #include "iconv.h"
// int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
// iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
// size_t ucs4_input_len = ucs4_max_len;
// size_t outLen = ucs4_max_len;
// if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
// iconv_close(cd);
// return -1;
// }
// iconv_close(cd);
// return (int32_t)(ucs4_max_len - outLen);
// }
// bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
// memset(ucs4, 0, ucs4_max_len);
// iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
// size_t ucs4_input_len = mbsLength;
// size_t outLeft = ucs4_max_len;
// if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) {
// iconv_close(cd);
// return false;
// }
// iconv_close(cd);
// if (len != NULL) {
// *len = (int32_t)(ucs4_max_len - outLeft);
// if (*len < 0) {
// return false;
// }
// }
// return true;
// }
// bool taosValidateEncodec(const char *encodec) {
// iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC);
// if (cd == (iconv_t)(-1)) {
// return false;
// }
// iconv_close(cd);
// return true;
// }
// #else
// int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
// mbstate_t state = {0};
// int32_t len = (int32_t)wcsnrtombs(NULL, (const wchar_t **)&ucs4, ucs4_max_len / 4, 0, &state);
// if (len < 0) {
// return -1;
// }
// memset(&state, 0, sizeof(state));
// len = wcsnrtombs(mbs, (const wchar_t **)&ucs4, ucs4_max_len / 4, (size_t)len, &state);
// if (len < 0) {
// return -1;
// }
// return len;
// }
// bool taosMbsToUcs4(const char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
// memset(ucs4, 0, ucs4_max_len);
// mbstate_t state = {0};
// int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state);
// *len = retlen;
// return retlen >= 0;
// }
// bool taosValidateEncodec(const char *encodec) {
// return true;
// }
// #endif
// #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
// /*
// * windows implementation
// */
// #ifdef HAVE_CONFIG_H
// #include <config.h>
// #endif
// #include <sys/types.h>
// #include <stdio.h>
// #include <assert.h>
// #include <errno.h>
// #include <stdint.h>
// #if STDC_HEADERS
// #include <stdlib.h>
// #else
// char *malloc(), *realloc();
// #endif
// /* Always add at least this many bytes when extending the buffer. */
// #define MIN_CHUNK 64
// /* Read up to (and including) a TERMINATOR from STREAM into *LINEPTR
// + OFFSET (and null-terminate it). *LINEPTR is a pointer returned from
// malloc (or NULL), pointing to *N characters of space. It is realloc'd
// as necessary. Return the number of characters read (not including the
// null terminator), or -1 on error or EOF. On a -1 return, the caller
// should check feof(), if not then errno has been set to indicate
// the error. */
// int32_t getstr(char **lineptr, size_t *n, FILE *stream, char terminator, int32_t offset) {
// int32_t nchars_avail; /* Allocated but unused chars in *LINEPTR. */
// char * read_pos; /* Where we're reading into *LINEPTR. */
// int32_t ret;
// if (!lineptr || !n || !stream) {
// errno = EINVAL;
// return -1;
// }
// if (!*lineptr) {
// *n = MIN_CHUNK;
// *lineptr = malloc(*n);
// if (!*lineptr) {
// errno = ENOMEM;
// return -1;
// }
// }
// nchars_avail = (int32_t)(*n - offset);
// read_pos = *lineptr + offset;
// for (;;) {
// int32_t save_errno;
// register int32_t c = getc(stream);
// save_errno = errno;
// /* We always want at least one char left in the buffer, since we
// always (unless we get an error while reading the first char)
// NUL-terminate the line buffer. */
// assert((*lineptr + *n) == (read_pos + nchars_avail));
// if (nchars_avail < 2) {
// if (*n > MIN_CHUNK)
// *n *= 2;
// else
// *n += MIN_CHUNK;
// nchars_avail = (int32_t)(*n + *lineptr - read_pos);
// char* lineptr1 = realloc(*lineptr, *n);
// if (!lineptr1) {
// errno = ENOMEM;
// return -1;
// }
// *lineptr = lineptr1;
// read_pos = *n - nchars_avail + *lineptr;
// assert((*lineptr + *n) == (read_pos + nchars_avail));
// }
// if (ferror(stream)) {
// /* Might like to return partial line, but there is no
// place for us to store errno. And we don't want to just
// lose errno. */
// errno = save_errno;
// return -1;
// }
// if (c == EOF) {
// /* Return partial line, if any. */
// if (read_pos == *lineptr)
// return -1;
// else
// break;
// }
// *read_pos++ = c;
// nchars_avail--;
// if (c == terminator) /* Return the line. */
// break;
// }
// /* Done - NUL terminate and return the number of chars read. */
// *read_pos = '\0';
// ret = (int32_t)(read_pos - (*lineptr + offset));
// return ret;
// }
// int32_t tgetline(char **lineptr, size_t *n, FILE *stream) { return getstr(lineptr, n, stream, '\n', 0); }
// /*
// * Get next token from string *stringp, where tokens are possibly-empty
// * strings separated by characters from delim.
// *
// * Writes NULs into the string at *stringp to end tokens.
// * delim need not remain constant from call to call.
// * On return, *stringp points past the last NUL written (if there might
// * be further tokens), or is NULL (if there are definitely no moretokens).
// *
// * If *stringp is NULL, strsep returns NULL.
// */
// char *strsep(char **stringp, const char *delim) {
// char * s;
// const char *spanp;
// int32_t c, sc;
// char *tok;
// if ((s = *stringp) == NULL)
// return (NULL);
// for (tok = s;;) {
// c = *s++;
// spanp = delim;
// do {
// if ((sc = *spanp++) == c) {
// if (c == 0)
// s = NULL;
// else
// s[-1] = 0;
// *stringp = s;
// return (tok);
// }
// } while (sc != 0);
// }
// /* NOTREACHED */
// }
// char *getpass(const char *prefix) {
// static char passwd[TSDB_PASSWORD_LEN] = {0};
// memset(passwd, 0, TSDB_PASSWORD_LEN);
// //printf("%s", prefix);
// int32_t index = 0;
// char ch;
// while (index < TSDB_PASSWORD_LEN) {
// ch = getch();
// if (ch == '\n' || ch == '\r') {
// break;
// } else {
// passwd[index++] = ch;
// }
// }
// return passwd;
// }
// int32_t twcslen(const wchar_t *wcs) {
// int32_t *wstr = (int32_t *)wcs;
// if (NULL == wstr) {
// return 0;
// }
// int32_t n = 0;
// while (1) {
// if (0 == *wstr++) {
// break;
// }
// n++;
// }
// return n;
// }
// int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes) {
// for (int32_t i = 0; i < bytes; i += TSDB_NCHAR_SIZE) {
// int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i);
// int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i);
// if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) {
// return f1 - f2;
// } else if (f1 == 0 && f2 == 0) {
// return 0;
// }
// if (f1 != f2) {
// return f1 - f2;
// }
// }
// return 0;
// #if 0
// int32_t ucs4_max_len = bytes + 4;
// char *f1_mbs = calloc(bytes, 1);
// char *f2_mbs = calloc(bytes, 1);
// if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) {
// return -1;
// }
// if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) {
// return -1;
// }
// int32_t ret = strcmp(f1_mbs, f2_mbs);
// free(f1_mbs);
// free(f2_mbs);
// return ret;
// #endif
// }
// /* Copy memory to memory until the specified number of bytes
// has been copied, return pointer to following byte.
// Overlap is NOT handled correctly. */
// void *mempcpy(void *dest, const void *src, size_t len) {
// return (char*)memcpy(dest, src, len) + len;
// }
// /* Copy SRC to DEST, returning the address of the terminating '\0' in DEST. */
// char *stpcpy (char *dest, const char *src) {
// size_t len = strlen (src);
// return (char*)memcpy(dest, src, len + 1) + len;
// }
// /* Copy no more than N characters of SRC to DEST, returning the address of
// the terminating '\0' in DEST, if any, or else DEST + N. */
// char *stpncpy (char *dest, const char *src, size_t n) {
// size_t size = strnlen (src, n);
// memcpy (dest, src, size);
// dest += size;
// if (size == n)
// return dest;
// return memset (dest, '\0', n - size);
// }
// #else
// /*
// * linux and darwin implementation
// */
// int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize) {
// return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / ncharSize);
// }
// #endif

28
source/os/src/osTimezone.c
View File

@ -48,16 +48,11 @@
void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) {
if (inTimezone == NULL || inTimezone[0] == 0) return;
#ifdef WINDOWS
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
char winStr[TD_LOCALE_LEN * 2];
sprintf(winStr, "TZ=%s", inTimezone);
putenv(winStr);
#else
setenv("TZ", inTimezone, 1);
#endif
tzset();
/*
* get CURRENT time zone.
* system current time zone is affected by daylight saving time(DST)
*
@ -75,15 +70,34 @@ void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *ou
int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR);
tz += daylight;
/*
* format:
* (CST, +0800)
* (BST, +0100)
*/
sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
*outDaylight = daylight;
#elif defined(_TD_DARWIN_64)
setenv("TZ", inTimezone, 1);
tzset();
int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR);
tz += daylight;
sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
*outDaylight = daylight;
#else
setenv("TZ", inTimezone, 1);
tzset();
int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR);
tz += daylight;
sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
*outDaylight = daylight;
#endif
}
void taosGetSystemTimezone(char *outTimezone) {

23
source/util/src/tjson.c
View File

@ -202,6 +202,13 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV
return (errno == ERANGE||errno == EINVAL) ? TSDB_CODE_FAILED:TSDB_CODE_SUCCESS;
}
int32_t tjsonGetUIntValue(const SJson* pJson, const char* pName, uint32_t* pVal) {
uint64_t val = 0;
int32_t code = tjsonGetUBigIntValue(pJson, pName, &val);
*pVal = val;
return code;
}
int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal) {
uint64_t val = 0;
int32_t code = tjsonGetUBigIntValue(pJson, pName, &val);
@ -239,6 +246,22 @@ int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, voi
return func(pJsonObj, pObj);
}
int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, void** pObj, int32_t objSize) {
if (objSize <= 0) {
return TSDB_CODE_SUCCESS;
}
SJson* pJsonObj = tjsonGetObjectItem(pJson, pName);
if (NULL == pJsonObj) {
return TSDB_CODE_FAILED;
}
*pObj = calloc(1, objSize);
if (NULL == *pObj) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return func(pJsonObj, *pObj);
}
int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize) {
const cJSON* jArray = tjsonGetObjectItem(pJson, pName);
int32_t size = (NULL == jArray ? 0 : tjsonGetArraySize(jArray));

5
tests/script/tsim/insert/basic0.sim
View File

@ -26,9 +26,10 @@ endi
print =============== create child table
sql create table ct1 using stb tags(1000)
sql create table ct2 using stb tags(2000)
sql create table ct3 using stb tags(3000)
sql show tables
if $rows != 2 then
if $rows != 3 then
return -1
endi
@ -45,6 +46,8 @@ sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 1
sql_error insert into ct1 values(now+4s, -14, -2.4, -3.4) ct2 values(now+4s, -14, -2.4, -3.4)
sql_error insert into ct1 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) ct2 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6)
sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0)
#===================================================================
#===================================================================
print =============== query data from child table

305
tests/script/tsim/query/interval-offset.sim Normal file
View File

@ -0,0 +1,305 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 500
sql connect
print =============== create database
sql drop database d0 -x step1
step1:
sql create database d0
sql show databases
if $rows != 1 then
return -1
endi
sql use d0
print =============== create super table and child table
sql create table stb (ts timestamp, tbcol int) tags (t1 int)
sql show stables
print $rows $data00 $data01 $data02
if $rows != 1 then
return -1
endi
sql create table ct1 using stb tags ( 1 )
sql create table ct2 using stb tags ( 2 )
sql show tables
print $rows $data00 $data10 $data20
if $rows != 2 then
return -1
endi
print =============== insert data into child table
sql insert into ct1 values ( '2022-01-01 01:01:01.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:06.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:10.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:16.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:20.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:26.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:30.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:36.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:40.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:46.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:50.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:01:56.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:00.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:06.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:10.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:16.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:20.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:26.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:30.000', 1 )
sql insert into ct1 values ( '2022-01-01 01:02:36.000', 1 )
sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s)
print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s)
print ===> $rows $data00 $data01 $data02 $data03 $data04
if $rows != 10 then
return -1
endi
if $data00 != 2 then
return -1
endi
if $data04 != 2 then
return -1
endi
sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-01 12:00:01.000', 2 )
sql insert into ct2 values ( '2022-01-01 23:00:01.000', 3 )
sql insert into ct2 values ( '2022-01-02 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-03 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-04 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-05 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-06 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-07 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-08 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-09 10:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-10 10:00:01.000', 1 )
sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h)
print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2w)
print ===> rows0: $data00 $data01 $data02 $data03 $data04
print ===> rows1: $data10 $data11 $data12 $data13 $data14
print ===> rows2: $data20 $data21 $data22 $data23 $data24
if $rows != 11 then
return -1
endi
if $data00 != 1 then
return -1
endi
if $data10 != 2 then
return -1
endi
return
sql select count(*) from car interval(1n, 10d) order by ts desc
# tdSql.checkData(0, 1, 1)
# tdSql.checkData(1, 1, 2)
# tdSql.checkData(2, 1, 3)
# tdSql.checkData(3, 1, 3)
# tdSql.checkData(4, 1, 6)
# tdSql.checkData(5, 1, 1)
# tdSql.checkData(6, 1, 1)
#
sql select count(*) from car interval(2n, 5d)
# tdSql.checkData(0, 1, 1)
# tdSql.checkData(1, 1, 1)
# tdSql.checkData(2, 1, 6)
# tdSql.checkData(3, 1, 6)
# tdSql.checkData(4, 1, 3)
sql select count(*) from car interval(2n) order by ts desc
# tdSql.checkData(0, 1, 3)
# tdSql.checkData(1, 1, 6)
# tdSql.checkData(2, 1, 6)
# tdSql.checkData(3, 1, 1)
# tdSql.checkData(4, 1, 1)
#
sql select count(*) from car interval(1y, 1n)
# tdSql.checkData(0, 1, 1)
# tdSql.checkData(1, 1, 8)
# tdSql.checkData(2, 1, 8)
#
sql select count(*) from car interval(1y, 2n)
# tdSql.checkData(0, 1, 1)
# tdSql.checkData(1, 1, 11)
# tdSql.checkData(2, 1, 5)
sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d)
# tdSql.checkData(0, 1, 6)
# tdSql.checkData(1, 1, 9)
sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int)
print ====== start create child tables and insert data
$i = 0
while $i < $tbNum
$tb = $tbPrefix . $i
sql create table $tb using $mt tags( $i )
$x = 0
while $x < $rowNum
$cc = $x * 60000
$ms = 1601481600000 + $cc
sql insert into $tb values ($ms , $x )
$x = $x + 1
endw
$i = $i + 1
endw
print =============== step2
$i = 1
$tb = $tbPrefix . $i
sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m)
print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m)
print ===> $rows $data01 $data05
if $rows != $rowNum then
return -1
endi
if $data00 != 1 then
return -1
endi
if $data04 != 1 then
return -1
endi
#print =============== step3
#$cc = 4 * 60000
#$ms = 1601481600000 + $cc
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m)
#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m)
#print ===> $rows $data01 $data05
#if $rows != 5 then
# return -1
#endi
#if $data00 != 1 then
# return -1
#endi
#if $data04 != 1 then
# return -1
#endi
#print =============== step4
#$cc = 40 * 60000
#$ms = 1601481600000 + $cc
#$cc = 1 * 60000
#$ms2 = 1601481600000 - $cc
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m)
#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m)
#print ===> $rows $data01 $data05
#if $rows != 20 then
# return -1
#endi
#if $data00 != 1 then
# return -1
#endi
#if $data04 != 1 then
# return -1
#endi
#print =============== step5
#$cc = 40 * 60000
#$ms = 1601481600000 + $cc
#$cc = 1 * 60000
#$ms2 = 1601481600000 - $cc
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0)
#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0)
#print ===> $rows $data21 $data25
#if $rows != 42 then
# return -1
#endi
#if $data20 != 1 then
# return -1
#endi
#if $data24 != 1 then
# return -1
#endi
#print =============== step6
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m)
#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m)
#print ===> $rows $data11
#if $rows != 20 then
# return -1
#endi
#if $data11 != 10 then
# return -1
#endi
#print =============== step7
#$cc = 4 * 60000
#$ms = 1601481600000 + $cc
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m)
#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m)
#print ===> $rows $data11
#if $rows != 5 then
# return -1
#endi
#if $data11 != 10 then
# return -1
#endi
#print =============== step8
#$cc = 40 * 60000
#$ms1 = 1601481600000 + $cc
#
#$cc = 1 * 60000
#$ms2 = 1601481600000 - $cc
#
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m)
#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m)
#print ===> $rows $data11
#if $rows != 20 then
# return -1
#endi
#if $data11 != 10 then
# return -1
#endi
#
#print =============== step9
#$cc = 40 * 60000
#$ms1 = 1601481600000 + $cc
#
#$cc = 1 * 60000
#$ms2 = 1601481600000 - $cc
#
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0)
#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0)
#print ===> $rows $data11
#if $rows != 42 then
# return -1
#endi
#if $data11 != 10 then
# return -1
#endi
print =============== clear
#sql drop database $db
#sql show databases
#if $rows != 0 then
# return -1
#endi
#system sh/exec.sh -n dnode1 -s stop -x SIGINT

2
tests/script/tsim/query/interval.sim
View File

@ -3,7 +3,7 @@ system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c wal -v 1
system sh/exec.sh -n dnode1 -s start
sleep 2000
sleep 500
sql connect
$dbPrefix = m_in_db