xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

merge from 3.0

This commit is contained in:
Liu Jicong 2021-12-29 17:57:54 +08:00
parent 335de53594 1ba2f77cb6
commit f3067eb26c
96 changed files with 5561 additions and 4326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.devcontainer/devcontainer.json
View File

@ -28,5 +28,5 @@
// "postCreateCommand": "gcc -v", // "postCreateCommand": "gcc -v",
// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
"remoteUser": "vscode" "remoteUser": "root"
} }

48
contrib/test/tdev/src/main.c
View File

@ -14,6 +14,10 @@
#define tPutB(buf, val) \ #define tPutB(buf, val) \
({ \ ({ \
((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \
((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \
((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \
((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \
((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \ ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \
((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \ ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \
((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; \ ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; \
@ -23,11 +27,31 @@
#define tPutC(buf, val) \ #define tPutC(buf, val) \
({ \ ({ \
if (buf) { \
((uint64_t *)buf)[0] = (val); \ ((uint64_t *)buf)[0] = (val); \
POINTER_SHIFT(buf, sizeof(val)); \ POINTER_SHIFT(buf, sizeof(val)); \
} \
NULL; \
})
#define tPutD(buf, val) \
({ \
uint64_t tmp = val; \
for (size_t i = 0; i < sizeof(val); i++) { \
((uint8_t *)buf)[i] = tmp & 0xff; \
tmp >>= 8; \
} \
POINTER_SHIFT(buf, sizeof(val)); \
}) })
typedef enum { A, B, C } T; static inline void tPutE(void **buf, uint64_t val) {
if (buf) {
((uint64_t *)(*buf))[0] = val;
*buf = POINTER_SHIFT(*buf, sizeof(val));
}
}
typedef enum { A, B, C, D, E } T;
static void func(T t) { static void func(T t) {
uint64_t val = 198; uint64_t val = 198;
@ -59,6 +83,22 @@ static void func(T t) {
} }
} }
break; break;
case D:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
pBuf = tPutD(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf;
}
}
break;
case E:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
tPutE(&pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf;
}
}
break;
default: default:
break; break;
@ -83,5 +123,11 @@ int main(int argc, char const *argv[]) {
func(C); func(C);
uint64_t t4 = now(); uint64_t t4 = now();
printf("C: %ld\n", t4 - t3); printf("C: %ld\n", t4 - t3);
func(D);
uint64_t t5 = now();
printf("D: %ld\n", t5 - t4);
func(E);
uint64_t t6 = now();
printf("E: %ld\n", t6 - t5);
return 0; return 0;
} }

35
include/common/tmsg.h
View File

@ -20,6 +20,7 @@
extern "C" { extern "C" {
#endif #endif
#include "encode.h"
#include "taosdef.h" #include "taosdef.h"
#include "taoserror.h" #include "taoserror.h"
#include "tcoding.h" #include "tcoding.h"
@ -673,8 +674,6 @@ typedef struct {
typedef struct { typedef struct {
int32_t dnodeId; int32_t dnodeId;
int64_t clusterId; int64_t clusterId;
int8_t dropped;
char reserved[7];
} SDnodeCfg; } SDnodeCfg;
typedef struct { typedef struct {
@ -1105,10 +1104,9 @@ static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateT
} }
static FORCE_INLINE void* tDeserializeSCMCreateTopicReq(void* buf, SCMCreateTopicReq* pReq) { static FORCE_INLINE void* tDeserializeSCMCreateTopicReq(void* buf, SCMCreateTopicReq* pReq) {
buf = taosDecodeFixedI8(buf, &pReq->igExists); buf = taosDecodeFixedI8(buf, &(pReq->igExists));
buf = taosDecodeString(buf, &pReq->name); buf = taosDecodeString(buf, &(pReq->name));
buf = taosDecodeString(buf, &pReq->physicalPlan); buf = taosDecodeString(buf, &(pReq->phyPlan));
buf = taosDecodeString(buf, &pReq->logicalPlan);
return buf; return buf;
} }
@ -1339,6 +1337,31 @@ static FORCE_INLINE void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pR
typedef struct SVCreateTbRsp { typedef struct SVCreateTbRsp {
} SVCreateTbRsp; } SVCreateTbRsp;
typedef struct SVShowTablesReq {
SMsgHead head;
} SVShowTablesReq;
typedef struct SVShowTablesRsp {
int64_t id;
STableMetaMsg metaInfo;
} SVShowTablesRsp;
typedef struct SVShowTablesFetchReq {
SMsgHead head;
int64_t id;
} SVShowTablesFetchReq;
typedef struct SVShowTablesFetchRsp {
int64_t useconds;
int8_t completed; // all results are returned to client
int8_t precision;
int8_t compressed;
int32_t compLen;
int32_t numOfRows;
char data[];
} SVShowTablesFetchRsp;
#pragma pack(pop) #pragma pack(pop)
#ifdef __cplusplus #ifdef __cplusplus

4
include/common/tmsgdef.h
View File

@ -147,8 +147,12 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TOPIC, "vnode-create-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TOPIC, "vnode-create-topic", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TOPIC, "vnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TOPIC, "vnode-alter-topic", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_TOPIC, "vnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_TOPIC, "vnode-drop-topic", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES, "vnode-show-tables", SVShowTablesReq, SVShowTablesRsp)
TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES_FETCH, "vnode-show-tables-fetch", SVShowTablesFetchReq, SVShowTablesFetchRsp)
TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp)
// Requests handled by QNODE // Requests handled by QNODE
TD_NEW_MSG_SEG(TDMT_QND_MSG) TD_NEW_MSG_SEG(TDMT_QND_MSG)

388
include/common/ttokendef.h
View File

@ -13,205 +13,199 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef _TD_COMMON_TOKEN_DEF_H_ #ifndef TDENGINE_TTOKENDEF_H
#define _TD_COMMON_TOKEN_DEF_H_ #define TDENGINE_TTOKENDEF_H
#define TK_ID 1 #define TK_ID 1
#define TK_BOOL 2 #define TK_BOOL 2
#define TK_TINYINT 3 #define TK_INTEGER 3
#define TK_SMALLINT 4 #define TK_FLOAT 4
#define TK_INTEGER 5 #define TK_STRING 5
#define TK_BIGINT 6 #define TK_TIMESTAMP 6
#define TK_FLOAT 7 #define TK_OR 7
#define TK_DOUBLE 8 #define TK_AND 8
#define TK_STRING 9 #define TK_NOT 9
#define TK_TIMESTAMP 10 #define TK_EQ 10
#define TK_BINARY 11 #define TK_NE 11
#define TK_NCHAR 12 #define TK_ISNULL 12
#define TK_OR 13 #define TK_NOTNULL 13
#define TK_AND 14 #define TK_IS 14
#define TK_NOT 15 #define TK_LIKE 15
#define TK_EQ 16 #define TK_MATCH 16
#define TK_NE 17 #define TK_NMATCH 17
#define TK_ISNULL 18 #define TK_GLOB 18
#define TK_NOTNULL 19 #define TK_BETWEEN 19
#define TK_IS 20 #define TK_IN 20
#define TK_LIKE 21 #define TK_GT 21
#define TK_MATCH 22 #define TK_GE 22
#define TK_NMATCH 23 #define TK_LT 23
#define TK_GLOB 24 #define TK_LE 24
#define TK_BETWEEN 25 #define TK_BITAND 25
#define TK_IN 26 #define TK_BITOR 26
#define TK_GT 27 #define TK_LSHIFT 27
#define TK_GE 28 #define TK_RSHIFT 28
#define TK_LT 29 #define TK_PLUS 29
#define TK_LE 30 #define TK_MINUS 30
#define TK_BITAND 31 #define TK_DIVIDE 31
#define TK_BITOR 32 #define TK_TIMES 32
#define TK_LSHIFT 33 #define TK_STAR 33
#define TK_RSHIFT 34 #define TK_SLASH 34
#define TK_PLUS 35 #define TK_REM 35
#define TK_MINUS 36 #define TK_CONCAT 36
#define TK_DIVIDE 37 #define TK_UMINUS 37
#define TK_TIMES 38 #define TK_UPLUS 38
#define TK_STAR 39 #define TK_BITNOT 39
#define TK_SLASH 40 #define TK_SHOW 40
#define TK_REM 41 #define TK_DATABASES 41
#define TK_CONCAT 42 #define TK_TOPICS 42
#define TK_UMINUS 43 #define TK_FUNCTIONS 43
#define TK_UPLUS 44 #define TK_MNODES 44
#define TK_BITNOT 45 #define TK_DNODES 45
#define TK_SHOW 46 #define TK_ACCOUNTS 46
#define TK_DATABASES 47 #define TK_USERS 47
#define TK_TOPICS 48 #define TK_MODULES 48
#define TK_FUNCTIONS 49 #define TK_QUERIES 49
#define TK_MNODES 50 #define TK_CONNECTIONS 50
#define TK_DNODES 51 #define TK_STREAMS 51
#define TK_ACCOUNTS 52 #define TK_VARIABLES 52
#define TK_USERS 53 #define TK_SCORES 53
#define TK_MODULES 54 #define TK_GRANTS 54
#define TK_QUERIES 55 #define TK_VNODES 55
#define TK_CONNECTIONS 56 #define TK_DOT 56
#define TK_STREAMS 57 #define TK_CREATE 57
#define TK_VARIABLES 58 #define TK_TABLE 58
#define TK_SCORES 59 #define TK_STABLE 59
#define TK_GRANTS 60 #define TK_DATABASE 60
#define TK_VNODES 61 #define TK_TABLES 61
#define TK_DOT 62 #define TK_STABLES 62
#define TK_CREATE 63 #define TK_VGROUPS 63
#define TK_TABLE 64 #define TK_DROP 64
#define TK_STABLE 65 #define TK_TOPIC 65
#define TK_DATABASE 66 #define TK_FUNCTION 66
#define TK_TABLES 67 #define TK_DNODE 67
#define TK_STABLES 68 #define TK_USER 68
#define TK_VGROUPS 69 #define TK_ACCOUNT 69
#define TK_DROP 70 #define TK_USE 70
#define TK_TOPIC 71 #define TK_DESCRIBE 71
#define TK_FUNCTION 72 #define TK_DESC 72
#define TK_DNODE 73 #define TK_ALTER 73
#define TK_USER 74 #define TK_PASS 74
#define TK_ACCOUNT 75 #define TK_PRIVILEGE 75
#define TK_USE 76 #define TK_LOCAL 76
#define TK_DESCRIBE 77 #define TK_COMPACT 77
#define TK_DESC 78 #define TK_LP 78
#define TK_ALTER 79 #define TK_RP 79
#define TK_PASS 80 #define TK_IF 80
#define TK_PRIVILEGE 81 #define TK_EXISTS 81
#define TK_LOCAL 82 #define TK_PORT 82
#define TK_COMPACT 83 #define TK_IPTOKEN 83
#define TK_LP 84 #define TK_AS 84
#define TK_RP 85 #define TK_OUTPUTTYPE 85
#define TK_IF 86 #define TK_AGGREGATE 86
#define TK_EXISTS 87 #define TK_BUFSIZE 87
#define TK_AS 88 #define TK_PPS 88
#define TK_OUTPUTTYPE 89 #define TK_TSERIES 89
#define TK_AGGREGATE 90 #define TK_DBS 90
#define TK_BUFSIZE 91 #define TK_STORAGE 91
#define TK_PPS 92 #define TK_QTIME 92
#define TK_TSERIES 93 #define TK_CONNS 93
#define TK_DBS 94 #define TK_STATE 94
#define TK_STORAGE 95 #define TK_COMMA 95
#define TK_QTIME 96 #define TK_KEEP 96
#define TK_CONNS 97 #define TK_CACHE 97
#define TK_STATE 98 #define TK_REPLICA 98
#define TK_COMMA 99 #define TK_QUORUM 99
#define TK_KEEP 100 #define TK_DAYS 100
#define TK_CACHE 101 #define TK_MINROWS 101
#define TK_REPLICA 102 #define TK_MAXROWS 102
#define TK_QUORUM 103 #define TK_BLOCKS 103
#define TK_DAYS 104 #define TK_CTIME 104
#define TK_MINROWS 105 #define TK_WAL 105
#define TK_MAXROWS 106 #define TK_FSYNC 106
#define TK_BLOCKS 107 #define TK_COMP 107
#define TK_CTIME 108 #define TK_PRECISION 108
#define TK_WAL 109 #define TK_UPDATE 109
#define TK_FSYNC 110 #define TK_CACHELAST 110
#define TK_COMP 111 #define TK_UNSIGNED 111
#define TK_PRECISION 112 #define TK_TAGS 112
#define TK_UPDATE 113 #define TK_USING 113
#define TK_CACHELAST 114 #define TK_NULL 114
#define TK_PARTITIONS 115 #define TK_NOW 115
#define TK_UNSIGNED 116 #define TK_SELECT 116
#define TK_TAGS 117 #define TK_UNION 117
#define TK_USING 118 #define TK_ALL 118
#define TK_NULL 119 #define TK_DISTINCT 119
#define TK_NOW 120 #define TK_FROM 120
#define TK_SELECT 121 #define TK_VARIABLE 121
#define TK_UNION 122 #define TK_INTERVAL 122
#define TK_ALL 123 #define TK_EVERY 123
#define TK_DISTINCT 124 #define TK_SESSION 124
#define TK_FROM 125 #define TK_STATE_WINDOW 125
#define TK_VARIABLE 126 #define TK_FILL 126
#define TK_INTERVAL 127 #define TK_SLIDING 127
#define TK_EVERY 128 #define TK_ORDER 128
#define TK_SESSION 129 #define TK_BY 129
#define TK_STATE_WINDOW 130 #define TK_ASC 130
#define TK_FILL 131 #define TK_GROUP 131
#define TK_SLIDING 132 #define TK_HAVING 132
#define TK_ORDER 133 #define TK_LIMIT 133
#define TK_BY 134 #define TK_OFFSET 134
#define TK_ASC 135 #define TK_SLIMIT 135
#define TK_GROUP 136 #define TK_SOFFSET 136
#define TK_HAVING 137 #define TK_WHERE 137
#define TK_LIMIT 138 #define TK_RESET 138
#define TK_OFFSET 139 #define TK_QUERY 139
#define TK_SLIMIT 140 #define TK_SYNCDB 140
#define TK_SOFFSET 141 #define TK_ADD 141
#define TK_WHERE 142 #define TK_COLUMN 142
#define TK_RESET 143 #define TK_MODIFY 143
#define TK_QUERY 144 #define TK_TAG 144
#define TK_SYNCDB 145 #define TK_CHANGE 145
#define TK_ADD 146 #define TK_SET 146
#define TK_COLUMN 147 #define TK_KILL 147
#define TK_MODIFY 148 #define TK_CONNECTION 148
#define TK_TAG 149 #define TK_STREAM 149
#define TK_CHANGE 150 #define TK_COLON 150
#define TK_SET 151 #define TK_ABORT 151
#define TK_KILL 152 #define TK_AFTER 152
#define TK_CONNECTION 153 #define TK_ATTACH 153
#define TK_STREAM 154 #define TK_BEFORE 154
#define TK_COLON 155 #define TK_BEGIN 155
#define TK_ABORT 156 #define TK_CASCADE 156
#define TK_AFTER 157 #define TK_CLUSTER 157
#define TK_ATTACH 158 #define TK_CONFLICT 158
#define TK_BEFORE 159 #define TK_COPY 159
#define TK_BEGIN 160 #define TK_DEFERRED 160
#define TK_CASCADE 161 #define TK_DELIMITERS 161
#define TK_CLUSTER 162 #define TK_DETACH 162
#define TK_CONFLICT 163 #define TK_EACH 163
#define TK_COPY 164 #define TK_END 164
#define TK_DEFERRED 165 #define TK_EXPLAIN 165
#define TK_DELIMITERS 166 #define TK_FAIL 166
#define TK_DETACH 167 #define TK_FOR 167
#define TK_EACH 168 #define TK_IGNORE 168
#define TK_END 169 #define TK_IMMEDIATE 169
#define TK_EXPLAIN 170 #define TK_INITIALLY 170
#define TK_FAIL 171 #define TK_INSTEAD 171
#define TK_FOR 172 #define TK_KEY 172
#define TK_IGNORE 173 #define TK_OF 173
#define TK_IMMEDIATE 174 #define TK_RAISE 174
#define TK_INITIALLY 175 #define TK_REPLACE 175
#define TK_INSTEAD 176 #define TK_RESTRICT 176
#define TK_KEY 177 #define TK_ROW 177
#define TK_OF 178 #define TK_STATEMENT 178
#define TK_RAISE 179 #define TK_TRIGGER 179
#define TK_REPLACE 180 #define TK_VIEW 180
#define TK_RESTRICT 181 #define TK_SEMI 181
#define TK_ROW 182 #define TK_NONE 182
#define TK_STATEMENT 183 #define TK_PREV 183
#define TK_TRIGGER 184 #define TK_LINEAR 184
#define TK_VIEW 185 #define TK_IMPORT 185
#define TK_IPTOKEN 186 #define TK_TBNAME 186
#define TK_SEMI 187 #define TK_JOIN 187
#define TK_NONE 188 #define TK_INSERT 188
#define TK_PREV 189 #define TK_INTO 189
#define TK_LINEAR 190 #define TK_VALUES 190
#define TK_IMPORT 191
#define TK_TBNAME 192
#define TK_JOIN 193
#define TK_INSERT 194
#define TK_INTO 195
#define TK_VALUES 196
#define TK_SPACE 300 #define TK_SPACE 300
@ -223,6 +217,6 @@
#define TK_FILE 306 #define TK_FILE 306
#define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query #define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query
#endif /*_TD_COMMON_TOKEN_DEF_H_*/ #endif

10
include/dnode/bnode/bnode.h
View File

@ -49,10 +49,11 @@ typedef struct {
/** /**
* @brief Start one Bnode in Dnode. * @brief Start one Bnode in Dnode.
* *
* @param path Path of the bnode.
* @param pOption Option of the bnode. * @param pOption Option of the bnode.
* @return SBnode* The bnode object. * @return SBnode* The bnode object.
*/ */
SBnode *bndOpen(const SBnodeOpt *pOption); SBnode *bndOpen(const char *path, const SBnodeOpt *pOption);
/** /**
* @brief Stop Bnode in Dnode. * @brief Stop Bnode in Dnode.
@ -79,6 +80,13 @@ int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad);
*/ */
int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs); int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs);
/**
* @brief Drop a bnode.
*
* @param path Path of the bnode.
*/
void bndDestroy(const char *path);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

20
include/dnode/mnode/mnode.h
View File

@ -147,28 +147,12 @@ void mndCleanupMsg(SMnodeMsg *pMsg);
void mndSendRsp(SMnodeMsg *pMsg, int32_t code); void mndSendRsp(SMnodeMsg *pMsg, int32_t code);
/** /**
* @brief Process the read request. * @brief Process the read, write, sync request.
* *
* @param pMsg The request msg. * @param pMsg The request msg.
* @return int32_t 0 for success, -1 for failure. * @return int32_t 0 for success, -1 for failure.
*/ */
void mndProcessReadMsg(SMnodeMsg *pMsg); void mndProcessMsg(SMnodeMsg *pMsg);
/**
* @brief Process the write request.
*
* @param pMsg The request msg.
* @return int32_t 0 for success, -1 for failure.
*/
void mndProcessWriteMsg(SMnodeMsg *pMsg);
/**
* @brief Process the sync request.
*
* @param pMsg The request msg.
* @return int32_t 0 for success, -1 for failure.
*/
void mndProcessSyncMsg(SMnodeMsg *pMsg);
#ifdef __cplusplus #ifdef __cplusplus
} }

14
include/dnode/mnode/sdb/sdb.h
View File

@ -25,7 +25,7 @@ extern "C" {
#define SDB_GET_INT64(pData, pRow, dataPos, val) \ #define SDB_GET_INT64(pData, pRow, dataPos, val) \
{ \ { \
if (sdbGetRawInt64(pRaw, dataPos, val) != 0) { \ if (sdbGetRawInt64(pRaw, dataPos, val) != 0) { \
sdbFreeRow(pRow); \ tfree(pRow); \
return NULL; \ return NULL; \
} \ } \
dataPos += sizeof(int64_t); \ dataPos += sizeof(int64_t); \
@ -34,7 +34,7 @@ extern "C" {
#define SDB_GET_INT32(pData, pRow, dataPos, val) \ #define SDB_GET_INT32(pData, pRow, dataPos, val) \
{ \ { \
if (sdbGetRawInt32(pRaw, dataPos, val) != 0) { \ if (sdbGetRawInt32(pRaw, dataPos, val) != 0) { \
sdbFreeRow(pRow); \ tfree(pRow); \
return NULL; \ return NULL; \
} \ } \
dataPos += sizeof(int32_t); \ dataPos += sizeof(int32_t); \
@ -43,7 +43,7 @@ extern "C" {
#define SDB_GET_INT16(pData, pRow, dataPos, val) \ #define SDB_GET_INT16(pData, pRow, dataPos, val) \
{ \ { \
if (sdbGetRawInt16(pRaw, dataPos, val) != 0) { \ if (sdbGetRawInt16(pRaw, dataPos, val) != 0) { \
sdbFreeRow(pRow); \ tfree(pRow); \
return NULL; \ return NULL; \
} \ } \
dataPos += sizeof(int16_t); \ dataPos += sizeof(int16_t); \
@ -52,7 +52,7 @@ extern "C" {
#define SDB_GET_INT8(pData, pRow, dataPos, val) \ #define SDB_GET_INT8(pData, pRow, dataPos, val) \
{ \ { \
if (sdbGetRawInt8(pRaw, dataPos, val) != 0) { \ if (sdbGetRawInt8(pRaw, dataPos, val) != 0) { \
sdbFreeRow(pRow); \ tfree(pRow); \
return NULL; \ return NULL; \
} \ } \
dataPos += sizeof(int8_t); \ dataPos += sizeof(int8_t); \
@ -61,7 +61,7 @@ extern "C" {
#define SDB_GET_BINARY(pRaw, pRow, dataPos, val, valLen) \ #define SDB_GET_BINARY(pRaw, pRow, dataPos, val, valLen) \
{ \ { \
if (sdbGetRawBinary(pRaw, dataPos, val, valLen) != 0) { \ if (sdbGetRawBinary(pRaw, dataPos, val, valLen) != 0) { \
sdbFreeRow(pRow); \ tfree(pRow); \
return NULL; \ return NULL; \
} \ } \
dataPos += valLen; \ dataPos += valLen; \
@ -71,7 +71,7 @@ extern "C" {
{ \ { \
char val[valLen] = {0}; \ char val[valLen] = {0}; \
if (sdbGetRawBinary(pRaw, dataPos, val, valLen) != 0) { \ if (sdbGetRawBinary(pRaw, dataPos, val, valLen) != 0) { \
sdbFreeRow(pRow); \ tfree(pRow); \
return NULL; \ return NULL; \
} \ } \
dataPos += valLen; \ dataPos += valLen; \
@ -327,7 +327,7 @@ int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver);
int32_t sdbGetRawTotalSize(SSdbRaw *pRaw); int32_t sdbGetRawTotalSize(SSdbRaw *pRaw);
SSdbRow *sdbAllocRow(int32_t objSize); SSdbRow *sdbAllocRow(int32_t objSize);
void sdbFreeRow(SSdbRow *pRow); void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow);
void *sdbGetRowObj(SSdbRow *pRow); void *sdbGetRowObj(SSdbRow *pRow);
#ifdef __cplusplus #ifdef __cplusplus

12
include/dnode/snode/snode.h
View File

@ -49,10 +49,11 @@ typedef struct {
/** /**
* @brief Start one Snode in Dnode. * @brief Start one Snode in Dnode.
* *
* @param path Path of the snode.
* @param pOption Option of the snode. * @param pOption Option of the snode.
* @return SSnode* The snode object. * @return SSnode* The snode object.
*/ */
SSnode *sndOpen(const SSnodeOpt *pOption); SSnode *sndOpen(const char *path, const SSnodeOpt *pOption);
/** /**
* @brief Stop Snode in Dnode. * @brief Stop Snode in Dnode.
@ -78,7 +79,14 @@ int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad);
* @param pRsp The response message * @param pRsp The response message
* @return int32_t 0 for success, -1 for failure * @return int32_t 0 for success, -1 for failure
*/ */
int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
/**
* @brief Drop a snode.
*
* @param path Path of the snode.
*/
void sndDestroy(const char *path);
#ifdef __cplusplus #ifdef __cplusplus
} }

30
include/dnode/vnode/meta/meta.h
View File

@ -18,8 +18,8 @@
#include "mallocator.h" #include "mallocator.h"
#include "os.h" #include "os.h"
#include "trow.h"
#include "tmsg.h" #include "tmsg.h"
#include "trow.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -47,35 +47,13 @@ int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg);
int metaDropTable(SMeta *pMeta, tb_uid_t uid); int metaDropTable(SMeta *pMeta, tb_uid_t uid);
int metaCommit(SMeta *pMeta); int metaCommit(SMeta *pMeta);
// For Query
int metaGetTableInfo(SMeta *pMeta, char *tbname, STableMetaMsg **ppMsg);
// Options // Options
void metaOptionsInit(SMetaCfg *pMetaCfg); void metaOptionsInit(SMetaCfg *pMetaCfg);
void metaOptionsClear(SMetaCfg *pMetaCfg); void metaOptionsClear(SMetaCfg *pMetaCfg);
// STbCfg
#define META_INIT_STB_CFG(NAME, TTL, KEEP, SUID, PSCHEMA, PTAGSCHEMA) \
{ \
.name = (NAME), .ttl = (TTL), .keep = (KEEP), .type = META_SUPER_TABLE, .stbCfg = { \
.suid = (SUID), \
.pSchema = (PSCHEMA), \
.pTagSchema = (PTAGSCHEMA) \
} \
}
#define META_INIT_CTB_CFG(NAME, TTL, KEEP, SUID, PTAG) \
{ \
.name = (NAME), .ttl = (TTL), .keep = (KEEP), .type = META_CHILD_TABLE, .ctbCfg = {.suid = (SUID), .pTag = PTAG } \
}
#define META_INIT_NTB_CFG(NAME, TTL, KEEP, SUID, PSCHEMA) \
{ \
.name = (NAME), .ttl = (TTL), .keep = (KEEP), .type = META_NORMAL_TABLE, .ntbCfg = {.pSchema = (PSCHEMA) } \
}
#define META_CLEAR_TB_CFG(pTbCfg)
int metaEncodeTbCfg(void **pBuf, STbCfg *pTbCfg);
void *metaDecodeTbCfg(void *pBuf, STbCfg *pTbCfg);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

72
include/dnode/vnode/vnode.h
View File

@ -162,16 +162,6 @@ int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
*/ */
int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
/**
* @brief Process a consume message.
*
* @param pVnode The vnode object.
* @param pMsg The request message
* @param pRsp The response message
* @return int 0 for success, -1 for failure
*/
int vnodeProcessConsumeReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
/* ------------------------ SVnodeCfg ------------------------ */ /* ------------------------ SVnodeCfg ------------------------ */
/** /**
* @brief Initialize VNODE options. * @brief Initialize VNODE options.
@ -187,68 +177,6 @@ void vnodeOptionsInit(SVnodeCfg *pOptions);
*/ */
void vnodeOptionsClear(SVnodeCfg *pOptions); void vnodeOptionsClear(SVnodeCfg *pOptions);
/* ------------------------ REQUESTS ------------------------ */
typedef STbCfg SVCreateTableReq;
typedef struct {
tb_uid_t uid;
} SVDropTableReq;
typedef struct {
// TODO
} SVSubmitReq;
typedef struct {
uint64_t ver;
union {
SVCreateTableReq ctReq;
SVDropTableReq dtReq;
};
} SVnodeReq;
typedef struct {
int err;
char info[];
} SVnodeRsp;
static FORCE_INLINE void vnodeSetCreateStbReq(SVnodeReq *pReq, char *name, uint32_t ttl, uint32_t keep, tb_uid_t suid,
STSchema *pSchema, STSchema *pTagSchema) {
pReq->ver = 0;
pReq->ctReq.name = name;
pReq->ctReq.ttl = ttl;
pReq->ctReq.keep = keep;
pReq->ctReq.type = META_SUPER_TABLE;
pReq->ctReq.stbCfg.suid = suid;
pReq->ctReq.stbCfg.pSchema = pSchema;
pReq->ctReq.stbCfg.pTagSchema = pTagSchema;
}
static FORCE_INLINE void vnodeSetCreateCtbReq(SVnodeReq *pReq, char *name, uint32_t ttl, uint32_t keep, tb_uid_t suid,
SKVRow pTag) {
pReq->ver = 0;
pReq->ctReq.name = name;
pReq->ctReq.ttl = ttl;
pReq->ctReq.keep = keep;
pReq->ctReq.type = META_CHILD_TABLE;
pReq->ctReq.ctbCfg.suid = suid;
pReq->ctReq.ctbCfg.pTag = pTag;
}
static FORCE_INLINE void vnodeSetCreateNtbReq(SVnodeReq *pReq, char *name, uint32_t ttl, uint32_t keep,
STSchema *pSchema) {
pReq->ver = 0;
pReq->ctReq.name = name;
pReq->ctReq.ttl = ttl;
pReq->ctReq.keep = keep;
pReq->ctReq.type = META_NORMAL_TABLE;
pReq->ctReq.ntbCfg.pSchema = pSchema;
}
int vnodeBuildReq(void **buf, const SVnodeReq *pReq, tmsg_t type);
void *vnodeParseReq(void *buf, SVnodeReq *pReq, tmsg_t type);
/* ------------------------ FOR COMPILE ------------------------ */ /* ------------------------ FOR COMPILE ------------------------ */
int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg); int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg);

31
include/libs/catalog/catalog.h
View File

@ -21,13 +21,14 @@ extern "C" {
#endif #endif
#include "os.h" #include "os.h"
#include "thash.h"
#include "tarray.h"
#include "taosdef.h" #include "taosdef.h"
#include "transport.h"
#include "common.h"
#include "tmsg.h"
#include "query.h" #include "query.h"
#include "tname.h"
#include "common.h"
#include "tarray.h"
#include "thash.h"
#include "tmsg.h"
#include "transport.h"
struct SCatalog; struct SCatalog;
@ -45,7 +46,6 @@ typedef struct SMetaData {
} SMetaData; } SMetaData;
typedef struct SCatalogCfg { typedef struct SCatalogCfg {
bool enableVgroupCache;
uint32_t maxTblCacheNum; uint32_t maxTblCacheNum;
uint32_t maxDBCacheNum; uint32_t maxDBCacheNum;
} SCatalogCfg; } SCatalogCfg;
@ -61,43 +61,40 @@ int32_t catalogInit(SCatalogCfg *cfg);
int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle);
int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version);
int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo);
int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo);
/** /**
* Get a table's meta data. * Get a table's meta data.
* @param pCatalog (input, got with catalogGetHandle) * @param pCatalog (input, got with catalogGetHandle)
* @param pTransporter (input, rpc object) * @param pTransporter (input, rpc object)
* @param pMgmtEps (input, mnode EPs) * @param pMgmtEps (input, mnode EPs)
* @param pDBName (input, full db name)
* @param pTableName (input, table name, NOT including db name) * @param pTableName (input, table name, NOT including db name)
* @param pTableMeta(output, table meta data, NEED to free it by calller) * @param pTableMeta(output, table meta data, NEED to free it by calller)
* @return error code * @return error code
*/ */
int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta);
/** /**
* Force renew a table's local cached meta data. * Force renew a table's local cached meta data.
* @param pCatalog (input, got with catalogGetHandle) * @param pCatalog (input, got with catalogGetHandle)
* @param pRpc (input, rpc object) * @param pRpc (input, rpc object)
* @param pMgmtEps (input, mnode EPs) * @param pMgmtEps (input, mnode EPs)
* @param pDBName (input, full db name)
* @param pTableName (input, table name, NOT including db name) * @param pTableName (input, table name, NOT including db name)
* @return error code * @return error code
*/ */
int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName); int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName);
/** /**
* Force renew a table's local cached meta data and get the new one. * Force renew a table's local cached meta data and get the new one.
* @param pCatalog (input, got with catalogGetHandle) * @param pCatalog (input, got with catalogGetHandle)
* @param pRpc (input, rpc object) * @param pRpc (input, rpc object)
* @param pMgmtEps (input, mnode EPs) * @param pMgmtEps (input, mnode EPs)
* @param pDBName (input, full db name)
* @param pTableName (input, table name, NOT including db name) * @param pTableName (input, table name, NOT including db name)
* @param pTableMeta(output, table meta data, NEED to free it by calller) * @param pTableMeta(output, table meta data, NEED to free it by calller)
* @return error code * @return error code
*/ */
int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta);
/** /**
@ -105,24 +102,22 @@ int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const
* @param pCatalog (input, got with catalogGetHandle) * @param pCatalog (input, got with catalogGetHandle)
* @param pRpc (input, rpc object) * @param pRpc (input, rpc object)
* @param pMgmtEps (input, mnode EPs) * @param pMgmtEps (input, mnode EPs)
* @param pDBName (input, full db name)
* @param pTableName (input, table name, NOT including db name) * @param pTableName (input, table name, NOT including db name)
* @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller)
* @return error code * @return error code
*/ */
int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray** pVgroupList); int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList);
/** /**
* Get a table's vgroup from its name's hash value. * Get a table's vgroup from its name's hash value.
* @param pCatalog (input, got with catalogGetHandle) * @param pCatalog (input, got with catalogGetHandle)
* @param pTransporter (input, rpc object) * @param pTransporter (input, rpc object)
* @param pMgmtEps (input, mnode EPs) * @param pMgmtEps (input, mnode EPs)
* @param pDBName (input, full db name)
* @param pTableName (input, table name, NOT including db name) * @param pTableName (input, table name, NOT including db name)
* @param vgInfo (output, vgroup info) * @param vgInfo (output, vgroup info)
* @return error code * @return error code
*/ */
int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo); int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pName, SVgroupInfo* vgInfo);
/** /**

1
include/libs/parser/parsenodes.h
View File

@ -166,6 +166,7 @@ typedef struct SInsertStmtInfo {
typedef struct SDclStmtInfo { typedef struct SDclStmtInfo {
int16_t nodeType; int16_t nodeType;
int16_t msgType; int16_t msgType;
SEpSet epSet;
char* pMsg; char* pMsg;
int32_t msgLen; int32_t msgLen;
} SDclStmtInfo; } SDclStmtInfo;

2
include/libs/parser/parser.h
View File

@ -27,7 +27,7 @@ typedef struct SParseContext {
int8_t schemaAttached; // denote if submit block is built with table schema or not int8_t schemaAttached; // denote if submit block is built with table schema or not
const char *pSql; // sql string const char *pSql; // sql string
size_t sqlLen; // length of the sql string size_t sqlLen; // length of the sql string
char *pMsg; // extended error message if exists to help avoid the problem in sql statement. char *pMsg; // extended error message if exists to help identifying the problem in sql statement.
int32_t msgLen; // max length of the msg int32_t msgLen; // max length of the msg
} SParseContext; } SParseContext;

2
include/libs/qcom/query.h
View File

@ -23,6 +23,7 @@ extern "C" {
#include "tarray.h" #include "tarray.h"
#include "thash.h" #include "thash.h"
#include "tlog.h" #include "tlog.h"
#include "tmsg.h"
enum { enum {
JOB_TASK_STATUS_NULL = 0, JOB_TASK_STATUS_NULL = 0,
@ -73,7 +74,6 @@ typedef struct STableMeta {
SSchema schema[]; SSchema schema[];
} STableMeta; } STableMeta;
typedef struct SDBVgroupInfo { typedef struct SDBVgroupInfo {
int32_t vgVersion; int32_t vgVersion;
int8_t hashMethod; int8_t hashMethod;

1
include/libs/scheduler/scheduler.h
View File

@ -24,7 +24,6 @@ extern "C" {
#include "catalog.h" #include "catalog.h"
typedef struct SSchedulerCfg { typedef struct SSchedulerCfg {
int32_t clusterType;
int32_t maxJobNum; int32_t maxJobNum;
} SSchedulerCfg; } SSchedulerCfg;

3
include/os/osEndian.h
View File

@ -20,8 +20,11 @@
extern "C" { extern "C" {
#endif #endif
typedef enum { TD_LITTLE_ENDIAN = 0, TD_BIG_ENDIAN } td_endian_t;
static const int32_t endian_test_var = 1; static const int32_t endian_test_var = 1;
#define IS_LITTLE_ENDIAN() (*(uint8_t *)(&endian_test_var) != 0) #define IS_LITTLE_ENDIAN() (*(uint8_t *)(&endian_test_var) != 0)
#define TD_RT_ENDIAN() (IS_LITTLE_ENDIAN() ? TD_LITTLE_ENDIAN : TD_BIG_ENDIAN)
#ifdef __cplusplus #ifdef __cplusplus
} }

476
include/util/encode.h Normal file
View File

@ -0,0 +1,476 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_ENCODE_H_
#define _TD_UTIL_ENCODE_H_
#include "tcoding.h"
#include "tmacro.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
td_endian_t endian;
uint8_t* data;
int64_t size;
int64_t pos;
} SEncoder, SDecoder;
#define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL)
#define tGet(TYPE, BUF, VAL) (VAL) = ((TYPE*)(BUF))[0]
#define tRPut16(PDEST, PSRC) \
((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[1]; \
((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[0];
#define tRPut32(PDEST, PSRC) \
((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[3]; \
((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[2]; \
((uint8_t*)(PDEST))[2] = ((uint8_t*)(PSRC))[1]; \
((uint8_t*)(PDEST))[3] = ((uint8_t*)(PSRC))[0];
#define tRPut64(PDEST, PSRC) \
((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[7]; \
((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[6]; \
((uint8_t*)(PDEST))[2] = ((uint8_t*)(PSRC))[5]; \
((uint8_t*)(PDEST))[3] = ((uint8_t*)(PSRC))[4]; \
((uint8_t*)(PDEST))[4] = ((uint8_t*)(PSRC))[3]; \
((uint8_t*)(PDEST))[5] = ((uint8_t*)(PSRC))[2]; \
((uint8_t*)(PDEST))[6] = ((uint8_t*)(PSRC))[1]; \
((uint8_t*)(PDEST))[7] = ((uint8_t*)(PSRC))[0];
#define tRGet16 tRPut16
#define tRGet32 tRPut32
#define tRGet64 tRPut64
#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos)
#define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE))
#define TD_CHECK_CODER_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
/* ------------------------ FOR ENCODER ------------------------ */
static FORCE_INLINE void tInitEncoder(SEncoder* pEncoder, td_endian_t endian, uint8_t* data, int64_t size) {
pEncoder->endian = endian;
pEncoder->data = data;
pEncoder->size = (data) ? size : 0;
pEncoder->pos = 0;
}
// 8
static FORCE_INLINE int tEncodeU8(SEncoder* pEncoder, uint8_t val) {
if (pEncoder->data) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
tPut(uint8_t, TD_CODER_CURRENT(pEncoder), val);
}
TD_CODER_MOVE_POS(pEncoder, sizeof(val));
return 0;
}
static FORCE_INLINE int tEncodeI8(SEncoder* pEncoder, int8_t val) {
if (pEncoder->data) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
tPut(int8_t, TD_CODER_CURRENT(pEncoder), val);
}
TD_CODER_MOVE_POS(pEncoder, sizeof(val));
return 0;
}
// 16
static FORCE_INLINE int tEncodeU16(SEncoder* pEncoder, uint16_t val) {
if (pEncoder->data) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
if (TD_RT_ENDIAN() == pEncoder->endian) {
tPut(uint16_t, TD_CODER_CURRENT(pEncoder), val);
} else {
tRPut16(TD_CODER_CURRENT(pEncoder), &val);
}
}
TD_CODER_MOVE_POS(pEncoder, sizeof(val));
return 0;
}
static FORCE_INLINE int tEncodeI16(SEncoder* pEncoder, int16_t val) {
if (pEncoder->data) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
if (TD_RT_ENDIAN() == pEncoder->endian) {
tPut(int16_t, TD_CODER_CURRENT(pEncoder), val);
} else {
tRPut16(TD_CODER_CURRENT(pEncoder), &val);
}
}
TD_CODER_MOVE_POS(pEncoder, sizeof(val));
return 0;
}
// 32
static FORCE_INLINE int tEncodeU32(SEncoder* pEncoder, uint32_t val) {
if (pEncoder->data) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
if (TD_RT_ENDIAN() == pEncoder->endian) {
tPut(uint32_t, TD_CODER_CURRENT(pEncoder), val);
} else {
tRPut32(TD_CODER_CURRENT(pEncoder), &val);
}
}
TD_CODER_MOVE_POS(pEncoder, sizeof(val));
return 0;
}
static FORCE_INLINE int tEncodeI32(SEncoder* pEncoder, int32_t val) {
if (pEncoder->data) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
if (TD_RT_ENDIAN() == pEncoder->endian) {
tPut(int32_t, TD_CODER_CURRENT(pEncoder), val);
} else {
tRPut32(TD_CODER_CURRENT(pEncoder), &val);
}
}
TD_CODER_MOVE_POS(pEncoder, sizeof(val));
return 0;
}
// 64
static FORCE_INLINE int tEncodeU64(SEncoder* pEncoder, uint64_t val) {
if (pEncoder->data) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
if (TD_RT_ENDIAN() == pEncoder->endian) {
tPut(uint64_t, TD_CODER_CURRENT(pEncoder), val);
} else {
tRPut64(TD_CODER_CURRENT(pEncoder), &val);
}
}
TD_CODER_MOVE_POS(pEncoder, sizeof(val));
return 0;
}
static FORCE_INLINE int tEncodeI64(SEncoder* pEncoder, int64_t val) {
if (pEncoder->data) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
if (TD_RT_ENDIAN() == pEncoder->endian) {
tPut(int64_t, TD_CODER_CURRENT(pEncoder), val);
} else {
tRPut64(TD_CODER_CURRENT(pEncoder), &val);
}
}
TD_CODER_MOVE_POS(pEncoder, sizeof(val));
return 0;
}
// 16v
static FORCE_INLINE int tEncodeU16v(SEncoder* pEncoder, uint16_t val) {
int64_t i = 0;
while (val >= ENCODE_LIMIT) {
if (pEncoder->data) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1;
TD_CODER_CURRENT(pEncoder)[i] = (val | ENCODE_LIMIT) & 0xff;
}
val >>= 7;
i++;
}
if (pEncoder->data) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1;
TD_CODER_CURRENT(pEncoder)[i] = (uint8_t)val;
}
TD_CODER_MOVE_POS(pEncoder, i + 1);
return 0;
}
static FORCE_INLINE int tEncodeI16v(SEncoder* pEncoder, int16_t val) {
return tEncodeU16v(pEncoder, ZIGZAGE(int16_t, val));
}
// 32v
static FORCE_INLINE int tEncodeU32v(SEncoder* pEncoder, uint32_t val) {
int64_t i = 0;
while (val >= ENCODE_LIMIT) {
if (pEncoder->data) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1;
TD_CODER_CURRENT(pEncoder)[i] = (val | ENCODE_LIMIT) & 0xff;
}
val >>= 7;
i++;
}
if (pEncoder->data) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1;
TD_CODER_CURRENT(pEncoder)[i] = (uint8_t)val;
}
TD_CODER_MOVE_POS(pEncoder, i + 1);
return 0;
}
static FORCE_INLINE int tEncodeI32v(SEncoder* pEncoder, int32_t val) {
return tEncodeU32v(pEncoder, ZIGZAGE(int32_t, val));
}
// 64v
static FORCE_INLINE int tEncodeU64v(SEncoder* pEncoder, uint64_t val) {
int64_t i = 0;
while (val >= ENCODE_LIMIT) {
if (pEncoder->data) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1;
TD_CODER_CURRENT(pEncoder)[i] = (val | ENCODE_LIMIT) & 0xff;
}
val >>= 7;
i++;
}
if (pEncoder->data) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1;
TD_CODER_CURRENT(pEncoder)[i] = (uint8_t)val;
}
TD_CODER_MOVE_POS(pEncoder, i + 1);
return 0;
}
static FORCE_INLINE int tEncodeI64v(SEncoder* pEncoder, int64_t val) {
return tEncodeU64v(pEncoder, ZIGZAGE(int64_t, val));
}
static FORCE_INLINE int tEncodeFloat(SEncoder* pEncoder, float val) {
// TODO
return 0;
}
static FORCE_INLINE int tEncodeDouble(SEncoder* pEncoder, double val) {
// TODO
return 0;
}
static FORCE_INLINE int tEncodeCStr(SEncoder* pEncoder, const char* val) {
// TODO
return 0;
}
/* ------------------------ FOR DECODER ------------------------ */
static FORCE_INLINE void tInitDecoder(SDecoder* pDecoder, td_endian_t endian, uint8_t* data, int64_t size) {
ASSERT(!TD_IS_NULL(data));
pDecoder->endian = endian;
pDecoder->data = data;
pDecoder->size = size;
pDecoder->pos = 0;
}
// 8
static FORCE_INLINE int tDecodeU8(SDecoder* pDecoder, uint8_t* val) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
tGet(uint8_t, TD_CODER_CURRENT(pDecoder), *val);
TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
return 0;
}
static FORCE_INLINE int tDecodeI8(SDecoder* pDecoder, int8_t* val) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
tGet(int8_t, TD_CODER_CURRENT(pDecoder), *val);
TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
return 0;
}
// 16
static FORCE_INLINE int tDecodeU16(SDecoder* pDecoder, uint16_t* val) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
if (TD_RT_ENDIAN() == pDecoder->endian) {
tGet(uint16_t, TD_CODER_CURRENT(pDecoder), *val);
} else {
tRGet16(val, TD_CODER_CURRENT(pDecoder));
}
TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
return 0;
}
static FORCE_INLINE int tDecodeI16(SDecoder* pDecoder, int16_t* val) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
if (TD_RT_ENDIAN() == pDecoder->endian) {
tGet(int16_t, TD_CODER_CURRENT(pDecoder), *val);
} else {
tRGet16(val, TD_CODER_CURRENT(pDecoder));
}
TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
return 0;
}
// 32
static FORCE_INLINE int tDecodeU32(SDecoder* pDecoder, uint32_t* val) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
if (TD_RT_ENDIAN() == pDecoder->endian) {
tGet(uint32_t, TD_CODER_CURRENT(pDecoder), *val);
} else {
tRGet32(val, TD_CODER_CURRENT(pDecoder));
}
TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
return 0;
}
static FORCE_INLINE int tDecodeI32(SDecoder* pDecoder, int32_t* val) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
if (TD_RT_ENDIAN() == pDecoder->endian) {
tGet(int32_t, TD_CODER_CURRENT(pDecoder), *val);
} else {
tRGet32(val, TD_CODER_CURRENT(pDecoder));
}
TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
return 0;
}
// 64
static FORCE_INLINE int tDecodeU64(SDecoder* pDecoder, uint64_t* val) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
if (TD_RT_ENDIAN() == pDecoder->endian) {
tGet(uint64_t, TD_CODER_CURRENT(pDecoder), *val);
} else {
tRGet64(val, TD_CODER_CURRENT(pDecoder));
}
TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
return 0;
}
static FORCE_INLINE int tDecodeI64(SDecoder* pDecoder, int64_t* val) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
if (TD_RT_ENDIAN() == pDecoder->endian) {
tGet(int64_t, TD_CODER_CURRENT(pDecoder), *val);
} else {
tRGet64(val, TD_CODER_CURRENT(pDecoder));
}
TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
return 0;
}
// 16v
static FORCE_INLINE int tDecodeU16v(SDecoder* pDecoder, uint16_t* val) {
int64_t i = 0;
*val = 0;
for (;;) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, 1)) return -1;
uint16_t tval = TD_CODER_CURRENT(pDecoder)[i];
if (tval < ENCODE_LIMIT) {
(*val) |= (tval << (7 * i));
break;
} else {
(*val) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i));
i++;
}
}
TD_CODER_MOVE_POS(pDecoder, i);
return 0;
}
static FORCE_INLINE int tDecodeI16v(SDecoder* pDecoder, int16_t* val) {
uint16_t tval;
if (tDecodeU16v(pDecoder, &tval) < 0) {
return -1;
}
*val = ZIGZAGD(int16_t, tval);
return 0;
}
// 32v
static FORCE_INLINE int tDecodeU32v(SDecoder* pDecoder, uint32_t* val) {
int64_t i = 0;
*val = 0;
for (;;) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, 1)) return -1;
uint32_t tval = TD_CODER_CURRENT(pDecoder)[i];
if (tval < ENCODE_LIMIT) {
(*val) |= (tval << (7 * i));
break;
} else {
(*val) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i));
i++;
}
}
TD_CODER_MOVE_POS(pDecoder, i);
return 0;
}
static FORCE_INLINE int tDecodeI32v(SDecoder* pDecoder, int32_t* val) {
uint32_t tval;
if (tDecodeU32v(pDecoder, &tval) < 0) {
return -1;
}
*val = ZIGZAGD(int32_t, tval);
return 0;
}
// 64v
static FORCE_INLINE int tDecodeU64v(SDecoder* pDecoder, uint64_t* val) {
int64_t i = 0;
*val = 0;
for (;;) {
if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, 1)) return -1;
uint64_t tval = TD_CODER_CURRENT(pDecoder)[i];
if (tval < ENCODE_LIMIT) {
(*val) |= (tval << (7 * i));
break;
} else {
(*val) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i));
i++;
}
}
TD_CODER_MOVE_POS(pDecoder, i);
return 0;
}
static FORCE_INLINE int tDecodeI64v(SDecoder* pDecoder, int64_t* val) {
uint64_t tval;
if (tDecodeU64v(pDecoder, &tval) < 0) {
return -1;
}
*val = ZIGZAGD(int64_t, tval);
return 0;
}
static FORCE_INLINE int tDecodeFloat(SDecoder* pDecoder, float* val) {
// TODO
return 0;
}
static FORCE_INLINE int tDecodeDouble(SDecoder* pDecoder, double* val) {
// TODO
return 0;
}
static FORCE_INLINE int tDecodeCStr(SDecoder* pEncoder, const char** val) {
// TODO
return 0;
}
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_ENCODE_H_*/

67
include/util/tbuffer.h
View File

@ -16,6 +16,8 @@
#ifndef _TD_UTIL_BUFFER_H #ifndef _TD_UTIL_BUFFER_H
#define _TD_UTIL_BUFFER_H #define _TD_UTIL_BUFFER_H
#include "os.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
@ -90,48 +92,17 @@ typedef struct SBufferWriter {
#define tbufTell(buf) ((buf)->pos) #define tbufTell(buf) ((buf)->pos)
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
// reader functions & macros /* ------------------------ BUFFER WRITER FUNCTIONS AND MACROS ------------------------ */
// *Endian*, if true, reader functions of primitive types will do 'ntoh' automatically
#define tbufInitReader( Data, Size, Endian ) {.endian = (Endian), .data = (Data), .pos = 0, .size = ((Data) == NULL ? 0 :(Size))}
size_t tbufSkip( SBufferReader* buf, size_t size );
const char* tbufRead( SBufferReader* buf, size_t size );
void tbufReadToBuffer( SBufferReader* buf, void* dst, size_t size );
const char* tbufReadString( SBufferReader* buf, size_t* len );
size_t tbufReadToString( SBufferReader* buf, char* dst, size_t size );
const char* tbufReadBinary( SBufferReader* buf, size_t *len );
size_t tbufReadToBinary( SBufferReader* buf, void* dst, size_t size );
bool tbufReadBool( SBufferReader* buf );
char tbufReadChar( SBufferReader* buf );
int8_t tbufReadInt8( SBufferReader* buf );
uint8_t tbufReadUint8( SBufferReader* buf );
int16_t tbufReadInt16( SBufferReader* buf );
uint16_t tbufReadUint16( SBufferReader* buf );
int32_t tbufReadInt32( SBufferReader* buf );
uint32_t tbufReadUint32( SBufferReader* buf );
int64_t tbufReadInt64( SBufferReader* buf );
uint64_t tbufReadUint64( SBufferReader* buf );
float tbufReadFloat( SBufferReader* buf );
double tbufReadDouble( SBufferReader* buf );
////////////////////////////////////////////////////////////////////////////////
// writer functions & macros
// *Allocator*, function to allocate memory, will use 'realloc' if NULL // *Allocator*, function to allocate memory, will use 'realloc' if NULL
// *Endian*, if true, writer functions of primitive types will do 'hton' automatically // *Endian*, if true, writer functions of primitive types will do 'hton' automatically
#define tbufInitWriter( Allocator, Endian ) {.endian = (Endian), .data = NULL, .pos = 0, .size = 0, .allocator = ((Allocator) == NULL ? realloc : (Allocator))} #define tbufInitWriter(Allocator, Endian) \
void tbufCloseWriter( SBufferWriter* buf ); { .endian = (Endian), .data = NULL, .pos = 0, .size = 0, .allocator = ((Allocator) == NULL ? realloc : (Allocator)) }
void tbufCloseWriter(SBufferWriter* buf);
void tbufEnsureCapacity(SBufferWriter* buf, size_t size); void tbufEnsureCapacity(SBufferWriter* buf, size_t size);
size_t tbufReserve(SBufferWriter* buf, size_t size); size_t tbufReserve(SBufferWriter* buf, size_t size);
char* tbufGetData(SBufferWriter* buf, bool takeOver); char* tbufGetData(SBufferWriter* buf, bool takeOver);
void tbufWrite(SBufferWriter* buf, const void* data, size_t size); void tbufWrite(SBufferWriter* buf, const void* data, size_t size);
void tbufWriteAt(SBufferWriter* buf, size_t pos, const void* data, size_t size); void tbufWriteAt(SBufferWriter* buf, size_t pos, const void* data, size_t size);
void tbufWriteStringLen(SBufferWriter* buf, const char* str, size_t len); void tbufWriteStringLen(SBufferWriter* buf, const char* str, size_t len);
@ -142,7 +113,6 @@ void tbufWriteString( SBufferWriter* buf, const char* str );
// size before read. Write only write the data itself, which means the reader // size before read. Write only write the data itself, which means the reader
// need to know data size before read. // need to know data size before read.
void tbufWriteBinary(SBufferWriter* buf, const void* data, size_t len); void tbufWriteBinary(SBufferWriter* buf, const void* data, size_t len);
void tbufWriteBool(SBufferWriter* buf, bool data); void tbufWriteBool(SBufferWriter* buf, bool data);
void tbufWriteBoolAt(SBufferWriter* buf, size_t pos, bool data); void tbufWriteBoolAt(SBufferWriter* buf, size_t pos, bool data);
void tbufWriteChar(SBufferWriter* buf, char data); void tbufWriteChar(SBufferWriter* buf, char data);
@ -168,6 +138,31 @@ void tbufWriteFloatAt( SBufferWriter* buf, size_t pos, float data );
void tbufWriteDouble(SBufferWriter* buf, double data); void tbufWriteDouble(SBufferWriter* buf, double data);
void tbufWriteDoubleAt(SBufferWriter* buf, size_t pos, double data); void tbufWriteDoubleAt(SBufferWriter* buf, size_t pos, double data);
/* ------------------------ BUFFER READER FUNCTIONS AND MACROS ------------------------ */
// *Endian*, if true, reader functions of primitive types will do 'ntoh' automatically
#define tbufInitReader(Data, Size, Endian) \
{ .endian = (Endian), .data = (Data), .pos = 0, .size = ((Data) == NULL ? 0 : (Size)) }
size_t tbufSkip(SBufferReader* buf, size_t size);
const char* tbufRead(SBufferReader* buf, size_t size);
void tbufReadToBuffer(SBufferReader* buf, void* dst, size_t size);
const char* tbufReadString(SBufferReader* buf, size_t* len);
size_t tbufReadToString(SBufferReader* buf, char* dst, size_t size);
const char* tbufReadBinary(SBufferReader* buf, size_t* len);
size_t tbufReadToBinary(SBufferReader* buf, void* dst, size_t size);
bool tbufReadBool(SBufferReader* buf);
char tbufReadChar(SBufferReader* buf);
int8_t tbufReadInt8(SBufferReader* buf);
uint8_t tbufReadUint8(SBufferReader* buf);
int16_t tbufReadInt16(SBufferReader* buf);
uint16_t tbufReadUint16(SBufferReader* buf);
int32_t tbufReadInt32(SBufferReader* buf);
uint32_t tbufReadUint32(SBufferReader* buf);
int64_t tbufReadInt64(SBufferReader* buf);
uint64_t tbufReadUint64(SBufferReader* buf);
float tbufReadFloat(SBufferReader* buf);
double tbufReadDouble(SBufferReader* buf);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

4
include/util/tcoding.h
View File

@ -25,6 +25,8 @@ extern "C" {
#define ZIGZAGE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode #define ZIGZAGE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode
#define ZIGZAGD(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode #define ZIGZAGD(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode
/* ------------------------ LEGACY CODES ------------------------ */
#if 1
// ---- Fixed U8 // ---- Fixed U8
static FORCE_INLINE int taosEncodeFixedU8(void **buf, uint8_t value) { static FORCE_INLINE int taosEncodeFixedU8(void **buf, uint8_t value) {
if (buf != NULL) { if (buf != NULL) {
@ -368,6 +370,8 @@ static FORCE_INLINE void *taosDecodeStringTo(void *buf, char *value) {
return POINTER_SHIFT(buf, size); return POINTER_SHIFT(buf, size);
} }
#endif
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

2
include/util/tdef.h
View File

@ -317,7 +317,7 @@ do { \
#define TSDB_MAX_FIELD_LEN 16384 #define TSDB_MAX_FIELD_LEN 16384
#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 #define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384
#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 #define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384
#define PRIMARYKEY_TIMESTAMP_COL_ID 0 #define PRIMARYKEY_TIMESTAMP_COL_ID 1
#define TSDB_MAX_RPC_THREADS 5 #define TSDB_MAX_RPC_THREADS 5

2
include/util/tmacro.h
View File

@ -35,6 +35,8 @@ typedef int8_t td_mode_flag_t;
#define TD_CHECK_AND_SET_MOD_CLEAR(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNCLEARD, TD_MOD_CLEARD) #define TD_CHECK_AND_SET_MOD_CLEAR(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNCLEARD, TD_MOD_CLEARD)
#define TD_IS_NULL(PTR) ((PTR) == NULL)
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

38
source/client/src/clientImpl.c
View File

@ -181,43 +181,13 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) {
pRequest->body.requestMsg = (SDataBuf){.pData = pDcl->pMsg, .len = pDcl->msgLen}; pRequest->body.requestMsg = (SDataBuf){.pData = pDcl->pMsg, .len = pDcl->msgLen};
STscObj* pTscObj = pRequest->pTscObj; STscObj* pTscObj = pRequest->pTscObj;
SMsgSendInfo* pSendMsg = buildSendMsgInfoImpl(pRequest); SMsgSendInfo* pSendMsg = buildSendMsgInfoImpl(pRequest);
SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet;
int64_t transporterId = 0;
if (pDcl->msgType == TDMT_VND_CREATE_TABLE) { if (pDcl->msgType == TDMT_VND_CREATE_TABLE) {
struct SCatalog* pCatalog = NULL; asyncSendMsgToServer(pTscObj->pTransporter, &pDcl->epSet, &transporterId, pSendMsg);
char buf[18] = {0};
sprintf(buf, "%" PRId64, pRequest->pTscObj->pAppInfo->clusterId);
int32_t code = catalogGetHandle(buf, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
SCreateTableMsg* pMsg = pSendMsg->msgInfo.pData;
SName t = {0};
tNameFromString(&t, pMsg->name, T_NAME_ACCT|T_NAME_DB|T_NAME_TABLE);
char db[TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN + TSDB_ACCT_ID_LEN] = {0};
tNameGetFullDbName(&t, db);
SVgroupInfo info = {0};
catalogGetTableHashVgroup(pCatalog, pRequest->pTscObj->pTransporter, pEpSet, db, tNameGetTableName(&t), &info);
int64_t transporterId = 0;
SEpSet ep = {0};
ep.inUse = info.inUse;
ep.numOfEps = info.numOfEps;
for(int32_t i = 0; i < ep.numOfEps; ++i) {
ep.port[i] = info.epAddr[i].port;
tstrncpy(ep.fqdn[i], info.epAddr[i].fqdn, tListLen(ep.fqdn[i]));
}
asyncSendMsgToServer(pTscObj->pTransporter, &ep, &transporterId, pSendMsg);
} else { } else {
int64_t transporterId = 0; SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet;
asyncSendMsgToServer(pTscObj->pTransporter, pEpSet, &transporterId, pSendMsg); asyncSendMsgToServer(pTscObj->pTransporter, pEpSet, &transporterId, pSendMsg);
} }
@ -410,7 +380,7 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest) {
pMsgSendInfo->msgInfo.len = sizeof(SConnectMsg); pMsgSendInfo->msgInfo.len = sizeof(SConnectMsg);
pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->requestObjRefId = pRequest->self;
pMsgSendInfo->requestId = pRequest->requestId; pMsgSendInfo->requestId = pRequest->requestId;
pMsgSendInfo->fp = handleRequestRspFp[pMsgSendInfo->msgType]; pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)];
pMsgSendInfo->param = pRequest; pMsgSendInfo->param = pRequest;
SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg)); SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg));

27
source/client/src/clientMsgHandler.c
View File

@ -79,7 +79,7 @@ static int32_t buildRetrieveMnodeMsg(SRequestObj *pRequest, SMsgSendInfo* pMsgSe
pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableMsg); pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableMsg);
pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->requestObjRefId = pRequest->self;
pMsgSendInfo->param = pRequest; pMsgSendInfo->param = pRequest;
pMsgSendInfo->fp = handleRequestRspFp[pMsgSendInfo->msgType]; pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)];
SRetrieveTableMsg *pRetrieveMsg = calloc(1, sizeof(SRetrieveTableMsg)); SRetrieveTableMsg *pRetrieveMsg = calloc(1, sizeof(SRetrieveTableMsg));
if (pRetrieveMsg == NULL) { if (pRetrieveMsg == NULL) {
@ -104,7 +104,7 @@ SMsgSendInfo* buildSendMsgInfoImpl(SRequestObj *pRequest) {
pMsgSendInfo->requestId = pRequest->requestId; pMsgSendInfo->requestId = pRequest->requestId;
pMsgSendInfo->param = pRequest; pMsgSendInfo->param = pRequest;
pMsgSendInfo->fp = (handleRequestRspFp[pRequest->type] == NULL)? genericRspCallback:handleRequestRspFp[pRequest->type]; pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)? genericRspCallback:handleRequestRspFp[TMSG_INDEX(pRequest->type)];
} }
return pMsgSendInfo; return pMsgSendInfo;
@ -188,6 +188,14 @@ int32_t processCreateDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
} }
int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
SRequestObj* pRequest = param;
if (code != TSDB_CODE_SUCCESS) {
pRequest->code = code;
tsem_post(&pRequest->body.rspSem);
return code;
}
SUseDbRsp* pUseDbRsp = (SUseDbRsp*)pMsg->pData; SUseDbRsp* pUseDbRsp = (SUseDbRsp*)pMsg->pData;
SName name = {0}; SName name = {0};
tNameFromString(&name, pUseDbRsp->db, T_NAME_ACCT | T_NAME_DB); tNameFromString(&name, pUseDbRsp->db, T_NAME_ACCT | T_NAME_DB);
@ -195,7 +203,6 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
char db[TSDB_DB_NAME_LEN] = {0}; char db[TSDB_DB_NAME_LEN] = {0};
tNameGetDbName(&name, db); tNameGetDbName(&name, db);
SRequestObj* pRequest = param;
setConnectionDB(pRequest->pTscObj, db); setConnectionDB(pRequest->pTscObj, db);
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);
@ -290,11 +297,11 @@ void initMsgHandleFp() {
tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp;
#endif #endif
handleRequestRspFp[TDMT_MND_CONNECT] = processConnectRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_CONNECT)] = processConnectRsp;
handleRequestRspFp[TDMT_MND_SHOW] = processShowRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_SHOW)] = processShowRsp;
handleRequestRspFp[TDMT_MND_SHOW_RETRIEVE] = processRetrieveMnodeRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_SHOW_RETRIEVE)] = processRetrieveMnodeRsp;
handleRequestRspFp[TDMT_MND_CREATE_DB] = processCreateDbRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = processCreateDbRsp;
handleRequestRspFp[TDMT_MND_USE_DB] = processUseDbRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_USE_DB)] = processUseDbRsp;
handleRequestRspFp[TDMT_MND_CREATE_STB] = processCreateTableRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = processCreateTableRsp;
handleRequestRspFp[TDMT_MND_DROP_DB] = processDropDbRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = processDropDbRsp;
} }

91
source/client/test/clientTests.cpp
View File

@ -163,7 +163,7 @@ TEST(testCase, create_db_Test) {
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create database abc1"); pRes = taos_query(pConn, "create database abc1 vgroups 4");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("error in create db, reason:%s\n", taos_errstr(pRes)); printf("error in create db, reason:%s\n", taos_errstr(pRes));
} }
@ -178,14 +178,14 @@ TEST(testCase, create_dnode_Test) {
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); printf("error in create dnode, reason:%s\n", taos_errstr(pRes));
} }
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
ASSERT_TRUE(pFields == NULL);
int32_t numOfFields = taos_num_fields(pRes);
ASSERT_EQ(numOfFields, 0);
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000");
if (taos_errno(pRes) != 0) {
printf("failed to create dnode, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
taos_close(pConn); taos_close(pConn);
} }
@ -401,50 +401,49 @@ TEST(testCase, drop_stable_Test) {
} }
#endif #endif
TEST(testCase, create_topic_Test) { //TEST(testCase, create_topic_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "create database abc1");
if (taos_errno(pRes) != 0) {
printf("error in create db, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) {
printf("error in use db, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)");
if (taos_errno(pRes) != 0) {
printf("error in create stable, reason:%s\n", taos_errstr(pRes));
}
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
ASSERT_TRUE(pFields == NULL);
int32_t numOfFields = taos_num_fields(pRes);
ASSERT_EQ(numOfFields, 0);
taos_free_result(pRes);
char* sql = "select * from st1";
tmq_create_topic(pConn, "test_topic_1", sql, strlen(sql));
taos_close(pConn);
}
//TEST(testCase, show_table_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL); // assert(pConn != NULL);
// //
// TAOS_RES* pRes = taos_query(pConn, "use abc1"); // TAOS_RES* pRes = taos_query(pConn, "create database abc1");
// if (taos_errno(pRes) != 0) {
// printf("error in create db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes); // taos_free_result(pRes);
// //
// pRes = taos_query(pConn, "show tables"); // pRes = taos_query(pConn, "use abc1");
// if (taos_errno(pRes) != 0) {
// printf("error in use db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes); // taos_free_result(pRes);
// //
// pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)");
// if (taos_errno(pRes) != 0) {
// printf("error in create stable, reason:%s\n", taos_errstr(pRes));
// }
//
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// ASSERT_TRUE(pFields == NULL);
//
// int32_t numOfFields = taos_num_fields(pRes);
// ASSERT_EQ(numOfFields, 0);
//
// taos_free_result(pRes);
//
// char* sql = "select * from st1";
// tmq_create_topic(pConn, "test_topic_1", sql, strlen(sql));
// taos_close(pConn); // taos_close(pConn);
//} //}
TEST(testCase, show_table_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "use abc1");
taos_free_result(pRes);
pRes = taos_query(pConn, "show tables");
taos_free_result(pRes);
taos_close(pConn);
}

1
source/common/src/tvariant.c
View File

@ -16,7 +16,6 @@
#include "taos.h" #include "taos.h"
#include "taosdef.h" #include "taosdef.h"
#include "thash.h"
#include "ttime.h" #include "ttime.h"
#include "ttokendef.h" #include "ttokendef.h"
#include "ttypes.h" #include "ttypes.h"

4
source/dnode/bnode/src/bnode.c
View File

@ -15,7 +15,7 @@
#include "bndInt.h" #include "bndInt.h"
SBnode *bndOpen(const SBnodeOpt *pOption) { SBnode *bndOpen(const char *path, const SBnodeOpt *pOption) {
SBnode *pBnode = calloc(1, sizeof(SBnode)); SBnode *pBnode = calloc(1, sizeof(SBnode));
return pBnode; return pBnode;
} }
@ -25,3 +25,5 @@ void bndClose(SBnode *pBnode) { free(pBnode); }
int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad) { return 0; } int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad) { return 0; }
int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs) { return 0; } int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs) { return 0; }
void bndDestroy(const char *path) {}

2
source/dnode/mgmt/daemon/src/daemon.c
View File

@ -139,7 +139,7 @@ void dmnWaitSignal() {
void dmnInitOption(SDnodeOpt *pOption) { void dmnInitOption(SDnodeOpt *pOption) {
pOption->sver = 30000000; //3.0.0.0 pOption->sver = 30000000; //3.0.0.0
pOption->numOfCores = tsNumOfCores; pOption->numOfCores = tsNumOfCores;
pOption->numOfSupportVnodes = 1; pOption->numOfSupportVnodes = 16;
pOption->numOfCommitThreads = 1; pOption->numOfCommitThreads = 1;
pOption->statusInterval = tsStatusInterval; pOption->statusInterval = tsStatusInterval;
pOption->numOfThreadsPerCore = tsNumOfThreadsPerCore; pOption->numOfThreadsPerCore = tsNumOfThreadsPerCore;

2
source/dnode/mgmt/impl/inc/dndBnode.h
View File

@ -24,7 +24,7 @@ extern "C" {
int32_t dndInitBnode(SDnode *pDnode); int32_t dndInitBnode(SDnode *pDnode);
void dndCleanupBnode(SDnode *pDnode); void dndCleanupBnode(SDnode *pDnode);
ioid dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);

31
source/dnode/mgmt/impl/inc/dndInt.h
View File

@ -54,18 +54,21 @@ extern int32_t dDebugFlag;
#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat; typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat;
typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EDndWorkerType; typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType;
typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps); typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps);
typedef struct { typedef struct {
EDndWorkerType type; EWorkerType type;
const char *name; const char *name;
int32_t minNum; int32_t minNum;
int32_t maxNum; int32_t maxNum;
FProcessItem fp; void *queueFp;
SDnode *pDnode; SDnode *pDnode;
taos_queue queue; taos_queue queue;
union {
SWorkerPool pool; SWorkerPool pool;
SMWorkerPool mpool;
};
} SDnodeWorker; } SDnodeWorker;
typedef struct { typedef struct {
@ -97,18 +100,14 @@ typedef struct {
int32_t refCount; int32_t refCount;
int8_t deployed; int8_t deployed;
int8_t dropped; int8_t dropped;
SMnode *pMnode;
SRWLatch latch;
SDnodeWorker readWorker;
SDnodeWorker writeWorker;
SDnodeWorker syncWorker;
int8_t replica; int8_t replica;
int8_t selfIndex; int8_t selfIndex;
SReplica replicas[TSDB_MAX_REPLICA]; SReplica replicas[TSDB_MAX_REPLICA];
char *file;
SMnode *pMnode;
SRWLatch latch;
taos_queue pReadQ;
taos_queue pWriteQ;
taos_queue pSyncQ;
SWorkerPool readPool;
SWorkerPool writePool;
SWorkerPool syncPool;
} SMnodeMgmt; } SMnodeMgmt;
typedef struct { typedef struct {
@ -125,22 +124,18 @@ typedef struct {
int32_t refCount; int32_t refCount;
int8_t deployed; int8_t deployed;
int8_t dropped; int8_t dropped;
char *file;
SSnode *pSnode; SSnode *pSnode;
SRWLatch latch; SRWLatch latch;
taos_queue pWriteQ; SDnodeWorker writeWorker;
SWorkerPool writePool;
} SSnodeMgmt; } SSnodeMgmt;
typedef struct { typedef struct {
int32_t refCount; int32_t refCount;
int8_t deployed; int8_t deployed;
int8_t dropped; int8_t dropped;
char *file;
SBnode *pBnode; SBnode *pBnode;
SRWLatch latch; SRWLatch latch;
taos_queue pWriteQ; SDnodeWorker writeWorker;
SMWorkerPool writePool;
} SBnodeMgmt; } SBnodeMgmt;
typedef struct { typedef struct {

4
source/dnode/mgmt/impl/inc/dndWorker.h
View File

@ -21,8 +21,8 @@ extern "C" {
#endif #endif
#include "dndInt.h" #include "dndInt.h"
int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EDndWorkerType type, const char *name, int32_t minNum, int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum,
int32_t maxNum, FProcessItem fp); int32_t maxNum, void *queueFp);
void dndCleanupWorker(SDnodeWorker *pWorker); void dndCleanupWorker(SDnodeWorker *pWorker);
int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen); int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen);

379
source/dnode/mgmt/impl/src/dndBnode.c Normal file
View File

@ -0,0 +1,379 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http:www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dndBnode.h"
#include "dndDnode.h"
#include "dndTransport.h"
#include "dndWorker.h"
static void dndProcessBnodeQueue(SDnode *pDnode, taos_qall qall, int32_t numOfMsgs);
static SBnode *dndAcquireBnode(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
SBnode *pBnode = NULL;
int32_t refCount = 0;
taosRLockLatch(&pMgmt->latch);
if (pMgmt->deployed && !pMgmt->dropped) {
refCount = atomic_add_fetch_32(&pMgmt->refCount, 1);
pBnode = pMgmt->pBnode;
} else {
terrno = TSDB_CODE_DND_BNODE_NOT_DEPLOYED;
}
taosRUnLockLatch(&pMgmt->latch);
if (pBnode != NULL) {
dTrace("acquire bnode, refCount:%d", refCount);
}
return pBnode;
}
static void dndReleaseBnode(SDnode *pDnode, SBnode *pBnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
int32_t refCount = 0;
taosRLockLatch(&pMgmt->latch);
if (pBnode != NULL) {
refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1);
}
taosRUnLockLatch(&pMgmt->latch);
if (pBnode != NULL) {
dTrace("release bnode, refCount:%d", refCount);
}
}
static int32_t dndReadBnodeFile(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
int32_t code = TSDB_CODE_DND_BNODE_READ_FILE_ERROR;
int32_t len = 0;
int32_t maxLen = 4096;
char *content = calloc(1, maxLen + 1);
cJSON *root = NULL;
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_BNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_BNODE_OVER;
}
content[len] = 0;
root = cJSON_Parse(content);
if (root == NULL) {
dError("failed to read %s since invalid json format", file);
goto PRASE_BNODE_OVER;
}
cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
if (!deployed || deployed->type != cJSON_Number) {
dError("failed to read %s since deployed not found", file);
goto PRASE_BNODE_OVER;
}
pMgmt->deployed = deployed->valueint;
cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
if (!dropped || dropped->type != cJSON_Number) {
dError("failed to read %s since dropped not found", file);
goto PRASE_BNODE_OVER;
}
pMgmt->dropped = dropped->valueint;
code = 0;
dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped);
PRASE_BNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
terrno = code;
return code;
}
static int32_t dndWriteBnodeFile(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
}
int32_t len = 0;
int32_t maxLen = 4096;
char *content = calloc(1, maxLen + 1);
len += snprintf(content + len, maxLen - len, "{\n");
len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed);
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
free(content);
char realfile[PATH_MAX + 20];
snprintf(realfile, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
if (taosRenameFile(file, realfile) != 0) {
terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR;
dError("failed to rename %s since %s", file, terrstr());
return -1;
}
dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
return 0;
}
static int32_t dndStartBnodeWorker(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_MULTI, "bnode-write", 0, 1, dndProcessBnodeQueue) != 0) {
dError("failed to start bnode write worker since %s", terrstr());
return -1;
}
return 0;
}
static void dndStopBnodeWorker(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
taosWLockLatch(&pMgmt->latch);
pMgmt->deployed = 0;
taosWUnLockLatch(&pMgmt->latch);
while (pMgmt->refCount > 1) {
taosMsleep(10);
}
dndCleanupWorker(&pMgmt->writeWorker);
}
static void dndBuildBnodeOption(SDnode *pDnode, SBnodeOpt *pOption) {
pOption->pDnode = pDnode;
pOption->sendMsgToDnodeFp = dndSendMsgToDnode;
pOption->sendMsgToMnodeFp = dndSendMsgToMnode;
pOption->sendRedirectMsgFp = dndSendRedirectMsg;
pOption->dnodeId = dndGetDnodeId(pDnode);
pOption->clusterId = dndGetClusterId(pDnode);
pOption->cfg.sver = pDnode->opt.sver;
}
static int32_t dndOpenBnode(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
SBnodeOpt option = {0};
dndBuildBnodeOption(pDnode, &option);
SBnode *pBnode = bndOpen(pDnode->dir.bnode, &option);
if (pBnode == NULL) {
dError("failed to open bnode since %s", terrstr());
return -1;
}
if (dndStartBnodeWorker(pDnode) != 0) {
dError("failed to start bnode worker since %s", terrstr());
bndClose(pBnode);
return -1;
}
pMgmt->deployed = 1;
if (dndWriteBnodeFile(pDnode) != 0) {
pMgmt->deployed = 0;
dError("failed to write bnode file since %s", terrstr());
dndStopBnodeWorker(pDnode);
bndClose(pBnode);
return -1;
}
taosWLockLatch(&pMgmt->latch);
pMgmt->pBnode = pBnode;
taosWUnLockLatch(&pMgmt->latch);
dInfo("bnode open successfully");
return 0;
}
static int32_t dndDropBnode(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
SBnode *pBnode = dndAcquireBnode(pDnode);
if (pBnode == NULL) {
dError("failed to drop bnode since %s", terrstr());
return -1;
}
taosRLockLatch(&pMgmt->latch);
pMgmt->dropped = 1;
taosRUnLockLatch(&pMgmt->latch);
if (dndWriteBnodeFile(pDnode) != 0) {
taosRLockLatch(&pMgmt->latch);
pMgmt->dropped = 0;
taosRUnLockLatch(&pMgmt->latch);
dndReleaseBnode(pDnode, pBnode);
dError("failed to drop bnode since %s", terrstr());
return -1;
}
dndReleaseBnode(pDnode, pBnode);
dndStopBnodeWorker(pDnode);
pMgmt->deployed = 0;
dndWriteBnodeFile(pDnode);
bndClose(pBnode);
pMgmt->pBnode = NULL;
bndDestroy(pDnode->dir.bnode);
return 0;
}
int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
SCreateBnodeInMsg *pMsg = pRpcMsg->pCont;
pMsg->dnodeId = htonl(pMsg->dnodeId);
if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
terrno = TSDB_CODE_DND_BNODE_ID_INVALID;
return -1;
} else {
return dndOpenBnode(pDnode);
}
}
int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
SDropBnodeInMsg *pMsg = pRpcMsg->pCont;
pMsg->dnodeId = htonl(pMsg->dnodeId);
if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
terrno = TSDB_CODE_DND_BNODE_ID_INVALID;
return -1;
} else {
return dndDropBnode(pDnode);
}
}
static void dndSendBnodeErrorRsp(SRpcMsg *pMsg, int32_t code) {
SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
rpcSendResponse(&rpcRsp);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
static void dndSendBnodeErrorRsps(taos_qall qall, int32_t numOfMsgs, int32_t code) {
for (int32_t i = 0; i < numOfMsgs; ++i) {
SRpcMsg *pMsg = NULL;
taosGetQitem(qall, (void **)&pMsg);
dndSendBnodeErrorRsp(pMsg, code);
}
}
static void dndProcessBnodeQueue(SDnode *pDnode, taos_qall qall, int32_t numOfMsgs) {
SBnode *pBnode = dndAcquireBnode(pDnode);
if (pBnode == NULL) {
dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY);
return;
}
SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *));
if (pArray == NULL) {
dndReleaseBnode(pDnode, pBnode);
dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY);
return;
}
for (int32_t i = 0; i < numOfMsgs; ++i) {
SRpcMsg *pMsg = NULL;
taosGetQitem(qall, (void **)&pMsg);
void *ptr = taosArrayPush(pArray, &pMsg);
if (ptr == NULL) {
dndSendBnodeErrorRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY);
}
}
bndProcessWMsgs(pBnode, pArray);
for (size_t i = 0; i < numOfMsgs; i++) {
SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
taosArrayDestroy(pArray);
dndReleaseBnode(pDnode, pBnode);
}
static void dndWriteBnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) {
int32_t code = TSDB_CODE_DND_BNODE_NOT_DEPLOYED;
SBnode *pBnode = dndAcquireBnode(pDnode);
if (pBnode != NULL) {
code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
}
dndReleaseBnode(pDnode, pBnode);
if (code != 0) {
if (pMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
rpcSendResponse(&rsp);
}
rpcFreeCont(pMsg->pCont);
}
}
void dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
dndWriteBnodeMsgToWorker(pDnode, &pDnode->bmgmt.writeWorker, pMsg);
}
int32_t dndInitBnode(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
taosInitRWLatch(&pMgmt->latch);
if (dndReadBnodeFile(pDnode) != 0) {
return -1;
}
if (pMgmt->dropped) {
dInfo("bnode has been deployed and needs to be deleted");
bndDestroy(pDnode->dir.bnode);
return 0;
}
if (!pMgmt->deployed) return 0;
return dndOpenBnode(pDnode);
}
void dndCleanupBnode(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
if (pMgmt->pBnode) {
dndStopBnodeWorker(pDnode);
bndClose(pMgmt->pBnode);
pMgmt->pBnode = NULL;
}
}

23
source/dnode/mgmt/impl/src/dndDnode.c
View File

@ -393,13 +393,11 @@ void dndSendStatusMsg(SDnode *pDnode) {
static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) { static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) {
SDnodeMgmt *pMgmt = &pDnode->dmgmt; SDnodeMgmt *pMgmt = &pDnode->dmgmt;
if (pMgmt->dnodeId == 0 || pMgmt->dropped != pCfg->dropped) { if (pMgmt->dnodeId == 0) {
dInfo("set dnodeId:%d clusterId:% " PRId64 " dropped:%d", pCfg->dnodeId, pCfg->clusterId, pCfg->dropped); dInfo("set dnodeId:%d clusterId:% " PRId64, pCfg->dnodeId, pCfg->clusterId);
taosWLockLatch(&pMgmt->latch); taosWLockLatch(&pMgmt->latch);
pMgmt->dnodeId = pCfg->dnodeId; pMgmt->dnodeId = pCfg->dnodeId;
pMgmt->clusterId = pCfg->clusterId; pMgmt->clusterId = pCfg->clusterId;
pMgmt->dropped = pCfg->dropped;
dndWriteDnodes(pDnode); dndWriteDnodes(pDnode);
taosWUnLockLatch(&pMgmt->latch); taosWUnLockLatch(&pMgmt->latch);
} }
@ -430,6 +428,11 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg) {
if (pMsg->code != TSDB_CODE_SUCCESS) { if (pMsg->code != TSDB_CODE_SUCCESS) {
pMgmt->statusSent = 0; pMgmt->statusSent = 0;
if (pMsg->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->dropped && pMgmt->dnodeId > 0) {
dInfo("dnode:%d, set to dropped since not exist in mnode", pMgmt->dnodeId);
pMgmt->dropped = 1;
dndWriteDnodes(pDnode);
}
return; return;
} }
@ -439,11 +442,6 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg) {
pCfg->clusterId = htobe64(pCfg->clusterId); pCfg->clusterId = htobe64(pCfg->clusterId);
dndUpdateDnodeCfg(pDnode, pCfg); dndUpdateDnodeCfg(pDnode, pCfg);
if (pCfg->dropped) {
pMgmt->statusSent = 0;
return;
}
SDnodeEps *pDnodeEps = &pRsp->dnodeEps; SDnodeEps *pDnodeEps = &pRsp->dnodeEps;
pDnodeEps->num = htonl(pDnodeEps->num); pDnodeEps->num = htonl(pDnodeEps->num);
for (int32_t i = 0; i < pDnodeEps->num; ++i) { for (int32_t i = 0; i < pDnodeEps->num; ++i) {
@ -487,7 +485,7 @@ static void *dnodeThreadRoutine(void *param) {
pthread_testcancel(); pthread_testcancel();
taosMsleep(ms); taosMsleep(ms);
if (dndGetStat(pDnode) == DND_STAT_RUNNING && !pMgmt->statusSent) { if (dndGetStat(pDnode) == DND_STAT_RUNNING && !pMgmt->statusSent && !pMgmt->dropped) {
dndSendStatusMsg(pDnode); dndSendStatusMsg(pDnode);
} }
} }
@ -522,6 +520,11 @@ int32_t dndInitDnode(SDnode *pDnode) {
return -1; return -1;
} }
if (pMgmt->dropped) {
dError("dnode will not start for its already dropped");
return -1;
}
if (dndInitMgmtWorker(pDnode) != 0) { if (dndInitMgmtWorker(pDnode) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;

356
source/dnode/mgmt/impl/src/dndMnode.c
View File

@ -17,42 +17,9 @@
#include "dndMnode.h" #include "dndMnode.h"
#include "dndDnode.h" #include "dndDnode.h"
#include "dndTransport.h" #include "dndTransport.h"
#include "dndWorker.h"
static int32_t dndInitMnodeReadWorker(SDnode *pDnode); static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg);
static int32_t dndInitMnodeWriteWorker(SDnode *pDnode);
static int32_t dndInitMnodeSyncWorker(SDnode *pDnode);
static void dndCleanupMnodeReadWorker(SDnode *pDnode);
static void dndCleanupMnodeWriteWorker(SDnode *pDnode);
static void dndCleanupMnodeSyncWorker(SDnode *pDnode);
static void dndCleanupMnodeMgmtWorker(SDnode *pDnode);
static int32_t dndAllocMnodeReadQueue(SDnode *pDnode);
static int32_t dndAllocMnodeWriteQueue(SDnode *pDnode);
static int32_t dndAllocMnodeSyncQueue(SDnode *pDnode);
static void dndFreeMnodeReadQueue(SDnode *pDnode);
static void dndFreeMnodeWriteQueue(SDnode *pDnode);
static void dndFreeMnodeSyncQueue(SDnode *pDnode);
static void dndFreeMnodeMgmtQueue(SDnode *pDnode);
static void dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg);
static void dndProcessMnodeWriteQueue(SDnode *pDnode, SMnodeMsg *pMsg);
static void dndProcessMnodeSyncQueue(SDnode *pDnode, SMnodeMsg *pMsg);
static int32_t dndWriteMnodeMsgToQueue(SMnode *pMnode, taos_queue pQueue, SRpcMsg *pRpcMsg);
void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
static int32_t dndStartMnodeWorker(SDnode *pDnode);
static void dndStopMnodeWorker(SDnode *pDnode);
static SMnode *dndAcquireMnode(SDnode *pDnode);
static void dndReleaseMnode(SDnode *pDnode, SMnode *pMnode);
static int32_t dndReadMnodeFile(SDnode *pDnode);
static int32_t dndWriteMnodeFile(SDnode *pDnode);
static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOpt *pOption);
static int32_t dndAlterMnode(SDnode *pDnode, SMnodeOpt *pOption);
static int32_t dndDropMnode(SDnode *pDnode);
static SMnode *dndAcquireMnode(SDnode *pDnode) { static SMnode *dndAcquireMnode(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt; SMnodeMgmt *pMgmt = &pDnode->mmgmt;
@ -97,49 +64,52 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) {
char *content = calloc(1, maxLen + 1); char *content = calloc(1, maxLen + 1);
cJSON *root = NULL; cJSON *root = NULL;
FILE *fp = fopen(pMgmt->file, "r"); char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) { if (fp == NULL) {
dDebug("file %s not exist", pMgmt->file); dDebug("file %s not exist", file);
code = 0; code = 0;
goto PRASE_MNODE_OVER; goto PRASE_MNODE_OVER;
} }
len = (int32_t)fread(content, 1, maxLen, fp); len = (int32_t)fread(content, 1, maxLen, fp);
if (len <= 0) { if (len <= 0) {
dError("failed to read %s since content is null", pMgmt->file); dError("failed to read %s since content is null", file);
goto PRASE_MNODE_OVER; goto PRASE_MNODE_OVER;
} }
content[len] = 0; content[len] = 0;
root = cJSON_Parse(content); root = cJSON_Parse(content);
if (root == NULL) { if (root == NULL) {
dError("failed to read %s since invalid json format", pMgmt->file); dError("failed to read %s since invalid json format", file);
goto PRASE_MNODE_OVER; goto PRASE_MNODE_OVER;
} }
cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
if (!deployed || deployed->type != cJSON_Number) { if (!deployed || deployed->type != cJSON_Number) {
dError("failed to read %s since deployed not found", pMgmt->file); dError("failed to read %s since deployed not found", file);
goto PRASE_MNODE_OVER; goto PRASE_MNODE_OVER;
} }
pMgmt->deployed = deployed->valueint; pMgmt->deployed = deployed->valueint;
cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
if (!dropped || dropped->type != cJSON_Number) { if (!dropped || dropped->type != cJSON_Number) {
dError("failed to read %s since dropped not found", pMgmt->file); dError("failed to read %s since dropped not found", file);
goto PRASE_MNODE_OVER; goto PRASE_MNODE_OVER;
} }
pMgmt->dropped = dropped->valueint; pMgmt->dropped = dropped->valueint;
cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes"); cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes");
if (!mnodes || mnodes->type != cJSON_Array) { if (!mnodes || mnodes->type != cJSON_Array) {
dError("failed to read %s since nodes not found", pMgmt->file); dError("failed to read %s since nodes not found", file);
goto PRASE_MNODE_OVER; goto PRASE_MNODE_OVER;
} }
pMgmt->replica = cJSON_GetArraySize(mnodes); pMgmt->replica = cJSON_GetArraySize(mnodes);
if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) { if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) {
dError("failed to read %s since mnodes size %d invalid", pMgmt->file, pMgmt->replica); dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica);
goto PRASE_MNODE_OVER; goto PRASE_MNODE_OVER;
} }
@ -151,28 +121,28 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) {
cJSON *id = cJSON_GetObjectItem(node, "id"); cJSON *id = cJSON_GetObjectItem(node, "id");
if (!id || id->type != cJSON_Number) { if (!id || id->type != cJSON_Number) {
dError("failed to read %s since id not found", pMgmt->file); dError("failed to read %s since id not found", file);
goto PRASE_MNODE_OVER; goto PRASE_MNODE_OVER;
} }
pReplica->id = id->valueint; pReplica->id = id->valueint;
cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn"); cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn");
if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) { if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) {
dError("failed to read %s since fqdn not found", pMgmt->file); dError("failed to read %s since fqdn not found", file);
goto PRASE_MNODE_OVER; goto PRASE_MNODE_OVER;
} }
tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN);
cJSON *port = cJSON_GetObjectItem(node, "port"); cJSON *port = cJSON_GetObjectItem(node, "port");
if (!port || port->type != cJSON_Number) { if (!port || port->type != cJSON_Number) {
dError("failed to read %s since port not found", pMgmt->file); dError("failed to read %s since port not found", file);
goto PRASE_MNODE_OVER; goto PRASE_MNODE_OVER;
} }
pReplica->port = port->valueint; pReplica->port = port->valueint;
} }
code = 0; code = 0;
dDebug("succcessed to read file %s, deployed:%d dropped:%d", pMgmt->file, pMgmt->deployed, pMgmt->dropped); dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped);
PRASE_MNODE_OVER: PRASE_MNODE_OVER:
if (content != NULL) free(content); if (content != NULL) free(content);
@ -186,8 +156,8 @@ PRASE_MNODE_OVER:
static int32_t dndWriteMnodeFile(SDnode *pDnode) { static int32_t dndWriteMnodeFile(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt; SMnodeMgmt *pMgmt = &pDnode->mmgmt;
char file[PATH_MAX + 20] = {0}; char file[PATH_MAX + 20];
snprintf(file, sizeof(file), "%s.bak", pMgmt->file); snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode);
FILE *fp = fopen(file, "w"); FILE *fp = fopen(file, "w");
if (fp == NULL) { if (fp == NULL) {
@ -223,47 +193,36 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) {
fclose(fp); fclose(fp);
free(content); free(content);
if (taosRenameFile(file, pMgmt->file) != 0) { char realfile[PATH_MAX + 20];
snprintf(realfile, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode);
if (taosRenameFile(file, realfile) != 0) {
terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR;
dError("failed to rename %s since %s", pMgmt->file, terrstr()); dError("failed to rename %s since %s", file, terrstr());
return -1; return -1;
} }
dInfo("successed to write %s, deployed:%d dropped:%d", pMgmt->file, pMgmt->deployed, pMgmt->dropped); dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
return 0; return 0;
} }
static int32_t dndStartMnodeWorker(SDnode *pDnode) { static int32_t dndStartMnodeWorker(SDnode *pDnode) {
if (dndInitMnodeReadWorker(pDnode) != 0) { SMnodeMgmt *pMgmt = &pDnode->mmgmt;
if (dndInitWorker(pDnode, &pMgmt->readWorker, DND_WORKER_SINGLE, "mnode-read", 0, 1, dndProcessMnodeQueue) != 0) {
dError("failed to start mnode read worker since %s", terrstr()); dError("failed to start mnode read worker since %s", terrstr());
return -1; return -1;
} }
if (dndInitMnodeWriteWorker(pDnode) != 0) { if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "mnode-write", 0, 1, dndProcessMnodeQueue) != 0) {
dError("failed to start mnode write worker since %s", terrstr()); dError("failed to start mnode write worker since %s", terrstr());
return -1; return -1;
} }
if (dndInitMnodeSyncWorker(pDnode) != 0) { if (dndInitWorker(pDnode, &pMgmt->syncWorker, DND_WORKER_SINGLE, "mnode-sync", 0, 1, dndProcessMnodeQueue) != 0) {
dError("failed to start mnode sync worker since %s", terrstr()); dError("failed to start mnode sync worker since %s", terrstr());
return -1; return -1;
} }
if (dndAllocMnodeReadQueue(pDnode) != 0) {
dError("failed to alloc mnode read queue since %s", terrstr());
return -1;
}
if (dndAllocMnodeWriteQueue(pDnode) != 0) {
dError("failed to alloc mnode write queue since %s", terrstr());
return -1;
}
if (dndAllocMnodeSyncQueue(pDnode) != 0) {
dError("failed to alloc mnode sync queue since %s", terrstr());
return -1;
}
return 0; return 0;
} }
@ -274,18 +233,13 @@ static void dndStopMnodeWorker(SDnode *pDnode) {
pMgmt->deployed = 0; pMgmt->deployed = 0;
taosWUnLockLatch(&pMgmt->latch); taosWUnLockLatch(&pMgmt->latch);
while (pMgmt->refCount > 1) taosMsleep(10); while (pMgmt->refCount > 1) {
while (!taosQueueEmpty(pMgmt->pReadQ)) taosMsleep(10); taosMsleep(10);
while (!taosQueueEmpty(pMgmt->pWriteQ)) taosMsleep(10); }
while (!taosQueueEmpty(pMgmt->pSyncQ)) taosMsleep(10);
dndCleanupMnodeReadWorker(pDnode); dndCleanupWorker(&pMgmt->readWorker);
dndCleanupMnodeWriteWorker(pDnode); dndCleanupWorker(&pMgmt->writeWorker);
dndCleanupMnodeSyncWorker(pDnode); dndCleanupWorker(&pMgmt->syncWorker);
dndFreeMnodeReadQueue(pDnode);
dndFreeMnodeWriteQueue(pDnode);
dndFreeMnodeSyncQueue(pDnode);
} }
static bool dndNeedDeployMnode(SDnode *pDnode) { static bool dndNeedDeployMnode(SDnode *pDnode) {
@ -383,28 +337,21 @@ static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOpt *pOption) {
dError("failed to open mnode since %s", terrstr()); dError("failed to open mnode since %s", terrstr());
return -1; return -1;
} }
pMgmt->deployed = 1;
int32_t code = dndWriteMnodeFile(pDnode); if (dndStartMnodeWorker(pDnode) != 0) {
if (code != 0) { dError("failed to start mnode worker since %s", terrstr());
dError("failed to write mnode file since %s", terrstr());
code = terrno;
pMgmt->deployed = 0;
mndClose(pMnode); mndClose(pMnode);
mndDestroy(pDnode->dir.mnode); mndDestroy(pDnode->dir.mnode);
terrno = code;
return -1; return -1;
} }
code = dndStartMnodeWorker(pDnode); pMgmt->deployed = 1;
if (code != 0) { if (dndWriteMnodeFile(pDnode) != 0) {
dError("failed to start mnode worker since %s", terrstr()); dError("failed to write mnode file since %s", terrstr());
code = terrno;
pMgmt->deployed = 0; pMgmt->deployed = 0;
dndStopMnodeWorker(pDnode); dndStopMnodeWorker(pDnode);
mndClose(pMnode); mndClose(pMnode);
mndDestroy(pDnode->dir.mnode); mndDestroy(pDnode->dir.mnode);
terrno = code;
return -1; return -1;
} }
@ -461,6 +408,7 @@ static int32_t dndDropMnode(SDnode *pDnode) {
dndReleaseMnode(pDnode, pMnode); dndReleaseMnode(pDnode, pMnode);
dndStopMnodeWorker(pDnode); dndStopMnodeWorker(pDnode);
pMgmt->deployed = 0;
dndWriteMnodeFile(pDnode); dndWriteMnodeFile(pDnode);
mndClose(pMnode); mndClose(pMnode);
pMgmt->pMnode = NULL; pMgmt->pMnode = NULL;
@ -528,13 +476,12 @@ int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
} }
} }
static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
static void dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt; SMnodeMgmt *pMgmt = &pDnode->mmgmt;
SMnode *pMnode = dndAcquireMnode(pDnode); SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode != NULL) { if (pMnode != NULL) {
mndProcessReadMsg(pMsg); mndProcessMsg(pMsg);
dndReleaseMnode(pDnode, pMnode); dndReleaseMnode(pDnode, pMnode);
} else { } else {
mndSendRsp(pMsg, terrno); mndSendRsp(pMsg, terrno);
@ -543,208 +490,43 @@ static void dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
mndCleanupMsg(pMsg); mndCleanupMsg(pMsg);
} }
static void dndProcessMnodeWriteQueue(SDnode *pDnode, SMnodeMsg *pMsg) { static void dndWriteMnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pRpcMsg) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt; int32_t code = TSDB_CODE_DND_MNODE_NOT_DEPLOYED;
SMnode *pMnode = dndAcquireMnode(pDnode); SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode != NULL) { if (pMnode != NULL) {
mndProcessWriteMsg(pMsg);
dndReleaseMnode(pDnode, pMnode);
} else {
mndSendRsp(pMsg, terrno);
}
mndCleanupMsg(pMsg);
}
static void dndProcessMnodeSyncQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode != NULL) {
mndProcessSyncMsg(pMsg);
dndReleaseMnode(pDnode, pMnode);
} else {
mndSendRsp(pMsg, terrno);
}
mndCleanupMsg(pMsg);
}
static int32_t dndWriteMnodeMsgToQueue(SMnode *pMnode, taos_queue pQueue, SRpcMsg *pRpcMsg) {
SMnodeMsg *pMsg = mndInitMsg(pMnode, pRpcMsg); SMnodeMsg *pMsg = mndInitMsg(pMnode, pRpcMsg);
if (pMsg == NULL) { if (pMsg == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; } else {
code = dndWriteMsgToWorker(pWorker, pMsg, 0);
} }
if (taosWriteQitem(pQueue, pMsg) != 0) { if (code != 0) {
mndCleanupMsg(pMsg); mndCleanupMsg(pMsg);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
}
dndReleaseMnode(pDnode, pMnode);
return 0; if (code != 0) {
if (pRpcMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pRpcMsg->handle, .ahandle = pRpcMsg->ahandle, .code = code};
rpcSendResponse(&rsp);
}
rpcFreeCont(pRpcMsg->pCont);
}
} }
void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt; dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.writeWorker, pMsg);
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pWriteQ, pMsg) != 0) {
if (pMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
rpcSendResponse(&rsp);
}
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
}
dndReleaseMnode(pDnode, pMnode);
} }
void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt; dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.syncWorker, pMsg);
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pSyncQ, pMsg) != 0) {
if (pMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
rpcSendResponse(&rsp);
}
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
}
dndReleaseMnode(pDnode, pMnode);
} }
void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt; dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.readWorker, pMsg);
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pReadQ, pMsg) != 0) {
if (pMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
rpcSendResponse(&rsp);
}
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
}
dndReleaseMnode(pDnode, pMnode);
}
static int32_t dndAllocMnodeReadQueue(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
pMgmt->pReadQ = tWorkerAllocQueue(&pMgmt->readPool, pDnode, (FProcessItem)dndProcessMnodeReadQueue);
if (pMgmt->pReadQ == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
static void dndFreeMnodeReadQueue(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
tWorkerFreeQueue(&pMgmt->readPool, pMgmt->pReadQ);
pMgmt->pReadQ = NULL;
}
static int32_t dndInitMnodeReadWorker(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
SWorkerPool *pPool = &pMgmt->readPool;
pPool->name = "mnode-read";
pPool->min = 0;
pPool->max = 1;
if (tWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
dDebug("mnode read worker is initialized");
return 0;
}
static void dndCleanupMnodeReadWorker(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
tWorkerCleanup(&pMgmt->readPool);
dDebug("mnode read worker is closed");
}
static int32_t dndAllocMnodeWriteQueue(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
pMgmt->pWriteQ = tWorkerAllocQueue(&pMgmt->writePool, pDnode, (FProcessItem)dndProcessMnodeWriteQueue);
if (pMgmt->pWriteQ == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
static void dndFreeMnodeWriteQueue(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
tWorkerFreeQueue(&pMgmt->writePool, pMgmt->pWriteQ);
pMgmt->pWriteQ = NULL;
}
static int32_t dndInitMnodeWriteWorker(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
SWorkerPool *pPool = &pMgmt->writePool;
pPool->name = "mnode-write";
pPool->min = 0;
pPool->max = 1;
if (tWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
dDebug("mnode write worker is initialized");
return 0;
}
static void dndCleanupMnodeWriteWorker(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
tWorkerCleanup(&pMgmt->writePool);
dDebug("mnode write worker is closed");
}
static int32_t dndAllocMnodeSyncQueue(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
pMgmt->pSyncQ = tWorkerAllocQueue(&pMgmt->syncPool, pDnode, (FProcessItem)dndProcessMnodeSyncQueue);
if (pMgmt->pSyncQ == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
static void dndFreeMnodeSyncQueue(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
tWorkerFreeQueue(&pMgmt->syncPool, pMgmt->pSyncQ);
pMgmt->pSyncQ = NULL;
}
static int32_t dndInitMnodeSyncWorker(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
SWorkerPool *pPool = &pMgmt->syncPool;
pPool->name = "mnode-sync";
pPool->min = 0;
pPool->max = 1;
if (tWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
dDebug("mnode sync worker is initialized");
return 0;
}
static void dndCleanupMnodeSyncWorker(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
tWorkerCleanup(&pMgmt->syncPool);
dDebug("mnode sync worker is closed");
} }
int32_t dndInitMnode(SDnode *pDnode) { int32_t dndInitMnode(SDnode *pDnode) {
@ -752,14 +534,6 @@ int32_t dndInitMnode(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt; SMnodeMgmt *pMgmt = &pDnode->mmgmt;
taosInitRWLatch(&pMgmt->latch); taosInitRWLatch(&pMgmt->latch);
char path[PATH_MAX];
snprintf(path, PATH_MAX, "%s/mnode.json", pDnode->dir.dnode);
pMgmt->file = strdup(path);
if (pMgmt->file == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (dndReadMnodeFile(pDnode) != 0) { if (dndReadMnodeFile(pDnode) != 0) {
return -1; return -1;
} }
@ -790,13 +564,13 @@ int32_t dndInitMnode(SDnode *pDnode) {
} }
void dndCleanupMnode(SDnode *pDnode) { void dndCleanupMnode(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
dInfo("dnode-mnode start to clean up"); dInfo("dnode-mnode start to clean up");
if (pMgmt->pMnode) dndStopMnodeWorker(pDnode); SMnodeMgmt *pMgmt = &pDnode->mmgmt;
tfree(pMgmt->file); if (pMgmt->pMnode) {
dndStopMnodeWorker(pDnode);
mndClose(pMgmt->pMnode); mndClose(pMgmt->pMnode);
pMgmt->pMnode = NULL; pMgmt->pMnode = NULL;
}
dInfo("dnode-mnode is cleaned up"); dInfo("dnode-mnode is cleaned up");
} }

19
source/dnode/mgmt/impl/src/dndQnode.c
View File

@ -140,26 +140,27 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) {
fclose(fp); fclose(fp);
free(content); free(content);
if (taosRenameFile(file, file) != 0) { char realfile[PATH_MAX + 20];
snprintf(realfile, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
if (taosRenameFile(file, realfile) != 0) {
terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR;
dError("failed to rename %s since %s", file, terrstr()); dError("failed to rename %s since %s", file, terrstr());
return -1; return -1;
} }
dInfo("successed to write %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped); dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
return 0; return 0;
} }
static int32_t dndStartQnodeWorker(SDnode *pDnode) { static int32_t dndStartQnodeWorker(SDnode *pDnode) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt; SQnodeMgmt *pMgmt = &pDnode->qmgmt;
if (dndInitWorker(pDnode, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, if (dndInitWorker(pDnode, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, dndProcessQnodeQueue) != 0) {
(FProcessItem)dndProcessQnodeQueue) != 0) {
dError("failed to start qnode query worker since %s", terrstr()); dError("failed to start qnode query worker since %s", terrstr());
return -1; return -1;
} }
if (dndInitWorker(pDnode, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, if (dndInitWorker(pDnode, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, dndProcessQnodeQueue) != 0) {
(FProcessItem)dndProcessQnodeQueue) != 0) {
dError("failed to start qnode fetch worker since %s", terrstr()); dError("failed to start qnode fetch worker since %s", terrstr());
return -1; return -1;
} }
@ -209,7 +210,9 @@ static int32_t dndOpenQnode(SDnode *pDnode) {
return -1; return -1;
} }
pMgmt->deployed = 1;
if (dndWriteQnodeFile(pDnode) != 0) { if (dndWriteQnodeFile(pDnode) != 0) {
pMgmt->deployed = 0;
dError("failed to write qnode file since %s", terrstr()); dError("failed to write qnode file since %s", terrstr());
dndStopQnodeWorker(pDnode); dndStopQnodeWorker(pDnode);
qndClose(pQnode); qndClose(pQnode);
@ -218,7 +221,6 @@ static int32_t dndOpenQnode(SDnode *pDnode) {
taosWLockLatch(&pMgmt->latch); taosWLockLatch(&pMgmt->latch);
pMgmt->pQnode = pQnode; pMgmt->pQnode = pQnode;
pMgmt->deployed = 1;
taosWUnLockLatch(&pMgmt->latch); taosWUnLockLatch(&pMgmt->latch);
dInfo("qnode open successfully"); dInfo("qnode open successfully");
@ -250,9 +252,10 @@ static int32_t dndDropQnode(SDnode *pDnode) {
dndReleaseQnode(pDnode, pQnode); dndReleaseQnode(pDnode, pQnode);
dndStopQnodeWorker(pDnode); dndStopQnodeWorker(pDnode);
pMgmt->deployed = 0;
dndWriteQnodeFile(pDnode);
qndClose(pQnode); qndClose(pQnode);
pMgmt->pQnode = NULL; pMgmt->pQnode = NULL;
// qndDestroy(pDnode->dir.qnode);
return 0; return 0;
} }

354
source/dnode/mgmt/impl/src/dndSnode.c Normal file
View File

@ -0,0 +1,354 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http:www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dndSnode.h"
#include "dndDnode.h"
#include "dndTransport.h"
#include "dndWorker.h"
static void dndProcessSnodeQueue(SDnode *pDnode, SRpcMsg *pMsg);
static SSnode *dndAcquireSnode(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
SSnode *pSnode = NULL;
int32_t refCount = 0;
taosRLockLatch(&pMgmt->latch);
if (pMgmt->deployed && !pMgmt->dropped) {
refCount = atomic_add_fetch_32(&pMgmt->refCount, 1);
pSnode = pMgmt->pSnode;
} else {
terrno = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
}
taosRUnLockLatch(&pMgmt->latch);
if (pSnode != NULL) {
dTrace("acquire snode, refCount:%d", refCount);
}
return pSnode;
}
static void dndReleaseSnode(SDnode *pDnode, SSnode *pSnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
int32_t refCount = 0;
taosRLockLatch(&pMgmt->latch);
if (pSnode != NULL) {
refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1);
}
taosRUnLockLatch(&pMgmt->latch);
if (pSnode != NULL) {
dTrace("release snode, refCount:%d", refCount);
}
}
static int32_t dndReadSnodeFile(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
int32_t code = TSDB_CODE_DND_SNODE_READ_FILE_ERROR;
int32_t len = 0;
int32_t maxLen = 4096;
char *content = calloc(1, maxLen + 1);
cJSON *root = NULL;
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_SNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_SNODE_OVER;
}
content[len] = 0;
root = cJSON_Parse(content);
if (root == NULL) {
dError("failed to read %s since invalid json format", file);
goto PRASE_SNODE_OVER;
}
cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
if (!deployed || deployed->type != cJSON_Number) {
dError("failed to read %s since deployed not found", file);
goto PRASE_SNODE_OVER;
}
pMgmt->deployed = deployed->valueint;
cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
if (!dropped || dropped->type != cJSON_Number) {
dError("failed to read %s since dropped not found", file);
goto PRASE_SNODE_OVER;
}
pMgmt->dropped = dropped->valueint;
code = 0;
dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped);
PRASE_SNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
terrno = code;
return code;
}
static int32_t dndWriteSnodeFile(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
}
int32_t len = 0;
int32_t maxLen = 4096;
char *content = calloc(1, maxLen + 1);
len += snprintf(content + len, maxLen - len, "{\n");
len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed);
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
free(content);
char realfile[PATH_MAX + 20];
snprintf(realfile, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
if (taosRenameFile(file, realfile) != 0) {
terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR;
dError("failed to rename %s since %s", file, terrstr());
return -1;
}
dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
return 0;
}
static int32_t dndStartSnodeWorker(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "snode-write", 0, 1, dndProcessSnodeQueue) != 0) {
dError("failed to start snode write worker since %s", terrstr());
return -1;
}
return 0;
}
static void dndStopSnodeWorker(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
taosWLockLatch(&pMgmt->latch);
pMgmt->deployed = 0;
taosWUnLockLatch(&pMgmt->latch);
while (pMgmt->refCount > 1) {
taosMsleep(10);
}
dndCleanupWorker(&pMgmt->writeWorker);
}
static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) {
pOption->pDnode = pDnode;
pOption->sendMsgToDnodeFp = dndSendMsgToDnode;
pOption->sendMsgToMnodeFp = dndSendMsgToMnode;
pOption->sendRedirectMsgFp = dndSendRedirectMsg;
pOption->dnodeId = dndGetDnodeId(pDnode);
pOption->clusterId = dndGetClusterId(pDnode);
pOption->cfg.sver = pDnode->opt.sver;
}
static int32_t dndOpenSnode(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
SSnodeOpt option = {0};
dndBuildSnodeOption(pDnode, &option);
SSnode *pSnode = sndOpen(pDnode->dir.snode, &option);
if (pSnode == NULL) {
dError("failed to open snode since %s", terrstr());
return -1;
}
if (dndStartSnodeWorker(pDnode) != 0) {
dError("failed to start snode worker since %s", terrstr());
sndClose(pSnode);
return -1;
}
pMgmt->deployed = 1;
if (dndWriteSnodeFile(pDnode) != 0) {
pMgmt->deployed = 0;
dError("failed to write snode file since %s", terrstr());
dndStopSnodeWorker(pDnode);
sndClose(pSnode);
return -1;
}
taosWLockLatch(&pMgmt->latch);
pMgmt->pSnode = pSnode;
taosWUnLockLatch(&pMgmt->latch);
dInfo("snode open successfully");
return 0;
}
static int32_t dndDropSnode(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
SSnode *pSnode = dndAcquireSnode(pDnode);
if (pSnode == NULL) {
dError("failed to drop snode since %s", terrstr());
return -1;
}
taosRLockLatch(&pMgmt->latch);
pMgmt->dropped = 1;
taosRUnLockLatch(&pMgmt->latch);
if (dndWriteSnodeFile(pDnode) != 0) {
taosRLockLatch(&pMgmt->latch);
pMgmt->dropped = 0;
taosRUnLockLatch(&pMgmt->latch);
dndReleaseSnode(pDnode, pSnode);
dError("failed to drop snode since %s", terrstr());
return -1;
}
dndReleaseSnode(pDnode, pSnode);
dndStopSnodeWorker(pDnode);
pMgmt->deployed = 0;
dndWriteSnodeFile(pDnode);
sndClose(pSnode);
pMgmt->pSnode = NULL;
sndDestroy(pDnode->dir.snode);
return 0;
}
int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
SCreateSnodeInMsg *pMsg = pRpcMsg->pCont;
pMsg->dnodeId = htonl(pMsg->dnodeId);
if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
terrno = TSDB_CODE_DND_SNODE_ID_INVALID;
return -1;
} else {
return dndOpenSnode(pDnode);
}
}
int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
SDropSnodeInMsg *pMsg = pRpcMsg->pCont;
pMsg->dnodeId = htonl(pMsg->dnodeId);
if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
terrno = TSDB_CODE_DND_SNODE_ID_INVALID;
return -1;
} else {
return dndDropSnode(pDnode);
}
}
static void dndProcessSnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
SRpcMsg *pRsp = NULL;
int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
SSnode *pSnode = dndAcquireSnode(pDnode);
if (pSnode != NULL) {
code = sndProcessMsg(pSnode, pMsg, &pRsp);
}
if (pRsp != NULL) {
pRsp->ahandle = pMsg->ahandle;
rpcSendResponse(pRsp);
free(pRsp);
} else {
if (code != 0) code = terrno;
SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
rpcSendResponse(&rpcRsp);
}
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
static void dndWriteSnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) {
int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
SSnode *pSnode = dndAcquireSnode(pDnode);
if (pSnode != NULL) {
code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
}
dndReleaseSnode(pDnode, pSnode);
if (code != 0) {
if (pMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
rpcSendResponse(&rsp);
}
rpcFreeCont(pMsg->pCont);
}
}
void dndProcessSnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.writeWorker, pMsg);
}
int32_t dndInitSnode(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
taosInitRWLatch(&pMgmt->latch);
if (dndReadSnodeFile(pDnode) != 0) {
return -1;
}
if (pMgmt->dropped) {
dInfo("snode has been deployed and needs to be deleted");
sndDestroy(pDnode->dir.snode);
return 0;
}
if (!pMgmt->deployed) return 0;
return dndOpenSnode(pDnode);
}
void dndCleanupSnode(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
if (pMgmt->pSnode) {
dndStopSnodeWorker(pDnode);
sndClose(pMgmt->pSnode);
pMgmt->pSnode = NULL;
}
}

49
source/dnode/mgmt/impl/src/dndWorker.c
View File

@ -16,9 +16,9 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "dndWorker.h" #include "dndWorker.h"
int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EDndWorkerType type, const char *name, int32_t minNum, int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum,
int32_t maxNum, FProcessItem fp) { int32_t maxNum, void *queueFp) {
if (pDnode == NULL || pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || fp == NULL) { if (pDnode == NULL || pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || queueFp == NULL) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return -1; return -1;
} }
@ -27,19 +27,32 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EDndWorkerType type
pWorker->name = name; pWorker->name = name;
pWorker->minNum = minNum; pWorker->minNum = minNum;
pWorker->maxNum = maxNum; pWorker->maxNum = maxNum;
pWorker->fp = fp; pWorker->queueFp = queueFp;
pWorker->pDnode = pDnode; pWorker->pDnode = pDnode;
if (pWorker->type == DND_WORKER_SINGLE) { if (pWorker->type == DND_WORKER_SINGLE) {
SWorkerPool *pPool = &pWorker->pool; SWorkerPool *pPool = &pWorker->pool;
pPool->name = name;
pPool->min = minNum; pPool->min = minNum;
pPool->max = maxNum; pPool->max = maxNum;
if (tWorkerInit(pPool) != 0) { if (tWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
pWorker->queue = tWorkerAllocQueue(pPool, pDnode, (FProcessItem)queueFp);
pWorker->queue = tWorkerAllocQueue(&pPool, pDnode, fp); if (pWorker->queue == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
} else if (pWorker->type == DND_WORKER_MULTI) {
SMWorkerPool *pPool = &pWorker->mpool;
pPool->name = name;
pPool->max = maxNum;
if (tMWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pWorker->queue = tMWorkerAllocQueue(pPool, pDnode, (FProcessItems)queueFp);
if (pWorker->queue == NULL) { if (pWorker->queue == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
@ -52,12 +65,17 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EDndWorkerType type
} }
void dndCleanupWorker(SDnodeWorker *pWorker) { void dndCleanupWorker(SDnodeWorker *pWorker) {
if (pWorker->type == DND_WORKER_SINGLE) {
while (!taosQueueEmpty(pWorker->queue)) { while (!taosQueueEmpty(pWorker->queue)) {
taosMsleep(10); taosMsleep(10);
} }
if (pWorker->type == DND_WORKER_SINGLE) {
tWorkerCleanup(&pWorker->pool); tWorkerCleanup(&pWorker->pool);
tWorkerFreeQueue(&pWorker->pool, pWorker->queue); tWorkerFreeQueue(&pWorker->pool, pWorker->queue);
} else if (pWorker->type == DND_WORKER_MULTI) {
tMWorkerCleanup(&pWorker->mpool);
tMWorkerFreeQueue(&pWorker->mpool, pWorker->queue);
} else {
} }
} }
@ -67,16 +85,23 @@ int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen)
return -1; return -1;
} }
void *pMsg = taosAllocateQitem(contLen); void *pMsg = NULL;
if (contLen != 0) {
pMsg = taosAllocateQitem(contLen);
if (pMsg != NULL) {
memcpy(pMsg, pCont, contLen);
}
} else {
pMsg = pCont;
}
if (pMsg == NULL) { if (pMsg == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
memcpy(pMsg, pCont, contLen); if (taosWriteQitem(pWorker->queue, pMsg) != 0) {
taosFreeQitem(pMsg);
if (taosWriteQitem(pWorker, pMsg) != 0) {
taosFreeItem(pMsg);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }

2
source/dnode/mgmt/impl/test/dnode/dnode.cpp
View File

@ -162,7 +162,7 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) {
SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen);
strcpy(pReq->fqdn, "localhost"); strcpy(pReq->fqdn, "localhost");
pReq->port = htonl(904); pReq->port = htonl(9044);
SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen);
ASSERT_NE(pMsg, nullptr); ASSERT_NE(pMsg, nullptr);

39
source/dnode/mgmt/impl/test/stb/stb.cpp
View File

@ -68,7 +68,6 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
{ {
SSchema* pSchema = &pReq->pSchema[0]; SSchema* pSchema = &pReq->pSchema[0];
pSchema->colId = htonl(0);
pSchema->bytes = htonl(8); pSchema->bytes = htonl(8);
pSchema->type = TSDB_DATA_TYPE_TIMESTAMP; pSchema->type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema->name, "ts"); strcpy(pSchema->name, "ts");
@ -76,7 +75,6 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
{ {
SSchema* pSchema = &pReq->pSchema[1]; SSchema* pSchema = &pReq->pSchema[1];
pSchema->colId = htonl(1);
pSchema->bytes = htonl(4); pSchema->bytes = htonl(4);
pSchema->type = TSDB_DATA_TYPE_INT; pSchema->type = TSDB_DATA_TYPE_INT;
strcpy(pSchema->name, "col1"); strcpy(pSchema->name, "col1");
@ -84,7 +82,6 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
{ {
SSchema* pSchema = &pReq->pSchema[2]; SSchema* pSchema = &pReq->pSchema[2];
pSchema->colId = htonl(2);
pSchema->bytes = htonl(2); pSchema->bytes = htonl(2);
pSchema->type = TSDB_DATA_TYPE_TINYINT; pSchema->type = TSDB_DATA_TYPE_TINYINT;
strcpy(pSchema->name, "tag1"); strcpy(pSchema->name, "tag1");
@ -92,7 +89,6 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
{ {
SSchema* pSchema = &pReq->pSchema[3]; SSchema* pSchema = &pReq->pSchema[3];
pSchema->colId = htonl(3);
pSchema->bytes = htonl(8); pSchema->bytes = htonl(8);
pSchema->type = TSDB_DATA_TYPE_BIGINT; pSchema->type = TSDB_DATA_TYPE_BIGINT;
strcpy(pSchema->name, "tag2"); strcpy(pSchema->name, "tag2");
@ -100,7 +96,6 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
{ {
SSchema* pSchema = &pReq->pSchema[4]; SSchema* pSchema = &pReq->pSchema[4];
pSchema->colId = htonl(4);
pSchema->bytes = htonl(16); pSchema->bytes = htonl(16);
pSchema->type = TSDB_DATA_TYPE_BINARY; pSchema->type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema->name, "tag3"); strcpy(pSchema->name, "tag3");
@ -167,10 +162,42 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
{ {
SSchema* pSchema = &pRsp->pSchema[0]; SSchema* pSchema = &pRsp->pSchema[0];
EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP);
EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->colId, 1);
EXPECT_EQ(pSchema->bytes, 8); EXPECT_EQ(pSchema->bytes, 8);
EXPECT_STREQ(pSchema->name, "ts"); EXPECT_STREQ(pSchema->name, "ts");
} }
{
SSchema* pSchema = &pRsp->pSchema[1];
EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT);
EXPECT_EQ(pSchema->colId, 2);
EXPECT_EQ(pSchema->bytes, 4);
EXPECT_STREQ(pSchema->name, "col1");
}
{
SSchema* pSchema = &pRsp->pSchema[2];
EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TINYINT);
EXPECT_EQ(pSchema->colId, 3);
EXPECT_EQ(pSchema->bytes, 2);
EXPECT_STREQ(pSchema->name, "tag1");
}
{
SSchema* pSchema = &pRsp->pSchema[3];
EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BIGINT);
EXPECT_EQ(pSchema->colId, 4);
EXPECT_EQ(pSchema->bytes, 8);
EXPECT_STREQ(pSchema->name, "tag2");
}
{
SSchema* pSchema = &pRsp->pSchema[4];
EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY);
EXPECT_EQ(pSchema->colId, 5);
EXPECT_EQ(pSchema->bytes, 16);
EXPECT_STREQ(pSchema->name, "tag3");
}
} }
// restart // restart

4
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -124,12 +124,8 @@ typedef struct {
int64_t rebootTime; int64_t rebootTime;
int64_t lastAccessTime; int64_t lastAccessTime;
int32_t accessTimes; int32_t accessTimes;
int16_t numOfMnodes;
int16_t numOfVnodes; int16_t numOfVnodes;
int16_t numOfQnodes;
int16_t numOfSupportMnodes;
int16_t numOfSupportVnodes; int16_t numOfSupportVnodes;
int16_t numOfSupportQnodes;
int16_t numOfCores; int16_t numOfCores;
EDndStatus status; EDndStatus status;
EDndReason offlineReason; EDndReason offlineReason;

1
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -19,6 +19,7 @@
#include "mndDef.h" #include "mndDef.h"
#include "sdb.h" #include "sdb.h"
#include "tcache.h" #include "tcache.h"
#include "tep.h"
#include "tqueue.h" #include "tqueue.h"
#include "ttime.h" #include "ttime.h"

1
source/dnode/mnode/impl/inc/mndVgroup.h
View File

@ -29,6 +29,7 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup);
SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup);
int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups);
SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup); SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup);
int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId);
SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup);
SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup);

8
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -18,8 +18,7 @@
#include "mndMnode.h" #include "mndMnode.h"
#include "mndShow.h" #include "mndShow.h"
#include "mndTrans.h" #include "mndTrans.h"
#include "tep.h" #include "mndVgroup.h"
#include "ttime.h"
#define TSDB_DNODE_VER_NUMBER 1 #define TSDB_DNODE_VER_NUMBER 1
#define TSDB_DNODE_RESERVE_SIZE 64 #define TSDB_DNODE_RESERVE_SIZE 64
@ -370,7 +369,6 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) {
} }
pRsp->dnodeCfg.dnodeId = htonl(pDnode->id); pRsp->dnodeCfg.dnodeId = htonl(pDnode->id);
pRsp->dnodeCfg.dropped = 0;
pRsp->dnodeCfg.clusterId = htobe64(pMnode->clusterId); pRsp->dnodeCfg.clusterId = htobe64(pMnode->clusterId);
mndGetDnodeData(pMnode, &pRsp->dnodeEps, numOfEps); mndGetDnodeData(pMnode, &pRsp->dnodeEps, numOfEps);
@ -388,7 +386,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg *
dnodeObj.updateTime = dnodeObj.createdTime; dnodeObj.updateTime = dnodeObj.createdTime;
dnodeObj.port = pCreate->port; dnodeObj.port = pCreate->port;
memcpy(dnodeObj.fqdn, pCreate->fqdn, TSDB_FQDN_LEN); memcpy(dnodeObj.fqdn, pCreate->fqdn, TSDB_FQDN_LEN);
snprintf(dnodeObj.ep, "%s:%u", dnodeObj.fqdn, dnodeObj.port); snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg);
if (pTrans == NULL) { if (pTrans == NULL) {
@ -700,7 +698,7 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i
cols++; cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *)pWrite = pDnode->numOfVnodes; *(int16_t *)pWrite = mndGetVnodesNum(pMnode, pDnode->id);
cols++; cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

10
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -118,17 +118,17 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid,
SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000); SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000);
if (pConn == NULL) { if (pConn == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("conn:%d, data:%p failed to put into cache since %s, user:%s", connId, pConn, pInfo->user, terrstr()); mError("conn:%d, failed to put into cache since %s, user:%s", connId, pInfo->user, terrstr());
return NULL; return NULL;
} else { } else {
mTrace("conn:%d, data:%p created, user:%s", pConn->id, pConn, pInfo->user); mTrace("conn:%d, is created, data:%p user:%s", pConn->id, pConn, pInfo->user);
return pConn; return pConn;
} }
} }
static void mndFreeConn(SConnObj *pConn) { static void mndFreeConn(SConnObj *pConn) {
tfree(pConn->pQueries); tfree(pConn->pQueries);
mTrace("conn:%d, data:%p destroyed", pConn->id, pConn); mTrace("conn:%d, is destroyed, data:%p", pConn->id, pConn);
} }
static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId) { static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId) {
@ -143,13 +143,13 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId) {
int32_t keepTime = pMnode->cfg.shellActivityTimer * 3; int32_t keepTime = pMnode->cfg.shellActivityTimer * 3;
pConn->lastAccessTimeMs = keepTime * 1000 + (uint64_t)taosGetTimestampMs(); pConn->lastAccessTimeMs = keepTime * 1000 + (uint64_t)taosGetTimestampMs();
mTrace("conn:%d, data:%p acquired from cache", pConn->id, pConn); mTrace("conn:%d, acquired from cache, data:%p", pConn->id, pConn);
return pConn; return pConn;
} }
static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) { static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) {
if (pConn == NULL) return; if (pConn == NULL) return;
mTrace("conn:%d, data:%p released from cache", pConn->id, pConn); mTrace("conn:%d, released from cache, data:%p", pConn->id, pConn);
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
taosCacheRelease(pMgmt->cache, (void **)&pConn, false); taosCacheRelease(pMgmt->cache, (void **)&pConn, false);

58
source/dnode/mnode/impl/src/mndShow.c
View File

@ -56,31 +56,24 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg) {
if (showId == 0) atomic_add_fetch_32(&pMgmt->showId, 1); if (showId == 0) atomic_add_fetch_32(&pMgmt->showId, 1);
int32_t size = sizeof(SShowObj) + pMsg->payloadLen; int32_t size = sizeof(SShowObj) + pMsg->payloadLen;
SShowObj *pShow = calloc(1, size); SShowObj showObj = {0};
if (pShow != NULL) { showObj.id = showId;
pShow->id = showId; showObj.pMnode = pMnode;
pShow->pMnode = pMnode; showObj.type = pMsg->type;
pShow->type = pMsg->type; showObj.payloadLen = pMsg->payloadLen;
pShow->payloadLen = pMsg->payloadLen; memcpy(showObj.db, pMsg->db, TSDB_DB_FNAME_LEN);
memcpy(pShow->db, pMsg->db, TSDB_DB_FNAME_LEN); memcpy(showObj.payload, pMsg->payload, pMsg->payloadLen);
memcpy(pShow->payload, pMsg->payload, pMsg->payloadLen);
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to process show-meta msg:%s since %s", mndShowStr(pMsg->type), terrstr());
return NULL;
}
int32_t keepTime = pMnode->cfg.shellActivityTimer * 6 * 1000; int32_t keepTime = pMnode->cfg.shellActivityTimer * 6 * 1000;
SShowObj *pShowRet = taosCachePut(pMgmt->cache, &showId, sizeof(int32_t), pShow, size, keepTime); SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int32_t), &showObj, size, keepTime);
free(pShow); if (pShow == NULL) {
if (pShowRet == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("show:%d, failed to put into cache since %s", showId, terrstr()); mError("show:%d, failed to put into cache since %s", showId, terrstr());
return NULL; return NULL;
} else {
mTrace("show:%d, data:%p created", showId, pShowRet);
return pShowRet;
} }
mTrace("show:%d, is created, data:%p", showId, pShow);
return pShow;
} }
static void mndFreeShowObj(SShowObj *pShow) { static void mndFreeShowObj(SShowObj *pShow) {
@ -94,7 +87,7 @@ static void mndFreeShowObj(SShowObj *pShow) {
} }
} }
mTrace("show:%d, data:%p destroyed", pShow->id, pShow); mTrace("show:%d, is destroyed, data:%p", pShow->id, pShow);
} }
static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId) { static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId) {
@ -106,13 +99,13 @@ static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId) {
return NULL; return NULL;
} }
mTrace("show:%d, data:%p acquired from cache", pShow->id, pShow); mTrace("show:%d, acquired from cache, data:%p", pShow->id, pShow);
return pShow; return pShow;
} }
static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) {
if (pShow == NULL) return; if (pShow == NULL) return;
mTrace("show:%d, data:%p released from cache, force:%d", pShow->id, pShow, forceRemove); mTrace("show:%d, released from cache, data:%p force:%d", pShow->id, pShow, forceRemove);
// A bug in tcache.c // A bug in tcache.c
forceRemove = 0; forceRemove = 0;
@ -158,8 +151,8 @@ static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) {
} }
int32_t code = (*metaFp)(pMnodeMsg, pShow, &pRsp->tableMeta); int32_t code = (*metaFp)(pMnodeMsg, pShow, &pRsp->tableMeta);
mDebug("show:%d, data:%p get meta finished, numOfRows:%d cols:%d type:%s result:%s", pShow->id, pShow, mDebug("show:%d, get meta finished, numOfRows:%d cols:%d type:%s result:%s", pShow->id, pShow->numOfRows,
pShow->numOfRows, pShow->numOfColumns, mndShowStr(type), tstrerror(code)); pShow->numOfColumns, mndShowStr(type), tstrerror(code));
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
pMnodeMsg->contLen = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns; pMnodeMsg->contLen = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns;
@ -195,16 +188,15 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) {
if (retrieveFp == NULL) { if (retrieveFp == NULL) {
mndReleaseShowObj(pShow, false); mndReleaseShowObj(pShow, false);
terrno = TSDB_CODE_MSG_NOT_PROCESSED; terrno = TSDB_CODE_MSG_NOT_PROCESSED;
mError("show:%d, data:%p failed to retrieve data since %s", pShow->id, pShow, terrstr()); mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr());
return -1; return -1;
} }
mDebug("show:%d, data:%p start retrieve data, numOfReads:%d numOfRows:%d type:%s", pShow->id, pShow, mDebug("show:%d, start retrieve data, numOfReads:%d numOfRows:%d type:%s", pShow->id, pShow->numOfReads,
pShow->numOfReads, pShow->numOfRows, mndShowStr(pShow->type)); pShow->numOfRows, mndShowStr(pShow->type));
if (mndCheckRetrieveFinished(pShow)) { if (mndCheckRetrieveFinished(pShow)) {
mDebug("show:%d, data:%p read finished, numOfReads:%d numOfRows:%d", pShow->id, pShow, pShow->numOfReads, mDebug("show:%d, read finished, numOfReads:%d numOfRows:%d", pShow->id, pShow->numOfReads, pShow->numOfRows);
pShow->numOfRows);
pShow->numOfReads = pShow->numOfRows; pShow->numOfReads = pShow->numOfRows;
} }
@ -227,7 +219,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) {
if (pRsp == NULL) { if (pRsp == NULL) {
mndReleaseShowObj(pShow, false); mndReleaseShowObj(pShow, false);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("show:%d, data:%p failed to retrieve data since %s", pShow->id, pShow, terrstr()); mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr());
return -1; return -1;
} }
@ -236,7 +228,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) {
rowsRead = (*retrieveFp)(pMnodeMsg, pShow, pRsp->data, rowsToRead); rowsRead = (*retrieveFp)(pMnodeMsg, pShow, pRsp->data, rowsToRead);
} }
mDebug("show:%d, data:%p stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, pShow, rowsRead, rowsToRead); mDebug("show:%d, stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead);
pRsp->numOfRows = htonl(rowsRead); pRsp->numOfRows = htonl(rowsRead);
pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision
@ -246,10 +238,10 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) {
if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) {
pRsp->completed = 1; pRsp->completed = 1;
mDebug("show:%d, data:%p retrieve completed", pShow->id, pShow); mDebug("show:%d, retrieve completed", pShow->id);
mndReleaseShowObj(pShow, true); mndReleaseShowObj(pShow, true);
} else { } else {
mDebug("show:%d, data:%p retrieve not completed yet", pShow->id, pShow); mDebug("show:%d, retrieve not completed yet", pShow->id);
mndReleaseShowObj(pShow, false); mndReleaseShowObj(pShow, false);
} }

11
source/dnode/mnode/impl/src/mndStb.c
View File

@ -291,7 +291,6 @@ static int32_t mndCheckCreateStbMsg(SCreateStbMsg *pCreate) {
int32_t totalCols = pCreate->numOfColumns + pCreate->numOfTags; int32_t totalCols = pCreate->numOfColumns + pCreate->numOfTags;
for (int32_t i = 0; i < totalCols; ++i) { for (int32_t i = 0; i < totalCols; ++i) {
SSchema *pSchema = &pCreate->pSchema[i]; SSchema *pSchema = &pCreate->pSchema[i];
pSchema->colId = htonl(pSchema->colId);
pSchema->bytes = htonl(pSchema->bytes); pSchema->bytes = htonl(pSchema->bytes);
} }
@ -317,10 +316,6 @@ static int32_t mndCheckCreateStbMsg(SCreateStbMsg *pCreate) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION; terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1; return -1;
} }
if (pSchema->colId < 0 || pSchema->colId >= maxColId) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1;
}
if (pSchema->bytes <= 0) { if (pSchema->bytes <= 0) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION; terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1; return -1;
@ -453,6 +448,10 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre
} }
memcpy(stbObj.pSchema, pCreate->pSchema, totalSize); memcpy(stbObj.pSchema, pCreate->pSchema, totalSize);
for (int32_t i = 0; i < totalCols; ++i) {
stbObj.pSchema[i].colId = i + 1;
}
int32_t code = 0; int32_t code = 0;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg);
if (pTrans == NULL) { if (pTrans == NULL) {
@ -767,7 +766,7 @@ static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) {
return -1; return -1;
} }
memcpy(pMeta->stbFname, pStb->name, TSDB_TABLE_FNAME_LEN); memcpy(pMeta->tbFname, pStb->name, TSDB_TABLE_FNAME_LEN);
pMeta->numOfTags = htonl(pStb->numOfTags); pMeta->numOfTags = htonl(pStb->numOfTags);
pMeta->numOfColumns = htonl(pStb->numOfColumns); pMeta->numOfColumns = htonl(pStb->numOfColumns);
pMeta->precision = pDb->cfg.precision; pMeta->precision = pDb->cfg.precision;

18
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -294,18 +294,18 @@ TRANS_DECODE_OVER:
return NULL; return NULL;
} }
mTrace("trans:%d, decode from raw:%p", pTrans->id, pRaw); mTrace("trans:%d, decode from raw:%p, data:%p", pTrans->id, pRaw, pTrans);
return pRow; return pRow;
} }
static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) {
pTrans->stage = TRN_STAGE_PREPARE; pTrans->stage = TRN_STAGE_PREPARE;
mTrace("trans:%d, perform insert action", pTrans->id); mTrace("trans:%d, perform insert action, data:%p", pTrans->id, pTrans);
return 0; return 0;
} }
static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) { static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) {
mTrace("trans:%d, perform delete action", pTrans->id); mTrace("trans:%d, perform delete action, data:%p", pTrans->id, pTrans);
mndTransDropLogs(pTrans->redoLogs); mndTransDropLogs(pTrans->redoLogs);
mndTransDropLogs(pTrans->undoLogs); mndTransDropLogs(pTrans->undoLogs);
@ -317,7 +317,7 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) {
} }
static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOldTrans, STrans *pNewTrans) { static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOldTrans, STrans *pNewTrans) {
mTrace("trans:%d, perform update action", pOldTrans->id); mTrace("trans:%d, perform update action, data:%p", pOldTrans->id, pOldTrans);
pOldTrans->stage = pNewTrans->stage; pOldTrans->stage = pNewTrans->stage;
return 0; return 0;
} }
@ -362,14 +362,14 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, SRpcMsg *pMsg) {
return NULL; return NULL;
} }
mDebug("trans:%d, is created", pTrans->id); mDebug("trans:%d, is created, data:%p", pTrans->id, pTrans);
return pTrans; return pTrans;
} }
static void mndTransDropLogs(SArray *pArray) { static void mndTransDropLogs(SArray *pArray) {
for (int32_t i = 0; i < pArray->size; ++i) { for (int32_t i = 0; i < pArray->size; ++i) {
SSdbRaw *pRaw = taosArrayGetP(pArray, i); SSdbRaw *pRaw = taosArrayGetP(pArray, i);
tfree(pRaw); sdbFreeRaw(pRaw);
} }
taosArrayDestroy(pArray); taosArrayDestroy(pArray);
@ -391,7 +391,7 @@ void mndTransDrop(STrans *pTrans) {
mndTransDropActions(pTrans->redoActions); mndTransDropActions(pTrans->redoActions);
mndTransDropActions(pTrans->undoActions); mndTransDropActions(pTrans->undoActions);
// mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans);
tfree(pTrans); tfree(pTrans);
} }
@ -442,7 +442,7 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) {
} }
sdbSetRawStatus(pRaw, SDB_STATUS_READY); sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mTrace("trans:%d, sync to other nodes", pTrans->id); mDebug("trans:%d, sync to other nodes", pTrans->id);
int32_t code = mndSyncPropose(pMnode, pRaw); int32_t code = mndSyncPropose(pMnode, pRaw);
if (code != 0) { if (code != 0) {
mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); mError("trans:%d, failed to sync since %s", pTrans->id, terrstr());
@ -450,7 +450,7 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) {
return -1; return -1;
} }
mTrace("trans:%d, sync finished", pTrans->id); mDebug("trans:%d, sync finished", pTrans->id);
code = sdbWrite(pMnode->pSdb, pRaw); code = sdbWrite(pMnode->pSdb, pRaw);
if (code != 0) { if (code != 0) {

102
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -86,7 +86,6 @@ SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) {
for (int8_t i = 0; i < pVgroup->replica; ++i) { for (int8_t i = 0; i < pVgroup->replica; ++i) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; SVnodeGid *pVgid = &pVgroup->vnodeGid[i];
SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId) SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId)
SDB_SET_INT8(pRaw, dataPos, pVgid->role)
} }
SDB_SET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE) SDB_SET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE)
SDB_SET_DATALEN(pRaw, dataPos); SDB_SET_DATALEN(pRaw, dataPos);
@ -121,7 +120,6 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
for (int8_t i = 0; i < pVgroup->replica; ++i) { for (int8_t i = 0; i < pVgroup->replica; ++i) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; SVnodeGid *pVgid = &pVgroup->vnodeGid[i];
SDB_GET_INT32(pRaw, pRow, dataPos, &pVgid->dnodeId) SDB_GET_INT32(pRaw, pRow, dataPos, &pVgid->dnodeId)
SDB_GET_INT8(pRaw, pRow, dataPos, (int8_t *)&pVgid->role)
} }
SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_VGROUP_RESERVE_SIZE) SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_VGROUP_RESERVE_SIZE)
@ -237,44 +235,95 @@ SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *p
return pDrop; return pDrop;
} }
static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup) { static SArray *mndBuildDnodesArray(SMnode *pMnode) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int32_t allocedVnodes = 0; int32_t numOfDnodes = mndGetDnodeSize(pMnode);
void *pIter = NULL; SArray *pArray = taosArrayInit(numOfDnodes, sizeof(SDnodeObj));
if (pArray == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
while (allocedVnodes < pVgroup->replica) { void *pIter = NULL;
while (1) {
SDnodeObj *pDnode = NULL; SDnodeObj *pDnode = NULL;
pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode);
if (pIter == NULL) break; if (pIter == NULL) break;
// todo int32_t numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id);
if (mndIsDnodeInReadyStatus(pMnode, pDnode)) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[allocedVnodes]; bool isMnode = mndIsMnode(pMnode, pDnode->id);
if (isMnode) {
pDnode->numOfVnodes++;
}
bool isReady = mndIsDnodeInReadyStatus(pMnode, pDnode);
if (isReady) {
taosArrayPush(pArray, pDnode);
}
mDebug("dnode:%d, numOfVnodes:%d numOfSupportVnodes:%d isMnode:%d ready:%d", pDnode->id, numOfVnodes,
pDnode->numOfSupportVnodes, isMnode, isReady);
sdbRelease(pSdb, pDnode);
}
return pArray;
}
static int32_t mndCompareDnodeVnodes(SDnodeObj *pDnode1, SDnodeObj *pDnode2) {
float d1Score = (float)pDnode1->numOfVnodes / pDnode1->numOfSupportVnodes;
float d2Score = (float)pDnode2->numOfVnodes / pDnode2->numOfSupportVnodes;
return d1Score > d2Score ? 0 : 1;
}
static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) {
SSdb *pSdb = pMnode->pSdb;
int32_t allocedVnodes = 0;
void *pIter = NULL;
taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
for (int32_t v = 0; v < pVgroup->replica; ++v) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[v];
SDnodeObj *pDnode = taosArrayGet(pArray, v);
if (pDnode == NULL || pDnode->numOfVnodes > pDnode->numOfSupportVnodes) {
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
return -1;
}
pVgid->dnodeId = pDnode->id; pVgid->dnodeId = pDnode->id;
if (pVgroup->replica == 1) { if (pVgroup->replica == 1) {
pVgid->role = TAOS_SYNC_STATE_LEADER; pVgid->role = TAOS_SYNC_STATE_LEADER;
} else { } else {
pVgid->role = TAOS_SYNC_STATE_FOLLOWER; pVgid->role = TAOS_SYNC_STATE_FOLLOWER;
} }
allocedVnodes++;
} mDebug("db:%s, vgId:%d, vindex:%d dnodeId:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId);
sdbRelease(pSdb, pDnode); pDnode->numOfVnodes++;
} }
if (allocedVnodes != pVgroup->replica) {
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
return -1;
}
return 0; return 0;
} }
int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
SVgObj *pVgroups = calloc(pDb->cfg.numOfVgroups, sizeof(SVgObj)); int32_t code = -1;
SArray *pArray = NULL;
SVgObj *pVgroups = NULL;
pVgroups = calloc(pDb->cfg.numOfVgroups, sizeof(SVgObj));
if (pVgroups == NULL) { if (pVgroups == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; goto ALLOC_VGROUP_OVER;
} }
pArray = mndBuildDnodesArray(pMnode);
if (pArray == NULL) {
goto ALLOC_VGROUP_OVER;
}
mDebug("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray),
pDb->cfg.numOfVgroups, pDb->cfg.numOfVgroups * pDb->cfg.replications);
int32_t allocedVgroups = 0; int32_t allocedVgroups = 0;
int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP);
uint32_t hashMin = 0; uint32_t hashMin = 0;
@ -298,17 +347,23 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
pVgroup->dbUid = pDb->uid; pVgroup->dbUid = pDb->uid;
pVgroup->replica = pDb->cfg.replications; pVgroup->replica = pDb->cfg.replications;
if (mndGetAvailableDnode(pMnode, pVgroup) != 0) { if (mndGetAvailableDnode(pMnode, pVgroup, pArray) != 0) {
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
free(pVgroups); goto ALLOC_VGROUP_OVER;
return -1;
} }
allocedVgroups++; allocedVgroups++;
} }
*ppVgroups = pVgroups; *ppVgroups = pVgroups;
return 0; code = 0;
mDebug("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications);
ALLOC_VGROUP_OVER:
if (code != 0) free(pVgroups);
taosArrayDestroy(pArray);
return code;
} }
SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup) { SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup) {
@ -348,6 +403,7 @@ static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) {
} }
static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) { static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) {
@ -478,7 +534,7 @@ static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
} }
static int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int32_t numOfVnodes = 0; int32_t numOfVnodes = 0;
void *pIter = NULL; void *pIter = NULL;

18
source/dnode/mnode/impl/src/mnode.c
View File

@ -146,15 +146,15 @@ static int32_t mndInitSteps(SMnode *pMnode) {
if (mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans) != 0) return -1; if (mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster) != 0) return -1; if (mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1; if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1;
if (pMnode->clusterId <= 0) { if (pMnode->clusterId <= 0) {
if (mndAllocStep(pMnode, "mnode-sdb-deploy", mndDeploySdb, NULL) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sdb-deploy", mndDeploySdb, NULL) != 0) return -1;
@ -390,7 +390,7 @@ void mndSendRsp(SMnodeMsg *pMsg, int32_t code) {
rpcSendResponse(&rpcRsp); rpcSendResponse(&rpcRsp);
} }
static void mndProcessRpcMsg(SMnodeMsg *pMsg) { void mndProcessMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode; SMnode *pMnode = pMsg->pMnode;
int32_t code = 0; int32_t code = 0;
tmsg_t msgType = pMsg->rpcMsg.msgType; tmsg_t msgType = pMsg->rpcMsg.msgType;
@ -451,12 +451,6 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) {
} }
} }
void mndProcessReadMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); }
void mndProcessWriteMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); }
void mndProcessSyncMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); }
uint64_t mndGenerateUid(char *name, int32_t len) { uint64_t mndGenerateUid(char *name, int32_t len) {
int64_t us = taosGetTimestampUs(); int64_t us = taosGetTimestampUs();
int32_t hashval = MurmurHash3_32(name, len); int32_t hashval = MurmurHash3_32(name, len);

1
source/dnode/mnode/sdb/inc/sdbInt.h
View File

@ -72,6 +72,7 @@ typedef struct SSdb {
} SSdb; } SSdb;
int32_t sdbWriteFile(SSdb *pSdb); int32_t sdbWriteFile(SSdb *pSdb);
void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper);
#ifdef __cplusplus #ifdef __cplusplus
} }

6
source/dnode/mnode/sdb/src/sdb.c
View File

@ -80,16 +80,12 @@ void sdbCleanup(SSdb *pSdb) {
SHashObj *hash = pSdb->hashObjs[i]; SHashObj *hash = pSdb->hashObjs[i];
if (hash == NULL) continue; if (hash == NULL) continue;
SdbDeleteFp deleteFp = pSdb->deleteFps[i];
SSdbRow **ppRow = taosHashIterate(hash, NULL); SSdbRow **ppRow = taosHashIterate(hash, NULL);
while (ppRow != NULL) { while (ppRow != NULL) {
SSdbRow *pRow = *ppRow; SSdbRow *pRow = *ppRow;
if (pRow == NULL) continue; if (pRow == NULL) continue;
if (deleteFp != NULL) { sdbFreeRow(pSdb, pRow);
(*deleteFp)(pSdb, pRow->pObj);
}
sdbFreeRow(pRow);
ppRow = taosHashIterate(hash, ppRow); ppRow = taosHashIterate(hash, ppRow);
} }
} }

6
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -151,7 +151,7 @@ int32_t sdbWriteFile(SSdb *pSdb) {
if (taosWriteFile(fd, pRaw, writeLen) != writeLen) { if (taosWriteFile(fd, pRaw, writeLen) != writeLen) {
code = TAOS_SYSTEM_ERROR(terrno); code = TAOS_SYSTEM_ERROR(terrno);
taosHashCancelIterate(hash, ppRow); taosHashCancelIterate(hash, ppRow);
free(pRaw); sdbFreeRaw(pRaw);
break; break;
} }
@ -159,7 +159,7 @@ int32_t sdbWriteFile(SSdb *pSdb) {
if (taosWriteFile(fd, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { if (taosWriteFile(fd, &cksum, sizeof(int32_t)) != sizeof(int32_t)) {
code = TAOS_SYSTEM_ERROR(terrno); code = TAOS_SYSTEM_ERROR(terrno);
taosHashCancelIterate(hash, ppRow); taosHashCancelIterate(hash, ppRow);
free(pRaw); sdbFreeRaw(pRaw);
break; break;
} }
} else { } else {
@ -168,7 +168,7 @@ int32_t sdbWriteFile(SSdb *pSdb) {
break; break;
} }
free(pRaw); sdbFreeRaw(pRaw);
ppRow = taosHashIterate(hash, ppRow); ppRow = taosHashIterate(hash, ppRow);
} }
taosWUnLockLatch(pLock); taosWUnLockLatch(pLock);

79
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -16,6 +16,50 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "sdbInt.h" #include "sdbInt.h"
static const char *sdbTableName(ESdbType type) {
switch (type) {
case SDB_TRANS:
return "trans";
case SDB_CLUSTER:
return "cluster";
case SDB_MNODE:
return "mnode";
case SDB_DNODE:
return "dnode";
case SDB_USER:
return "user";
case SDB_AUTH:
return "auth";
case SDB_ACCT:
return "acct";
case SDB_TOPIC:
return "topic";
case SDB_VGROUP:
return "vgId";
case SDB_STB:
return "stb";
case SDB_DB:
return "db";
case SDB_FUNC:
return "func";
default:
return "undefine";
}
}
void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper) {
EKeyType keyType = pSdb->keyTypes[pRow->type];
if (keyType == SDB_KEY_BINARY) {
mTrace("%s:%s, refCount:%d oper:%s", sdbTableName(pRow->type), (char *)pRow->pObj, pRow->refCount, oper);
} else if (keyType == SDB_KEY_INT32) {
mTrace("%s:%d, refCount:%d oper:%s", sdbTableName(pRow->type), *(int32_t *)pRow->pObj, pRow->refCount, oper);
} else if (keyType == SDB_KEY_INT64) {
mTrace("%s:%" PRId64 ", refCount:%d oper:%s", sdbTableName(pRow->type), *(int64_t *)pRow->pObj, pRow->refCount, oper);
} else {
}
}
static SHashObj *sdbGetHash(SSdb *pSdb, int32_t type) { static SHashObj *sdbGetHash(SSdb *pSdb, int32_t type) {
if (type >= SDB_MAX || type <= SDB_START) { if (type >= SDB_MAX || type <= SDB_START) {
terrno = TSDB_CODE_SDB_INVALID_TABLE_TYPE; terrno = TSDB_CODE_SDB_INVALID_TABLE_TYPE;
@ -55,17 +99,18 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize); SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize);
if (pOldRow != NULL) { if (pOldRow != NULL) {
taosWUnLockLatch(pLock); taosWUnLockLatch(pLock);
sdbFreeRow(pRow); sdbFreeRow(pSdb, pRow);
terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE;
return terrno; return terrno;
} }
pRow->refCount = 1; pRow->refCount = 0;
pRow->status = pRaw->status; pRow->status = pRaw->status;
sdbPrintOper(pSdb, pRow, "insertRow");
if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) {
taosWUnLockLatch(pLock); taosWUnLockLatch(pLock);
sdbFreeRow(pRow); sdbFreeRow(pSdb, pRow);
terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE;
return terrno; return terrno;
} }
@ -83,7 +128,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
taosWLockLatch(pLock); taosWLockLatch(pLock);
taosHashRemove(hash, pRow->pObj, keySize); taosHashRemove(hash, pRow->pObj, keySize);
taosWUnLockLatch(pLock); taosWUnLockLatch(pLock);
sdbFreeRow(pRow); sdbFreeRow(pSdb, pRow);
terrno = code; terrno = code;
return terrno; return terrno;
} }
@ -113,7 +158,7 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj); code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj);
} }
sdbFreeRow(pNewRow); sdbFreeRow(pSdb, pNewRow);
return code; return code;
} }
@ -123,14 +168,10 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
SRWLatch *pLock = &pSdb->locks[pRow->type]; SRWLatch *pLock = &pSdb->locks[pRow->type];
taosWLockLatch(pLock); taosWLockLatch(pLock);
// remove attached object such as trans
SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type];
if (deleteFp != NULL) (*deleteFp)(pSdb, pRow->pObj);
SSdbRow **ppOldRow = taosHashGet(hash, pRow->pObj, keySize); SSdbRow **ppOldRow = taosHashGet(hash, pRow->pObj, keySize);
if (ppOldRow == NULL || *ppOldRow == NULL) { if (ppOldRow == NULL || *ppOldRow == NULL) {
taosWUnLockLatch(pLock); taosWUnLockLatch(pLock);
sdbFreeRow(pRow); sdbFreeRow(pSdb, pRow);
terrno = TSDB_CODE_SDB_OBJ_NOT_THERE; terrno = TSDB_CODE_SDB_OBJ_NOT_THERE;
return terrno; return terrno;
} }
@ -140,8 +181,8 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
taosHashRemove(hash, pOldRow->pObj, keySize); taosHashRemove(hash, pOldRow->pObj, keySize);
taosWUnLockLatch(pLock); taosWUnLockLatch(pLock);
sdbRelease(pSdb, pOldRow->pObj); // sdbRelease(pSdb, pOldRow->pObj);
sdbFreeRow(pRow); sdbFreeRow(pSdb, pRow);
return code; return code;
} }
@ -206,6 +247,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, void *pKey) {
case SDB_STATUS_UPDATING: case SDB_STATUS_UPDATING:
atomic_add_fetch_32(&pRow->refCount, 1); atomic_add_fetch_32(&pRow->refCount, 1);
pRet = pRow->pObj; pRet = pRow->pObj;
sdbPrintOper(pSdb, pRow, "acquireRow");
break; break;
case SDB_STATUS_CREATING: case SDB_STATUS_CREATING:
terrno = TSDB_CODE_SDB_OBJ_CREATING; terrno = TSDB_CODE_SDB_OBJ_CREATING;
@ -232,13 +274,9 @@ void sdbRelease(SSdb *pSdb, void *pObj) {
taosRLockLatch(pLock); taosRLockLatch(pLock);
int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1);
sdbPrintOper(pSdb, pRow, "releaseRow");
if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) { if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) {
SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type]; sdbFreeRow(pSdb, pRow);
if (deleteFp != NULL) {
(*deleteFp)(pSdb, pRow->pObj);
}
sdbFreeRow(pRow);
} }
taosRUnLockLatch(pLock); taosRUnLockLatch(pLock);
@ -255,9 +293,9 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
if (pIter != NULL) { if (pIter != NULL) {
SSdbRow *pLastRow = *(SSdbRow **)pIter; SSdbRow *pLastRow = *(SSdbRow **)pIter;
int32_t ref = atomic_sub_fetch_32(&pLastRow->refCount, 1); int32_t ref = atomic_load_32(&pLastRow->refCount);
if (ref <= 0 && pLastRow->status == SDB_STATUS_DROPPED) { if (ref <= 0 && pLastRow->status == SDB_STATUS_DROPPED) {
sdbFreeRow(pLastRow); sdbFreeRow(pSdb, pLastRow);
} }
} }
@ -270,6 +308,7 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
} }
atomic_add_fetch_32(&pRow->refCount, 1); atomic_add_fetch_32(&pRow->refCount, 1);
sdbPrintOper(pSdb, pRow, "fetchRow");
*ppObj = pRow->pObj; *ppObj = pRow->pObj;
break; break;
} }

4
source/dnode/mnode/sdb/src/sdbRaw.c
View File

@ -27,12 +27,12 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) {
pRaw->sver = sver; pRaw->sver = sver;
pRaw->dataLen = dataLen; pRaw->dataLen = dataLen;
// mTrace("raw:%p, is created, len:%d", pRaw, dataLen); mTrace("raw:%p, is created, len:%d", pRaw, dataLen);
return pRaw; return pRaw;
} }
void sdbFreeRaw(SSdbRaw *pRaw) { void sdbFreeRaw(SSdbRaw *pRaw) {
// mTrace("raw:%p, is freed", pRaw); mTrace("raw:%p, is freed", pRaw);
free(pRaw); free(pRaw);
} }

11
source/dnode/mnode/sdb/src/sdbRow.c
View File

@ -35,4 +35,13 @@ void *sdbGetRowObj(SSdbRow *pRow) {
return pRow->pObj; return pRow->pObj;
} }
void sdbFreeRow(SSdbRow *pRow) { tfree(pRow); } void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow) {
// remove attached object such as trans
SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type];
if (deleteFp != NULL) {
(*deleteFp)(pSdb, pRow->pObj);
}
sdbPrintOper(pSdb, pRow, "freeRow");
tfree(pRow);
}

4
source/dnode/snode/src/snode.c
View File

@ -15,7 +15,7 @@
#include "sndInt.h" #include "sndInt.h"
SSnode *sndOpen(const SSnodeOpt *pOption) { SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
SSnode *pSnode = calloc(1, sizeof(SSnode)); SSnode *pSnode = calloc(1, sizeof(SSnode));
return pSnode; return pSnode;
} }
@ -28,3 +28,5 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
*pRsp = NULL; *pRsp = NULL;
return 0; return 0;
} }
void sndDestroy(const char *path) {}

4
source/dnode/vnode/impl/inc/vnodeRequest.h
View File

@ -23,8 +23,8 @@ extern "C" {
#endif #endif
// SVDropTableReq // SVDropTableReq
int vnodeBuildDropTableReq(void **buf, const SVDropTableReq *pReq); // int vnodeBuildDropTableReq(void **buf, const SVDropTableReq *pReq);
void *vnodeParseDropTableReq(void *buf, SVDropTableReq *pReq); // void *vnodeParseDropTableReq(void *buf, SVDropTableReq *pReq);
#ifdef __cplusplus #ifdef __cplusplus
} }

5
source/dnode/vnode/impl/src/vnodeInt.c
View File

@ -28,8 +28,3 @@ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vInfo("sync message is processed"); vInfo("sync message is processed");
return 0; return 0;
} }
int vnodeProcessConsumeReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vInfo("consume message is processed");
return 0;
}

26
source/dnode/vnode/impl/src/vnodeQuery.c
View File

@ -13,12 +13,10 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "vnodeDef.h"
#include "vnodeQuery.h" #include "vnodeQuery.h"
#include "vnodeDef.h"
int vnodeQueryOpen(SVnode *pVnode) { int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery); }
return qWorkerInit(NULL, &pVnode->pQuery);
}
int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vInfo("query message is processed"); vInfo("query message is processed");
@ -46,4 +44,24 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
return 0; return 0;
} }
static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
STableInfoMsg *pReq = (STableInfoMsg *)(pMsg->pCont);
STableMetaMsg *pRspMsg;
int ret;
if (metaGetTableInfo(pVnode->pMeta, pReq->tableFname, &pRspMsg) < 0) {
return -1;
}
*pRsp = malloc(sizeof(SRpcMsg));
if (TD_IS_NULL(*pRsp)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
free(pMsg);
return -1;
}
// TODO
(*pRsp)->pCont = pRspMsg;
return 0;
}

3
source/dnode/vnode/impl/src/vnodeRequest.c
View File

@ -15,6 +15,8 @@
#include "vnodeDef.h" #include "vnodeDef.h"
#if 0
static int vnodeBuildCreateTableReq(void **buf, const SVCreateTableReq *pReq); static int vnodeBuildCreateTableReq(void **buf, const SVCreateTableReq *pReq);
static void *vnodeParseCreateTableReq(void *buf, SVCreateTableReq *pReq); static void *vnodeParseCreateTableReq(void *buf, SVCreateTableReq *pReq);
@ -114,3 +116,4 @@ int vnodeBuildDropTableReq(void **buf, const SVDropTableReq *pReq) {
void *vnodeParseDropTableReq(void *buf, SVDropTableReq *pReq) { void *vnodeParseDropTableReq(void *buf, SVDropTableReq *pReq) {
// TODO // TODO
} }
#endif

9
source/dnode/vnode/impl/src/vnodeWrite.c
View File

@ -28,7 +28,6 @@ int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) {
int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
SRpcMsg * pMsg; SRpcMsg * pMsg;
SVnodeReq *pVnodeReq;
for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { for (int i = 0; i < taosArrayGetSize(pMsgs); i++) {
pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i);
@ -51,7 +50,6 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
} }
int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
SVnodeReq vReq;
SVCreateTbReq vCreateTbReq; SVCreateTbReq vCreateTbReq;
void * ptr = vnodeMalloc(pVnode, pMsg->contLen); void * ptr = vnodeMalloc(pVnode, pMsg->contLen);
if (ptr == NULL) { if (ptr == NULL) {
@ -70,6 +68,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
switch (pMsg->msgType) { switch (pMsg->msgType) {
case TDMT_VND_CREATE_STB: case TDMT_VND_CREATE_STB:
case TDMT_VND_CREATE_TABLE:
tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq); tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq);
if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq)) < 0) { if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq)) < 0) {
// TODO: handle error // TODO: handle error
@ -79,9 +78,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
break; break;
case TDMT_VND_DROP_STB: case TDMT_VND_DROP_STB:
case TDMT_VND_DROP_TABLE: case TDMT_VND_DROP_TABLE:
if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) { // if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) {
// TODO: handle error // // TODO: handle error
} // }
break; break;
case TDMT_VND_SUBMIT: case TDMT_VND_SUBMIT:
if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr) < 0) { if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr) < 0) {

84
source/dnode/vnode/meta/src/metaBDBImpl.c
View File

@ -351,7 +351,7 @@ static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey
pDbt[0].size = sizeof(pTbCfg->ctbCfg.suid); pDbt[0].size = sizeof(pTbCfg->ctbCfg.suid);
// Second key is the first tag // Second key is the first tag
void *pTagVal = tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, 0); void *pTagVal = tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId);
pDbt[1].data = varDataVal(pTagVal); pDbt[1].data = varDataVal(pTagVal);
pDbt[1].size = varDataLen(pTagVal); pDbt[1].size = varDataLen(pTagVal);
@ -403,10 +403,10 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) {
buf = taosDecodeFixedU8(buf, &(pTbCfg->type)); buf = taosDecodeFixedU8(buf, &(pTbCfg->type));
if (pTbCfg->type == META_SUPER_TABLE) { if (pTbCfg->type == META_SUPER_TABLE) {
buf = taosDecodeVariantU32(buf, pTbCfg->stbCfg.nTagCols); buf = taosDecodeVariantU32(buf, &(pTbCfg->stbCfg.nTagCols));
pTbCfg->stbCfg.pTagSchema = (SSchema *)malloc(sizeof(SSchema) * pTbCfg->stbCfg.nTagCols); pTbCfg->stbCfg.pTagSchema = (SSchema *)malloc(sizeof(SSchema) * pTbCfg->stbCfg.nTagCols);
for (uint32_t i = 0; i < pTbCfg->stbCfg.nTagCols; i++) { for (uint32_t i = 0; i < pTbCfg->stbCfg.nTagCols; i++) {
buf = taosDecodeFixedI8(buf, &pTbCfg->stbCfg.pSchema[i].type); buf = taosDecodeFixedI8(buf, &(pTbCfg->stbCfg.pSchema[i].type));
buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pSchema[i].colId); buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pSchema[i].colId);
buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pSchema[i].bytes); buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pSchema[i].bytes);
buf = taosDecodeStringTo(buf, pTbCfg->stbCfg.pSchema[i].name); buf = taosDecodeStringTo(buf, pTbCfg->stbCfg.pSchema[i].name);
@ -429,3 +429,81 @@ static void metaClearTbCfg(STbCfg *pTbCfg) {
tfree(pTbCfg->ctbCfg.pTag); tfree(pTbCfg->ctbCfg.pTag);
} }
} }
/* ------------------------ FOR QUERY ------------------------ */
int metaGetTableInfo(SMeta *pMeta, char *tbname, STableMetaMsg **ppMsg) {
DBT key = {0};
DBT value = {0};
SMetaDB * pMetaDB = pMeta->pDB;
int ret;
STbCfg tbCfg;
SSchemaKey schemaKey;
DBT key1 = {0};
DBT value1 = {0};
uint32_t ncols;
void * pBuf;
int tlen;
STableMetaMsg *pMsg;
key.data = tbname;
key.size = strlen(tbname) + 1;
ret = pMetaDB->pNameIdx->get(pMetaDB->pNameIdx, NULL, &key, &value, 0);
if (ret != 0) {
// TODO
return -1;
}
metaDecodeTbInfo(value.data, &tbCfg);
switch (tbCfg.type) {
case META_SUPER_TABLE:
schemaKey.uid = tbCfg.stbCfg.suid;
schemaKey.sver = 0;
key1.data = &schemaKey;
key1.size = sizeof(schemaKey);
ret = pMetaDB->pSchemaDB->get(pMetaDB->pSchemaDB, &key1, &value1, NULL, 0);
if (ret != 0) {
// TODO
return -1;
}
pBuf = value1.data;
pBuf = taosDecodeFixedU32(pBuf, &ncols);
tlen = sizeof(STableMetaMsg) + (tbCfg.stbCfg.nTagCols + ncols) * sizeof(SSchema);
pMsg = calloc(1, tlen);
if (pMsg == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
strcpy(pMsg->tbFname, tbCfg.name);
pMsg->numOfTags = tbCfg.stbCfg.nTagCols;
pMsg->numOfColumns = ncols;
pMsg->tableType = tbCfg.type;
pMsg->sversion = 0;
pMsg->tversion = 0;
pMsg->suid = tbCfg.stbCfg.suid;
pMsg->tuid = tbCfg.stbCfg.suid;
for (size_t i = 0; i < tbCfg.stbCfg.nTagCols; i++) {
}
break;
case META_CHILD_TABLE:
ASSERT(0);
break;
case META_NORMAL_TABLE:
ASSERT(0);
break;
default:
ASSERT(0);
break;
}
*ppMsg = pMsg;
return 0;
}

54
source/dnode/vnode/meta/src/metaTbCfg.c
View File

@ -47,57 +47,3 @@ size_t metaEncodeTbObjFromTbOptions(const STbCfg *pTbOptions, void *pBuf, size_t
return tlen; return tlen;
} }
int metaEncodeTbCfg(void **pBuf, STbCfg *pTbCfg) {
int tsize = 0;
tsize += taosEncodeString(pBuf, pTbCfg->name);
tsize += taosEncodeFixedU32(pBuf, pTbCfg->ttl);
tsize += taosEncodeFixedU32(pBuf, pTbCfg->keep);
tsize += taosEncodeFixedU8(pBuf, pTbCfg->type);
switch (pTbCfg->type) {
case META_SUPER_TABLE:
tsize += taosEncodeFixedU64(pBuf, pTbCfg->stbCfg.suid);
tsize += tdEncodeSchema(pBuf, pTbCfg->stbCfg.pSchema);
tsize += tdEncodeSchema(pBuf, pTbCfg->stbCfg.pTagSchema);
break;
case META_CHILD_TABLE:
tsize += taosEncodeFixedU64(pBuf, pTbCfg->ctbCfg.suid);
tsize += tdEncodeKVRow(pBuf, pTbCfg->ctbCfg.pTag);
break;
case META_NORMAL_TABLE:
tsize += tdEncodeSchema(pBuf, pTbCfg->ntbCfg.pSchema);
break;
default:
break;
}
return tsize;
}
void *metaDecodeTbCfg(void *pBuf, STbCfg *pTbCfg) {
pBuf = taosDecodeString(pBuf, &(pTbCfg->name));
pBuf = taosDecodeFixedU32(pBuf, &(pTbCfg->ttl));
pBuf = taosDecodeFixedU32(pBuf, &(pTbCfg->keep));
pBuf = taosDecodeFixedU8(pBuf, &(pTbCfg->type));
switch (pTbCfg->type) {
case META_SUPER_TABLE:
pBuf = taosDecodeFixedU64(pBuf, &(pTbCfg->stbCfg.suid));
pBuf = tdDecodeSchema(pBuf, &(pTbCfg->stbCfg.pSchema));
pBuf = tdDecodeSchema(pBuf, &(pTbCfg->stbCfg.pTagSchema));
break;
case META_CHILD_TABLE:
pBuf = taosDecodeFixedU64(pBuf, &(pTbCfg->ctbCfg.suid));
pBuf = tdDecodeKVRow(pBuf, &(pTbCfg->ctbCfg.pTag));
break;
case META_NORMAL_TABLE:
pBuf = tdDecodeSchema(pBuf, &(pTbCfg->ntbCfg.pSchema));
break;
default:
break;
}
return pBuf;
}

2
source/libs/catalog/inc/catalogInt.h
View File

@ -66,8 +66,6 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t);
#define ctgTrace(...) do { if (ctgDebugFlag & DEBUG_TRACE) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) #define ctgTrace(...) do { if (ctgDebugFlag & DEBUG_TRACE) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0)
#define ctgDebugL(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLongString("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) #define ctgDebugL(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLongString("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0)
#define CTG_CACHE_ENABLED() (ctgMgmt.cfg.maxDBCacheNum > 0 || ctgMgmt.cfg.maxTblCacheNum > 0)
#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
#define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
#define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); terrno = _code; return _code; } } while (0) #define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); terrno = _code; return _code; } } while (0)

209
source/libs/catalog/src/catalog.c
View File

@ -13,10 +13,10 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "catalogInt.h"
#include "trpc.h" #include "trpc.h"
#include "query.h" #include "query.h"
#include "tname.h" #include "tname.h"
#include "catalogInt.h"
SCatalogMgmt ctgMgmt = {0}; SCatalogMgmt ctgMgmt = {0};
@ -49,22 +49,11 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp
SEpSet *pVnodeEpSet = NULL; SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0; int32_t msgLen = 0;
CTG_ERR_RET(queryBuildMsg[TDMT_MND_USE_DB](input, &msg, 0, &msgLen)); CTG_ERR_RET(queryBuildMsg[TMSG_INDEX(TDMT_MND_USE_DB)](input, &msg, 0, &msgLen));
char *pMsg = rpcMallocCont(msgLen);
if (NULL == pMsg) {
ctgError("rpc malloc %d failed", msgLen);
tfree(msg);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
memcpy(pMsg, msg, msgLen);
tfree(msg);
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
.msgType = TDMT_MND_USE_DB, .msgType = TDMT_MND_USE_DB,
.pCont = pMsg, .pCont = msg,
.contLen = msgLen, .contLen = msgLen,
}; };
@ -76,21 +65,20 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp
CTG_ERR_RET(rpcRsp.code); CTG_ERR_RET(rpcRsp.code);
} }
CTG_ERR_RET(queryProcessMsgRsp[TDMT_MND_USE_DB](out, rpcRsp.pCont, rpcRsp.contLen)); CTG_ERR_RET(queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)](out, rpcRsp.pCont, rpcRsp.contLen));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const char *dbName, const char* pTableName, STableMeta** pTableMeta, int32_t *exist) { int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist) {
if (NULL == pCatalog->tableCache.cache) { if (NULL == pCatalog->tableCache.cache) {
*exist = 0; *exist = 0;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
char tbFullName[TSDB_TABLE_FNAME_LEN]; char tbFullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pTableName, tbFullName);
snprintf(tbFullName, sizeof(tbFullName), "%s.%s", dbName, pTableName);
STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName)); STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName));
@ -146,21 +134,20 @@ void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) {
} }
} }
int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, STableMetaOutput* output) { int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) {
if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == output) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == output) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
} }
char tbFullName[TSDB_TABLE_FNAME_LEN]; char tbFullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pTableName, tbFullName);
snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName);
SBuildTableMetaInput bInput = {.vgId = 0, .tableFullName = tbFullName}; SBuildTableMetaInput bInput = {.vgId = 0, .tableFullName = tbFullName};
char *msg = NULL; char *msg = NULL;
SEpSet *pVnodeEpSet = NULL; SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0; int32_t msgLen = 0;
CTG_ERR_RET(queryBuildMsg[TDMT_MND_STB_META](&bInput, &msg, 0, &msgLen)); CTG_ERR_RET(queryBuildMsg[TMSG_INDEX(TDMT_MND_STB_META)](&bInput, &msg, 0, &msgLen));
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
.msgType = TDMT_MND_STB_META, .msgType = TDMT_MND_STB_META,
@ -177,7 +164,7 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE
CTG_ERR_RET(rpcRsp.code); CTG_ERR_RET(rpcRsp.code);
} }
CTG_ERR_RET(queryProcessMsgRsp[TDMT_MND_STB_META](output, rpcRsp.pCont, rpcRsp.contLen)); CTG_ERR_RET(queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_STB_META)](output, rpcRsp.pCont, rpcRsp.contLen));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -197,7 +184,7 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SE
SEpSet *pVnodeEpSet = NULL; SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0; int32_t msgLen = 0;
CTG_ERR_RET(queryBuildMsg[TDMT_VND_TABLE_META](&bInput, &msg, 0, &msgLen)); CTG_ERR_RET(queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)](&bInput, &msg, 0, &msgLen));
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
.msgType = TDMT_VND_TABLE_META, .msgType = TDMT_VND_TABLE_META,
@ -217,7 +204,7 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SE
CTG_ERR_RET(rpcRsp.code); CTG_ERR_RET(rpcRsp.code);
} }
CTG_ERR_RET(queryProcessMsgRsp[TDMT_VND_TABLE_META](output, rpcRsp.pCont, rpcRsp.contLen)); CTG_ERR_RET(queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -259,10 +246,13 @@ int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet *
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) {
int32_t vgNum = taosHashGetSize(dbInfo->vgInfo); int32_t vgNum = taosHashGetSize(dbInfo->vgInfo);
char db[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, db);
if (vgNum <= 0) { if (vgNum <= 0) {
ctgError("db[%s] vgroup cache invalid, vgroup number:%d", pDBName, vgNum); ctgError("db[%s] vgroup cache invalid, vgroup number:%d", db, vgNum);
CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED);
} }
@ -272,8 +262,7 @@ int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const char *pDBName, co
CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp));
char tbFullName[TSDB_TABLE_FNAME_LEN]; char tbFullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pTableName, tbFullName);
snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName);
uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName)); uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName));
@ -298,24 +287,24 @@ int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const char *pDBName, co
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, bool forceUpdate, STableMeta** pTableMeta) { int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta) {
if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
} }
int32_t exist = 0; int32_t exist = 0;
if (!forceUpdate) { if (!forceUpdate) {
CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pDBName, pTableName, pTableMeta, &exist)); CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist));
if (exist) { if (exist) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
} }
CTG_ERR_RET(catalogRenewTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName)); CTG_ERR_RET(catalogRenewTableMeta(pCatalog, pRpc, pMgmtEps, pTableName));
CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pDBName, pTableName, pTableMeta, &exist)); CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist));
if (0 == exist) { if (0 == exist) {
ctgError("get table meta from cache failed, but fetch succeed"); ctgError("get table meta from cache failed, but fetch succeed");
@ -381,6 +370,41 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo) {
if (NULL == pCatalog || NULL == dbName || NULL == pRpc || NULL == pMgmtEps) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t exist = 0;
if (0 == forceUpdate) {
CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &exist));
if (exist) {
return TSDB_CODE_SUCCESS;
}
}
SUseDbOutput DbOut = {0};
SBuildUseDBInput input = {0};
strncpy(input.db, dbName, sizeof(input.db));
input.db[sizeof(input.db) - 1] = 0;
input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut));
CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbName, &DbOut.dbVgroup));
if (dbInfo) {
*dbInfo = DbOut.dbVgroup;
}
return TSDB_CODE_SUCCESS;
}
int32_t catalogInit(SCatalogCfg *cfg) { int32_t catalogInit(SCatalogCfg *cfg) {
if (ctgMgmt.pCluster) { if (ctgMgmt.pCluster) {
ctgError("catalog already init"); ctgError("catalog already init");
@ -389,17 +413,23 @@ int32_t catalogInit(SCatalogCfg *cfg) {
if (cfg) { if (cfg) {
memcpy(&ctgMgmt.cfg, cfg, sizeof(*cfg)); memcpy(&ctgMgmt.cfg, cfg, sizeof(*cfg));
if (ctgMgmt.cfg.maxDBCacheNum == 0) {
ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER;
}
if (ctgMgmt.cfg.maxTblCacheNum == 0) {
ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER;
}
} else { } else {
ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER;
ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER;
} }
if (CTG_CACHE_ENABLED()) {
ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == ctgMgmt.pCluster) { if (NULL == ctgMgmt.pCluster) {
CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER);
} }
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -460,13 +490,19 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName,
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) {
if (NULL == pCatalog || NULL == dbName || NULL == dbInfo) { if (NULL == pCatalog || NULL == dbName || NULL == dbInfo) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
} }
if (dbInfo->vgVersion < 0) { if (dbInfo->vgVersion < 0) {
if (pCatalog->dbCache.cache) { if (pCatalog->dbCache.cache) {
SDBVgroupInfo *oldInfo = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName));
if (oldInfo && oldInfo->vgInfo) {
taosHashCleanup(oldInfo->vgInfo);
oldInfo->vgInfo = NULL;
}
taosHashRemove(pCatalog->dbCache.cache, dbName, strlen(dbName)); taosHashRemove(pCatalog->dbCache.cache, dbName, strlen(dbName));
} }
@ -496,74 +532,41 @@ int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) {
return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta);
int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo) {
if (NULL == pCatalog || NULL == dbName || NULL == pRpc || NULL == pMgmtEps) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
} }
int32_t exist = 0; int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName) {
if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName) {
if (0 == forceUpdate) {
CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &exist));
if (exist) {
return TSDB_CODE_SUCCESS;
}
}
SUseDbOutput DbOut = {0};
SBuildUseDBInput input = {0};
strncpy(input.db, dbName, sizeof(input.db));
input.db[sizeof(input.db) - 1] = 0;
input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut));
CTG_ERR_RET(catalogUpdateDBVgroupCache(pCatalog, dbName, &DbOut.dbVgroup));
if (dbInfo) {
*dbInfo = DbOut.dbVgroup;
}
return TSDB_CODE_SUCCESS;
}
int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) {
return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pDBName, pTableName, false, pTableMeta);
}
int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName) {
if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
} }
SVgroupInfo vgroupInfo = {0}; SVgroupInfo vgroupInfo = {0};
int32_t code = 0;
CTG_ERR_RET(catalogGetTableHashVgroup(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo)); CTG_ERR_RET(catalogGetTableHashVgroup(pCatalog, pRpc, pMgmtEps, pTableName, &vgroupInfo));
STableMetaOutput output = {0}; STableMetaOutput output = {0};
//CTG_ERR_RET(ctgGetTableMetaFromVnode(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo, &output)); //CTG_ERR_RET(ctgGetTableMetaFromVnode(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo, &output));
CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &output)); CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pTableName, &output));
CTG_ERR_RET(ctgUpdateTableMetaCache(pCatalog, &output)); CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, &output));
_return:
tfree(output.tbMeta); tfree(output.tbMeta);
return TSDB_CODE_SUCCESS; CTG_RET(code);
} }
int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) {
return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, true, pTableMeta); return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pTableName, true, pTableMeta);
} }
int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray** pVgroupList) { int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList) {
if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == pVgroupList) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgroupList) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
} }
@ -572,9 +575,11 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S
SVgroupInfo vgroupInfo = {0}; SVgroupInfo vgroupInfo = {0};
SDBVgroupInfo dbVgroup = {0}; SDBVgroupInfo dbVgroup = {0};
CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &tbMeta)); CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pTableName, &tbMeta));
CTG_ERR_JRET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbVgroup)); char db[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, db);
CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, db, false, &dbVgroup));
if (tbMeta->tableType == TSDB_SUPER_TABLE) { if (tbMeta->tableType == TSDB_SUPER_TABLE) {
CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, &dbVgroup, pVgroupList)); CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, &dbVgroup, pVgroupList));
@ -598,31 +603,30 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S
} }
tfree(tbMeta); tfree(tbMeta);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return: _return:
tfree(tbMeta); tfree(tbMeta);
taosArrayDestroy(*pVgroupList); taosArrayDestroy(*pVgroupList);
CTG_RET(code); CTG_RET(code);
} }
int32_t catalogGetTableHashVgroup(struct SCatalog *pCatalog, void *pTransporter, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { int32_t catalogGetTableHashVgroup(struct SCatalog *pCatalog, void *pTransporter, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) {
SDBVgroupInfo dbInfo = {0}; SDBVgroupInfo dbInfo = {0};
int32_t code = 0; int32_t code = 0;
int32_t vgId = 0;
CTG_ERR_RET(catalogGetDBVgroup(pCatalog, pTransporter, pMgmtEps, pDBName, false, &dbInfo)); char db[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, db);
CTG_ERR_RET(ctgGetDBVgroup(pCatalog, pTransporter, pMgmtEps, db, false, &dbInfo));
if (dbInfo.vgVersion < 0 || NULL == dbInfo.vgInfo) { if (dbInfo.vgVersion < 0 || NULL == dbInfo.vgInfo) {
ctgError("db[%s] vgroup cache invalid, vgroup version:%d, vgInfo:%p", pDBName, dbInfo.vgVersion, dbInfo.vgInfo); ctgError("db[%s] vgroup cache invalid, vgroup version:%d, vgInfo:%p", db, dbInfo.vgVersion, dbInfo.vgInfo);
CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED);
} }
CTG_ERR_RET(ctgGetVgInfoFromHashValue(&dbInfo, pDBName, pTableName, pVgroup)); CTG_ERR_RET(ctgGetVgInfoFromHashValue(&dbInfo, pTableName, pVgroup));
CTG_RET(code); CTG_RET(code);
} }
@ -636,23 +640,23 @@ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* p
int32_t code = 0; int32_t code = 0;
if (pReq->pTableName) { if (pReq->pTableName) {
char dbName[TSDB_DB_FNAME_LEN];
int32_t tbNum = (int32_t)taosArrayGetSize(pReq->pTableName); int32_t tbNum = (int32_t)taosArrayGetSize(pReq->pTableName);
if (tbNum > 0) { if (tbNum <= 0) {
ctgError("empty table name list");
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
pRsp->pTableMeta = taosArrayInit(tbNum, POINTER_BYTES); pRsp->pTableMeta = taosArrayInit(tbNum, POINTER_BYTES);
if (NULL == pRsp->pTableMeta) { if (NULL == pRsp->pTableMeta) {
ctgError("taosArrayInit num[%d] failed", tbNum); ctgError("taosArrayInit num[%d] failed", tbNum);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
} }
}
for (int32_t i = 0; i < tbNum; ++i) { for (int32_t i = 0; i < tbNum; ++i) {
SName *name = taosArrayGet(pReq->pTableName, i); SName *name = taosArrayGet(pReq->pTableName, i);
STableMeta *pTableMeta = NULL; STableMeta *pTableMeta = NULL;
snprintf(dbName, sizeof(dbName), "%d.%s", name->acctId, name->dbname); CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, name, &pTableMeta));
CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, dbName, name->tname, &pTableMeta));
if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) {
ctgError("taosArrayPush failed, idx:%d", i); ctgError("taosArrayPush failed, idx:%d", i);
@ -674,6 +678,7 @@ _return:
} }
taosArrayDestroy(pRsp->pTableMeta); taosArrayDestroy(pRsp->pTableMeta);
pRsp->pTableMeta = NULL;
} }
CTG_RET(code); CTG_RET(code);

68
source/libs/catalog/test/catalogTests.cpp
View File

@ -388,7 +388,11 @@ TEST(tableMeta, normalTable) {
code = catalogGetHandle(ctgTestClusterId, &pCtg); code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestTablename, &vgInfo); SName n = {.type = T_NAME_TABLE, .acctId = 1};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 8); ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.numOfEps, 3); ASSERT_EQ(vgInfo.numOfEps, 3);
@ -396,7 +400,7 @@ TEST(tableMeta, normalTable) {
ctgTestSetPrepareTableMeta(); ctgTestSetPrepareTableMeta();
STableMeta *tableMeta = NULL; STableMeta *tableMeta = NULL;
code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestTablename, &tableMeta); code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 8); ASSERT_EQ(tableMeta->vgId, 8);
ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE);
@ -408,7 +412,7 @@ TEST(tableMeta, normalTable) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
tableMeta = NULL; tableMeta = NULL;
code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestTablename, &tableMeta); code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 8); ASSERT_EQ(tableMeta->vgId, 8);
ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE);
@ -433,14 +437,15 @@ TEST(tableMeta, childTableCase) {
//sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
int32_t code = catalogInit(NULL); int32_t code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg); SName n = {.type = T_NAME_TABLE, .acctId = 1};
ASSERT_EQ(code, 0); strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestCTablename);
STableMeta *tableMeta = NULL; STableMeta *tableMeta = NULL;
code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestCTablename, &tableMeta); code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 9); ASSERT_EQ(tableMeta->vgId, 9);
ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE);
@ -452,7 +457,7 @@ TEST(tableMeta, childTableCase) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
tableMeta = NULL; tableMeta = NULL;
code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestCTablename, &tableMeta); code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 9); ASSERT_EQ(tableMeta->vgId, 9);
ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE);
@ -464,7 +469,9 @@ TEST(tableMeta, childTableCase) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
tableMeta = NULL; tableMeta = NULL;
code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestSTablename, &tableMeta);
strcpy(n.tname, ctgTestSTablename);
code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 0); ASSERT_EQ(tableMeta->vgId, 0);
ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
@ -488,15 +495,15 @@ TEST(tableMeta, superTableCase) {
initQueryModuleMsgHandle(); initQueryModuleMsgHandle();
//sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
int32_t code = catalogGetHandle(ctgTestClusterId, &pCtg);
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg); SName n = {.type = T_NAME_TABLE, .acctId = 1};
ASSERT_EQ(code, 0); strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestSTablename);
STableMeta *tableMeta = NULL; STableMeta *tableMeta = NULL;
code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestSTablename, &tableMeta); code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 0); ASSERT_EQ(tableMeta->vgId, 0);
ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
@ -510,7 +517,10 @@ TEST(tableMeta, superTableCase) {
ctgTestSetPrepareCTableMeta(); ctgTestSetPrepareCTableMeta();
tableMeta = NULL; tableMeta = NULL;
code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestCTablename, &tableMeta);
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestCTablename);
code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 9); ASSERT_EQ(tableMeta->vgId, 9);
ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE);
@ -522,7 +532,7 @@ TEST(tableMeta, superTableCase) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
tableMeta = NULL; tableMeta = NULL;
code = catalogRenewAndGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestCTablename, &tableMeta); code = catalogRenewAndGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 9); ASSERT_EQ(tableMeta->vgId, 9);
ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE);
@ -550,14 +560,14 @@ TEST(tableDistVgroup, normalTable) {
//sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
int32_t code = catalogInit(NULL); int32_t code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg); SName n = {.type = T_NAME_TABLE, .acctId = 1};
ASSERT_EQ(code, 0); strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestTablename, &vgList);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
@ -585,7 +595,11 @@ TEST(tableDistVgroup, childTableCase) {
code = catalogGetHandle(ctgTestClusterId, &pCtg); code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestCTablename, &vgList); SName n = {.type = T_NAME_TABLE, .acctId = 1};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestCTablename);
code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
@ -607,14 +621,14 @@ TEST(tableDistVgroup, superTableCase) {
initQueryModuleMsgHandle(); initQueryModuleMsgHandle();
//sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
int32_t code = catalogGetHandle(ctgTestClusterId, &pCtg);
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
code = catalogGetHandle(ctgTestClusterId, &pCtg); SName n = {.type = T_NAME_TABLE, .acctId = 1};
ASSERT_EQ(code, 0); strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestSTablename);
code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, ctgTestSTablename, &vgList); code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 10); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 10);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);

7
source/libs/index/inc/index_cache.h
View File

@ -30,9 +30,13 @@
extern "C" { extern "C" {
#endif #endif
typedef struct MemTable {
T_REF_DECLARE()
SSkipList* mem;
} MemTable;
typedef struct IndexCache { typedef struct IndexCache {
T_REF_DECLARE() T_REF_DECLARE()
SSkipList *mem, *imm; MemTable *mem, *imm;
SIndex* index; SIndex* index;
char* colName; char* colName;
int32_t version; int32_t version;
@ -45,7 +49,6 @@ typedef struct IndexCache {
#define CACHE_VERSION(cache) atomic_load_32(&cache->version) #define CACHE_VERSION(cache) atomic_load_32(&cache->version)
typedef struct CacheTerm { typedef struct CacheTerm {
// key // key
int32_t nColVal;
char* colVal; char* colVal;
int32_t version; int32_t version;
// value // value

60
source/libs/index/src/index.c
View File

@ -34,9 +34,7 @@ int32_t indexInit() {
return indexQhandle == NULL ? -1 : 0; return indexQhandle == NULL ? -1 : 0;
// do nothing // do nothing
} }
void indexCleanUp() { void indexCleanUp() { taosCleanUpScheduler(indexQhandle); }
taosCleanUpScheduler(indexQhandle);
}
static int uidCompare(const void* a, const void* b) { static int uidCompare(const void* a, const void* b) {
uint64_t u1 = *(uint64_t*)a; uint64_t u1 = *(uint64_t*)a;
@ -63,7 +61,9 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oTyp
int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
// pthread_once(&isInit, indexInit); // pthread_once(&isInit, indexInit);
SIndex* sIdx = calloc(1, sizeof(SIndex)); SIndex* sIdx = calloc(1, sizeof(SIndex));
if (sIdx == NULL) { return -1; } if (sIdx == NULL) {
return -1;
}
#ifdef USE_LUCENE #ifdef USE_LUCENE
index_t* index = index_open(path); index_t* index = index_open(path);
@ -99,7 +99,9 @@ void indexClose(SIndex* sIdx) {
void* iter = taosHashIterate(sIdx->colObj, NULL); void* iter = taosHashIterate(sIdx->colObj, NULL);
while (iter) { while (iter) {
IndexCache** pCache = iter; IndexCache** pCache = iter;
if (*pCache) { indexCacheUnRef(*pCache); } if (*pCache) {
indexCacheUnRef(*pCache);
}
iter = taosHashIterate(sIdx->colObj, iter); iter = taosHashIterate(sIdx->colObj, iter);
} }
taosHashCleanup(sIdx->colObj); taosHashCleanup(sIdx->colObj);
@ -133,7 +135,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
for (int i = 0; i < taosArrayGetSize(fVals); i++) { for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm* p = taosArrayGetP(fVals, i); SIndexTerm* p = taosArrayGetP(fVals, i);
IndexCache** cache = taosHashGet(index->colObj, p->colName, p->nColName); IndexCache** cache = taosHashGet(index->colObj, p->colName, p->nColName);
if (*cache == NULL) { if (cache == NULL) {
IndexCache* pCache = indexCacheCreate(index, p->colName, p->colType); IndexCache* pCache = indexCacheCreate(index, p->colName, p->colType);
taosHashPut(index->colObj, p->colName, p->nColName, &pCache, sizeof(void*)); taosHashPut(index->colObj, p->colName, p->nColName, &pCache, sizeof(void*));
} }
@ -143,10 +145,11 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
for (int i = 0; i < taosArrayGetSize(fVals); i++) { for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm* p = taosArrayGetP(fVals, i); SIndexTerm* p = taosArrayGetP(fVals, i);
IndexCache** cache = taosHashGet(index->colObj, p->colName, p->nColName); IndexCache** cache = taosHashGet(index->colObj, p->colName, p->nColName);
assert(*cache != NULL); assert(*cache != NULL);
int ret = indexCachePut(*cache, p, uid); int ret = indexCachePut(*cache, p, uid);
if (ret != 0) { return ret; } if (ret != 0) {
return ret;
}
} }
#endif #endif
@ -227,14 +230,17 @@ SIndexOpts* indexOptsCreate() {
void indexOptsDestroy(SIndexOpts* opts) { void indexOptsDestroy(SIndexOpts* opts) {
#ifdef USE_LUCENE #ifdef USE_LUCENE
#endif #endif
} /* return;
}
/*
* @param: oper * @param: oper
* *
*/ */
SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) {
SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery));
if (p == NULL) { return NULL; } if (p == NULL) {
return NULL;
}
p->opera = opera; p->opera = opera;
p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); p->query = taosArrayInit(4, sizeof(SIndexTermQuery));
return p; return p;
@ -253,15 +259,12 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde
return 0; return 0;
} }
SIndexTerm* indexTermCreate(int64_t suid, SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName,
SIndexOperOnColumn oper, int32_t nColName, const char* colVal, int32_t nColVal) {
uint8_t colType,
const char* colName,
int32_t nColName,
const char* colVal,
int32_t nColVal) {
SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm)));
if (t == NULL) { return NULL; } if (t == NULL) {
return NULL;
}
t->suid = suid; t->suid = suid;
t->operType = oper; t->operType = oper;
@ -282,9 +285,7 @@ void indexTermDestroy(SIndexTerm* p) {
free(p); free(p);
} }
SIndexMultiTerm* indexMultiTermCreate() { SIndexMultiTerm* indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm*)); }
return taosArrayInit(4, sizeof(SIndexTerm*));
}
int indexMultiTermAdd(SIndexMultiTerm* terms, SIndexTerm* term) { int indexMultiTermAdd(SIndexMultiTerm* terms, SIndexTerm* term) {
taosArrayPush(terms, &term); taosArrayPush(terms, &term);
@ -307,7 +308,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result
IndexCache* cache = NULL; IndexCache* cache = NULL;
pthread_mutex_lock(&sIdx->mtx); pthread_mutex_lock(&sIdx->mtx);
IndexCache** pCache = taosHashGet(sIdx->colObj, colName, nColName); IndexCache** pCache = taosHashGet(sIdx->colObj, colName, nColName);
if (*pCache == NULL) { if (pCache == NULL) {
pthread_mutex_unlock(&sIdx->mtx); pthread_mutex_unlock(&sIdx->mtx);
return -1; return -1;
} }
@ -335,7 +336,9 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result
return 0; return 0;
} }
static void indexInterResultsDestroy(SArray* results) { static void indexInterResultsDestroy(SArray* results) {
if (results == NULL) { return; } if (results == NULL) {
return;
}
size_t sz = taosArrayGetSize(results); size_t sz = taosArrayGetSize(results);
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
@ -366,7 +369,9 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType
} }
int indexFlushCacheTFile(SIndex* sIdx, void* cache) { int indexFlushCacheTFile(SIndex* sIdx, void* cache) {
if (sIdx == NULL) { return -1; } if (sIdx == NULL) {
return -1;
}
indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid);
IndexCache* pCache = (IndexCache*)cache; IndexCache* pCache = (IndexCache*)cache;
@ -399,7 +404,6 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) {
TFileValue* tfv = tfileValueCreate(cv->colVal); TFileValue* tfv = tfileValueCreate(cv->colVal);
taosArrayAddAll(tfv->tableId, cv->val); taosArrayAddAll(tfv->tableId, cv->val);
taosArrayPush(result, &tfv); taosArrayPush(result, &tfv);
// copy to final Result; // copy to final Result;
cn = cacheIter->next(cacheIter); cn = cacheIter->next(cacheIter);
} else { } else {
@ -433,7 +437,9 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) {
indexError("faile to open file to write"); indexError("faile to open file to write");
} else { } else {
int ret = tfileWriterPut(tw, result); int ret = tfileWriterPut(tw, result);
if (ret != 0) { indexError("faile to write into tindex "); } if (ret != 0) {
indexError("faile to write into tindex ");
}
} }
// not free later, just put int table cache // not free later, just put int table cache
indexCacheDestroyImm(pCache); indexCacheDestroyImm(pCache);

256
source/libs/index/src/index_cache.c
View File

@ -23,46 +23,22 @@
#define MEM_TERM_LIMIT 1000000 #define MEM_TERM_LIMIT 1000000
// ref index_cache.h:22 // ref index_cache.h:22
//#define CACHE_KEY_LEN(p) \ //#define CACHE_KEY_LEN(p) \
// (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType)) // (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) +
// sizeof(p->operType))
static void cacheTermDestroy(CacheTerm* ct) { static void indexMemRef(MemTable* tbl);
if (ct == NULL) { return; } static void indexMemUnRef(MemTable* tbl);
free(ct->colVal); static void cacheTermDestroy(CacheTerm* ct);
free(ct); static char* getIndexKey(const void* pData);
} static int32_t compareKey(const void* l, const void* r);
static char* getIndexKey(const void* pData) {
CacheTerm* p = (CacheTerm*)pData;
return (char*)p;
}
static int32_t compareKey(const void* l, const void* r) { static MemTable* indexInternalCacheCreate(int8_t type);
CacheTerm* lt = (CacheTerm*)l;
CacheTerm* rt = (CacheTerm*)r;
// compare colVal static void doMergeWork(SSchedMsg* msg);
int i, j; static bool indexCacheIteratorNext(Iterate* itera);
for (i = 0, j = 0; i < lt->nColVal && j < rt->nColVal; i++, j++) {
if (lt->colVal[i] == rt->colVal[j]) {
continue;
} else {
return lt->colVal[i] < rt->colVal[j] ? -1 : 1;
}
}
if (i < lt->nColVal) {
return 1;
} else if (j < rt->nColVal) {
return -1;
}
// compare version
return rt->version - lt->version;
}
static SSkipList* indexInternalCacheCreate(int8_t type) { static IterateValue* indexCacheIteratorGetValue(Iterate* iter);
if (type == TSDB_DATA_TYPE_BINARY) {
return tSkipListCreate(MAX_SKIP_LIST_LEVEL, type, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey);
}
}
IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type) { IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type) {
IndexCache* cache = calloc(1, sizeof(IndexCache)); IndexCache* cache = calloc(1, sizeof(IndexCache));
@ -83,7 +59,15 @@ IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type) {
return cache; return cache;
} }
void indexCacheDebug(IndexCache* cache) { void indexCacheDebug(IndexCache* cache) {
SSkipListIterator* iter = tSkipListCreateIter(cache->mem); MemTable* tbl = NULL;
pthread_mutex_lock(&cache->mtx);
tbl = cache->mem;
indexMemRef(tbl);
pthread_mutex_unlock(&cache->mtx);
SSkipList* slt = tbl->mem;
SSkipListIterator* iter = tSkipListCreateIter(slt);
while (tSkipListIterNext(iter)) { while (tSkipListIterNext(iter)) {
SSkipListNode* node = tSkipListIterGet(iter); SSkipListNode* node = tSkipListIterGet(iter);
CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node);
@ -93,6 +77,8 @@ void indexCacheDebug(IndexCache* cache) {
} }
} }
tSkipListDestroyIter(iter); tSkipListDestroyIter(iter);
indexMemUnRef(tbl);
} }
void indexCacheDestroySkiplist(SSkipList* slt) { void indexCacheDestroySkiplist(SSkipList* slt) {
@ -100,71 +86,50 @@ void indexCacheDestroySkiplist(SSkipList* slt) {
while (tSkipListIterNext(iter)) { while (tSkipListIterNext(iter)) {
SSkipListNode* node = tSkipListIterGet(iter); SSkipListNode* node = tSkipListIterGet(iter);
CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node);
if (ct != NULL) {} if (ct != NULL) {
}
} }
tSkipListDestroyIter(iter); tSkipListDestroyIter(iter);
tSkipListDestroy(slt);
} }
void indexCacheDestroyImm(IndexCache* cache) { void indexCacheDestroyImm(IndexCache* cache) {
MemTable* tbl = NULL;
pthread_mutex_lock(&cache->mtx); pthread_mutex_lock(&cache->mtx);
SSkipList* timm = (SSkipList*)cache->imm; tbl = cache->imm;
cache->imm = NULL; // or throw int bg thread cache->imm = NULL; // or throw int bg thread
pthread_mutex_unlock(&cache->mtx); pthread_mutex_unlock(&cache->mtx);
indexMemUnRef(tbl);
indexCacheDestroySkiplist(timm);
} }
void indexCacheDestroy(void* cache) { void indexCacheDestroy(void* cache) {
IndexCache* pCache = cache; IndexCache* pCache = cache;
if (pCache == NULL) { return; } if (pCache == NULL) {
tSkipListDestroy(pCache->mem); return;
tSkipListDestroy(pCache->imm); }
indexMemUnRef(pCache->mem);
indexMemUnRef(pCache->imm);
free(pCache->colName); free(pCache->colName);
free(pCache); free(pCache);
} }
static void doMergeWork(SSchedMsg* msg) {
IndexCache* pCache = msg->ahandle;
SIndex* sidx = (SIndex*)pCache->index;
indexFlushCacheTFile(sidx, pCache);
}
static bool indexCacheIteratorNext(Iterate* itera) {
SSkipListIterator* iter = itera->iter;
if (iter == NULL) { return false; }
IterateValue* iv = &itera->val;
iterateValueDestroy(iv, false);
bool next = tSkipListIterNext(iter);
if (next) {
SSkipListNode* node = tSkipListIterGet(iter);
CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node);
iv->type = ct->operaType;
iv->colVal = ct->colVal;
taosArrayPush(iv->val, &ct->uid);
}
return next;
}
static IterateValue* indexCacheIteratorGetValue(Iterate* iter) {
return &iter->val;
}
Iterate* indexCacheIteratorCreate(IndexCache* cache) { Iterate* indexCacheIteratorCreate(IndexCache* cache) {
Iterate* iiter = calloc(1, sizeof(Iterate)); Iterate* iiter = calloc(1, sizeof(Iterate));
if (iiter == NULL) { return NULL; } if (iiter == NULL) {
return NULL;
}
MemTable* tbl = cache->imm;
iiter->val.val = taosArrayInit(1, sizeof(uint64_t)); iiter->val.val = taosArrayInit(1, sizeof(uint64_t));
iiter->iter = cache->imm != NULL ? tSkipListCreateIter(cache->imm) : NULL; iiter->iter = tbl != NULL ? tSkipListCreateIter(tbl->mem) : NULL;
iiter->next = indexCacheIteratorNext; iiter->next = indexCacheIteratorNext;
iiter->getValue = indexCacheIteratorGetValue; iiter->getValue = indexCacheIteratorGetValue;
return iiter; return iiter;
} }
void indexCacheIteratorDestroy(Iterate* iter) { void indexCacheIteratorDestroy(Iterate* iter) {
if (iter == NULL) { return; } if (iter == NULL) {
return;
}
tSkipListDestroyIter(iter->iter); tSkipListDestroyIter(iter->iter);
iterateValueDestroy(&iter->val, true); iterateValueDestroy(&iter->val, true);
free(iter); free(iter);
@ -201,18 +166,21 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) {
} }
int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
if (cache == NULL) { return -1; } if (cache == NULL) {
return -1;
}
IndexCache* pCache = cache; IndexCache* pCache = cache;
indexCacheRef(pCache); indexCacheRef(pCache);
// encode data // encode data
CacheTerm* ct = calloc(1, sizeof(CacheTerm)); CacheTerm* ct = calloc(1, sizeof(CacheTerm));
if (cache == NULL) { return -1; } if (cache == NULL) {
return -1;
}
// set up key // set up key
ct->colType = term->colType; ct->colType = term->colType;
ct->nColVal = term->nColVal; ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1));
ct->colVal = (char*)calloc(1, sizeof(char) * (ct->nColVal + 1)); memcpy(ct->colVal, term->colVal, term->nColVal);
memcpy(ct->colVal, term->colVal, ct->nColVal);
ct->version = atomic_add_fetch_32(&pCache->version, 1); ct->version = atomic_add_fetch_32(&pCache->version, 1);
// set value // set value
ct->uid = uid; ct->uid = uid;
@ -220,8 +188,13 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
// ugly code, refactor later // ugly code, refactor later
pthread_mutex_lock(&pCache->mtx); pthread_mutex_lock(&pCache->mtx);
indexCacheMakeRoomForWrite(pCache); indexCacheMakeRoomForWrite(pCache);
tSkipListPut(pCache->mem, (char*)ct); MemTable* tbl = pCache->mem;
indexMemRef(tbl);
tSkipListPut(tbl->mem, (char*)ct);
indexMemUnRef(tbl);
pthread_mutex_unlock(&pCache->mtx); pthread_mutex_unlock(&pCache->mtx);
indexCacheUnRef(pCache); indexCacheUnRef(pCache);
@ -233,27 +206,38 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u
return 0; return 0;
} }
int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) { int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) {
if (cache == NULL) { return -1; } if (cache == NULL) {
return -1;
}
IndexCache* pCache = cache; IndexCache* pCache = cache;
SIndexTerm* term = query->term; SIndexTerm* term = query->term;
EIndexQueryType qtype = query->qType; EIndexQueryType qtype = query->qType;
MemTable *mem = NULL, *imm = NULL;
pthread_mutex_lock(&pCache->mtx);
mem = pCache->mem;
imm = pCache->imm;
indexMemRef(mem);
indexMemRef(imm);
pthread_mutex_unlock(&pCache->mtx);
CacheTerm* ct = calloc(1, sizeof(CacheTerm)); CacheTerm* ct = calloc(1, sizeof(CacheTerm));
if (ct == NULL) { return -1; } if (ct == NULL) {
ct->nColVal = term->nColVal; return -1;
ct->colVal = calloc(1, sizeof(char) * (ct->nColVal + 1)); }
memcpy(ct->colVal, term->colVal, ct->nColVal); ct->colVal = calloc(1, sizeof(char) * (term->nColVal + 1));
memcpy(ct->colVal, term->colVal, term->nColVal);
ct->version = atomic_load_32(&pCache->version); ct->version = atomic_load_32(&pCache->version);
char* key = getIndexKey(ct); char* key = getIndexKey(ct);
// TODO handle multi situation later, and refactor // TODO handle multi situation later, and refactor
SSkipListIterator* iter = tSkipListCreateIterFromVal(pCache->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
while (tSkipListIterNext(iter)) { while (tSkipListIterNext(iter)) {
SSkipListNode* node = tSkipListIterGet(iter); SSkipListNode* node = tSkipListIterGet(iter);
if (node != NULL) { if (node != NULL) {
CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
if (c->operaType == ADD_VALUE || qtype == QUERY_TERM) { if (c->operaType == ADD_VALUE || qtype == QUERY_TERM) {
if (c->nColVal == ct->nColVal && strncmp(c->colVal, ct->colVal, c->nColVal) == 0) { if (strcmp(c->colVal, ct->colVal) == 0) {
taosArrayPush(result, &c->uid); taosArrayPush(result, &c->uid);
*s = kTypeValue; *s = kTypeValue;
} else { } else {
@ -279,14 +263,104 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV
} else if (qtype == QUERY_REGEX) { } else if (qtype == QUERY_REGEX) {
// //
} }
indexMemUnRef(mem);
indexMemUnRef(imm);
return 0; return 0;
} }
void indexCacheRef(IndexCache* cache) { void indexCacheRef(IndexCache* cache) {
if (cache == NULL) {
return;
}
int ref = T_REF_INC(cache); int ref = T_REF_INC(cache);
UNUSED(ref); UNUSED(ref);
} }
void indexCacheUnRef(IndexCache* cache) { void indexCacheUnRef(IndexCache* cache) {
int ref = T_REF_DEC(cache); if (cache == NULL) {
if (ref == 0) { indexCacheDestroy(cache); } return;
} }
int ref = T_REF_DEC(cache);
if (ref == 0) {
indexCacheDestroy(cache);
}
}
void indexMemRef(MemTable* tbl) {
if (tbl == NULL) {
return;
}
int ref = T_REF_INC(tbl);
UNUSED(ref);
}
void indexMemUnRef(MemTable* tbl) {
if (tbl == NULL) {
return;
}
int ref = T_REF_DEC(tbl);
if (ref == 0) {
SSkipList* slt = tbl->mem;
indexCacheDestroySkiplist(slt);
free(tbl);
}
}
static void cacheTermDestroy(CacheTerm* ct) {
if (ct == NULL) {
return;
}
free(ct->colVal);
free(ct);
}
static char* getIndexKey(const void* pData) {
CacheTerm* p = (CacheTerm*)pData;
return (char*)p;
}
static int32_t compareKey(const void* l, const void* r) {
CacheTerm* lt = (CacheTerm*)l;
CacheTerm* rt = (CacheTerm*)r;
// compare colVal
int32_t cmp = strcmp(lt->colVal, rt->colVal);
if (cmp == 0) {
return rt->version - lt->version;
}
return cmp;
}
static MemTable* indexInternalCacheCreate(int8_t type) {
MemTable* tbl = calloc(1, sizeof(MemTable));
indexMemRef(tbl);
if (type == TSDB_DATA_TYPE_BINARY) {
tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, type, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey);
}
return tbl;
}
static void doMergeWork(SSchedMsg* msg) {
IndexCache* pCache = msg->ahandle;
SIndex* sidx = (SIndex*)pCache->index;
indexFlushCacheTFile(sidx, pCache);
}
static bool indexCacheIteratorNext(Iterate* itera) {
SSkipListIterator* iter = itera->iter;
if (iter == NULL) {
return false;
}
IterateValue* iv = &itera->val;
iterateValueDestroy(iv, false);
bool next = tSkipListIterNext(iter);
if (next) {
SSkipListNode* node = tSkipListIterGet(iter);
CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node);
iv->type = ct->operaType;
iv->colVal = ct->colVal;
taosArrayPush(iv->val, &ct->uid);
}
return next;
}
static IterateValue* indexCacheIteratorGetValue(Iterate* iter) { return &iter->val; }

140
source/libs/index/src/index_tfile.c
View File

@ -54,7 +54,9 @@ static void tfileSerialCacheKey(TFileCacheKey* key, char* buf);
TFileCache* tfileCacheCreate(const char* path) { TFileCache* tfileCacheCreate(const char* path) {
TFileCache* tcache = calloc(1, sizeof(TFileCache)); TFileCache* tcache = calloc(1, sizeof(TFileCache));
if (tcache == NULL) { return NULL; } if (tcache == NULL) {
return NULL;
}
tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
tcache->capacity = 64; tcache->capacity = 64;
@ -83,7 +85,10 @@ TFileCache* tfileCacheCreate(const char* path) {
tfileReaderRef(reader); tfileReaderRef(reader);
// loader fst and validate it // loader fst and validate it
TFileHeader* header = &reader->header; TFileHeader* header = &reader->header;
TFileCacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = strlen(header->colName), .colType = header->colType}; TFileCacheKey key = {.suid = header->suid,
.colName = header->colName,
.nColName = strlen(header->colName),
.colType = header->colType};
char buf[128] = {0}; char buf[128] = {0};
tfileSerialCacheKey(&key, buf); tfileSerialCacheKey(&key, buf);
@ -97,13 +102,16 @@ End:
return NULL; return NULL;
} }
void tfileCacheDestroy(TFileCache* tcache) { void tfileCacheDestroy(TFileCache* tcache) {
if (tcache == NULL) { return; } if (tcache == NULL) {
return;
}
// free table cache // free table cache
TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); TFileReader** reader = taosHashIterate(tcache->tableCache, NULL);
while (reader) { while (reader) {
TFileReader* p = *reader; TFileReader* p = *reader;
indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, p->header.colType); indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName,
p->header.colType);
tfileReaderUnRef(p); tfileReaderUnRef(p);
reader = taosHashIterate(tcache->tableCache, reader); reader = taosHashIterate(tcache->tableCache, reader);
@ -116,10 +124,13 @@ TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key) {
char buf[128] = {0}; char buf[128] = {0};
tfileSerialCacheKey(key, buf); tfileSerialCacheKey(key, buf);
TFileReader* reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); TFileReader** reader = taosHashGet(tcache->tableCache, buf, strlen(buf));
tfileReaderRef(reader); if (reader == NULL) {
return NULL;
}
tfileReaderRef(*reader);
return reader; return *reader;
} }
void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) { void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) {
char buf[128] = {0}; char buf[128] = {0};
@ -138,14 +149,17 @@ void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader)
} }
TFileReader* tfileReaderCreate(WriterCtx* ctx) { TFileReader* tfileReaderCreate(WriterCtx* ctx) {
TFileReader* reader = calloc(1, sizeof(TFileReader)); TFileReader* reader = calloc(1, sizeof(TFileReader));
if (reader == NULL) { return NULL; } if (reader == NULL) {
return NULL;
}
// T_REF_INC(reader); // T_REF_INC(reader);
reader->ctx = ctx; reader->ctx = ctx;
if (0 != tfileReaderLoadHeader(reader)) { if (0 != tfileReaderLoadHeader(reader)) {
tfileReaderDestroy(reader); tfileReaderDestroy(reader);
indexError("failed to load index header, suid: %" PRIu64 ", colName: %s", reader->header.suid, reader->header.colName); indexError("failed to load index header, suid: %" PRIu64 ", colName: %s", reader->header.suid,
reader->header.colName);
return NULL; return NULL;
} }
@ -158,7 +172,9 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) {
return reader; return reader;
} }
void tfileReaderDestroy(TFileReader* reader) { void tfileReaderDestroy(TFileReader* reader) {
if (reader == NULL) { return; } if (reader == NULL) {
return;
}
// T_REF_INC(reader); // T_REF_INC(reader);
fstDestroy(reader->fst); fstDestroy(reader->fst);
writerCtxDestroy(reader->ctx); writerCtxDestroy(reader->ctx);
@ -175,10 +191,12 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* resul
uint64_t offset; uint64_t offset;
FstSlice key = fstSliceCreate(term->colVal, term->nColVal); FstSlice key = fstSliceCreate(term->colVal, term->nColVal);
if (fstGet(reader->fst, &key, &offset)) { if (fstGet(reader->fst, &key, &offset)) {
indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex", term->suid, term->colName, term->colVal); indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex", term->suid, term->colName,
term->colVal);
ret = tfileReaderLoadTableIds(reader, offset, result); ret = tfileReaderLoadTableIds(reader, offset, result);
} else { } else {
indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName, term->colVal); indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName,
term->colVal);
} }
fstSliceDestroy(&key); fstSliceDestroy(&key);
} else if (qtype == QUERY_PREFIX) { } else if (qtype == QUERY_PREFIX) {
@ -304,12 +322,16 @@ int tfileWriterPut(TFileWriter* tw, void* data) {
return 0; return 0;
} }
void tfileWriteClose(TFileWriter* tw) { void tfileWriteClose(TFileWriter* tw) {
if (tw == NULL) { return; } if (tw == NULL) {
return;
}
writerCtxDestroy(tw->ctx); writerCtxDestroy(tw->ctx);
free(tw); free(tw);
} }
void tfileWriterDestroy(TFileWriter* tw) { void tfileWriterDestroy(TFileWriter* tw) {
if (tw == NULL) { return; } if (tw == NULL) {
return;
}
writerCtxDestroy(tw->ctx); writerCtxDestroy(tw->ctx);
free(tw); free(tw);
@ -317,29 +339,35 @@ void tfileWriterDestroy(TFileWriter* tw) {
IndexTFile* indexTFileCreate(const char* path) { IndexTFile* indexTFileCreate(const char* path) {
IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); IndexTFile* tfile = calloc(1, sizeof(IndexTFile));
if (tfile == NULL) { return NULL; } if (tfile == NULL) {
return NULL;
}
tfile->cache = tfileCacheCreate(path); tfile->cache = tfileCacheCreate(path);
return tfile; return tfile;
} }
void IndexTFileDestroy(IndexTFile* tfile) { void IndexTFileDestroy(IndexTFile* tfile) { free(tfile); }
free(tfile);
}
int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) {
int ret = -1; int ret = -1;
if (tfile == NULL) { return ret; } if (tfile == NULL) {
return ret;
}
IndexTFile* pTfile = (IndexTFile*)tfile; IndexTFile* pTfile = (IndexTFile*)tfile;
SIndexTerm* term = query->term; SIndexTerm* term = query->term;
TFileCacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; TFileCacheKey key = {
.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName};
TFileReader* reader = tfileCacheGet(pTfile->cache, &key); TFileReader* reader = tfileCacheGet(pTfile->cache, &key);
if (reader == NULL) {
return 0;
}
return tfileReaderSearch(reader, query, result); return tfileReaderSearch(reader, query, result);
} }
int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) {
// TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = // TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName =
// 1}; // term->nColName, .version = 1};
return 0; return 0;
} }
@ -353,7 +381,9 @@ static bool tfileIteratorNext(Iterate* iiter) {
TFileFstIter* tIter = iiter->iter; TFileFstIter* tIter = iiter->iter;
StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL); StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL);
if (rt == NULL) { return false; } if (rt == NULL) {
return false;
}
int32_t sz = 0; int32_t sz = 0;
char* ch = (char*)fstSliceData(&rt->data, &sz); char* ch = (char*)fstSliceData(&rt->data, &sz);
@ -364,20 +394,22 @@ static bool tfileIteratorNext(Iterate* iiter) {
swsResultDestroy(rt); swsResultDestroy(rt);
// set up iterate value // set up iterate value
if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; } if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) {
return false;
}
iv->colVal = colVal; iv->colVal = colVal;
// std::string key(ch, sz); // std::string key(ch, sz);
} }
static IterateValue* tifileIterateGetValue(Iterate* iter) { static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; }
return &iter->val;
}
static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) {
TFileFstIter* tIter = calloc(1, sizeof(Iterate)); TFileFstIter* tIter = calloc(1, sizeof(Iterate));
if (tIter == NULL) { return NULL; } if (tIter == NULL) {
return NULL;
}
tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS);
tIter->fb = fstSearch(reader->fst, tIter->ctx); tIter->fb = fstSearch(reader->fst, tIter->ctx);
tIter->st = streamBuilderIntoStream(tIter->fb); tIter->st = streamBuilderIntoStream(tIter->fb);
@ -389,14 +421,18 @@ Iterate* tfileIteratorCreate(TFileReader* reader) {
Iterate* iter = calloc(1, sizeof(Iterate)); Iterate* iter = calloc(1, sizeof(Iterate));
iter->iter = tfileFstIteratorCreate(reader); iter->iter = tfileFstIteratorCreate(reader);
if (iter->iter == NULL) { return NULL; } if (iter->iter == NULL) {
return NULL;
}
iter->next = tfileIteratorNext; iter->next = tfileIteratorNext;
iter->getValue = tifileIterateGetValue; iter->getValue = tifileIterateGetValue;
return iter; return iter;
} }
void tfileIteratorDestroy(Iterate* iter) { void tfileIteratorDestroy(Iterate* iter) {
if (iter == NULL) { return; } if (iter == NULL) {
return;
}
IterateValue* iv = &iter->val; IterateValue* iv = &iter->val;
iterateValueDestroy(iv, true); iterateValueDestroy(iv, true);
@ -409,14 +445,18 @@ void tfileIteratorDestroy(Iterate* iter) {
} }
TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName) { TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName) {
if (tf == NULL) { return NULL; } if (tf == NULL) {
return NULL;
}
TFileCacheKey key = {.suid = 0, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; TFileCacheKey key = {.suid = 0, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)};
return tfileCacheGet(tf->cache, &key); return tfileCacheGet(tf->cache, &key);
} }
static int tfileStrCompare(const void* a, const void* b) { static int tfileStrCompare(const void* a, const void* b) {
int ret = strcmp((char*)a, (char*)b); int ret = strcmp((char*)a, (char*)b);
if (ret == 0) { return ret; } if (ret == 0) {
return ret;
}
return ret < 0 ? -1 : 1; return ret < 0 ? -1 : 1;
} }
@ -431,13 +471,17 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) {
TFileValue* tfileValueCreate(char* val) { TFileValue* tfileValueCreate(char* val) {
TFileValue* tf = calloc(1, sizeof(TFileValue)); TFileValue* tf = calloc(1, sizeof(TFileValue));
if (tf == NULL) { return NULL; } if (tf == NULL) {
return NULL;
}
tf->tableId = taosArrayInit(32, sizeof(uint64_t)); tf->tableId = taosArrayInit(32, sizeof(uint64_t));
return tf; return tf;
} }
int tfileValuePush(TFileValue* tf, uint64_t val) { int tfileValuePush(TFileValue* tf, uint64_t val) {
if (tf == NULL) { return -1; } if (tf == NULL) {
return -1;
}
taosArrayPush(tf->tableId, &val); taosArrayPush(tf->tableId, &val);
return 0; return 0;
} }
@ -457,7 +501,9 @@ static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) {
static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) { static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) {
int32_t fstOffset = offset + sizeof(tw->header.fstOffset); int32_t fstOffset = offset + sizeof(tw->header.fstOffset);
tw->header.fstOffset = fstOffset; tw->header.fstOffset = fstOffset;
if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; } if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) {
return -1;
}
tw->offset += sizeof(fstOffset); tw->offset += sizeof(fstOffset);
return 0; return 0;
} }
@ -468,7 +514,9 @@ static int tfileWriteHeader(TFileWriter* writer) {
memcpy(buf, (char*)header, sizeof(buf)); memcpy(buf, (char*)header, sizeof(buf));
int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf)); int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf));
if (sizeof(buf) != nwrite) { return -1; } if (sizeof(buf) != nwrite) {
return -1;
}
writer->offset = nwrite; writer->offset = nwrite;
return 0; return 0;
} }
@ -502,7 +550,9 @@ static int tfileReaderLoadFst(TFileReader* reader) {
static int FST_MAX_SIZE = 16 * 1024; static int FST_MAX_SIZE = 16 * 1024;
char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE); char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE);
if (buf == NULL) { return -1; } if (buf == NULL) {
return -1;
}
WriterCtx* ctx = reader->ctx; WriterCtx* ctx = reader->ctx;
int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset); int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset);
@ -525,7 +575,9 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray*
int32_t total = sizeof(uint64_t) * nid; int32_t total = sizeof(uint64_t) * nid;
char* buf = calloc(1, total); char* buf = calloc(1, total);
if (buf == NULL) { return -1; } if (buf == NULL) {
return -1;
}
nread = ctx->read(ctx, buf, total); nread = ctx->read(ctx, buf, total);
assert(total == nread); assert(total == nread);
@ -543,12 +595,16 @@ void tfileReaderRef(TFileReader* reader) {
void tfileReaderUnRef(TFileReader* reader) { void tfileReaderUnRef(TFileReader* reader) {
int ref = T_REF_DEC(reader); int ref = T_REF_DEC(reader);
if (ref == 0) { tfileReaderDestroy(reader); } if (ref == 0) {
tfileReaderDestroy(reader);
}
} }
static int tfileGetFileList(const char* path, SArray* result) { static int tfileGetFileList(const char* path, SArray* result) {
DIR* dir = opendir(path); DIR* dir = opendir(path);
if (NULL == dir) { return -1; } if (NULL == dir) {
return -1;
}
struct dirent* entry; struct dirent* entry;
while ((entry = readdir(dir)) != NULL) { while ((entry = readdir(dir)) != NULL) {
@ -576,7 +632,9 @@ static int tfileCompare(const void* a, const void* b) {
size_t bLen = strlen(bName); size_t bLen = strlen(bName);
int ret = strncmp(aName, bName, aLen > bLen ? aLen : bLen); int ret = strncmp(aName, bName, aLen > bLen ? aLen : bLen);
if (ret == 0) { return ret; } if (ret == 0) {
return ret;
}
return ret < 0 ? -1 : 1; return ret < 0 ? -1 : 1;
} }
// tfile name suid-colId-version.tindex // tfile name suid-colId-version.tindex

198
source/libs/index/test/indexTests.cc
View File

@ -2,7 +2,8 @@
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com> * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
* *
* This program is free software: you can use, redistribute, and/or modify * This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free Software Foundation. * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free
* Software Foundation.
* *
* This program is distributed in the hope that it will be useful, but WITHOUT * This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
@ -75,7 +76,9 @@ class FstReadMemory {
bool init() { bool init() {
char* buf = (char*)calloc(1, sizeof(char) * _size); char* buf = (char*)calloc(1, sizeof(char) * _size);
int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size);
if (nRead <= 0) { return false; } if (nRead <= 0) {
return false;
}
_size = nRead; _size = nRead;
_s = fstSliceCreate((uint8_t*)buf, _size); _s = fstSliceCreate((uint8_t*)buf, _size);
_fst = fstCreate(&_s); _fst = fstCreate(&_s);
@ -179,7 +182,9 @@ void checkFstPerf() {
delete fw; delete fw;
FstReadMemory* m = new FstReadMemory(1024 * 64); FstReadMemory* m = new FstReadMemory(1024 * 64);
if (m->init()) { printf("success to init fst read"); } if (m->init()) {
printf("success to init fst read");
}
Performance_fstReadRecords(m); Performance_fstReadRecords(m);
delete m; delete m;
} }
@ -283,7 +288,8 @@ class IndexEnv : public ::testing::Test {
// / { // / {
// / std::string colName("tag1"), colVal("Hello world"); // / std::string colName("tag1"), colVal("Hello world");
// / SIndexTerm* term = // / SIndexTerm* term =
// indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), / colVal.size()); // indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), /
// colVal.size());
// SIndexMultiTerm* terms = indexMultiTermCreate(); // SIndexMultiTerm* terms = indexMultiTermCreate();
// indexMultiTermAdd(terms, term); // indexMultiTermAdd(terms, term);
// / / for (size_t i = 0; i < 100; i++) { // / / for (size_t i = 0; i < 100; i++) {
@ -301,14 +307,16 @@ class IndexEnv : public ::testing::Test {
// / { // / {
// / std::string colName("tag1"), colVal("Hello world"); // / std::string colName("tag1"), colVal("Hello world");
// / SIndexTerm* term = // / SIndexTerm* term =
// / indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); // / indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(),
// colVal.size());
// / indexMultiTermAdd(terms, term); // / indexMultiTermAdd(terms, term);
// / // /
// } // }
// / { // / {
// / std::string colName("tag2"), colVal("Hello world"); // / std::string colName("tag2"), colVal("Hello world");
// / SIndexTerm* term = // / SIndexTerm* term =
// / indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); // / indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(),
// colVal.size());
// / indexMultiTermAdd(terms, term); // / indexMultiTermAdd(terms, term);
// / // /
// } // }
@ -327,7 +335,8 @@ class IndexEnv : public ::testing::Test {
class TFileObj { class TFileObj {
public: public:
TFileObj(const std::string& path = "/tmp/tindex", const std::string& colName = "voltage") : path_(path), colName_(colName) { TFileObj(const std::string& path = "/tmp/tindex", const std::string& colName = "voltage")
: path_(path), colName_(colName) {
colId_ = 10; colId_ = 10;
// Do Nothing // Do Nothing
// //
@ -337,7 +346,9 @@ class TFileObj {
tfileReaderDestroy(reader_); tfileReaderDestroy(reader_);
reader_ = NULL; reader_ = NULL;
} }
if (writer_ == NULL) { InitWriter(); } if (writer_ == NULL) {
InitWriter();
}
return tfileWriterPut(writer_, tv); return tfileWriterPut(writer_, tv);
} }
bool InitWriter() { bool InitWriter() {
@ -377,8 +388,12 @@ class TFileObj {
return tfileReaderSearch(reader_, query, result); return tfileReaderSearch(reader_, query, result);
} }
~TFileObj() { ~TFileObj() {
if (writer_) { tfileWriterDestroy(writer_); } if (writer_) {
if (reader_) { tfileReaderDestroy(reader_); } tfileWriterDestroy(writer_);
}
if (reader_) {
tfileReaderDestroy(reader_);
}
} }
private: private:
@ -457,7 +472,8 @@ TEST_F(IndexTFileEnv, test_tfile_write) {
std::string colName("voltage"); std::string colName("voltage");
std::string colVal("ab"); std::string colVal("ab");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; SIndexTermQuery query = {.term = term, .qType = QUERY_TERM};
SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
@ -525,46 +541,53 @@ TEST_F(IndexCacheEnv, cache_test) {
std::string colName("voltage"); std::string colName("voltage");
{ {
std::string colVal("v1"); std::string colVal("v1");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++); coj->Put(term, colId, version++, suid++);
} }
{ {
std::string colVal("v3"); std::string colVal("v3");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++); coj->Put(term, colId, version++, suid++);
} }
{ {
std::string colVal("v2"); std::string colVal("v2");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++); coj->Put(term, colId, version++, suid++);
} }
{ {
std::string colVal("v3"); std::string colVal("v3");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++); coj->Put(term, colId, version++, suid++);
} }
{ {
std::string colVal("v3"); std::string colVal("v3");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++); coj->Put(term, colId, version++, suid++);
} }
{ {
std::string colVal("v3"); std::string colVal("v3");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, othColId, version++, suid++); coj->Put(term, othColId, version++, suid++);
} }
{ {
std::string colVal("v4"); std::string colVal("v4");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, othColId, version++, suid++); coj->Put(term, othColId, version++, suid++);
} }
{ {
std::string colVal("v4"); std::string colVal("v4");
for (size_t i = 0; i < 10; i++) { for (size_t i = 0; i < 10; i++) {
colVal[colVal.size() - 1] = 'a' + i; colVal[colVal.size() - 1] = 'a' + i;
SIndexTerm* term = SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++); coj->Put(term, colId, version++, suid++);
} }
} }
@ -572,7 +595,8 @@ TEST_F(IndexCacheEnv, cache_test) {
// begin query // begin query
{ {
std::string colVal("v3"); std::string colVal("v3");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; SIndexTermQuery query = {.term = term, .qType = QUERY_TERM};
SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid));
STermValueType valType; STermValueType valType;
@ -583,7 +607,8 @@ TEST_F(IndexCacheEnv, cache_test) {
} }
{ {
std::string colVal("v2"); std::string colVal("v2");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; SIndexTermQuery query = {.term = term, .qType = QUERY_TERM};
SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid));
STermValueType valType; STermValueType valType;
@ -592,3 +617,132 @@ TEST_F(IndexCacheEnv, cache_test) {
assert(taosArrayGetSize(ret) == 1); assert(taosArrayGetSize(ret) == 1);
} }
} }
class IndexObj {
public:
IndexObj() {
// opt
numOfWrite = 0;
numOfRead = 0;
indexInit();
}
int Init(const std::string& dir) {
taosRemoveDir(dir.c_str());
taosMkDir(dir.c_str());
int ret = indexOpen(&opts, dir.c_str(), &idx);
if (ret != 0) {
// opt
std::cout << "failed to open index: %s" << dir << std::endl;
}
return ret;
}
int Put(SIndexMultiTerm* fvs, uint64_t uid) {
numOfWrite += taosArrayGetSize(fvs);
return indexPut(idx, fvs, uid);
}
int Search(SIndexMultiTermQuery* multiQ, SArray* result) {
SArray* query = multiQ->query;
numOfRead = taosArrayGetSize(query);
return indexSearch(idx, multiQ, result);
}
void Debug() {
std::cout << "numOfWrite:" << numOfWrite << std::endl;
std::cout << "numOfRead:" << numOfRead << std::endl;
}
~IndexObj() {
indexClose(idx);
indexCleanUp();
}
private:
SIndexOpts opts;
SIndex* idx;
int numOfWrite;
int numOfRead;
};
class IndexEnv2 : public ::testing::Test {
protected:
virtual void SetUp() {
index = new IndexObj();
//
}
virtual void TearDown() {
// r
delete index;
}
IndexObj* index;
};
TEST_F(IndexEnv2, testIndexOpen) {
std::string path = "/tmp";
if (index->Init(path) != 0) {
std::cout << "failed to init index" << std::endl;
exit(1);
}
int targetSize = 100;
{
std::string colName("tag1"), colVal("Hello world");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < targetSize; i++) {
int tableId = i;
int ret = index->Put(terms, tableId);
assert(ret == 0);
}
indexMultiTermDestroy(terms);
}
{
size_t size = 100;
std::string colName("tag1"), colVal("hello world");
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < size; i++) {
int tableId = i;
int ret = index->Put(terms, tableId);
assert(ret == 0);
}
indexMultiTermDestroy(terms);
}
{
std::string colName("tag1"), colVal("Hello world");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
indexMultiTermQueryAdd(mq, term, QUERY_TERM);
SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
index->Search(mq, result);
assert(taosArrayGetSize(result) == targetSize);
}
}
TEST_F(IndexEnv2, testIndex_CachePut) {
std::string path = "/tmp";
if (index->Init(path) != 0) {
}
}
TEST_F(IndexEnv2, testIndexr_TFilePut) {
std::string path = "/tmp";
if (index->Init(path) != 0) {
}
}
TEST_F(IndexEnv2, testIndex_CacheSearch) {
std::string path = "/tmp";
if (index->Init(path) != 0) {
}
}
TEST_F(IndexEnv2, testIndex_TFileSearch) {
std::string path = "/tmp";
if (index->Init(path) != 0) {
}
}

2
source/libs/parser/inc/astGenerator.h
View File

@ -156,7 +156,7 @@ typedef struct SCreateDbInfo {
SToken dbname; SToken dbname;
int32_t replica; int32_t replica;
int32_t cacheBlockSize; int32_t cacheBlockSize;
int32_t maxTablesPerVnode; int32_t numOfVgroups;
int32_t numOfBlocks; int32_t numOfBlocks;
int32_t daysPerFile; int32_t daysPerFile;
int32_t minRowsPerBlock; int32_t minRowsPerBlock;

1
source/libs/parser/inc/astToMsg.h
View File

@ -4,7 +4,6 @@
#include "parserInt.h" #include "parserInt.h"
#include "tmsg.h" #include "tmsg.h"
int32_t createSName(SName* pName, SToken* pTableName, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf);
SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen);
SCreateAcctMsg* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SCreateAcctMsg* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen);

13
source/libs/parser/inc/parserUtil.h
View File

@ -70,6 +70,19 @@ bool isDclSqlStatement(SSqlInfo* pSqlInfo);
bool isDdlSqlStatement(SSqlInfo* pSqlInfo); bool isDdlSqlStatement(SSqlInfo* pSqlInfo);
bool isDqlSqlStatement(SSqlInfo* pSqlInfo); bool isDqlSqlStatement(SSqlInfo* pSqlInfo);
typedef struct SKvParam {
SKVRowBuilder *builder;
SSchema *schema;
char buf[TSDB_MAX_TAGS_LEN];
} SKvParam;
int32_t KvRowAppend(const void *value, int32_t len, void *param);
typedef int32_t (*_row_append_fn_t)(const void *value, int32_t len, void *param);
int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, _row_append_fn_t func, void* param, SMsgBuf* pMsgBuf);
int32_t createSName(SName* pName, SToken* pTableName, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

5
source/libs/parser/inc/sql.y
View File

@ -6,7 +6,7 @@
%default_type {SToken} %default_type {SToken}
%extra_argument {SSqlInfo* pInfo} %extra_argument {SSqlInfo* pInfo}
%fallback ID BOOL TINYINT SMALLINT INTEGER BIGINT FLOAT DOUBLE STRING TIMESTAMP BINARY NCHAR. %fallback ID BOOL INTEGER FLOAT STRING TIMESTAMP.
%left OR. %left OR.
%left AND. %left AND.
@ -201,6 +201,7 @@ ifnotexists(X) ::= . { X.n = 0;}
/////////////////////////////////THE CREATE STATEMENT/////////////////////////////////////// /////////////////////////////////THE CREATE STATEMENT///////////////////////////////////////
//create option for dnode/db/user/account //create option for dnode/db/user/account
cmd ::= CREATE DNODE ids(X) PORT ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 2, &X, &Y);} cmd ::= CREATE DNODE ids(X) PORT ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 2, &X, &Y);}
cmd ::= CREATE DNODE IPTOKEN(X) PORT ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 2, &X, &Y);}
cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z).
{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);} { setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);}
cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
@ -279,6 +280,7 @@ comp(Y) ::= COMP INTEGER(X). { Y = X; }
prec(Y) ::= PRECISION STRING(X). { Y = X; } prec(Y) ::= PRECISION STRING(X). { Y = X; }
update(Y) ::= UPDATE INTEGER(X). { Y = X; } update(Y) ::= UPDATE INTEGER(X). { Y = X; }
cachelast(Y) ::= CACHELAST INTEGER(X). { Y = X; } cachelast(Y) ::= CACHELAST INTEGER(X). { Y = X; }
vgroups(Y) ::= VGROUPS INTEGER(X). { Y = X; }
//partitions(Y) ::= PARTITIONS INTEGER(X). { Y = X; } //partitions(Y) ::= PARTITIONS INTEGER(X). { Y = X; }
%type db_optr {SCreateDbInfo} %type db_optr {SCreateDbInfo}
@ -299,6 +301,7 @@ db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; }
db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; } db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; }
db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } db_optr(Y) ::= db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) vgroups(X). { Y = Z; Y.numOfVgroups = strtol(X.z, NULL, 10); }
//%type topic_optr {SCreateDbInfo} //%type topic_optr {SCreateDbInfo}
// //

230
source/libs/parser/inc/ttokendef.h
View File

@ -1,230 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TTOKENDEF_H
#define TDENGINE_TTOKENDEF_H
#define TK_ID 1
#define TK_BOOL 2
#define TK_TINYINT 3
#define TK_SMALLINT 4
#define TK_INTEGER 5
#define TK_BIGINT 6
#define TK_FLOAT 7
#define TK_DOUBLE 8
#define TK_STRING 9
#define TK_TIMESTAMP 10
#define TK_BINARY 11
#define TK_NCHAR 12
#define TK_OR 13
#define TK_AND 14
#define TK_NOT 15
#define TK_EQ 16
#define TK_NE 17
#define TK_ISNULL 18
#define TK_NOTNULL 19
#define TK_IS 20
#define TK_LIKE 21
#define TK_MATCH 22
#define TK_NMATCH 23
#define TK_GLOB 24
#define TK_BETWEEN 25
#define TK_IN 26
#define TK_GT 27
#define TK_GE 28
#define TK_LT 29
#define TK_LE 30
#define TK_BITAND 31
#define TK_BITOR 32
#define TK_LSHIFT 33
#define TK_RSHIFT 34
#define TK_PLUS 35
#define TK_MINUS 36
#define TK_DIVIDE 37
#define TK_TIMES 38
#define TK_STAR 39
#define TK_SLASH 40
#define TK_REM 41
#define TK_CONCAT 42
#define TK_UMINUS 43
#define TK_UPLUS 44
#define TK_BITNOT 45
#define TK_SHOW 46
#define TK_DATABASES 47
#define TK_TOPICS 48
#define TK_FUNCTIONS 49
#define TK_MNODES 50
#define TK_DNODES 51
#define TK_ACCOUNTS 52
#define TK_USERS 53
#define TK_MODULES 54
#define TK_QUERIES 55
#define TK_CONNECTIONS 56
#define TK_STREAMS 57
#define TK_VARIABLES 58
#define TK_SCORES 59
#define TK_GRANTS 60
#define TK_VNODES 61
#define TK_DOT 62
#define TK_CREATE 63
#define TK_TABLE 64
#define TK_STABLE 65
#define TK_DATABASE 66
#define TK_TABLES 67
#define TK_STABLES 68
#define TK_VGROUPS 69
#define TK_DROP 70
#define TK_TOPIC 71
#define TK_FUNCTION 72
#define TK_DNODE 73
#define TK_USER 74
#define TK_ACCOUNT 75
#define TK_USE 76
#define TK_DESCRIBE 77
#define TK_DESC 78
#define TK_ALTER 79
#define TK_PASS 80
#define TK_PRIVILEGE 81
#define TK_LOCAL 82
#define TK_COMPACT 83
#define TK_LP 84
#define TK_RP 85
#define TK_IF 86
#define TK_EXISTS 87
#define TK_PORT 88
#define TK_AS 89
#define TK_OUTPUTTYPE 90
#define TK_AGGREGATE 91
#define TK_BUFSIZE 92
#define TK_PPS 93
#define TK_TSERIES 94
#define TK_DBS 95
#define TK_STORAGE 96
#define TK_QTIME 97
#define TK_CONNS 98
#define TK_STATE 99
#define TK_COMMA 100
#define TK_KEEP 101
#define TK_CACHE 102
#define TK_REPLICA 103
#define TK_QUORUM 104
#define TK_DAYS 105
#define TK_MINROWS 106
#define TK_MAXROWS 107
#define TK_BLOCKS 108
#define TK_CTIME 109
#define TK_WAL 110
#define TK_FSYNC 111
#define TK_COMP 112
#define TK_PRECISION 113
#define TK_UPDATE 114
#define TK_CACHELAST 115
#define TK_UNSIGNED 116
#define TK_TAGS 117
#define TK_USING 118
#define TK_NULL 119
#define TK_NOW 120
#define TK_SELECT 121
#define TK_UNION 122
#define TK_ALL 123
#define TK_DISTINCT 124
#define TK_FROM 125
#define TK_VARIABLE 126
#define TK_INTERVAL 127
#define TK_EVERY 128
#define TK_SESSION 129
#define TK_STATE_WINDOW 130
#define TK_FILL 131
#define TK_SLIDING 132
#define TK_ORDER 133
#define TK_BY 134
#define TK_ASC 135
#define TK_GROUP 136
#define TK_HAVING 137
#define TK_LIMIT 138
#define TK_OFFSET 139
#define TK_SLIMIT 140
#define TK_SOFFSET 141
#define TK_WHERE 142
#define TK_RESET 143
#define TK_QUERY 144
#define TK_SYNCDB 145
#define TK_ADD 146
#define TK_COLUMN 147
#define TK_MODIFY 148
#define TK_TAG 149
#define TK_CHANGE 150
#define TK_SET 151
#define TK_KILL 152
#define TK_CONNECTION 153
#define TK_STREAM 154
#define TK_COLON 155
#define TK_ABORT 156
#define TK_AFTER 157
#define TK_ATTACH 158
#define TK_BEFORE 159
#define TK_BEGIN 160
#define TK_CASCADE 161
#define TK_CLUSTER 162
#define TK_CONFLICT 163
#define TK_COPY 164
#define TK_DEFERRED 165
#define TK_DELIMITERS 166
#define TK_DETACH 167
#define TK_EACH 168
#define TK_END 169
#define TK_EXPLAIN 170
#define TK_FAIL 171
#define TK_FOR 172
#define TK_IGNORE 173
#define TK_IMMEDIATE 174
#define TK_INITIALLY 175
#define TK_INSTEAD 176
#define TK_KEY 177
#define TK_OF 178
#define TK_RAISE 179
#define TK_REPLACE 180
#define TK_RESTRICT 181
#define TK_ROW 182
#define TK_STATEMENT 183
#define TK_TRIGGER 184
#define TK_VIEW 185
#define TK_IPTOKEN 186
#define TK_SEMI 187
#define TK_NONE 188
#define TK_PREV 189
#define TK_LINEAR 190
#define TK_IMPORT 191
#define TK_TBNAME 192
#define TK_JOIN 193
#define TK_INSERT 194
#define TK_INTO 195
#define TK_VALUES 196
#define TK_SPACE 300
#define TK_COMMENT 301
#define TK_ILLEGAL 302
#define TK_HEX 303 // hex number 0x123
#define TK_OCT 304 // oct number
#define TK_BIN 305 // bin format data 0b111
#define TK_FILE 306
#define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query
#endif

13
source/libs/parser/src/astGenerator.c
View File

@ -13,11 +13,12 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "astGenerator.h"
#include <parserInt.h>
#include "os.h" #include "os.h"
#include "taos.h" #include "taos.h"
#include "tmsg.h"
#include "parserInt.h"
#include "tmsgtype.h" #include "tmsgtype.h"
#include "astGenerator.h"
SArray *tListItemAppend(SArray *pList, SVariant *pVar, uint8_t sortOrder) { SArray *tListItemAppend(SArray *pList, SVariant *pVar, uint8_t sortOrder) {
if (pList == NULL) { if (pList == NULL) {
@ -276,7 +277,7 @@ bool tSqlExprIsLeaf(tSqlExpr *pExpr) {
return (pExpr->pRight == NULL && pExpr->pLeft == NULL) && return (pExpr->pRight == NULL && pExpr->pLeft == NULL) &&
(pExpr->tokenId == 0 || (pExpr->tokenId == 0 ||
(pExpr->tokenId == TK_ID) || (pExpr->tokenId == TK_ID) ||
(pExpr->tokenId >= TK_BOOL && pExpr->tokenId <= TK_NCHAR) || (pExpr->tokenId == TK_BOOL || pExpr->tokenId == TK_STRING || pExpr->tokenId == TK_FLOAT) ||
(pExpr->tokenId == TK_NULL) || (pExpr->tokenId == TK_NULL) ||
(pExpr->tokenId == TK_SET)); (pExpr->tokenId == TK_SET));
} }
@ -948,22 +949,18 @@ void setCompactVnodeSql(SSqlInfo *pInfo, int32_t type, SArray *pParam) {
void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) { void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) {
pDBInfo->compressionLevel= -1; pDBInfo->compressionLevel= -1;
pDBInfo->walLevel = -1; pDBInfo->walLevel = -1;
pDBInfo->fsyncPeriod = -1; pDBInfo->fsyncPeriod = -1;
pDBInfo->commitTime = -1; pDBInfo->commitTime = -1;
pDBInfo->maxTablesPerVnode = -1; pDBInfo->numOfVgroups = 2;
pDBInfo->cacheBlockSize = -1; pDBInfo->cacheBlockSize = -1;
pDBInfo->numOfBlocks = -1; pDBInfo->numOfBlocks = -1;
pDBInfo->maxRowsPerBlock = -1; pDBInfo->maxRowsPerBlock = -1;
pDBInfo->minRowsPerBlock = -1; pDBInfo->minRowsPerBlock = -1;
pDBInfo->daysPerFile = -1; pDBInfo->daysPerFile = -1;
pDBInfo->replica = -1; pDBInfo->replica = -1;
pDBInfo->quorum = -1; pDBInfo->quorum = -1;
pDBInfo->keep = NULL; pDBInfo->keep = NULL;
pDBInfo->update = -1; pDBInfo->update = -1;
pDBInfo->cachelast = -1; pDBInfo->cachelast = -1;

47
source/libs/parser/src/astToMsg.c
View File

@ -188,13 +188,14 @@ static void doSetDbOptions(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb) {
pMsg->minRows = htonl(pCreateDb->minRowsPerBlock); pMsg->minRows = htonl(pCreateDb->minRowsPerBlock);
pMsg->maxRows = htonl(pCreateDb->maxRowsPerBlock); pMsg->maxRows = htonl(pCreateDb->maxRowsPerBlock);
pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod); pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod);
pMsg->compression = pCreateDb->compressionLevel; pMsg->compression = (int8_t) pCreateDb->compressionLevel;
pMsg->walLevel = (char)pCreateDb->walLevel; pMsg->walLevel = (char)pCreateDb->walLevel;
pMsg->replications = pCreateDb->replica; pMsg->replications = pCreateDb->replica;
pMsg->quorum = pCreateDb->quorum; pMsg->quorum = pCreateDb->quorum;
pMsg->ignoreExist = pCreateDb->ignoreExists; pMsg->ignoreExist = pCreateDb->ignoreExists;
pMsg->update = pCreateDb->update; pMsg->update = pCreateDb->update;
pMsg->cacheLastRow = pCreateDb->cachelast; pMsg->cacheLastRow = pCreateDb->cachelast;
pMsg->numOfVgroups = htonl(pCreateDb->numOfVgroups);
} }
int32_t setDbOptions(SCreateDbMsg* pCreateDbMsg, const SCreateDbInfo* pCreateDbSql, SMsgBuf* pMsgBuf) { int32_t setDbOptions(SCreateDbMsg* pCreateDbMsg, const SCreateDbInfo* pCreateDbSql, SMsgBuf* pMsgBuf) {
@ -208,8 +209,6 @@ int32_t setDbOptions(SCreateDbMsg* pCreateDbMsg, const SCreateDbInfo* pCreateDbS
return TSDB_CODE_TSC_INVALID_OPERATION; return TSDB_CODE_TSC_INVALID_OPERATION;
} }
// todo configurable
pCreateDbMsg->numOfVgroups = htonl(2);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -233,45 +232,6 @@ SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCt
return pCreateMsg; return pCreateMsg;
} }
int32_t createSName(SName* pName, SToken* pTableName, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) {
const char* msg1 = "name too long";
const char* msg2 = "acctId too long";
int32_t code = TSDB_CODE_SUCCESS;
char* p = strnchr(pTableName->z, TS_PATH_DELIMITER[0], pTableName->n, false);
if (p != NULL) { // db has been specified in sql string so we ignore current db path
code = tNameSetAcctId(pName, pParseCtx->acctId);
if (code != 0) {
return buildInvalidOperationMsg(pMsgBuf, msg2);
}
char name[TSDB_TABLE_FNAME_LEN] = {0};
strncpy(name, pTableName->z, pTableName->n);
code = tNameFromString(pName, name, T_NAME_DB|T_NAME_TABLE);
if (code != 0) {
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
} else { // get current DB name first, and then set it into path
if (pTableName->n >= TSDB_TABLE_NAME_LEN) {
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
tNameSetDbName(pName, pParseCtx->acctId, pParseCtx->db, strlen(pParseCtx->db));
char name[TSDB_TABLE_FNAME_LEN] = {0};
strncpy(name, pTableName->z, pTableName->n);
code = tNameFromString(pName, name, T_NAME_TABLE);
if (code != 0) {
code = buildInvalidOperationMsg(pMsgBuf, msg1);
}
}
return code;
}
SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) {
SSchema* pSchema; SSchema* pSchema;
@ -388,7 +348,7 @@ SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMs
} }
SToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0); SToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0);
if (id->type != TK_ID) { if (id->type != TK_ID && id->type != TK_IPTOKEN) {
buildInvalidOperationMsg(pMsgBuf, msg2); buildInvalidOperationMsg(pMsgBuf, msg2);
return NULL; return NULL;
} }
@ -428,6 +388,7 @@ SDropDnodeMsg *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf
char* end = NULL; char* end = NULL;
SDropDnodeMsg * pDrop = (SDropDnodeMsg *)calloc(1, sizeof(SDropDnodeMsg)); SDropDnodeMsg * pDrop = (SDropDnodeMsg *)calloc(1, sizeof(SDropDnodeMsg));
pDrop->dnodeId = strtoll(pzName->z, &end, 10); pDrop->dnodeId = strtoll(pzName->z, &end, 10);
pDrop->dnodeId = htonl(pDrop->dnodeId);
*len = sizeof(SDropDnodeMsg); *len = sizeof(SDropDnodeMsg);
if (end - pzName->z != pzName->n) { if (end - pzName->z != pzName->n) {

137
source/libs/parser/src/dCDAstProcess.c
View File

@ -1,3 +1,4 @@
#include <tmsg.h>
#include <ttime.h> #include <ttime.h>
#include "astToMsg.h" #include "astToMsg.h"
#include "parserInt.h" #include "parserInt.h"
@ -16,7 +17,8 @@ static bool has(SArray* pFieldList, int32_t startIndex, const char* name) {
return false; return false;
} }
static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx *pCtx, void** output, int32_t* outputLen, SMsgBuf* pMsgBuf) { static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** output, int32_t* outputLen,
SMsgBuf* pMsgBuf) {
const char* msg1 = "invalid name"; const char* msg1 = "invalid name";
const char* msg2 = "wildcard string should be less than %d characters"; const char* msg2 = "wildcard string should be less than %d characters";
const char* msg3 = "database name too long"; const char* msg3 = "database name too long";
@ -115,8 +117,8 @@ static int32_t doCheckDbOptions(SCreateDbMsg* pCreate, SMsgBuf* pMsgBuf) {
int32_t val = htonl(pCreate->daysPerFile); int32_t val = htonl(pCreate->daysPerFile);
if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) {
snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, TSDB_MIN_DAYS_PER_FILE,
TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); TSDB_MAX_DAYS_PER_FILE);
return buildInvalidOperationMsg(pMsgBuf, msg); return buildInvalidOperationMsg(pMsgBuf, msg);
} }
@ -136,15 +138,15 @@ static int32_t doCheckDbOptions(SCreateDbMsg* pCreate, SMsgBuf* pMsgBuf) {
val = htonl(pCreate->commitTime); val = htonl(pCreate->commitTime);
if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) {
snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, TSDB_MIN_COMMIT_TIME,
TSDB_MIN_COMMIT_TIME, TSDB_MAX_COMMIT_TIME); TSDB_MAX_COMMIT_TIME);
return buildInvalidOperationMsg(pMsgBuf, msg); return buildInvalidOperationMsg(pMsgBuf, msg);
} }
val = htonl(pCreate->fsyncPeriod); val = htonl(pCreate->fsyncPeriod);
if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) { if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) {
snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val, snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val, TSDB_MIN_FSYNC_PERIOD,
TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD); TSDB_MAX_FSYNC_PERIOD);
return buildInvalidOperationMsg(pMsgBuf, msg); return buildInvalidOperationMsg(pMsgBuf, msg);
} }
@ -155,6 +157,12 @@ static int32_t doCheckDbOptions(SCreateDbMsg* pCreate, SMsgBuf* pMsgBuf) {
return buildInvalidOperationMsg(pMsgBuf, msg); return buildInvalidOperationMsg(pMsgBuf, msg);
} }
val = htonl(pCreate->numOfVgroups);
if (val < TSDB_MIN_VNODES_PER_DB || val > TSDB_MAX_VNODES_PER_DB) {
snprintf(msg, tListLen(msg), "invalid number of vgroups for DB:%d valid range: [%d, %d]", val,
TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -283,7 +291,8 @@ int32_t doCheckForCreateTable(SSqlInfo* pInfo, SMsgBuf* pMsgBuf) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf) { int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len,
SEpSet* pEpSet) {
const char* msg1 = "invalid table name"; const char* msg1 = "invalid table name";
const char* msg2 = "tags number not matched"; const char* msg2 = "tags number not matched";
const char* msg3 = "tag value too long"; const char* msg3 = "tag value too long";
@ -313,20 +322,13 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx *pCtx, SMsgBuf* p
return code; return code;
} }
const char* pSTableName = tNameGetTableName(&name);
SArray* pValList = pCreateTableInfo->pTagVals; SArray* pValList = pCreateTableInfo->pTagVals;
if (code != TSDB_CODE_SUCCESS) {
return code;
}
size_t valSize = taosArrayGetSize(pValList); size_t numOfInputTag = taosArrayGetSize(pValList);
STableMeta* pSuperTableMeta = NULL; STableMeta* pSuperTableMeta = NULL;
char dbName[TSDB_DB_FNAME_LEN] = {0}; catalogGetTableMeta(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &name, &pSuperTableMeta);
tNameGetFullDbName(&name, dbName); assert(pSuperTableMeta != NULL);
catalogGetTableMeta(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbName, pSTableName, &pSuperTableMeta);
// too long tag values will return invalid sql, not be truncated automatically // too long tag values will return invalid sql, not be truncated automatically
SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta); SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta);
@ -346,7 +348,7 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx *pCtx, SMsgBuf* p
pNameList = pCreateTableInfo->pTagNames; pNameList = pCreateTableInfo->pTagNames;
nameSize = taosArrayGetSize(pNameList); nameSize = taosArrayGetSize(pNameList);
if (valSize != nameSize || schemaSize < valSize) { if (numOfInputTag != nameSize || schemaSize < numOfInputTag) {
tdDestroyKVRowBuilder(&kvRowBuilder); tdDestroyKVRowBuilder(&kvRowBuilder);
return buildInvalidOperationMsg(pMsgBuf, msg2); return buildInvalidOperationMsg(pMsgBuf, msg2);
} }
@ -422,49 +424,41 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx *pCtx, SMsgBuf* p
} }
} }
} else { } else {
if (schemaSize != valSize) { if (schemaSize != numOfInputTag) {
tdDestroyKVRowBuilder(&kvRowBuilder); tdDestroyKVRowBuilder(&kvRowBuilder);
return buildInvalidOperationMsg(pMsgBuf, msg2); return buildInvalidOperationMsg(pMsgBuf, msg2);
} }
for (int32_t i = 0; i < valSize; ++i) { for (int32_t i = 0; i < numOfInputTag; ++i) {
SSchema* pSchema = &pTagSchema[i]; SSchema* pSchema = &pTagSchema[i];
SListItem *pItem = taosArrayGet(pValList, i); SToken* pItem = taosArrayGet(pValList, i);
char tagVal[TSDB_MAX_TAGS_LEN];
if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
if (pItem->pVar.nLen > pSchema->bytes) { if (pItem->n > pSchema->bytes) {
tdDestroyKVRowBuilder(&kvRowBuilder); tdDestroyKVRowBuilder(&kvRowBuilder);
return buildInvalidOperationMsg(pMsgBuf, msg3); return buildInvalidOperationMsg(pMsgBuf, msg3);
} }
} else if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { } else if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) {
if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) { // if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) {
// code = convertTimestampStrToInt64(&(pItem->pVar), tinfo.precision); //// code = convertTimestampStrToInt64(&(pItem->pVar), tinfo.precision);
if (code != TSDB_CODE_SUCCESS) { // if (code != TSDB_CODE_SUCCESS) {
return buildInvalidOperationMsg(pMsgBuf, msg4); // return buildInvalidOperationMsg(pMsgBuf, msg4);
} // }
} else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) { // } else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) {
pItem->pVar.i = convertTimePrecision(pItem->pVar.i, TSDB_TIME_PRECISION_NANO, tinfo.precision); // pItem->pVar.i = convertTimePrecision(pItem->pVar.i, TSDB_TIME_PRECISION_NANO, tinfo.precision);
} // }
} }
code = taosVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); char tmpTokenBuf[TSDB_MAX_TAGS_LEN] = {0};
SKvParam param = {.builder = &kvRowBuilder, .schema = pSchema};
// check again after the convert since it may be converted from binary to nchar. char* endPtr = NULL;
if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { code = parseValueToken(&endPtr, pItem, pSchema, tinfo.precision, tmpTokenBuf, KvRowAppend, &param, pMsgBuf);
int16_t len = varDataTLen(tagVal);
if (len > pSchema->bytes) {
tdDestroyKVRowBuilder(&kvRowBuilder);
return buildInvalidOperationMsg(pMsgBuf, msg3);
}
}
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tdDestroyKVRowBuilder(&kvRowBuilder); tdDestroyKVRowBuilder(&kvRowBuilder);
return buildInvalidOperationMsg(pMsgBuf, msg4); return buildInvalidOperationMsg(pMsgBuf, msg4);
} }
tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal);
} }
} }
@ -473,39 +467,47 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx *pCtx, SMsgBuf* p
if (row == NULL) { if (row == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
tdSortKVRowByColIdx(row);
pTag->dataLen = kvRowLen(row);
if (pTag->data == NULL) { tdSortKVRowByColIdx(row);
pTag->data = malloc(pTag->dataLen);
SName tableName = {0};
code = createSName(&tableName, &pCreateTableInfo->name, pCtx, pMsgBuf);
if (code != TSDB_CODE_SUCCESS) {
return code;
} }
kvRowCpy(pTag->data, row); struct SVCreateTbReq req = {0};
free(row); req.type = TD_CHILD_TABLE;
req.name = strdup(tNameGetTableName(&tableName));
req.ctbCfg.suid = pSuperTableMeta->suid;
req.ctbCfg.pTag = row;
bool dbIncluded2 = false; int32_t serLen = sizeof(SMsgHead) + tSerializeSVCreateTbReq(NULL, &req);
// table name char* buf1 = calloc(1, serLen);
// if (tscValidateName(&(pCreateTableInfo->name), true, &dbIncluded2) != TSDB_CODE_SUCCESS) { *pOutput = buf1;
// return buildInvalidOperationMsg(pMsgBuf, msg1); buf1 += sizeof(SMsgHead);
// } tSerializeSVCreateTbReq((void*)&buf1, &req);
*len = serLen;
// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); SVgroupInfo info = {0};
// code = tscSetTableFullName(&pTableMetaInfo->name, &pCreateTableInfo->name, pSql, dbIncluded2); catalogGetTableHashVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &tableName, &info);
// if (code != TSDB_CODE_SUCCESS) {
// return code;
// }
// pCreateTableInfo->fullname = calloc(1, tNameLen(&pTableMetaInfo->name) + 1); pEpSet->inUse = info.inUse;
// code = tNameExtractFullName(&pTableMetaInfo->name, pCreateTableInfo->fullname); pEpSet->numOfEps = info.numOfEps;
// if (code != TSDB_CODE_SUCCESS) { for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {
// return buildInvalidOperationMsg(pMsgBuf, msg1); pEpSet->port[i] = info.epAddr[i].port;
// } tstrncpy(pEpSet->fqdn[i], info.epAddr[i].fqdn, tListLen(pEpSet->fqdn[i]));
}
((SMsgHead*)(*pOutput))->vgId = htonl(info.vgId);
((SMsgHead*)(*pOutput))->contLen = htonl(serLen);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStmtInfo* pDcl, char* msgBuf, int32_t msgBufLen) { int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStmtInfo* pDcl, char* msgBuf,
int32_t msgBufLen) {
int32_t code = 0; int32_t code = 0;
SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf m = {.buf = msgBuf, .len = msgBufLen};
@ -696,10 +698,12 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
pDcl->pMsg = (char*)buildCreateTableMsg(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf); pDcl->pMsg = (char*)buildCreateTableMsg(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf);
pDcl->msgType = (pCreateTable->type == TSQL_CREATE_TABLE) ? TDMT_VND_CREATE_TABLE : TDMT_MND_CREATE_STB; pDcl->msgType = (pCreateTable->type == TSQL_CREATE_TABLE) ? TDMT_VND_CREATE_TABLE : TDMT_MND_CREATE_STB;
} else if (pCreateTable->type == TSQL_CREATE_CTABLE) { } else if (pCreateTable->type == TSQL_CREATE_CTABLE) {
if ((code = doCheckForCreateCTable(pInfo, pCtx, pMsgBuf)) != TSDB_CODE_SUCCESS) { if ((code = doCheckForCreateCTable(pInfo, pCtx, pMsgBuf, &pDcl->pMsg, &pDcl->msgLen, &pDcl->epSet)) !=
TSDB_CODE_SUCCESS) {
return code; return code;
} }
pDcl->msgType = TDMT_VND_CREATE_TABLE;
} else if (pCreateTable->type == TSQL_CREATE_STREAM) { } else if (pCreateTable->type == TSQL_CREATE_STREAM) {
// if ((code = doCheckForStream(pSql, pInfo)) != TSDB_CODE_SUCCESS) { // if ((code = doCheckForStream(pSql, pInfo)) != TSDB_CODE_SUCCESS) {
// return code; // return code;
@ -744,4 +748,3 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
return code; return code;
} }

475
source/libs/parser/src/insertParser.c
View File

@ -51,7 +51,7 @@ enum {
typedef struct SInsertParseContext { typedef struct SInsertParseContext {
SParseContext* pComCxt; // input SParseContext* pComCxt; // input
const char* pSql; // input char *pSql; // input
SMsgBuf msg; // input SMsgBuf msg; // input
STableMeta* pTableMeta; // each table STableMeta* pTableMeta; // each table
SParsedDataColInfo tags; // each table SParsedDataColInfo tags; // each table
@ -64,22 +64,6 @@ typedef struct SInsertParseContext {
SInsertStmtInfo* pOutput; SInsertStmtInfo* pOutput;
} SInsertParseContext; } SInsertParseContext;
typedef int32_t (*FRowAppend)(const void *value, int32_t len, void *param);
typedef struct SKvParam {
char buf[TSDB_MAX_TAGS_LEN];
SKVRowBuilder* builder;
SSchema* schema;
} SKvParam;
static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE;
static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE;
static bool isNullStr(SToken *pToken) {
return (pToken->type == TK_NULL) || ((pToken->type == TK_STRING) && (pToken->n != 0) &&
(strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0));
}
static FORCE_INLINE int32_t toDouble(SToken *pToken, double *value, char **endPtr) { static FORCE_INLINE int32_t toDouble(SToken *pToken, double *value, char **endPtr) {
errno = 0; errno = 0;
*value = strtold(pToken->z, endPtr); *value = strtold(pToken->z, endPtr);
@ -170,12 +154,17 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD
} }
static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) {
char fullDbName[TSDB_DB_FNAME_LEN] = {0}; SName name = {0};
char tableName[TSDB_TABLE_NAME_LEN] = {0}; createSName(&name, pTname, &pCxt->pComCxt->ctx, &pCxt->msg);
CHECK_CODE(buildName(pCxt, pTname, fullDbName, tableName));
CHECK_CODE(catalogGetTableMeta(pCxt->pComCxt->ctx.pCatalog, pCxt->pComCxt->ctx.pTransporter, &pCxt->pComCxt->ctx.mgmtEpSet, fullDbName, tableName, &pCxt->pTableMeta)); char tableName[TSDB_TABLE_FNAME_LEN] = {0};
tNameExtractFullName(&name, tableName);
SParseBasicCtx* pBasicCtx = &pCxt->pComCxt->ctx;
CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta));
SVgroupInfo vg; SVgroupInfo vg;
CHECK_CODE(catalogGetTableHashVgroup(pCxt->pComCxt->ctx.pCatalog, pCxt->pComCxt->ctx.pTransporter, &pCxt->pComCxt->ctx.mgmtEpSet, fullDbName, tableName, &vg)); CHECK_CODE(catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg));
CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -263,23 +252,21 @@ static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start)
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int parseTime(SInsertParseContext* pCxt, SToken *pToken, int16_t timePrec, int64_t *time) { static int parseTime(char **end, SToken *pToken, int16_t timePrec, int64_t *time, SMsgBuf* pMsgBuf) {
int32_t index = 0; int32_t index = 0;
SToken sToken; SToken sToken;
int64_t interval; int64_t interval;
int64_t useconds = 0; int64_t ts = 0;
const char* pTokenEnd = pCxt->pSql; char* pTokenEnd = *end;
if (pToken->type == TK_NOW) { if (pToken->type == TK_NOW) {
useconds = taosGetTimestamp(timePrec); ts = taosGetTimestamp(timePrec);
} else if (strncmp(pToken->z, "0", 1) == 0 && pToken->n == 1) {
// do nothing
} else if (pToken->type == TK_INTEGER) { } else if (pToken->type == TK_INTEGER) {
useconds = taosStr2int64(pToken->z); bool isSigned = false;
} else { toInteger(pToken->z, pToken->n, 10, &ts, &isSigned);
// strptime("2001-11-12 18:31:01", "%Y-%m-%d %H:%M:%S", &tm); } else { // parse the RFC-3339/ISO-8601 timestamp format string
if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) {
return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp format", pToken->z); return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -288,8 +275,8 @@ static int parseTime(SInsertParseContext* pCxt, SToken *pToken, int16_t timePrec
for (int k = pToken->n; pToken->z[k] != '\0'; k++) { for (int k = pToken->n; pToken->z[k] != '\0'; k++) {
if (pToken->z[k] == ' ' || pToken->z[k] == '\t') continue; if (pToken->z[k] == ' ' || pToken->z[k] == '\t') continue;
if (pToken->z[k] == ',') { if (pToken->z[k] == ',') {
pCxt->pSql = pTokenEnd; *end = pTokenEnd;
*time = useconds; *time = ts;
return 0; return 0;
} }
@ -311,7 +298,7 @@ static int parseTime(SInsertParseContext* pCxt, SToken *pToken, int16_t timePrec
pTokenEnd += index; pTokenEnd += index;
if (valueToken.n < 2) { if (valueToken.n < 2) {
return buildSyntaxErrMsg(&pCxt->msg, "value expected in timestamp", sToken.z); return buildSyntaxErrMsg(pMsgBuf, "value expected in timestamp", sToken.z);
} }
char unit = 0; char unit = 0;
@ -320,34 +307,15 @@ static int parseTime(SInsertParseContext* pCxt, SToken *pToken, int16_t timePrec
} }
if (sToken.type == TK_PLUS) { if (sToken.type == TK_PLUS) {
useconds += interval; ts += interval;
} else { } else {
useconds = useconds - interval; ts = ts - interval;
} }
pCxt->pSql = pTokenEnd; *end = pTokenEnd;
} }
*time = useconds; *time = ts;
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE int32_t KvRowAppend(const void *value, int32_t len, void *param) {
SKvParam* pa = (SKvParam*)param;
if (TSDB_DATA_TYPE_BINARY == pa->schema->type) {
STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len);
tdAddColToKVRow(pa->builder, pa->schema->colId, pa->schema->type, pa->buf);
} else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) {
// if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
int32_t output = 0;
if (!taosMbsToUcs4(value, len, varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) {
return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
}
varDataSetLen(pa->buf, output);
tdAddColToKVRow(pa->builder, pa->schema->colId, pa->schema->type, pa->buf);
} else {
tdAddColToKVRow(pa->builder, pa->schema->colId, pa->schema->type, value);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -381,193 +349,206 @@ static FORCE_INLINE int32_t MemRowAppend(const void *value, int32_t len, void *p
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE int32_t checkAndTrimValue(SInsertParseContext* pCxt, SToken* pToken, SSchema* pSchema, char* tmpTokenBuf) { //static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, char* tmpTokenBuf, SMsgBuf* pMsgBuf) {
int16_t type = pToken->type; // if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL &&
if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && // type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) ||
type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || // (pToken->n == 0) || (type == TK_RP)) {
(pToken->n == 0) || (type == TK_RP)) { // return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "invalid data or symbol", pToken->z); // }
} //
// if (IS_NUMERIC_TYPE(type) && pToken->n == 0) {
// return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z);
// }
//
// // Remove quotation marks
// if (TK_STRING == type) {
// if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) {
// return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z);
// }
//
// // delete escape character: \\, \', \"
// char delim = pToken->z[0];
// int32_t cnt = 0;
// int32_t j = 0;
// for (uint32_t k = 1; k < pToken->n - 1; ++k) {
// if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) {
// tmpTokenBuf[j] = pToken->z[k + 1];
// cnt++;
// j++;
// k++;
// continue;
// }
// tmpTokenBuf[j] = pToken->z[k];
// j++;
// }
//
// tmpTokenBuf[j] = 0;
// pToken->z = tmpTokenBuf;
// pToken->n -= 2 + cnt;
// }
//
// return TSDB_CODE_SUCCESS;
//}
if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { //static FORCE_INLINE int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, _row_append_fn_t func, void* param, SMsgBuf* pMsgBuf) {
return buildSyntaxErrMsg(&pCxt->msg, "invalid numeric data", pToken->z); // int64_t iv;
} // char *endptr = NULL;
// bool isSigned = false;
// Remove quotation marks //
if (TK_STRING == type) { // CHECK_CODE(checkAndTrimValue(pToken, pSchema->type, tmpTokenBuf, pMsgBuf));
if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { //
return buildSyntaxErrMsg(&pCxt->msg, "too long string", pToken->z); // if (isNullStr(pToken)) {
} // if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
// delete escape character: \\, \', \" // int64_t tmpVal = 0;
char delim = pToken->z[0]; // return func(&tmpVal, pSchema->bytes, param);
int32_t cnt = 0; // }
int32_t j = 0; //
for (uint32_t k = 1; k < pToken->n - 1; ++k) { // return func(getNullValue(pSchema->type), 0, param);
if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) { // }
tmpTokenBuf[j] = pToken->z[k + 1]; //
cnt++; // switch (pSchema->type) {
j++; // case TSDB_DATA_TYPE_BOOL: {
k++; // if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) {
continue; // if (strncmp(pToken->z, "true", pToken->n) == 0) {
} // return func(&TRUE_VALUE, pSchema->bytes, param);
tmpTokenBuf[j] = pToken->z[k]; // } else if (strncmp(pToken->z, "false", pToken->n) == 0) {
j++; // return func(&FALSE_VALUE, pSchema->bytes, param);
} // } else {
tmpTokenBuf[j] = 0; // return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z);
pToken->z = tmpTokenBuf; // }
pToken->n -= 2 + cnt; // } else if (pToken->type == TK_INTEGER) {
} // return func(((strtoll(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param);
// } else if (pToken->type == TK_FLOAT) {
return TSDB_CODE_SUCCESS; // return func(((strtod(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param);
} // } else {
// return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z);
static FORCE_INLINE int32_t parseOneValue(SInsertParseContext* pCxt, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, FRowAppend func, void* param) { // }
int64_t iv; // }
int32_t ret; //
char * endptr = NULL; // case TSDB_DATA_TYPE_TINYINT: {
// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
CHECK_CODE(checkAndTrimValue(pCxt, pToken, pSchema, tmpTokenBuf)); // return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z);
// } else if (!IS_VALID_TINYINT(iv)) {
if (isNullStr(pToken)) { // return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z);
if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { // }
int64_t tmpVal = 0; //
return func(&tmpVal, pSchema->bytes, param); // uint8_t tmpVal = (uint8_t)iv;
} // return func(&tmpVal, pSchema->bytes, param);
return func(getNullValue(pSchema->type), 0, param); // }
} //
// case TSDB_DATA_TYPE_UTINYINT:{
switch (pSchema->type) { // if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
case TSDB_DATA_TYPE_BOOL: { // return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z);
if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { // } else if (!IS_VALID_UTINYINT(iv)) {
if (strncmp(pToken->z, "true", pToken->n) == 0) { // return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z);
return func(&TRUE_VALUE, pSchema->bytes, param); // }
} else if (strncmp(pToken->z, "false", pToken->n) == 0) { // uint8_t tmpVal = (uint8_t)iv;
return func(&FALSE_VALUE, pSchema->bytes, param); // return func(&tmpVal, pSchema->bytes, param);
} else { // }
return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); //
} // case TSDB_DATA_TYPE_SMALLINT: {
} else if (pToken->type == TK_INTEGER) { // if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
return func(((strtoll(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); // return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z);
} else if (pToken->type == TK_FLOAT) { // } else if (!IS_VALID_SMALLINT(iv)) {
return func(((strtod(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); // return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z);
} else { // }
return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); // int16_t tmpVal = (int16_t)iv;
} // return func(&tmpVal, pSchema->bytes, param);
break; // }
} //
case TSDB_DATA_TYPE_TINYINT: { // case TSDB_DATA_TYPE_USMALLINT: {
if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) { // if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); // return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z);
} else if (!IS_VALID_TINYINT(iv)) { // } else if (!IS_VALID_USMALLINT(iv)) {
return buildSyntaxErrMsg(&pCxt->msg, "data overflow", pToken->z); // return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z);
} // }
uint8_t tmpVal = (uint8_t)iv; // uint16_t tmpVal = (uint16_t)iv;
return func(&tmpVal, pSchema->bytes, param); // return func(&tmpVal, pSchema->bytes, param);
} // }
case TSDB_DATA_TYPE_UTINYINT:{ //
if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) { // case TSDB_DATA_TYPE_INT: {
return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); // if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
} else if (!IS_VALID_UTINYINT(iv)) { // return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); // } else if (!IS_VALID_INT(iv)) {
} // return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z);
uint8_t tmpVal = (uint8_t)iv; // }
return func(&tmpVal, pSchema->bytes, param); // int32_t tmpVal = (int32_t)iv;
} // return func(&tmpVal, pSchema->bytes, param);
case TSDB_DATA_TYPE_SMALLINT: { // }
if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) { //
return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); // case TSDB_DATA_TYPE_UINT: {
} else if (!IS_VALID_SMALLINT(iv)) { // if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); // return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z);
} // } else if (!IS_VALID_UINT(iv)) {
int16_t tmpVal = (int16_t)iv; // return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z);
return func(&tmpVal, pSchema->bytes, param); // }
} // uint32_t tmpVal = (uint32_t)iv;
case TSDB_DATA_TYPE_USMALLINT: { // return func(&tmpVal, pSchema->bytes, param);
if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) { // }
return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); //
} else if (!IS_VALID_USMALLINT(iv)) { // case TSDB_DATA_TYPE_BIGINT: {
return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); // if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
} // return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z);
uint16_t tmpVal = (uint16_t)iv; // } else if (!IS_VALID_BIGINT(iv)) {
return func(&tmpVal, pSchema->bytes, param); // return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z);
} // }
case TSDB_DATA_TYPE_INT: { // return func(&iv, pSchema->bytes, param);
if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) { // }
return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); //
} else if (!IS_VALID_INT(iv)) { // case TSDB_DATA_TYPE_UBIGINT: {
return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); // if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
} // return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z);
int32_t tmpVal = (int32_t)iv; // } else if (!IS_VALID_UBIGINT((uint64_t)iv)) {
return func(&tmpVal, pSchema->bytes, param); // return buildSyntaxErrMsg(pMsgBuf, "unsigned bigint data overflow", pToken->z);
} // }
case TSDB_DATA_TYPE_UINT: { // uint64_t tmpVal = (uint64_t)iv;
if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) { // return func(&tmpVal, pSchema->bytes, param);
return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); // }
} else if (!IS_VALID_UINT(iv)) { //
return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); // case TSDB_DATA_TYPE_FLOAT: {
} // double dv;
uint32_t tmpVal = (uint32_t)iv; // if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
return func(&tmpVal, pSchema->bytes, param); // return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z);
} // }
case TSDB_DATA_TYPE_BIGINT: { // if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) {
if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) { // return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z);
return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); // }
} else if (!IS_VALID_BIGINT(iv)) { // float tmpVal = (float)dv;
return buildSyntaxErrMsg(&pCxt->msg, "bigint data overflow", pToken->z); // return func(&tmpVal, pSchema->bytes, param);
} // }
return func(&iv, pSchema->bytes, param); //
} // case TSDB_DATA_TYPE_DOUBLE: {
case TSDB_DATA_TYPE_UBIGINT: { // double dv;
if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) { // if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); // return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z);
} else if (!IS_VALID_UBIGINT((uint64_t)iv)) { // }
return buildSyntaxErrMsg(&pCxt->msg, "unsigned bigint data overflow", pToken->z); // if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) {
} // return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z);
uint64_t tmpVal = (uint64_t)iv; // }
return func(&tmpVal, pSchema->bytes, param); // return func(&dv, pSchema->bytes, param);
} // }
case TSDB_DATA_TYPE_FLOAT: { //
double dv; // case TSDB_DATA_TYPE_BINARY: {
if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { // // too long values will return invalid sql, not be truncated automatically
return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); // if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) {
} // return buildSyntaxErrMsg(pMsgBuf, "string data overflow", pToken->z);
if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { // }
return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); // return func(pToken->z, pToken->n, param);
} // }
float tmpVal = (float)dv; // case TSDB_DATA_TYPE_NCHAR: {
return func(&tmpVal, pSchema->bytes, param); // return func(pToken->z, pToken->n, param);
} // }
case TSDB_DATA_TYPE_DOUBLE: { // case TSDB_DATA_TYPE_TIMESTAMP: {
double dv; // int64_t tmpVal;
if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { // if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) {
return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); // return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z);
} // }
if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { // return func(&tmpVal, pSchema->bytes, param);
return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); // }
} // }
return func(&dv, pSchema->bytes, param); //
} // return TSDB_CODE_FAILED;
case TSDB_DATA_TYPE_BINARY: { //}
// too long values will return invalid sql, not be truncated automatically
if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) {
return buildSyntaxErrMsg(&pCxt->msg, "string data overflow", pToken->z);
}
return func(pToken->z, pToken->n, param);
}
case TSDB_DATA_TYPE_NCHAR: {
return func(pToken->z, pToken->n, param);
}
case TSDB_DATA_TYPE_TIMESTAMP: {
int64_t tmpVal;
if (parseTime(pCxt, pToken, timePrec, &tmpVal) != TSDB_CODE_SUCCESS) {
return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z);
}
return func(&tmpVal, pSchema->bytes, param);
}
}
return TSDB_CODE_FAILED;
}
// pSql -> tag1_name, ...) // pSql -> tag1_name, ...)
static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) { static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) {
@ -644,7 +625,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema,
NEXT_TOKEN(pCxt->pSql, sToken); NEXT_TOKEN(pCxt->pSql, sToken);
SSchema* pSchema = &pTagsSchema[pCxt->tags.boundedColumns[i]]; SSchema* pSchema = &pTagsSchema[pCxt->tags.boundedColumns[i]];
param.schema = pSchema; param.schema = pSchema;
CHECK_CODE(parseOneValue(pCxt, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, &param)); CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, &param, &pCxt->msg));
} }
SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder); SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder);
@ -709,7 +690,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks,
param.schema = pSchema; param.schema = pSchema;
param.compareStat = pBuilder->compareStat; param.compareStat = pBuilder->compareStat;
getMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, &param.toffset); getMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, &param.toffset);
CHECK_CODE(parseOneValue(pCxt, &sToken, pSchema, timePrec, tmpTokenBuf, MemRowAppend, &param)); CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, timePrec, tmpTokenBuf, MemRowAppend, &param, &pCxt->msg));
if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
TSKEY tsKey = memRowKey(row); TSKEY tsKey = memRowKey(row);
@ -894,7 +875,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) {
SInsertParseContext context = { SInsertParseContext context = {
.pComCxt = pContext, .pComCxt = pContext,
.pSql = pContext->pSql, .pSql = (char*) pContext->pSql,
.msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, .msg = {.buf = pContext->pMsg, .len = pContext->msgLen},
.pTableMeta = NULL, .pTableMeta = NULL,
.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false), .pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false),

595
source/libs/parser/src/parserUtil.c
View File

@ -13,19 +13,20 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tmsg.h"
#include "parser.h"
#include "taoserror.h"
#include "tutil.h"
#include "ttypes.h"
#include "thash.h"
#include "tbuffer.h"
#include "parserInt.h"
#include "parserUtil.h" #include "parserUtil.h"
#include "tmsgtype.h" #include <tglobal.h>
#include "queryInfoUtil.h" #include <ttime.h>
#include "function.h" #include "function.h"
#include "parser.h"
#include "parserInt.h"
#include "queryInfoUtil.h"
#include "taoserror.h"
#include "tbuffer.h"
#include "thash.h"
#include "tmsg.h"
#include "tmsgtype.h"
#include "ttypes.h"
#include "tutil.h"
typedef struct STableFilterCond { typedef struct STableFilterCond {
uint64_t uid; uint64_t uid;
@ -1627,313 +1628,357 @@ bool isDqlSqlStatement(SSqlInfo* pSqlInfo) {
return pSqlInfo->type == TSDB_SQL_SELECT; return pSqlInfo->type == TSDB_SQL_SELECT;
} }
#if 0 static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE;
int32_t tscCreateQueryFromQueryInfo(SQueryStmtInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr) { static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE;
memset(pQueryAttr, 0, sizeof(SQueryAttr));
int16_t numOfCols = (int16_t) taosArrayGetSize(pQueryInfo->colList); static FORCE_INLINE int32_t toDouble(SToken *pToken, double *value, char **endPtr) {
int16_t numOfOutput = (int16_t) getNumOfExprs(pQueryInfo); errno = 0;
*value = strtold(pToken->z, endPtr);
pQueryAttr->topBotQuery = tscIsTopBotQuery(pQueryInfo); // not a valid integer number, return error
pQueryAttr->hasTagResults = hasTagValOutput(pQueryInfo); if ((*endPtr - pToken->z) != pToken->n) {
pQueryAttr->stabledev = isStabledev(pQueryInfo); return TK_ILLEGAL;
pQueryAttr->tsCompQuery = isTsCompQuery(pQueryInfo);
pQueryAttr->diffQuery = tscIsDiffDerivQuery(pQueryInfo);
pQueryAttr->simpleAgg = isSimpleAggregateRv(pQueryInfo);
pQueryAttr->needReverseScan = tscNeedReverseScan(pQueryInfo);
pQueryAttr->stableQuery = QUERY_IS_STABLE_QUERY(pQueryInfo->type);
pQueryAttr->groupbyColumn = (!pQueryInfo->stateWindow) && tscGroupbyColumn(pQueryInfo);
pQueryAttr->queryBlockDist = isBlockDistQuery(pQueryInfo);
pQueryAttr->pointInterpQuery = tscIsPointInterpQuery(pQueryInfo);
pQueryAttr->timeWindowInterpo = timeWindowInterpoRequired(pQueryInfo);
pQueryAttr->distinct = pQueryInfo->distinct;
pQueryAttr->sw = pQueryInfo->sessionWindow;
pQueryAttr->stateWindow = pQueryInfo->stateWindow;
pQueryAttr->multigroupResult = pQueryInfo->multigroupResult;
pQueryAttr->numOfCols = numOfCols;
pQueryAttr->numOfOutput = numOfOutput;
pQueryAttr->limit = pQueryInfo->limit;
pQueryAttr->slimit = pQueryInfo->slimit;
pQueryAttr->order = pQueryInfo->order;
pQueryAttr->fillType = pQueryInfo->fillType;
pQueryAttr->havingNum = pQueryInfo->havingFieldNum;
pQueryAttr->pUdfInfo = pQueryInfo->pUdfInfo;
if (pQueryInfo->order.order == TSDB_ORDER_ASC) { // TODO refactor
pQueryAttr->window = pQueryInfo->window;
} else {
pQueryAttr->window.skey = pQueryInfo->window.ekey;
pQueryAttr->window.ekey = pQueryInfo->window.skey;
} }
memcpy(&pQueryAttr->interval, &pQueryInfo->interval, sizeof(pQueryAttr->interval)); return pToken->type;
STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0];
if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) {
pQueryAttr->pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr));
*(pQueryAttr->pGroupbyExpr) = pQueryInfo->groupbyExpr;
pQueryAttr->pGroupbyExpr->columnInfo = taosArrayDup(pQueryInfo->groupbyExpr.columnInfo);
} else {
assert(pQueryInfo->groupbyExpr.columnInfo == NULL);
} }
pQueryAttr->pExpr1 = calloc(pQueryAttr->numOfOutput, sizeof(SExprInfo)); static bool isNullStr(SToken *pToken) {
for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { return (pToken->type == TK_NULL) || ((pToken->type == TK_STRING) && (pToken->n != 0) &&
SExprInfo* pExpr = getExprInfo(pQueryInfo, i); (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0));
ExprInfoCopy(&pQueryAttr->pExpr1[i], pExpr);
if (pQueryAttr->pExpr1[i].base.functionId == FUNCTION_ARITHM) {
for (int32_t j = 0; j < pQueryAttr->pExpr1[i].base.numOfParams; ++j) {
buildArithmeticExprFromMsg(&pQueryAttr->pExpr1[i], NULL);
}
}
} }
pQueryAttr->tableCols = calloc(numOfCols, sizeof(SColumnInfo)); static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, char* tmpTokenBuf, SMsgBuf* pMsgBuf) {
for(int32_t i = 0; i < numOfCols; ++i) { if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL &&
SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i); type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) ||
if (!isValidDataType(pCol->info.type) || pCol->info.type == TSDB_DATA_TYPE_NULL) { (pToken->n == 0) || (type == TK_RP)) {
assert(0); return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z);
} }
pQueryAttr->tableCols[i] = pCol->info; if (IS_NUMERIC_TYPE(type) && pToken->n == 0) {
pQueryAttr->tableCols[i].flist.filterInfo = tFilterInfoDup(pCol->info.flist.filterInfo, pQueryAttr->tableCols[i].flist.numOfFilters); return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z);
} }
// global aggregate query // Remove quotation marks
if (pQueryAttr->stableQuery && (pQueryAttr->simpleAgg || pQueryAttr->interval.interval > 0) && tscIsTwoStageSTableQuery(pQueryInfo, 0)) { if (TK_STRING == type) {
createGlobalAggregateExpr(pQueryAttr, pQueryInfo); if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) {
return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z);
} }
// for simple table, not for super table // delete escape character: \\, \', \"
if (pQueryInfo->arithmeticOnAgg) { char delim = pToken->z[0];
pQueryAttr->numOfExpr2 = (int32_t) taosArrayGetSize(pQueryInfo->exprList1); int32_t cnt = 0;
pQueryAttr->pExpr2 = calloc(pQueryAttr->numOfExpr2, sizeof(SExprInfo)); int32_t j = 0;
for(int32_t i = 0; i < pQueryAttr->numOfExpr2; ++i) { for (uint32_t k = 1; k < pToken->n - 1; ++k) {
SExprInfo* p = taosArrayGetP(pQueryInfo->exprList1, i); if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) {
ExprInfoCopy(&pQueryAttr->pExpr2[i], p); tmpTokenBuf[j] = pToken->z[k + 1];
cnt++;
j++;
k++;
continue;
} }
tmpTokenBuf[j] = pToken->z[k];
j++;
} }
// tag column info tmpTokenBuf[j] = 0;
int32_t code = createTagColumnInfo(pQueryAttr, pQueryInfo, pTableMetaInfo); pToken->z = tmpTokenBuf;
if (code != TSDB_CODE_SUCCESS) { pToken->n -= 2 + cnt;
return code;
}
if (pQueryAttr->fillType != TSDB_FILL_NONE) {
pQueryAttr->fillVal = calloc(pQueryAttr->numOfOutput, sizeof(int64_t));
memcpy(pQueryAttr->fillVal, pQueryInfo->fillVal, pQueryInfo->numOfFillVal * sizeof(int64_t));
}
pQueryAttr->srcRowSize = 0;
pQueryAttr->maxTableColumnWidth = 0;
for (int16_t i = 0; i < numOfCols; ++i) {
pQueryAttr->srcRowSize += pQueryAttr->tableCols[i].bytes;
if (pQueryAttr->maxTableColumnWidth < pQueryAttr->tableCols[i].bytes) {
pQueryAttr->maxTableColumnWidth = pQueryAttr->tableCols[i].bytes;
}
}
pQueryAttr->interBufSize = getOutputInterResultBufSize(pQueryAttr);
if (pQueryAttr->numOfCols <= 0 && !tscQueryTags(pQueryInfo) && !pQueryAttr->queryBlockDist) {
tscError("%p illegal value of numOfCols in query msg: %" PRIu64 ", table cols:%d", addr,
(uint64_t)pQueryAttr->numOfCols, numOfCols);
return TSDB_CODE_TSC_INVALID_OPERATION;
}
if (pQueryAttr->interval.interval < 0) {
tscError("%p illegal value of aggregation time interval in query msg: %" PRId64, addr,
(int64_t)pQueryInfo->interval.interval);
return TSDB_CODE_TSC_INVALID_OPERATION;
}
if (pQueryAttr->pGroupbyExpr != NULL && pQueryAttr->pGroupbyExpr->numOfGroupCols < 0) {
tscError("%p illegal value of numOfGroupCols in query msg: %d", addr, pQueryInfo->groupbyExpr.numOfGroupCols);
return TSDB_CODE_TSC_INVALID_OPERATION;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t doAddTableName(char* nextStr, char** str, SArray* pNameArray, SSqlObj* pSql) { static int parseTime(char **end, SToken *pToken, int16_t timePrec, int64_t *time, SMsgBuf* pMsgBuf) {
int32_t code = TSDB_CODE_SUCCESS; int32_t index = 0;
SSqlCmd* pCmd = &pSql->cmd; SToken sToken;
int64_t interval;
int64_t ts = 0;
char* pTokenEnd = *end;
char tablename[TSDB_TABLE_FNAME_LEN] = {0}; if (pToken->type == TK_NOW) {
int32_t len = 0; ts = taosGetTimestamp(timePrec);
} else if (pToken->type == TK_INTEGER) {
if (nextStr == NULL) { bool isSigned = false;
tstrncpy(tablename, *str, TSDB_TABLE_FNAME_LEN); toInteger(pToken->z, pToken->n, 10, &ts, &isSigned);
len = (int32_t) strlen(tablename); } else { // parse the RFC-3339/ISO-8601 timestamp format string
} else { if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) {
len = (int32_t)(nextStr - (*str)); return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
if (len >= TSDB_TABLE_NAME_LEN) {
sprintf(pCmd->payload, "table name too long");
return TSDB_CODE_TSC_INVALID_OPERATION;
} }
memcpy(tablename, *str, nextStr - (*str));
tablename[len] = '\0';
}
(*str) = nextStr + 1;
len = (int32_t)strtrim(tablename);
SToken sToken = {.n = len, .type = TK_ID, .z = tablename};
tGetToken(tablename, &sToken.type);
// Check if the table name available or not
if (tscValidateName(&sToken) != TSDB_CODE_SUCCESS) {
sprintf(pCmd->payload, "table name is invalid");
return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
}
SName name = {0};
if ((code = tscSetTableFullName(&name, &sToken, pSql)) != TSDB_CODE_SUCCESS) {
return code;
}
memset(tablename, 0, tListLen(tablename));
tNameExtractFullName(&name, tablename);
char* p = strdup(tablename);
taosArrayPush(pNameArray, &p);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t nameComparFn(const void* n1, const void* n2) { for (int k = pToken->n; pToken->z[k] != '\0'; k++) {
int32_t ret = strcmp(*(char**)n1, *(char**)n2); if (pToken->z[k] == ' ' || pToken->z[k] == '\t') continue;
if (ret == 0) { if (pToken->z[k] == ',') {
*end = pTokenEnd;
*time = ts;
return 0; return 0;
} else {
return ret > 0? 1:-1;
}
} }
static void freeContent(void* p) {
char* ptr = *(char**)p;
tfree(ptr);
}
int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray) {
SSqlCmd *pCmd = &pSql->cmd;
pCmd->command = TSDB_SQL_MULTI_META;
pCmd->msgType = TDMT_VND_TABLES_META;
int code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
char *str = (char *)pNameList;
SQueryStmtInfo *pQueryInfo = tscGetQueryInfoS(pCmd);
if (pQueryInfo == NULL) {
pSql->res.code = terrno;
return terrno;
}
char *nextStr;
while (1) {
nextStr = strchr(str, ',');
if (nextStr == NULL) {
code = doAddTableName(nextStr, &str, pNameArray, pSql);
break; break;
} }
code = doAddTableName(nextStr, &str, pNameArray, pSql); /*
* time expression:
* e.g., now+12a, now-5h
*/
SToken valueToken;
index = 0;
sToken = tStrGetToken(pTokenEnd, &index, false);
pTokenEnd += index;
if (sToken.type == TK_MINUS || sToken.type == TK_PLUS) {
index = 0;
valueToken = tStrGetToken(pTokenEnd, &index, false);
pTokenEnd += index;
if (valueToken.n < 2) {
return buildSyntaxErrMsg(pMsgBuf, "value expected in timestamp", sToken.z);
}
char unit = 0;
if (parseAbsoluteDuration(valueToken.z, valueToken.n, &interval, &unit, timePrec) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
if (sToken.type == TK_PLUS) {
ts += interval;
} else {
ts = ts - interval;
}
*end = pTokenEnd;
}
*time = ts;
return TSDB_CODE_SUCCESS;
}
int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, _row_append_fn_t func, void* param, SMsgBuf* pMsgBuf) {
int64_t iv;
char *endptr = NULL;
bool isSigned = false;
int32_t code = checkAndTrimValue(pToken, pSchema->type, tmpTokenBuf, pMsgBuf);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
if (taosArrayGetSize(pNameArray) > TSDB_MULTI_TABLEMETA_MAX_NUM) { if (isNullStr(pToken)) {
code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
sprintf(pCmd->payload, "tables over the max number"); int64_t tmpVal = 0;
return code; return func(&tmpVal, pSchema->bytes, param);
}
return func(getNullValue(pSchema->type), 0, param);
}
switch (pSchema->type) {
case TSDB_DATA_TYPE_BOOL: {
if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) {
if (strncmp(pToken->z, "true", pToken->n) == 0) {
return func(&TRUE_VALUE, pSchema->bytes, param);
} else if (strncmp(pToken->z, "false", pToken->n) == 0) {
return func(&FALSE_VALUE, pSchema->bytes, param);
} else {
return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z);
}
} else if (pToken->type == TK_INTEGER) {
return func(((strtoll(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param);
} else if (pToken->type == TK_FLOAT) {
return func(((strtod(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param);
} else {
return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z);
} }
} }
size_t len = taosArrayGetSize(pNameArray); case TSDB_DATA_TYPE_TINYINT: {
if (len == 1) { if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z);
} else if (!IS_VALID_TINYINT(iv)) {
return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z);
}
uint8_t tmpVal = (uint8_t)iv;
return func(&tmpVal, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_UTINYINT:{
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z);
} else if (!IS_VALID_UTINYINT(iv)) {
return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z);
}
uint8_t tmpVal = (uint8_t)iv;
return func(&tmpVal, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_SMALLINT: {
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z);
} else if (!IS_VALID_SMALLINT(iv)) {
return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z);
}
int16_t tmpVal = (int16_t)iv;
return func(&tmpVal, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_USMALLINT: {
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z);
} else if (!IS_VALID_USMALLINT(iv)) {
return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z);
}
uint16_t tmpVal = (uint16_t)iv;
return func(&tmpVal, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_INT: {
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z);
} else if (!IS_VALID_INT(iv)) {
return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z);
}
int32_t tmpVal = (int32_t)iv;
return func(&tmpVal, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_UINT: {
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z);
} else if (!IS_VALID_UINT(iv)) {
return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z);
}
uint32_t tmpVal = (uint32_t)iv;
return func(&tmpVal, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_BIGINT: {
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z);
} else if (!IS_VALID_BIGINT(iv)) {
return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z);
}
return func(&iv, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_UBIGINT: {
if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) {
return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z);
} else if (!IS_VALID_UBIGINT((uint64_t)iv)) {
return buildSyntaxErrMsg(pMsgBuf, "unsigned bigint data overflow", pToken->z);
}
uint64_t tmpVal = (uint64_t)iv;
return func(&tmpVal, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_FLOAT: {
double dv;
if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z);
}
if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) {
return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z);
}
float tmpVal = (float)dv;
return func(&tmpVal, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_DOUBLE: {
double dv;
if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z);
}
if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) {
return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z);
}
return func(&dv, pSchema->bytes, param);
}
case TSDB_DATA_TYPE_BINARY: {
// Too long values will raise the invalid sql error message
if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) {
return buildSyntaxErrMsg(pMsgBuf, "string data overflow", pToken->z);
}
return func(pToken->z, pToken->n, param);
}
case TSDB_DATA_TYPE_NCHAR: {
return func(pToken->z, pToken->n, param);
}
case TSDB_DATA_TYPE_TIMESTAMP: {
int64_t tmpVal;
if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) {
return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z);
}
return func(&tmpVal, pSchema->bytes, param);
}
}
return TSDB_CODE_FAILED;
}
int32_t KvRowAppend(const void *value, int32_t len, void *param) {
SKvParam* pa = (SKvParam*) param;
int32_t type = pa->schema->type;
int32_t colId = pa->schema->colId;
if (TSDB_DATA_TYPE_BINARY == type) {
STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len);
tdAddColToKVRow(pa->builder, colId, type, pa->buf);
} else if (TSDB_DATA_TYPE_NCHAR == type) {
// if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
int32_t output = 0;
if (!taosMbsToUcs4(value, len, varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) {
return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
}
varDataSetLen(pa->buf, output);
tdAddColToKVRow(pa->builder, colId, type, pa->buf);
} else {
tdAddColToKVRow(pa->builder, colId, type, value);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (len > TSDB_MULTI_TABLEMETA_MAX_NUM) { int32_t createSName(SName* pName, SToken* pTableName, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) {
code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; const char* msg1 = "name too long";
sprintf(pCmd->payload, "tables over the max number");
int32_t code = TSDB_CODE_SUCCESS;
char* p = strnchr(pTableName->z, TS_PATH_DELIMITER[0], pTableName->n, false);
if (p != NULL) { // db has been specified in sql string so we ignore current db path
tNameSetAcctId(pName, pParseCtx->acctId);
char name[TSDB_TABLE_FNAME_LEN] = {0};
strncpy(name, pTableName->z, pTableName->n);
code = tNameFromString(pName, name, T_NAME_DB|T_NAME_TABLE);
if (code != 0) {
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
} else { // get current DB name first, and then set it into path
if (pTableName->n >= TSDB_TABLE_NAME_LEN) {
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
tNameSetDbName(pName, pParseCtx->acctId, pParseCtx->db, strlen(pParseCtx->db));
char name[TSDB_TABLE_FNAME_LEN] = {0};
strncpy(name, pTableName->z, pTableName->n);
code = tNameFromString(pName, name, T_NAME_TABLE);
if (code != 0) {
code = buildInvalidOperationMsg(pMsgBuf, msg1);
}
}
return code; return code;
} }
taosArraySort(pNameArray, nameComparFn);
taosArrayRemoveDuplicate(pNameArray, nameComparFn, freeContent);
return TSDB_CODE_SUCCESS;
}
bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src) {
assert(pExisted != NULL && src != NULL);
if (pExisted->numOfEps != src->numOfEps) {
return false;
}
for(int32_t i = 0; i < pExisted->numOfEps; ++i) {
if (pExisted->ep[i].port != src->epAddr[i].port) {
return false;
}
if (strncmp(pExisted->ep[i].fqdn, src->epAddr[i].fqdn, tListLen(pExisted->ep[i].fqdn)) != 0) {
return false;
}
}
return true;
}
SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg) {
assert(pVgroupMsg != NULL);
SNewVgroupInfo info = {0};
info.numOfEps = pVgroupMsg->numOfEps;
info.vgId = pVgroupMsg->vgId;
info.inUse = 0; // 0 is the default value of inUse in case of multiple replica
assert(info.numOfEps >= 1 && info.vgId >= 1);
for(int32_t i = 0; i < pVgroupMsg->numOfEps; ++i) {
tstrncpy(info.ep[i].fqdn, pVgroupMsg->epAddr[i].fqdn, TSDB_FQDN_LEN);
info.ep[i].port = pVgroupMsg->epAddr[i].port;
}
return info;
}
char* cloneCurrentDBName(SSqlObj* pSql) {
char *p = NULL;
HttpContext *pCtx = NULL;
pthread_mutex_lock(&pSql->pTscObj->mutex);
STscObj *pTscObj = pSql->pTscObj;
switch (pTscObj->from) {
case TAOS_REQ_FROM_HTTP:
pCtx = pSql->param;
if (pCtx && pCtx->db[0] != '\0') {
char db[TSDB_DB_FNAME_LEN] = {0};
int32_t len = sprintf(db, "%s%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pCtx->db);
assert(len <= sizeof(db));
p = strdup(db);
}
break;
default:
break;
}
if (p == NULL) {
p = strdup(pSql->pTscObj->db);
}
pthread_mutex_unlock(&pSql->pTscObj->mutex);
return p;
}
#endif

2
source/libs/parser/src/queryInfoUtil.c
View File

@ -26,7 +26,7 @@ size_t getNumOfExprs(SQueryStmtInfo* pQueryInfo) {
} }
SSchema* getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) { SSchema* getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) {
assert(pTableMeta != NULL && pTableMeta->schema != NULL && colIndex >= 0 && colIndex < getNumOfColumns(pTableMeta)); assert(pTableMeta != NULL && pTableMeta->schema != NULL && colIndex >= 0 && colIndex < (getNumOfColumns(pTableMeta) + getNumOfTags(pTableMeta)));
SSchema* pSchema = (SSchema*) pTableMeta->schema; SSchema* pSchema = (SSchema*) pTableMeta->schema;
return &pSchema[colIndex]; return &pSchema[colIndex];

3354
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

12
source/libs/parser/src/ttokenizer.c
View File

@ -31,17 +31,17 @@ typedef struct SKeyword {
static SKeyword keywordTable[] = { static SKeyword keywordTable[] = {
{"ID", TK_ID}, {"ID", TK_ID},
{"BOOL", TK_BOOL}, {"BOOL", TK_BOOL},
{"TINYINT", TK_TINYINT}, // {"TINYINT", TK_TINYINT},
{"SMALLINT", TK_SMALLINT}, // {"SMALLINT", TK_SMALLINT},
{"INTEGER", TK_INTEGER}, {"INTEGER", TK_INTEGER},
{"INT", TK_INTEGER}, {"INT", TK_INTEGER},
{"BIGINT", TK_BIGINT}, // {"BIGINT", TK_BIGINT},
{"FLOAT", TK_FLOAT}, {"FLOAT", TK_FLOAT},
{"DOUBLE", TK_DOUBLE}, // {"DOUBLE", TK_DOUBLE},
{"STRING", TK_STRING}, {"STRING", TK_STRING},
{"TIMESTAMP", TK_TIMESTAMP}, {"TIMESTAMP", TK_TIMESTAMP},
{"BINARY", TK_BINARY}, // {"BINARY", TK_BINARY},
{"NCHAR", TK_NCHAR}, // {"NCHAR", TK_NCHAR},
{"OR", TK_OR}, {"OR", TK_OR},
{"AND", TK_AND}, {"AND", TK_AND},
{"NOT", TK_NOT}, {"NOT", TK_NOT},

8
source/libs/parser/test/mockCatalog.cpp
View File

@ -45,12 +45,12 @@ int32_t __catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandl
return 0; return 0;
} }
int32_t __catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { int32_t __catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) {
return mockCatalogService->catalogGetTableMeta(pDBName, pTableName, pTableMeta); return mockCatalogService->catalogGetTableMeta(pTableName, pTableMeta);
} }
int32_t __catalogGetTableHashVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) { int32_t __catalogGetTableHashVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo* vgInfo) {
return mockCatalogService->catalogGetTableHashVgroup(pDBName, pTableName, vgInfo); return mockCatalogService->catalogGetTableHashVgroup(pTableName, vgInfo);
} }
void initMetaDataEnv() { void initMetaDataEnv() {

19
source/libs/parser/test/mockCatalogService.cpp
View File

@ -94,9 +94,14 @@ public:
return 0; return 0;
} }
int32_t catalogGetTableMeta(const char* pDbFullName, const char* pTableName, STableMeta** pTableMeta) const { int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const {
std::unique_ptr<STableMeta> table; std::unique_ptr<STableMeta> table;
int32_t code = copyTableSchemaMeta(toDbname(pDbFullName), pTableName, &table);
char db[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, db);
const char* tname = tNameGetTableName(pTableName);
int32_t code = copyTableSchemaMeta(db, tname, &table);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return code; return code;
} }
@ -104,7 +109,7 @@ public:
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t catalogGetTableHashVgroup(const char* pDbFullName, const char* pTableName, SVgroupInfo* vgInfo) const { int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const {
// todo // todo
return 0; return 0;
} }
@ -283,10 +288,10 @@ std::shared_ptr<MockTableMeta> MockCatalogService::getTableMeta(const std::strin
return impl_->getTableMeta(db, tbname); return impl_->getTableMeta(db, tbname);
} }
int32_t MockCatalogService::catalogGetTableMeta(const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const {
return impl_->catalogGetTableMeta(pDBName, pTableName, pTableMeta); return impl_->catalogGetTableMeta(pTableName, pTableMeta);
} }
int32_t MockCatalogService::catalogGetTableHashVgroup(const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) const { int32_t MockCatalogService::catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const {
return impl_->catalogGetTableHashVgroup(pDBName, pTableName, vgInfo); return impl_->catalogGetTableHashVgroup(pTableName, vgInfo);
} }

4
source/libs/parser/test/mockCatalogService.h
View File

@ -57,8 +57,8 @@ public:
void showTables() const; void showTables() const;
std::shared_ptr<MockTableMeta> getTableMeta(const std::string& db, const std::string& tbname) const; std::shared_ptr<MockTableMeta> getTableMeta(const std::string& db, const std::string& tbname) const;
int32_t catalogGetTableMeta(const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const; int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const;
int32_t catalogGetTableHashVgroup(const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) const; int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const;
private: private:
std::unique_ptr<MockCatalogServiceImpl> impl_; std::unique_ptr<MockCatalogServiceImpl> impl_;

17
source/libs/qcom/src/querymsg.c
View File

@ -16,6 +16,7 @@
#include "tmsg.h" #include "tmsg.h"
#include "queryInt.h" #include "queryInt.h"
#include "query.h" #include "query.h"
#include "trpc.h"
int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0};
@ -31,7 +32,7 @@ int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int3
int32_t estimateSize = sizeof(STableInfoMsg); int32_t estimateSize = sizeof(STableInfoMsg);
if (NULL == *msg || msgSize < estimateSize) { if (NULL == *msg || msgSize < estimateSize) {
tfree(*msg); tfree(*msg);
*msg = calloc(1, estimateSize); *msg = rpcMallocCont(estimateSize);
if (NULL == *msg) { if (NULL == *msg) {
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
@ -59,7 +60,7 @@ int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *ms
int32_t estimateSize = sizeof(SUseDbMsg); int32_t estimateSize = sizeof(SUseDbMsg);
if (NULL == *msg || msgSize < estimateSize) { if (NULL == *msg || msgSize < estimateSize) {
tfree(*msg); tfree(*msg);
*msg = calloc(1, estimateSize); *msg = rpcMallocCont(estimateSize);
if (NULL == *msg) { if (NULL == *msg) {
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
@ -265,13 +266,13 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) {
void initQueryModuleMsgHandle() { void initQueryModuleMsgHandle() {
queryBuildMsg[TDMT_VND_TABLE_META] = queryBuildTableMetaReqMsg; queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg;
queryBuildMsg[TDMT_MND_STB_META] = queryBuildTableMetaReqMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_STB_META)] = queryBuildTableMetaReqMsg;
queryBuildMsg[TDMT_MND_USE_DB] = queryBuildUseDbMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_USE_DB)] = queryBuildUseDbMsg;
queryProcessMsgRsp[TDMT_VND_TABLE_META] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp;
queryProcessMsgRsp[TDMT_MND_STB_META] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_STB_META)] = queryProcessTableMetaRsp;
queryProcessMsgRsp[TDMT_MND_USE_DB] = queryProcessUseDBRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)] = queryProcessUseDBRsp;
} }

36
source/libs/scheduler/src/scheduler.c
View File

@ -791,9 +791,29 @@ void schDropJobAllTasks(SSchJob *job) {
} }
} }
uint64_t schGenSchId(void) {
uint64_t sId = 0;
// TODO
qDebug("Gen sId:0x%"PRIx64, sId);
return sId;
}
int32_t schedulerInit(SSchedulerCfg *cfg) { int32_t schedulerInit(SSchedulerCfg *cfg) {
if (schMgmt.jobs) {
qError("scheduler already init");
return TSDB_CODE_QRY_INVALID_INPUT;
}
if (cfg) { if (cfg) {
schMgmt.cfg = *cfg; schMgmt.cfg = *cfg;
if (schMgmt.cfg.maxJobNum <= 0) {
schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_JOB_NUMBER;
}
} else { } else {
schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_JOB_NUMBER; schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_JOB_NUMBER;
} }
@ -803,18 +823,14 @@ int32_t schedulerInit(SSchedulerCfg *cfg) {
SCH_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler jobs failed", schMgmt.cfg.maxJobNum); SCH_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler jobs failed", schMgmt.cfg.maxJobNum);
} }
schMgmt.sId = 1; //TODO GENERATE A UUID schMgmt.sId = schGenSchId();
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t scheduleExecJobImpl(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob, bool syncSchedule) { int32_t scheduleExecJobImpl(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob, bool syncSchedule) {
if (NULL == transport || NULL == transport ||NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { if (qnodeList && taosArrayGetSize(qnodeList) <= 0) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
if (taosArrayGetSize(qnodeList) <= 0) {
qInfo("qnodeList is empty"); qInfo("qnodeList is empty");
} }
@ -882,6 +898,10 @@ _return:
} }
int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob, uint64_t *numOfRows) { int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob, uint64_t *numOfRows) {
if (NULL == transport || /* NULL == qnodeList || */ NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == numOfRows) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
*numOfRows = 0; *numOfRows = 0;
SCH_ERR_RET(scheduleExecJobImpl(transport, qnodeList, pDag, pJob, true)); SCH_ERR_RET(scheduleExecJobImpl(transport, qnodeList, pDag, pJob, true));
@ -894,6 +914,10 @@ int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, voi
} }
int32_t scheduleAsyncExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob) { int32_t scheduleAsyncExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob) {
if (NULL == transport || NULL == qnodeList ||NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
return scheduleExecJobImpl(transport, qnodeList, pDag, pJob, false); return scheduleExecJobImpl(transport, qnodeList, pDag, pJob, false);
} }

38
source/util/src/tbuffer.c
View File

@ -13,9 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "os.h"
#include "tbuffer.h" #include "tbuffer.h"
#include "exception.h" #include "exception.h"
#include "os.h"
//#include "taoserror.h" //#include "taoserror.h"
typedef union Un4B { typedef union Un4B {
@ -281,46 +281,28 @@ void tbufWriteStringLen( SBufferWriter* buf, const char* str, size_t len ) {
tbufWriteChar(buf, '\0'); tbufWriteChar(buf, '\0');
} }
void tbufWriteString( SBufferWriter* buf, const char* str ) { void tbufWriteString(SBufferWriter* buf, const char* str) { tbufWriteStringLen(buf, str, strlen(str)); }
tbufWriteStringLen( buf, str, strlen(str) );
}
void tbufWriteBinary(SBufferWriter* buf, const void* data, size_t len) { void tbufWriteBinary(SBufferWriter* buf, const void* data, size_t len) {
tbufWriteLength(buf, len); tbufWriteLength(buf, len);
tbufWrite(buf, data, len); tbufWrite(buf, data, len);
} }
void tbufWriteBool( SBufferWriter* buf, bool data ) { void tbufWriteBool(SBufferWriter* buf, bool data) { tbufWrite(buf, &data, sizeof(data)); }
tbufWrite( buf, &data, sizeof(data) );
}
void tbufWriteBoolAt( SBufferWriter* buf, size_t pos, bool data ) { void tbufWriteBoolAt(SBufferWriter* buf, size_t pos, bool data) { tbufWriteAt(buf, pos, &data, sizeof(data)); }
tbufWriteAt( buf, pos, &data, sizeof(data) );
}
void tbufWriteChar( SBufferWriter* buf, char data ) { void tbufWriteChar(SBufferWriter* buf, char data) { tbufWrite(buf, &data, sizeof(data)); }
tbufWrite( buf, &data, sizeof(data) );
}
void tbufWriteCharAt( SBufferWriter* buf, size_t pos, char data ) { void tbufWriteCharAt(SBufferWriter* buf, size_t pos, char data) { tbufWriteAt(buf, pos, &data, sizeof(data)); }
tbufWriteAt( buf, pos, &data, sizeof(data) );
}
void tbufWriteInt8( SBufferWriter* buf, int8_t data ) { void tbufWriteInt8(SBufferWriter* buf, int8_t data) { tbufWrite(buf, &data, sizeof(data)); }
tbufWrite( buf, &data, sizeof(data) );
}
void tbufWriteInt8At( SBufferWriter* buf, size_t pos, int8_t data ) { void tbufWriteInt8At(SBufferWriter* buf, size_t pos, int8_t data) { tbufWriteAt(buf, pos, &data, sizeof(data)); }
tbufWriteAt( buf, pos, &data, sizeof(data) );
}
void tbufWriteUint8( SBufferWriter* buf, uint8_t data ) { void tbufWriteUint8(SBufferWriter* buf, uint8_t data) { tbufWrite(buf, &data, sizeof(data)); }
tbufWrite( buf, &data, sizeof(data) );
}
void tbufWriteUint8At( SBufferWriter* buf, size_t pos, uint8_t data ) { void tbufWriteUint8At(SBufferWriter* buf, size_t pos, uint8_t data) { tbufWriteAt(buf, pos, &data, sizeof(data)); }
tbufWriteAt( buf, pos, &data, sizeof(data) );
}
void tbufWriteInt16(SBufferWriter* buf, int16_t data) { void tbufWriteInt16(SBufferWriter* buf, int16_t data) {
if (buf->endian) { if (buf->endian) {

24
tests/script/general/db/basic1.sim
View File

@ -59,9 +59,29 @@ if $data03 != 0 then
endi endi
print =============== show vgroups print =============== show vgroups
sql use d4 sql show databases
if $rows == 0 then if $rows != 1 then
return -1
endi
sql_error use d1
sql use d4
sql show vgroups
if $rows != 2 then
return -1
endi
print =============== show dnodes
sql show dnodes
if $data00 != 1 then
return -1
endi
if $data02 != 2 then
return -1 return -1
endi endi

7
tests/script/general/table/basic1.sim
View File

@ -23,7 +23,7 @@ endi
print $data00 $data01 $data02 print $data00 $data01 $data02
sql create table st2 (ts timestamp, i float) tags (j bigint) sql create table st2 (ts timestamp, i float) tags (j int)
sql show stables sql show stables
if $rows != 2 then if $rows != 2 then
return -1 return -1
@ -39,15 +39,14 @@ if $rows != 1 then
return -1 return -1
endi endi
print -->
print $data00 $data01 $data02 print $data00 $data01 $data02
print $data10 $data11 $data12 print $data10 $data11 $data12
return
print =============== create child table print =============== create child table
sql create table c1 using st tags(1) sql create table c1 using st tags(1)
sql create table c2 using st tags(2) sql create table c2 using st tags(2)
return
sql show tables sql show tables
if $rows != 2 then if $rows != 2 then
return -1 return -1

4
tests/script/sh/exec.sh
View File

@ -17,7 +17,7 @@ OS_TYPE=`$UNAME_BIN`
NODE_NAME= NODE_NAME=
EXEC_OPTON= EXEC_OPTON=
CLEAR_OPTION="false" CLEAR_OPTION="false"
while getopts "n:s:u:x:ct" arg while getopts "n:s:u:x:cv" arg
do do
case $arg in case $arg in
n) n)
@ -29,7 +29,7 @@ do
c) c)
CLEAR_OPTION="clear" CLEAR_OPTION="clear"
;; ;;
t) v)
SHELL_OPTION="true" SHELL_OPTION="true"
;; ;;
u) u)

23
tests/script/tmp/dnodes.sim
View File

@ -2,20 +2,19 @@ system sh/stop_dnodes.sh
############## config parameter ##################### ############## config parameter #####################
$node1 = 192.168.101.174 $node1 = 192.168.0.201
$node2 = 192.168.0.202 $node2 = 192.168.0.202
$node2 = 192.168.0.203 $node3 = 192.168.0.203
$node3 = 192.168.0.204 $node4 = 192.168.0.204
$first = 1
$num = 5
$self = $node1 $self = $node1
$num = 25
############### deploy firstEp ##################### ############### deploy firstEp #####################
$firstEp = $node1 . :7100 $firstEp = $node1 . :7100
$firstPort = 7100 $firstPort = 7100
if $first == 1 then if $self == $node1 then
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c firstEp -v $firstEp system sh/cfg.sh -n dnode1 -c firstEp -v $firstEp
system sh/cfg.sh -n dnode1 -c secondEp -v $firstEp system sh/cfg.sh -n dnode1 -c secondEp -v $firstEp
@ -28,7 +27,7 @@ if $first == 1 then
$i = 0 $i = 0
while $i < $num while $i < $num
$port = $i * 100 $port = $i * 100
$port = $port + 8000 $port = $port + 8100
$i = $i + 1 $i = $i + 1
sql create dnode $node1 port $port sql create dnode $node1 port $port
endw endw
@ -36,7 +35,7 @@ if $first == 1 then
$i = 0 $i = 0
while $i < $num while $i < $num
$port = $i * 100 $port = $i * 100
$port = $port + 8000 $port = $port + 8100
$i = $i + 1 $i = $i + 1
sql create dnode $node2 port $port sql create dnode $node2 port $port
endw endw
@ -44,7 +43,7 @@ if $first == 1 then
$i = 0 $i = 0
while $i < $num while $i < $num
$port = $i * 100 $port = $i * 100
$port = $port + 8000 $port = $port + 8100
$i = $i + 1 $i = $i + 1
sql create dnode $node3 port $port sql create dnode $node3 port $port
endw endw
@ -52,7 +51,7 @@ if $first == 1 then
$i = 0 $i = 0
while $i < $num while $i < $num
$port = $i * 100 $port = $i * 100
$port = $port + 8000 $port = $port + 8100
$i = $i + 1 $i = $i + 1
sql create dnode $node4 port $port sql create dnode $node4 port $port
endw endw
@ -64,7 +63,7 @@ $i = 0
while $i < $num while $i < $num
$index = $i + 80 $index = $i + 80
$port = $i * 100 $port = $i * 100
$port = $port + 8000 $port = $port + 8100
$dnodename = dnode . $index $dnodename = dnode . $index
$i = $i + 1 $i = $i + 1
@ -74,5 +73,5 @@ while $i < $num
system sh/cfg.sh -n $dnodename -c fqdn -v $self system sh/cfg.sh -n $dnodename -c fqdn -v $self
system sh/cfg.sh -n $dnodename -c serverPort -v $port system sh/cfg.sh -n $dnodename -c serverPort -v $port
#system sh/exec.sh -n $dnodename -s start system sh/exec.sh -n $dnodename -s start
endw endw

11
tests/script/unique/dnode/basic1.sim
View File

@ -94,5 +94,16 @@ if $rows != 2 then
return -1 return -1
endi endi
print =============== drop dnode
sql drop dnode 2;
sql show dnodes;
if $rows != 1 then
return -1
endi
if $data00 != 1 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT