xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/qnode

This commit is contained in:
dapan1121 2022-04-08 08:45:55 +08:00
parent 0b6f147d9d dbda9ea052
commit 2ebb540d2e
123 changed files with 6655 additions and 4143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/client/taos.h
View File

@ -188,6 +188,9 @@ DLL_EXPORT void taos_stop_query(TAOS_RES *res);
DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col);
DLL_EXPORT bool taos_is_update_query(TAOS_RES *res); DLL_EXPORT bool taos_is_update_query(TAOS_RES *res);
DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int* numOfRows, TAOS_ROW *rows);
DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int* numOfRows, void** pData);
DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql);
DLL_EXPORT void taos_reset_current_db(TAOS *taos); DLL_EXPORT void taos_reset_current_db(TAOS *taos);

26
include/common/tglobal.h
View File

@ -41,9 +41,6 @@ extern int32_t tsMaxConnections;
extern int32_t tsMaxShellConns; extern int32_t tsMaxShellConns;
extern int32_t tsShellActivityTimer; extern int32_t tsShellActivityTimer;
extern int32_t tsMaxTmrCtrl; extern int32_t tsMaxTmrCtrl;
extern float tsNumOfThreadsPerCore;
extern int32_t tsNumOfCommitThreads;
extern float tsRatioOfQueryCores;
extern int32_t tsCompressMsgSize; extern int32_t tsCompressMsgSize;
extern int32_t tsCompressColData; extern int32_t tsCompressColData;
extern int32_t tsMaxNumOfDistinctResults; extern int32_t tsMaxNumOfDistinctResults;
@ -51,7 +48,30 @@ extern int32_t tsCompatibleModel;
extern bool tsEnableSlaveQuery; extern bool tsEnableSlaveQuery;
extern bool tsPrintAuth; extern bool tsPrintAuth;
extern int64_t tsTickPerDay[3]; extern int64_t tsTickPerDay[3];
// multi-process
extern bool tsMultiProcess; extern bool tsMultiProcess;
extern int32_t tsMnodeShmSize;
extern int32_t tsVnodeShmSize;
extern int32_t tsQnodeShmSize;
extern int32_t tsSnodeShmSize;
extern int32_t tsBnodeShmSize;
// queue & threads
extern int32_t tsNumOfRpcThreads;
extern int32_t tsNumOfCommitThreads;
extern int32_t tsNumOfTaskQueueThreads;
extern int32_t tsNumOfMnodeQueryThreads;
extern int32_t tsNumOfMnodeReadThreads;
extern int32_t tsNumOfVnodeQueryThreads;
extern int32_t tsNumOfVnodeFetchThreads;
extern int32_t tsNumOfVnodeWriteThreads;
extern int32_t tsNumOfVnodeSyncThreads;
extern int32_t tsNumOfVnodeMergeThreads;
extern int32_t tsNumOfQnodeQueryThreads;
extern int32_t tsNumOfQnodeFetchThreads;
extern int32_t tsNumOfSnodeSharedThreads;
extern int32_t tsNumOfSnodeUniqueThreads;
// monitor // monitor
extern bool tsEnableMonitor; extern bool tsEnableMonitor;

44
include/common/tmsg.h
View File

@ -483,7 +483,8 @@ typedef struct {
int32_t tz; // query client timezone int32_t tz; // query client timezone
char intervalUnit; char intervalUnit;
char slidingUnit; char slidingUnit;
char offsetUnit; // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration. char
offsetUnit; // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration.
int8_t precision; int8_t precision;
int64_t interval; int64_t interval;
int64_t sliding; int64_t sliding;
@ -934,12 +935,12 @@ typedef struct SExplainExecInfo {
uint64_t startupCost; uint64_t startupCost;
uint64_t totalCost; uint64_t totalCost;
uint64_t numOfRows; uint64_t numOfRows;
void *verboseInfo; void* verboseInfo;
} SExplainExecInfo; } SExplainExecInfo;
typedef struct { typedef struct {
int32_t numOfPlans; int32_t numOfPlans;
SExplainExecInfo *subplanInfo; SExplainExecInfo* subplanInfo;
} SExplainRsp; } SExplainRsp;
int32_t tSerializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp); int32_t tSerializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp);
@ -1827,6 +1828,7 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) {
typedef struct { typedef struct {
int64_t leftForVer; int64_t leftForVer;
int32_t vgId; int32_t vgId;
int32_t epoch;
int64_t consumerId; int64_t consumerId;
char topicName[TSDB_TOPIC_FNAME_LEN]; char topicName[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CGROUP_LEN]; char cgroup[TSDB_CGROUP_LEN];
@ -1840,6 +1842,7 @@ static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq*
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pReq->leftForVer); tlen += taosEncodeFixedI64(buf, pReq->leftForVer);
tlen += taosEncodeFixedI32(buf, pReq->vgId); tlen += taosEncodeFixedI32(buf, pReq->vgId);
tlen += taosEncodeFixedI32(buf, pReq->epoch);
tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeFixedI64(buf, pReq->consumerId);
tlen += taosEncodeString(buf, pReq->topicName); tlen += taosEncodeString(buf, pReq->topicName);
tlen += taosEncodeString(buf, pReq->cgroup); tlen += taosEncodeString(buf, pReq->cgroup);
@ -1853,6 +1856,7 @@ static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq*
static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) {
buf = taosDecodeFixedI64(buf, &pReq->leftForVer); buf = taosDecodeFixedI64(buf, &pReq->leftForVer);
buf = taosDecodeFixedI32(buf, &pReq->vgId); buf = taosDecodeFixedI32(buf, &pReq->vgId);
buf = taosDecodeFixedI32(buf, &pReq->epoch);
buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeFixedI64(buf, &pReq->consumerId);
buf = taosDecodeStringTo(buf, pReq->topicName); buf = taosDecodeStringTo(buf, pReq->topicName);
buf = taosDecodeStringTo(buf, pReq->cgroup); buf = taosDecodeStringTo(buf, pReq->cgroup);
@ -1863,11 +1867,43 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) {
return buf; return buf;
} }
typedef struct {
int64_t leftForVer;
int32_t vgId;
int32_t epoch;
int64_t consumerId;
char topicName[TSDB_TOPIC_FNAME_LEN];
} SMqCancelConnReq;
static FORCE_INLINE int32_t tEncodeSMqCancelConnReq(void** buf, const SMqCancelConnReq* pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pReq->leftForVer);
tlen += taosEncodeFixedI32(buf, pReq->vgId);
tlen += taosEncodeFixedI32(buf, pReq->epoch);
tlen += taosEncodeFixedI64(buf, pReq->consumerId);
tlen += taosEncodeString(buf, pReq->topicName);
return tlen;
}
static FORCE_INLINE void* tDecodeSMqCancelConnReq(void* buf, SMqCancelConnReq* pReq) {
buf = taosDecodeFixedI64(buf, &pReq->leftForVer);
buf = taosDecodeFixedI32(buf, &pReq->vgId);
buf = taosDecodeFixedI32(buf, &pReq->epoch);
buf = taosDecodeFixedI64(buf, &pReq->consumerId);
buf = taosDecodeStringTo(buf, pReq->topicName);
return buf;
}
typedef struct {
int8_t reserved;
} SMqCancelConnRsp;
typedef struct { typedef struct {
int64_t leftForVer; int64_t leftForVer;
int32_t vgId; int32_t vgId;
int64_t oldConsumerId; int64_t oldConsumerId;
int64_t newConsumerId; int64_t newConsumerId;
char* topic;
} SMqMVRebReq; } SMqMVRebReq;
static FORCE_INLINE int32_t tEncodeSMqMVRebReq(void** buf, const SMqMVRebReq* pReq) { static FORCE_INLINE int32_t tEncodeSMqMVRebReq(void** buf, const SMqMVRebReq* pReq) {
@ -1876,6 +1912,7 @@ static FORCE_INLINE int32_t tEncodeSMqMVRebReq(void** buf, const SMqMVRebReq* pR
tlen += taosEncodeFixedI32(buf, pReq->vgId); tlen += taosEncodeFixedI32(buf, pReq->vgId);
tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId);
tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); tlen += taosEncodeFixedI64(buf, pReq->newConsumerId);
tlen += taosEncodeString(buf, pReq->topic);
return tlen; return tlen;
} }
@ -1884,6 +1921,7 @@ static FORCE_INLINE void* tDecodeSMqMVRebReq(void* buf, SMqMVRebReq* pReq) {
buf = taosDecodeFixedI32(buf, &pReq->vgId); buf = taosDecodeFixedI32(buf, &pReq->vgId);
buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId);
buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); buf = taosDecodeFixedI64(buf, &pReq->newConsumerId);
buf = taosDecodeString(buf, &pReq->topic);
return buf; return buf;
} }

1
include/common/tmsgdef.h
View File

@ -176,6 +176,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_CANCEL_CONN, "vnode-mq-mv-cancel-conn", SMqCancelConnReq, SMqCancelConnRsp)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)

296
include/common/ttokendef.h
View File

@ -69,153 +69,154 @@
#define TK_CACHELAST 51 #define TK_CACHELAST 51
#define TK_COMP 52 #define TK_COMP 52
#define TK_DAYS 53 #define TK_DAYS 53
#define TK_FSYNC 54 #define TK_NK_VARIABLE 54
#define TK_MAXROWS 55 #define TK_FSYNC 55
#define TK_MINROWS 56 #define TK_MAXROWS 56
#define TK_KEEP 57 #define TK_MINROWS 57
#define TK_PRECISION 58 #define TK_KEEP 58
#define TK_QUORUM 59 #define TK_PRECISION 59
#define TK_REPLICA 60 #define TK_QUORUM 60
#define TK_TTL 61 #define TK_REPLICA 61
#define TK_WAL 62 #define TK_TTL 62
#define TK_VGROUPS 63 #define TK_WAL 63
#define TK_SINGLE_STABLE 64 #define TK_VGROUPS 64
#define TK_STREAM_MODE 65 #define TK_SINGLE_STABLE 65
#define TK_RETENTIONS 66 #define TK_STREAM_MODE 66
#define TK_NK_COMMA 67 #define TK_RETENTIONS 67
#define TK_TABLE 68 #define TK_NK_COMMA 68
#define TK_NK_LP 69 #define TK_NK_COLON 69
#define TK_NK_RP 70 #define TK_TABLE 70
#define TK_STABLE 71 #define TK_NK_LP 71
#define TK_ADD 72 #define TK_NK_RP 72
#define TK_COLUMN 73 #define TK_STABLE 73
#define TK_MODIFY 74 #define TK_ADD 74
#define TK_RENAME 75 #define TK_COLUMN 75
#define TK_TAG 76 #define TK_MODIFY 76
#define TK_SET 77 #define TK_RENAME 77
#define TK_NK_EQ 78 #define TK_TAG 78
#define TK_USING 79 #define TK_SET 79
#define TK_TAGS 80 #define TK_NK_EQ 80
#define TK_NK_DOT 81 #define TK_USING 81
#define TK_COMMENT 82 #define TK_TAGS 82
#define TK_BOOL 83 #define TK_NK_DOT 83
#define TK_TINYINT 84 #define TK_COMMENT 84
#define TK_SMALLINT 85 #define TK_BOOL 85
#define TK_INT 86 #define TK_TINYINT 86
#define TK_INTEGER 87 #define TK_SMALLINT 87
#define TK_BIGINT 88 #define TK_INT 88
#define TK_FLOAT 89 #define TK_INTEGER 89
#define TK_DOUBLE 90 #define TK_BIGINT 90
#define TK_BINARY 91 #define TK_FLOAT 91
#define TK_TIMESTAMP 92 #define TK_DOUBLE 92
#define TK_NCHAR 93 #define TK_BINARY 93
#define TK_UNSIGNED 94 #define TK_TIMESTAMP 94
#define TK_JSON 95 #define TK_NCHAR 95
#define TK_VARCHAR 96 #define TK_UNSIGNED 96
#define TK_MEDIUMBLOB 97 #define TK_JSON 97
#define TK_BLOB 98 #define TK_VARCHAR 98
#define TK_VARBINARY 99 #define TK_MEDIUMBLOB 99
#define TK_DECIMAL 100 #define TK_BLOB 100
#define TK_SMA 101 #define TK_VARBINARY 101
#define TK_ROLLUP 102 #define TK_DECIMAL 102
#define TK_FILE_FACTOR 103 #define TK_SMA 103
#define TK_NK_FLOAT 104 #define TK_ROLLUP 104
#define TK_DELAY 105 #define TK_FILE_FACTOR 105
#define TK_SHOW 106 #define TK_NK_FLOAT 106
#define TK_DATABASES 107 #define TK_DELAY 107
#define TK_TABLES 108 #define TK_SHOW 108
#define TK_STABLES 109 #define TK_DATABASES 109
#define TK_MNODES 110 #define TK_TABLES 110
#define TK_MODULES 111 #define TK_STABLES 111
#define TK_QNODES 112 #define TK_MNODES 112
#define TK_FUNCTIONS 113 #define TK_MODULES 113
#define TK_INDEXES 114 #define TK_QNODES 114
#define TK_FROM 115 #define TK_FUNCTIONS 115
#define TK_ACCOUNTS 116 #define TK_INDEXES 116
#define TK_APPS 117 #define TK_FROM 117
#define TK_CONNECTIONS 118 #define TK_ACCOUNTS 118
#define TK_LICENCE 119 #define TK_APPS 119
#define TK_QUERIES 120 #define TK_CONNECTIONS 120
#define TK_SCORES 121 #define TK_LICENCE 121
#define TK_TOPICS 122 #define TK_QUERIES 122
#define TK_VARIABLES 123 #define TK_SCORES 123
#define TK_LIKE 124 #define TK_TOPICS 124
#define TK_INDEX 125 #define TK_VARIABLES 125
#define TK_FULLTEXT 126 #define TK_LIKE 126
#define TK_FUNCTION 127 #define TK_INDEX 127
#define TK_INTERVAL 128 #define TK_FULLTEXT 128
#define TK_TOPIC 129 #define TK_FUNCTION 129
#define TK_AS 130 #define TK_INTERVAL 130
#define TK_DESC 131 #define TK_TOPIC 131
#define TK_DESCRIBE 132 #define TK_AS 132
#define TK_RESET 133 #define TK_DESC 133
#define TK_QUERY 134 #define TK_DESCRIBE 134
#define TK_EXPLAIN 135 #define TK_RESET 135
#define TK_ANALYZE 136 #define TK_QUERY 136
#define TK_VERBOSE 137 #define TK_EXPLAIN 137
#define TK_NK_BOOL 138 #define TK_ANALYZE 138
#define TK_RATIO 139 #define TK_VERBOSE 139
#define TK_COMPACT 140 #define TK_NK_BOOL 140
#define TK_VNODES 141 #define TK_RATIO 141
#define TK_IN 142 #define TK_COMPACT 142
#define TK_OUTPUTTYPE 143 #define TK_VNODES 143
#define TK_AGGREGATE 144 #define TK_IN 144
#define TK_BUFSIZE 145 #define TK_OUTPUTTYPE 145
#define TK_STREAM 146 #define TK_AGGREGATE 146
#define TK_INTO 147 #define TK_BUFSIZE 147
#define TK_KILL 148 #define TK_STREAM 148
#define TK_CONNECTION 149 #define TK_INTO 149
#define TK_MERGE 150 #define TK_KILL 150
#define TK_VGROUP 151 #define TK_CONNECTION 151
#define TK_REDISTRIBUTE 152 #define TK_MERGE 152
#define TK_SPLIT 153 #define TK_VGROUP 153
#define TK_SYNCDB 154 #define TK_REDISTRIBUTE 154
#define TK_NULL 155 #define TK_SPLIT 155
#define TK_NK_VARIABLE 156 #define TK_SYNCDB 156
#define TK_NOW 157 #define TK_NULL 157
#define TK_ROWTS 158 #define TK_FIRST 158
#define TK_TBNAME 159 #define TK_LAST 159
#define TK_QSTARTTS 160 #define TK_NOW 160
#define TK_QENDTS 161 #define TK_ROWTS 161
#define TK_WSTARTTS 162 #define TK_TBNAME 162
#define TK_WENDTS 163 #define TK_QSTARTTS 163
#define TK_WDURATION 164 #define TK_QENDTS 164
#define TK_BETWEEN 165 #define TK_WSTARTTS 165
#define TK_IS 166 #define TK_WENDTS 166
#define TK_NK_LT 167 #define TK_WDURATION 167
#define TK_NK_GT 168 #define TK_BETWEEN 168
#define TK_NK_LE 169 #define TK_IS 169
#define TK_NK_GE 170 #define TK_NK_LT 170
#define TK_NK_NE 171 #define TK_NK_GT 171
#define TK_MATCH 172 #define TK_NK_LE 172
#define TK_NMATCH 173 #define TK_NK_GE 173
#define TK_JOIN 174 #define TK_NK_NE 174
#define TK_INNER 175 #define TK_MATCH 175
#define TK_SELECT 176 #define TK_NMATCH 176
#define TK_DISTINCT 177 #define TK_JOIN 177
#define TK_WHERE 178 #define TK_INNER 178
#define TK_PARTITION 179 #define TK_SELECT 179
#define TK_BY 180 #define TK_DISTINCT 180
#define TK_SESSION 181 #define TK_WHERE 181
#define TK_STATE_WINDOW 182 #define TK_PARTITION 182
#define TK_SLIDING 183 #define TK_BY 183
#define TK_FILL 184 #define TK_SESSION 184
#define TK_VALUE 185 #define TK_STATE_WINDOW 185
#define TK_NONE 186 #define TK_SLIDING 186
#define TK_PREV 187 #define TK_FILL 187
#define TK_LINEAR 188 #define TK_VALUE 188
#define TK_NEXT 189 #define TK_NONE 189
#define TK_GROUP 190 #define TK_PREV 190
#define TK_HAVING 191 #define TK_LINEAR 191
#define TK_ORDER 192 #define TK_NEXT 192
#define TK_SLIMIT 193 #define TK_GROUP 193
#define TK_SOFFSET 194 #define TK_HAVING 194
#define TK_LIMIT 195 #define TK_ORDER 195
#define TK_OFFSET 196 #define TK_SLIMIT 196
#define TK_ASC 197 #define TK_SOFFSET 197
#define TK_NULLS 198 #define TK_LIMIT 198
#define TK_FIRST 199 #define TK_OFFSET 199
#define TK_LAST 200 #define TK_ASC 200
#define TK_NULLS 201
#define TK_NK_SPACE 300 #define TK_NK_SPACE 300
#define TK_NK_COMMENT 301 #define TK_NK_COMMENT 301
@ -226,7 +227,6 @@
#define TK_NK_FILE 306 #define TK_NK_FILE 306
#define TK_NK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query #define TK_NK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query
#define TK_NK_COLON 500
#define TK_NK_BITNOT 501 #define TK_NK_BITNOT 501
#define TK_INSERT 502 #define TK_INSERT 502
#define TK_VALUES 507 #define TK_VALUES 507

10
include/libs/function/function.h
View File

@ -110,10 +110,11 @@ typedef struct SFileBlockInfo {
#define FUNCTION_COV 38 #define FUNCTION_COV 38
typedef struct SResultRowEntryInfo { typedef struct SResultRowEntryInfo {
int8_t hasResult; // result generated, not NULL value // int8_t hasResult:6; // result generated, not NULL value
bool initialized; // output buffer has been initialized bool initialized:1; // output buffer has been initialized
bool complete; // query has completed bool complete:1; // query has completed
uint32_t numOfRes; // num of output result in current buffer uint8_t isNullRes:6; // the result is null
uint8_t numOfRes; // num of output result in current buffer
} SResultRowEntryInfo; } SResultRowEntryInfo;
// determine the real data need to calculated the result // determine the real data need to calculated the result
@ -156,7 +157,6 @@ typedef struct SResultDataInfo {
#define GET_RES_INFO(ctx) ((ctx)->resultInfo) #define GET_RES_INFO(ctx) ((ctx)->resultInfo)
#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowEntryInfo))) #define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowEntryInfo)))
#define DATA_SET_FLAG ',' // to denote the output area has data, not null value
typedef struct SInputColumnInfoData { typedef struct SInputColumnInfoData {
int32_t totalRows; // total rows in current columnar data int32_t totalRows; // total rows in current columnar data

49
include/libs/nodes/cmdnodes.h
View File

@ -30,25 +30,23 @@ extern "C" {
typedef struct SDatabaseOptions { typedef struct SDatabaseOptions {
ENodeType type; ENodeType type;
int32_t numOfBlocks; SValueNode* pNumOfBlocks;
int32_t cacheBlockSize; SValueNode* pCacheBlockSize;
int8_t cachelast; SValueNode* pCachelast;
int32_t compressionLevel; SValueNode* pCompressionLevel;
int32_t daysPerFile; SValueNode* pDaysPerFile;
int32_t fsyncPeriod; SValueNode* pFsyncPeriod;
int32_t maxRowsPerBlock; SValueNode* pMaxRowsPerBlock;
int32_t minRowsPerBlock; SValueNode* pMinRowsPerBlock;
int32_t keep0; SNodeList* pKeep;
int32_t keep1; SValueNode* pPrecision;
int32_t keep2; SValueNode* pQuorum;
int32_t precision; SValueNode* pReplica;
int32_t quorum; SValueNode* pTtl;
int32_t replica; SValueNode* pWalLevel;
int32_t ttl; SValueNode* pNumOfVgroups;
int32_t walLevel; SValueNode* pSingleStable;
int32_t numOfVgroups; SValueNode* pStreamMode;
int8_t singleStable;
int8_t streamMode;
SNodeList* pRetentions; SNodeList* pRetentions;
} SDatabaseOptions; } SDatabaseOptions;
@ -78,15 +76,13 @@ typedef struct SAlterDatabaseStmt {
typedef struct STableOptions { typedef struct STableOptions {
ENodeType type; ENodeType type;
int32_t keep0; SNodeList* pKeep;
int32_t keep1; SValueNode* pTtl;
int32_t keep2; SValueNode* pComments;
int32_t ttl;
char comments[TSDB_STB_COMMENT_LEN];
SNodeList* pSma; SNodeList* pSma;
SNodeList* pFuncs; SNodeList* pFuncs;
float filesFactor; SValueNode* pFilesFactor;
int32_t delay; SValueNode* pDelay;
} STableOptions; } STableOptions;
typedef struct SColumnDefNode { typedef struct SColumnDefNode {
@ -94,6 +90,7 @@ typedef struct SColumnDefNode {
char colName[TSDB_COL_NAME_LEN]; char colName[TSDB_COL_NAME_LEN];
SDataType dataType; SDataType dataType;
char comments[TSDB_STB_COMMENT_LEN]; char comments[TSDB_STB_COMMENT_LEN];
bool sma;
} SColumnDefNode; } SColumnDefNode;
typedef struct SCreateTableStmt { typedef struct SCreateTableStmt {

2
include/libs/nodes/nodes.h
View File

@ -163,6 +163,8 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_SORT,
QUERY_NODE_PHYSICAL_PLAN_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW,
QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW,
QUERY_NODE_PHYSICAL_PLAN_PARTITION,
QUERY_NODE_PHYSICAL_PLAN_DISPATCH, QUERY_NODE_PHYSICAL_PLAN_DISPATCH,
QUERY_NODE_PHYSICAL_PLAN_INSERT, QUERY_NODE_PHYSICAL_PLAN_INSERT,
QUERY_NODE_PHYSICAL_SUBPLAN, QUERY_NODE_PHYSICAL_SUBPLAN,

28
include/libs/nodes/plannodes.h
View File

@ -104,6 +104,7 @@ typedef struct SWindowLogicNode {
SFillNode* pFill; SFillNode* pFill;
int64_t sessionGap; int64_t sessionGap;
SNode* pTspk; SNode* pTspk;
SNode* pStateExpr;
} SWindowLogicNode; } SWindowLogicNode;
typedef struct SSortLogicNode { typedef struct SSortLogicNode {
@ -194,11 +195,20 @@ typedef struct SSystemTableScanPhysiNode {
int32_t accountId; int32_t accountId;
} SSystemTableScanPhysiNode; } SSystemTableScanPhysiNode;
typedef enum EScanRequired {
SCAN_REQUIRED_DATA_NO_NEEDED = 1,
SCAN_REQUIRED_DATA_STATIS_NEEDED,
SCAN_REQUIRED_DATA_ALL_NEEDED,
SCAN_REQUIRED_DATA_DISCARD,
} EScanRequired;
typedef struct STableScanPhysiNode { typedef struct STableScanPhysiNode {
SScanPhysiNode scan; SScanPhysiNode scan;
uint8_t scanFlag; // denotes reversed scan of data or not uint8_t scanFlag; // denotes reversed scan of data or not
STimeWindow scanRange; STimeWindow scanRange;
double ratio; double ratio;
EScanRequired scanRequired;
SNodeList* pScanReferFuncs;
} STableScanPhysiNode; } STableScanPhysiNode;
typedef STableScanPhysiNode STableSeqScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode;
@ -257,17 +267,35 @@ typedef struct SIntervalPhysiNode {
SFillNode* pFill; SFillNode* pFill;
} SIntervalPhysiNode; } SIntervalPhysiNode;
typedef struct SMultiTableIntervalPhysiNode {
SIntervalPhysiNode interval;
SNodeList* pPartitionKeys;
} SMultiTableIntervalPhysiNode;
typedef struct SSessionWinodwPhysiNode { typedef struct SSessionWinodwPhysiNode {
SWinodwPhysiNode window; SWinodwPhysiNode window;
int64_t gap; int64_t gap;
} SSessionWinodwPhysiNode; } SSessionWinodwPhysiNode;
typedef struct SStateWinodwPhysiNode {
SWinodwPhysiNode window;
SNode* pStateKey;
} SStateWinodwPhysiNode;
typedef struct SSortPhysiNode { typedef struct SSortPhysiNode {
SPhysiNode node; SPhysiNode node;
SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function
SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode
SNodeList* pTargets;
} SSortPhysiNode; } SSortPhysiNode;
typedef struct SPartitionPhysiNode {
SPhysiNode node;
SNodeList* pExprs; // these are expression list of partition_by_clause
SNodeList* pPartitionKeys;
SNodeList* pTargets;
} SPartitionPhysiNode;
typedef struct SDataSinkNode { typedef struct SDataSinkNode {
ENodeType type; ENodeType type;
SDataBlockDescNode* pInputDataBlockDesc; SDataBlockDescNode* pInputDataBlockDesc;

3
include/libs/nodes/querynodes.h
View File

@ -189,7 +189,7 @@ typedef struct SLimitNode {
typedef struct SStateWindowNode { typedef struct SStateWindowNode {
ENodeType type; // QUERY_NODE_STATE_WINDOW ENodeType type; // QUERY_NODE_STATE_WINDOW
SNode* pCol; SNode* pExpr;
} SStateWindowNode; } SStateWindowNode;
typedef struct SSessionWindowNode { typedef struct SSessionWindowNode {
@ -258,6 +258,7 @@ typedef enum ESqlClause {
SQL_CLAUSE_WINDOW, SQL_CLAUSE_WINDOW,
SQL_CLAUSE_GROUP_BY, SQL_CLAUSE_GROUP_BY,
SQL_CLAUSE_HAVING, SQL_CLAUSE_HAVING,
SQL_CLAUSE_DISTINCT,
SQL_CLAUSE_SELECT, SQL_CLAUSE_SELECT,
SQL_CLAUSE_ORDER_BY SQL_CLAUSE_ORDER_BY
} ESqlClause; } ESqlClause;

6
include/os/osMath.h
View File

@ -56,6 +56,12 @@ extern "C" {
__typeof(b) __b = (b); \ __typeof(b) __b = (b); \
(__a < __b) ? __a : __b; \ (__a < __b) ? __a : __b; \
}) })
#define TRANGE(a, b, c) \
({ \
a = TMAX(a, b); \
a = TMIN(a, c); \
})
#endif #endif
#ifdef __cplusplus #ifdef __cplusplus

14
include/util/taoserror.h
View File

@ -482,6 +482,20 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_DB_NOT_SPECIFIED TAOS_DEF_ERROR_CODE(0, 0x2616) #define TSDB_CODE_PAR_DB_NOT_SPECIFIED TAOS_DEF_ERROR_CODE(0, 0x2616)
#define TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME TAOS_DEF_ERROR_CODE(0, 0x2617) #define TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME TAOS_DEF_ERROR_CODE(0, 0x2617)
#define TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR TAOS_DEF_ERROR_CODE(0, 0x2618) #define TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR TAOS_DEF_ERROR_CODE(0, 0x2618)
#define TSDB_CODE_PAR_INVALID_RANGE_OPTION TAOS_DEF_ERROR_CODE(0, 0x2619)
#define TSDB_CODE_PAR_INVALID_STR_OPTION TAOS_DEF_ERROR_CODE(0, 0x2620)
#define TSDB_CODE_PAR_INVALID_ENUM_OPTION TAOS_DEF_ERROR_CODE(0, 0x2621)
#define TSDB_CODE_PAR_INVALID_TTL_OPTION TAOS_DEF_ERROR_CODE(0, 0x2622)
#define TSDB_CODE_PAR_INVALID_KEEP_NUM TAOS_DEF_ERROR_CODE(0, 0x2623)
#define TSDB_CODE_PAR_INVALID_KEEP_ORDER TAOS_DEF_ERROR_CODE(0, 0x2624)
#define TSDB_CODE_PAR_INVALID_KEEP_VALUE TAOS_DEF_ERROR_CODE(0, 0x2625)
#define TSDB_CODE_PAR_INVALID_COMMENT_OPTION TAOS_DEF_ERROR_CODE(0, 0x2626)
#define TSDB_CODE_PAR_INVALID_F_RANGE_OPTION TAOS_DEF_ERROR_CODE(0, 0x2627)
#define TSDB_CODE_PAR_INVALID_ROLLUP_OPTION TAOS_DEF_ERROR_CODE(0, 0x2628)
#define TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION TAOS_DEF_ERROR_CODE(0, 0x2629)
//planner
#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700)
#ifdef __cplusplus #ifdef __cplusplus
} }

3
include/util/tdef.h
View File

@ -472,9 +472,6 @@ typedef struct {
#define TMQ_SEPARATOR ':' #define TMQ_SEPARATOR ':'
#define SND_UNIQUE_THREAD_NUM 2
#define SND_SHARED_THREAD_NUM 2
enum { enum {
SND_WORKER_TYPE__SHARED = 1, SND_WORKER_TYPE__SHARED = 1,
SND_WORKER_TYPE__UNIQUE, SND_WORKER_TYPE__UNIQUE,

5
include/util/tprocess.h
View File

@ -22,12 +22,12 @@
extern "C" { extern "C" {
#endif #endif
typedef enum { PROC_QUEUE, PROC_REQ, PROC_RSP, PROC_REGIST, PROC_RELEASE } ProcFuncType; typedef enum { PROC_REQ = 1, PROC_RSP, PROC_REGIST, PROC_RELEASE } ProcFuncType;
typedef struct SProcObj SProcObj; typedef struct SProcObj SProcObj;
typedef void *(*ProcMallocFp)(int32_t contLen); typedef void *(*ProcMallocFp)(int32_t contLen);
typedef void *(*ProcFreeFp)(void *pCont); typedef void *(*ProcFreeFp)(void *pCont);
typedef void *(*ProcConsumeFp)(void *parent, void *pHead, int16_t headLen, void *pBody, int32_t bodyLen, typedef void (*ProcConsumeFp)(void *parent, void *pHead, int16_t headLen, void *pBody, int32_t bodyLen,
ProcFuncType ftype); ProcFuncType ftype);
typedef struct { typedef struct {
@ -50,6 +50,7 @@ typedef struct {
SProcObj *taosProcInit(const SProcCfg *pCfg); SProcObj *taosProcInit(const SProcCfg *pCfg);
void taosProcCleanup(SProcObj *pProc); void taosProcCleanup(SProcObj *pProc);
int32_t taosProcRun(SProcObj *pProc); int32_t taosProcRun(SProcObj *pProc);
void taosProcStop(SProcObj *pProc);
int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
void *handle, ProcFuncType ftype); void *handle, ProcFuncType ftype);

536
packaging/install.sh Executable file
View File

@ -0,0 +1,536 @@
#!/bin/bash
#
# This file is used to install database on linux systems. The operating system
# is required to use systemd to manage services at boot
set -e
#set -x
# -----------------------Variables definition---------------------
script_dir=$(dirname $(readlink -f "$0"))
# Dynamic directory
data_dir="/var/lib/taos"
log_dir="/var/log/taos"
data_link_dir="/usr/local/taos/data"
log_link_dir="/usr/local/taos/log"
cfg_install_dir="/etc/taos"
bin_link_dir="/usr/bin"
lib_link_dir="/usr/lib"
lib64_link_dir="/usr/lib64"
inc_link_dir="/usr/include"
#install main path
install_main_dir="/usr/local/taos"
# old bin dir
bin_dir="/usr/local/taos/bin"
service_config_dir="/etc/systemd/system"
# Color setting
RED='\033[0;31m'
GREEN='\033[1;32m'
GREEN_DARK='\033[0;32m'
GREEN_UNDERLINE='\033[4;32m'
NC='\033[0m'
csudo=""
if command -v sudo > /dev/null; then
csudo="sudo"
fi
update_flag=0
prompt_force=0
initd_mod=0
service_mod=2
if pidof systemd &> /dev/null; then
service_mod=0
elif $(which service &> /dev/null); then
service_mod=1
service_config_dir="/etc/init.d"
if $(which chkconfig &> /dev/null); then
initd_mod=1
elif $(which insserv &> /dev/null); then
initd_mod=2
elif $(which update-rc.d &> /dev/null); then
initd_mod=3
else
service_mod=2
fi
else
service_mod=2
fi
# get the operating system type for using the corresponding init file
# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification
#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release)
if [[ -e /etc/os-release ]]; then
osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||:
else
osinfo=""
fi
#echo "osinfo: ${osinfo}"
os_type=0
if echo $osinfo | grep -qwi "ubuntu" ; then
# echo "This is ubuntu system"
os_type=1
elif echo $osinfo | grep -qwi "debian" ; then
# echo "This is debian system"
os_type=1
elif echo $osinfo | grep -qwi "Kylin" ; then
# echo "This is Kylin system"
os_type=1
elif echo $osinfo | grep -qwi "centos" ; then
# echo "This is centos system"
os_type=2
elif echo $osinfo | grep -qwi "fedora" ; then
# echo "This is fedora system"
os_type=2
elif echo $osinfo | grep -qwi "Linx" ; then
# echo "This is Linx system"
os_type=1
service_mod=0
initd_mod=0
service_config_dir="/etc/systemd/system"
else
echo " osinfo: ${osinfo}"
echo " This is an officially unverified linux system,"
echo " if there are any problems with the installation and operation, "
echo " please feel free to contact taosdata.com for support."
os_type=1
fi
# ============================= get input parameters =================================================
# install.sh -v [server | client] -e [yes | no] -i [systemd | service | ...]
# set parameters by default value
interactiveFqdn=yes # [yes | no]
verType=server # [server | client]
initType=systemd # [systemd | service | ...]
while getopts "hv:e:i:" arg
do
case $arg in
e)
#echo "interactiveFqdn=$OPTARG"
interactiveFqdn=$( echo $OPTARG )
;;
v)
#echo "verType=$OPTARG"
verType=$(echo $OPTARG)
;;
i)
#echo "initType=$OPTARG"
initType=$(echo $OPTARG)
;;
h)
echo "Usage: `basename $0` -v [server | client] -e [yes | no]"
exit 0
;;
?) #unknow option
echo "unkonw argument"
exit 1
;;
esac
done
#echo "verType=${verType} interactiveFqdn=${interactiveFqdn}"
function kill_process() {
pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo} kill -9 $pid || :
fi
}
function install_main_path() {
#create install main dir and all sub dir
${csudo} rm -rf ${install_main_dir} || :
${csudo} mkdir -p ${install_main_dir}
${csudo} mkdir -p ${install_main_dir}/cfg
${csudo} mkdir -p ${install_main_dir}/bin
${csudo} mkdir -p ${install_main_dir}/connector
${csudo} mkdir -p ${install_main_dir}/driver
${csudo} mkdir -p ${install_main_dir}/examples
${csudo} mkdir -p ${install_main_dir}/include
${csudo} mkdir -p ${install_main_dir}/init.d
if [ "$verMode" == "cluster" ]; then
${csudo} mkdir -p ${nginx_dir}
fi
if [[ -e ${script_dir}/email ]]; then
${csudo} cp ${script_dir}/email ${install_main_dir}/ ||:
fi
}
function install_bin() {
# Remove links
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosd || :
${csudo} rm -f ${bin_link_dir}/taosadapter || :
${csudo} rm -f ${bin_link_dir}/create_table || :
${csudo} rm -f ${bin_link_dir}/tmq_sim || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${bin_link_dir}/rmtaos || :
#${csudo} rm -f ${bin_link_dir}/set_core || :
${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/*
#Make link
[ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || :
[ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || :
[ -x ${install_main_dir}/bin/create_table ] && ${csudo} ln -s ${install_main_dir}/bin/create_table ${bin_link_dir}/create_table || :
[ -x ${install_main_dir}/bin/tmq_sim ] && ${csudo} ln -s ${install_main_dir}/bin/tmq_sim ${bin_link_dir}/tmq_sim || :
# [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
# [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || :
# [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
}
function install_lib() {
# Remove links
${csudo} rm -f ${lib_link_dir}/libtaos.* || :
${csudo} rm -f ${lib64_link_dir}/libtaos.* || :
${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib_link_dir}/libtaos.so.1
${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
${csudo} ln -s ${install_main_dir}/lib/libtdb.* ${lib_link_dir}/libtdb.so.1
${csudo} ln -s ${lib_link_dir}/libtdb.so.1 ${lib_link_dir}/libtdb.so
if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
${csudo} ln -s ${install_main_dir}/lib/libtdb.* ${lib64_link_dir}/libtdb.so.1 || :
${csudo} ln -s ${lib64_link_dir}/libtdb.so.1 ${lib64_link_dir}/libtdb.so || :
fi
${csudo} ldconfig
}
function install_header() {
${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || :
${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/*
${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
# ${csudo} ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
}
function add_newHostname_to_hosts() {
localIp="127.0.0.1"
OLD_IFS="$IFS"
IFS=" "
iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}')
arr=($iphost)
IFS="$OLD_IFS"
for s in "${arr[@]}"
do
if [[ "$s" == "$localIp" ]]; then
return
fi
done
${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||:
}
function set_hostname() {
echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:"
read newHostname
while true; do
if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then
break
else
read -p "Please enter one hostname(must not be 'localhost'):" newHostname
fi
done
${csudo} hostname $newHostname ||:
retval=`echo $?`
if [[ $retval != 0 ]]; then
echo
echo "set hostname fail!"
return
fi
#echo -e -n "$(hostnamectl status --static)"
#echo -e -n "$(hostnamectl status --transient)"
#echo -e -n "$(hostnamectl status --pretty)"
#ubuntu/centos /etc/hostname
if [[ -e /etc/hostname ]]; then
${csudo} echo $newHostname > /etc/hostname ||:
fi
#debian: #HOSTNAME=yourname
if [[ -e /etc/sysconfig/network ]]; then
${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||:
fi
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg
serverFqdn=$newHostname
if [[ -e /etc/hosts ]]; then
add_newHostname_to_hosts $newHostname
fi
}
function is_correct_ipaddr() {
newIp=$1
OLD_IFS="$IFS"
IFS=" "
arr=($iplist)
IFS="$OLD_IFS"
for s in "${arr[@]}"
do
if [[ "$s" == "$newIp" ]]; then
return 0
fi
done
return 1
}
function set_ipAsFqdn() {
iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||:
if [ -z "$iplist" ]; then
iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||:
fi
if [ -z "$iplist" ]; then
echo
echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}"
localFqdn="127.0.0.1"
# Write the local FQDN to configuration file
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg
serverFqdn=$localFqdn
echo
return
fi
echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:"
echo
echo -e -n "${GREEN}$iplist${NC}"
echo
echo
echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:"
read localFqdn
while true; do
if [ ! -z "$localFqdn" ]; then
# Check if correct ip address
is_correct_ipaddr $localFqdn
retval=`echo $?`
if [[ $retval != 0 ]]; then
read -p "Please choose an IP from local IP list:" localFqdn
else
# Write the local FQDN to configuration file
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg
serverFqdn=$localFqdn
break
fi
else
read -p "Please choose an IP from local IP list:" localFqdn
fi
done
}
function local_fqdn_check() {
#serverFqdn=$(hostname)
echo
echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}"
echo
if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then
echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}"
echo
while true
do
read -r -p "Set hostname now? [Y/n] " input
if [ ! -n "$input" ]; then
set_hostname
break
else
case $input in
[yY][eE][sS]|[yY])
set_hostname
break
;;
[nN][oO]|[nN])
set_ipAsFqdn
break
;;
*)
echo "Invalid input..."
;;
esac
fi
done
fi
}
function install_log() {
${csudo} rm -rf ${log_dir} || :
${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir}
${csudo} ln -s ${log_dir} ${install_main_dir}/log
}
function install_data() {
${csudo} mkdir -p ${data_dir}
${csudo} ln -s ${data_dir} ${install_main_dir}/data
}
function clean_service_on_systemd() {
taosd_service_config="${service_config_dir}/taosd.service"
if systemctl is-active --quiet taosd; then
echo "TDengine is running, stopping it..."
${csudo} systemctl stop taosd &> /dev/null || echo &> /dev/null
fi
${csudo} systemctl disable taosd &> /dev/null || echo &> /dev/null
${csudo} rm -f ${taosd_service_config}
tarbitratord_service_config="${service_config_dir}/tarbitratord.service"
if systemctl is-active --quiet tarbitratord; then
echo "tarbitrator is running, stopping it..."
${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null
fi
${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null
${csudo} rm -f ${tarbitratord_service_config}
if [ "$verMode" == "cluster" ]; then
nginx_service_config="${service_config_dir}/nginxd.service"
if systemctl is-active --quiet nginxd; then
echo "Nginx for TDengine is running, stopping it..."
${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null
fi
${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null
${csudo} rm -f ${nginx_service_config}
fi
}
# taos:2345:respawn:/etc/init.d/taosd start
function install_service_on_systemd() {
clean_service_on_systemd
taosd_service_config="${service_config_dir}/taosd.service"
${csudo} bash -c "echo '[Unit]' >> ${taosd_service_config}"
${csudo} bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}"
${csudo} bash -c "echo 'After=network-online.target taosadapter.service' >> ${taosd_service_config}"
${csudo} bash -c "echo 'Wants=network-online.target taosadapter.service' >> ${taosd_service_config}"
${csudo} bash -c "echo >> ${taosd_service_config}"
${csudo} bash -c "echo '[Service]' >> ${taosd_service_config}"
${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}"
${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}"
${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}"
${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${taosd_service_config}"
${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}"
${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}"
${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}"
${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${taosd_service_config}"
${csudo} bash -c "echo 'StandardOutput=null' >> ${taosd_service_config}"
${csudo} bash -c "echo 'Restart=always' >> ${taosd_service_config}"
${csudo} bash -c "echo 'StartLimitBurst=3' >> ${taosd_service_config}"
${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${taosd_service_config}"
#${csudo} bash -c "echo 'StartLimitIntervalSec=60s' >> ${taosd_service_config}"
${csudo} bash -c "echo >> ${taosd_service_config}"
${csudo} bash -c "echo '[Install]' >> ${taosd_service_config}"
${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}"
${csudo} systemctl enable taosd
${csudo} systemctl daemon-reload
}
function install_service() {
if ((${service_mod}==0)); then
install_service_on_systemd
elif ((${service_mod}==1)); then
install_service_on_sysvinit
else
# must manual stop taosd
kill_process taosd
fi
}
function install_TDengine() {
# Start to install
echo -e "${GREEN}Start to install TDengine...${NC}"
install_main_path
install_data
install_log
install_header
install_lib
if [ -z $1 ]; then # install service and client
# For installing new
install_bin
install_service
#install_config
# Ask if to start the service
#echo
#echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
echo
echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg"
if ((${service_mod}==0)); then
echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} systemctl start taosd${NC}"
elif ((${service_mod}==1)); then
echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} service taosd start${NC}"
else
echo -e "${GREEN_DARK}To start TDengine ${NC}: taosd${NC}"
fi
if [ ! -z "$firstEp" ]; then
tmpFqdn=${firstEp%%:*}
substr=":"
if [[ $firstEp =~ $substr ]];then
tmpPort=${firstEp#*:}
else
tmpPort=""
fi
if [[ "$tmpPort" != "" ]];then
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
else
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
fi
echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}"
echo
elif [ ! -z "$serverFqdn" ]; then
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}"
echo
fi
echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
echo
else # Only install client
install_bin
#install_config
echo
echo -e "\033[44;32;1mTDengine client is installed successfully!${NC}"
fi
touch ~/.taos_history
}
## ==============================Main program starts from here============================
serverFqdn=$(hostname)
if [ "$verType" == "server" ]; then
# Install server and client
install_TDengine
elif [ "$verType" == "client" ]; then
interactiveFqdn=no
# Only install client
install_TDengine client
else
echo "please input correct verType"
fi

49
packaging/release.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash #!/bin/bash
# #
# Generate the deb package for ubuntu, or rpm package for centos, or tar.gz package for other linux os # Generate the tar.gz package for linux os
set -e set -e
#set -x #set -x
@ -17,9 +17,12 @@ echo "=======================new version number: ${verNumber}===================
build_time=$(date +"%F %R") build_time=$(date +"%F %R")
echo "script_dir: ${script_dir}"
echo "top_dir: ${top_dir}" echo "top_dir: ${top_dir}"
cd ${top_dir} cd ${top_dir}
git checkout -- .
git checkout 3.0
git pull || : git pull || :
echo "curr_dir: ${curr_dir}" echo "curr_dir: ${curr_dir}"
@ -49,28 +52,38 @@ cd ${release_dir}
install_dir="${release_dir}/TDengine-server-${version}" install_dir="${release_dir}/TDengine-server-${version}"
mkdir -p ${install_dir} mkdir -p ${install_dir}
mkdir -p ${install_dir}/bin
mkdir -p ${install_dir}/lib mkdir -p ${install_dir}/lib
mkdir -p ${install_dir}/inc
bin_files="${compile_dir}/source/dnode/mgmt/daemon/taosd ${compile_dir}/tools/shell/taos ${compile_dir}/tests/test/c/create_table" install_files="${script_dir}/install.sh"
cp ${bin_files} ${install_dir}/ && chmod a+x ${install_dir}/* || : chmod a+x ${script_dir}/install.sh || :
cp ${install_files} ${install_dir}
header_files="${top_dir}/include/client/taos.h ${top_dir}/include/util/taoserror.h"
cp ${header_files} ${install_dir}/inc
bin_files="${compile_dir}/source/dnode/mgmt/taosd ${compile_dir}/tools/shell/taos ${compile_dir}/tests/test/c/create_table ${compile_dir}/tests/test/c/tmq_sim ${script_dir}/remove.sh"
cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
cp ${compile_dir}/source/client/libtaos.so ${install_dir}/lib/ cp ${compile_dir}/source/client/libtaos.so ${install_dir}/lib/
cp ${compile_dir}/source/dnode/mnode/impl/libmnode.so ${install_dir}/lib/ cp ${compile_dir}/source/libs/tdb/libtdb.so ${install_dir}/lib/
cp ${compile_dir}/source/dnode/qnode/libqnode.so ${install_dir}/lib/
cp ${compile_dir}/source/dnode/snode/libsnode.so ${install_dir}/lib/ #cp ${compile_dir}/source/dnode/mnode/impl/libmnode.so ${install_dir}/lib/
cp ${compile_dir}/source/dnode/bnode/libbnode.so ${install_dir}/lib/ #cp ${compile_dir}/source/dnode/qnode/libqnode.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/wal/libwal.so ${install_dir}/lib/ #cp ${compile_dir}/source/dnode/snode/libsnode.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/scheduler/libscheduler.so ${install_dir}/lib/ #cp ${compile_dir}/source/dnode/bnode/libbnode.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/planner/libplanner.so ${install_dir}/lib/ #cp ${compile_dir}/source/libs/wal/libwal.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/parser/libparser.so ${install_dir}/lib/ #cp ${compile_dir}/source/libs/scheduler/libscheduler.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/qcom/libqcom.so ${install_dir}/lib/ #cp ${compile_dir}/source/libs/planner/libplanner.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/transport/libtransport.so ${install_dir}/lib/ #cp ${compile_dir}/source/libs/parser/libparser.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/function/libfunction.so ${install_dir}/lib/ #cp ${compile_dir}/source/libs/qcom/libqcom.so ${install_dir}/lib/
cp ${compile_dir}/source/common/libcommon.so ${install_dir}/lib/ #cp ${compile_dir}/source/libs/transport/libtransport.so ${install_dir}/lib/
cp ${compile_dir}/source/os/libos.so ${install_dir}/lib/ #cp ${compile_dir}/source/libs/function/libfunction.so ${install_dir}/lib/
cp ${compile_dir}/source/dnode/mnode/sdb/libsdb.so ${install_dir}/lib/ #cp ${compile_dir}/source/common/libcommon.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/catalog/libcatalog.so ${install_dir}/lib/ #cp ${compile_dir}/source/os/libos.so ${install_dir}/lib/
#cp ${compile_dir}/source/dnode/mnode/sdb/libsdb.so ${install_dir}/lib/
#cp ${compile_dir}/source/libs/catalog/libcatalog.so ${install_dir}/lib/
pkg_name=${install_dir}-Linux-x64 pkg_name=${install_dir}-Linux-x64

148
packaging/remove.sh Normal file
View File

@ -0,0 +1,148 @@
#!/bin/bash
#
# Script to stop the service and uninstall TDengine, but retain the config, data and log files.
set -e
#set -x
verMode=edge
RED='\033[0;31m'
GREEN='\033[1;32m'
NC='\033[0m'
#install main path
install_main_dir="/usr/local/taos"
data_link_dir="/usr/local/taos/data"
log_link_dir="/usr/local/taos/log"
cfg_link_dir="/usr/local/taos/cfg"
bin_link_dir="/usr/bin"
lib_link_dir="/usr/lib"
lib64_link_dir="/usr/lib64"
inc_link_dir="/usr/include"
service_config_dir="/etc/systemd/system"
taos_service_name="taosd"
csudo=""
if command -v sudo > /dev/null; then
csudo="sudo"
fi
initd_mod=0
service_mod=2
if pidof systemd &> /dev/null; then
service_mod=0
elif $(which service &> /dev/null); then
service_mod=1
service_config_dir="/etc/init.d"
if $(which chkconfig &> /dev/null); then
initd_mod=1
elif $(which insserv &> /dev/null); then
initd_mod=2
elif $(which update-rc.d &> /dev/null); then
initd_mod=3
else
service_mod=2
fi
else
service_mod=2
fi
function kill_taosd() {
pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo} kill -9 $pid || :
fi
}
function clean_bin() {
# Remove link
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosd || :
${csudo} rm -f ${bin_link_dir}/create_table || :
${csudo} rm -f ${bin_link_dir}/tmq_sim || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${bin_link_dir}/rmtaos || :
#${csudo} rm -f ${bin_link_dir}/set_core || :
}
function clean_lib() {
# Remove link
${csudo} rm -f ${lib_link_dir}/libtaos.* || :
${csudo} rm -f ${lib64_link_dir}/libtaos.* || :
${csudo} rm -f ${lib_link_dir}/libtdb.* || :
${csudo} rm -f ${lib64_link_dir}/libtdb.* || :
}
function clean_header() {
# Remove link
${csudo} rm -f ${inc_link_dir}/taos.h || :
${csudo} rm -f ${inc_link_dir}/taosdef.h || :
${csudo} rm -f ${inc_link_dir}/taoserror.h || :
}
function clean_config() {
# Remove link
${csudo} rm -f ${cfg_link_dir}/* || :
}
function clean_log() {
# Remove link
${csudo} rm -rf ${log_link_dir} || :
}
function clean_service_on_systemd() {
taosd_service_config="${service_config_dir}/${taos_service_name}.service"
if systemctl is-active --quiet ${taos_service_name}; then
echo "TDengine taosd is running, stopping it..."
${csudo} systemctl stop ${taos_service_name} &> /dev/null || echo &> /dev/null
fi
${csudo} systemctl disable ${taos_service_name} &> /dev/null || echo &> /dev/null
${csudo} rm -f ${taosd_service_config}
}
function clean_service_on_sysvinit() {
echo " "
}
function clean_service() {
if ((${service_mod}==0)); then
clean_service_on_systemd
elif ((${service_mod}==1)); then
clean_service_on_sysvinit
else
# must manual stop taosd
kill_taosd
fi
}
# Stop service and disable booting start.
clean_service
# Remove binary file and links
clean_bin
# Remove header file.
clean_header
# Remove lib file
clean_lib
# Remove link log directory
clean_log
# Remove link configuration file
clean_config
# Remove data link directory
${csudo} rm -rf ${data_link_dir} || :
${csudo} rm -rf ${install_main_dir}
if [[ -e /etc/os-release ]]; then
osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release)
else
osinfo=""
fi
echo -e "${GREEN}TDengine is removed successfully!${NC}"
echo

1
source/client/inc/clientInt.h
View File

@ -163,6 +163,7 @@ typedef struct SReqResultInfo {
uint64_t totalRows; uint64_t totalRows;
uint32_t current; uint32_t current;
bool completed; bool completed;
int32_t precision;
int32_t payloadLen; int32_t payloadLen;
} SReqResultInfo; } SReqResultInfo;

1
source/client/src/clientImpl.c
View File

@ -825,6 +825,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR
pResultInfo->current = 0; pResultInfo->current = 0;
pResultInfo->completed = (pRsp->completed == 1); pResultInfo->completed = (pRsp->completed == 1);
pResultInfo->payloadLen = htonl(pRsp->compLen); pResultInfo->payloadLen = htonl(pRsp->compLen);
pResultInfo->precision = pRsp->precision;
// TODO handle the compressed case // TODO handle the compressed case
pResultInfo->totalRows += pResultInfo->numOfRows; pResultInfo->totalRows += pResultInfo->numOfRows;

64
source/client/src/clientMain.c
View File

@ -323,7 +323,12 @@ int taos_affected_rows(TAOS_RES *res) {
} }
int taos_result_precision(TAOS_RES *res) { int taos_result_precision(TAOS_RES *res) {
SRequestObj* pRequest = (SRequestObj*) res;
if (pRequest == NULL) {
return TSDB_TIME_PRECISION_MILLI; return TSDB_TIME_PRECISION_MILLI;
}
return pRequest->body.resInfo.precision;
} }
int taos_select_db(TAOS *taos, const char *db) { int taos_select_db(TAOS *taos, const char *db) {
@ -380,11 +385,20 @@ bool taos_is_update_query(TAOS_RES *res) {
} }
int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
if (res == NULL) { int32_t numOfRows = 0;
/*int32_t code = */taos_fetch_block_s(res, &numOfRows, rows);
return numOfRows;
}
int taos_fetch_block_s(TAOS_RES *res, int* numOfRows, TAOS_ROW *rows) {
SRequestObj *pRequest = (SRequestObj *)res;
if (pRequest == NULL) {
return 0; return 0;
} }
SRequestObj *pRequest = (SRequestObj *)res; (*rows) = NULL;
(*numOfRows) = 0;
if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT || if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) { pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
return 0; return 0;
@ -395,9 +409,51 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
// TODO refactor // TODO refactor
SReqResultInfo *pResultInfo = &pRequest->body.resInfo; SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
pResultInfo->current = pResultInfo->numOfRows; pResultInfo->current = pResultInfo->numOfRows;
*rows = pResultInfo->row;
return pResultInfo->numOfRows; (*rows) = pResultInfo->row;
(*numOfRows) = pResultInfo->numOfRows;
return pRequest->code;
}
int taos_fetch_raw_block(TAOS_RES *res, int* numOfRows, void** pData) {
SRequestObj *pRequest = (SRequestObj *)res;
if (pRequest == NULL) {
return 0;
}
if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
return 0;
}
doFetchRow(pRequest, false);
SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
pResultInfo->current = pResultInfo->numOfRows;
(*numOfRows) = pResultInfo->numOfRows;
(*pData) = (void*) pResultInfo->pData;
return 0;
}
int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) {
SRequestObj *pRequest = (SRequestObj *)res;
if (pRequest == NULL) {
return 0;
}
int32_t numOfFields = taos_num_fields(pRequest);
if (columnIndex < 0 || columnIndex >= numOfFields || numOfFields == 0) {
return 0;
}
TAOS_FIELD* pField = &pRequest->body.resInfo.userFields[columnIndex];
if (!IS_VAR_DATA_TYPE(pField->type)) {
return 0;
}
return pRequest->body.resInfo.pCol[columnIndex].offset;
} }
int taos_validate_sql(TAOS *taos, const char *sql) { return true; } int taos_validate_sql(TAOS *taos, const char *sql) { return true; }

17
source/client/src/tmq.c
View File

@ -108,7 +108,7 @@ typedef struct {
// connection info // connection info
int32_t vgId; int32_t vgId;
int32_t vgStatus; int32_t vgStatus;
int64_t skipCnt; int32_t vgSkipCnt;
SEpSet epSet; SEpSet epSet;
} SMqClientVg; } SMqClientVg;
@ -849,7 +849,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
if (msgEpoch < tmqEpoch) { if (msgEpoch < tmqEpoch) {
/*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/ /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/
/*tsem_post(&tmq->rspSem);*/ /*tsem_post(&tmq->rspSem);*/
tscWarn("discard rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch);
return 0; return 0;
} }
@ -881,6 +881,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
/*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/ /*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/
tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));
if (pRsp == NULL) { if (pRsp == NULL) {
tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch);
goto CREATE_MSG_FAIL; goto CREATE_MSG_FAIL;
} }
memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
@ -969,14 +970,14 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
offset = *pOffset; offset = *pOffset;
tscDebug("consumer %ld epoch %d vg %d found %s", tmq->consumerId, epoch, pVgEp->vgId, vgKey); tscDebug("consumer %ld epoch %d vg %d found %s", tmq->consumerId, epoch, pVgEp->vgId, vgKey);
} }
tscDebug("consumer %ld epoch %d vg %d offset set to %ld\n", tmq->consumerId, epoch, pVgEp->vgId, offset); tscDebug("consumer %ld epoch %d vg %d offset set to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset);
SMqClientVg clientVg = { SMqClientVg clientVg = {
.pollCnt = 0, .pollCnt = 0,
.currentOffset = offset, .currentOffset = offset,
.vgId = pVgEp->vgId, .vgId = pVgEp->vgId,
.epSet = pVgEp->epSet, .epSet = pVgEp->epSet,
.vgStatus = TMQ_VG_STATUS__IDLE, .vgStatus = TMQ_VG_STATUS__IDLE,
.skipCnt = 0, .vgSkipCnt = 0,
}; };
taosArrayPush(topic.vgs, &clientVg); taosArrayPush(topic.vgs, &clientVg);
set = true; set = true;
@ -1232,9 +1233,10 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
if (vgStatus != TMQ_VG_STATUS__IDLE) { if (vgStatus != TMQ_VG_STATUS__IDLE) {
int64_t skipCnt = atomic_add_fetch_64(&pVg->skipCnt, 1); int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1);
tscDebug("consumer %ld epoch %d skip vg %d skip cnt %ld", tmq->consumerId, tmq->epoch, pVg->vgId, skipCnt); tscDebug("consumer %ld epoch %d skip vg %d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, vgSkipCnt);
continue; continue;
/*if (vgSkipCnt < 10000) continue;*/
#if 0 #if 0
if (skipCnt < 30000) { if (skipCnt < 30000) {
continue; continue;
@ -1243,7 +1245,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
} }
#endif #endif
} }
atomic_store_64(&pVg->skipCnt, 0); atomic_store_32(&pVg->vgSkipCnt, 0);
SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
if (pReq == NULL) { if (pReq == NULL) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
@ -1409,6 +1411,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
if (blocking_time != 0) { if (blocking_time != 0) {
int64_t endTime = taosGetTimestampMs(); int64_t endTime = taosGetTimestampMs();
if (endTime - startTime > blocking_time) { if (endTime - startTime > blocking_time) {
tscDebug("consumer %ld (epoch %d) timeout, no rsp", tmq->consumerId, tmq->epoch);
return NULL; return NULL;
} }
} }

4
source/client/test/clientTests.cpp
View File

@ -400,6 +400,7 @@ TEST(testCase, show_vgroup_Test) {
taos_free_result(pRes); taos_free_result(pRes);
taos_close(pConn); taos_close(pConn);
} }
#endif
TEST(testCase, create_multiple_tables) { TEST(testCase, create_multiple_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
@ -458,7 +459,7 @@ TEST(testCase, create_multiple_tables) {
taos_free_result(pRes); taos_free_result(pRes);
for (int32_t i = 0; i < 20; ++i) { for (int32_t i = 0; i < 25000; ++i) {
char sql[512] = {0}; char sql[512] = {0};
snprintf(sql, tListLen(sql), snprintf(sql, tListLen(sql),
"create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i,
@ -652,7 +653,6 @@ TEST(testCase, projection_query_stables) {
taos_free_result(pRes); taos_free_result(pRes);
taos_close(pConn); taos_close(pConn);
} }
#endif
TEST(testCase, agg_query_tables) { TEST(testCase, agg_query_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);

135
source/common/src/tglobal.c
View File

@ -30,25 +30,45 @@ char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port
uint16_t tsServerPort = 6030; uint16_t tsServerPort = 6030;
int32_t tsVersion = 30000000; int32_t tsVersion = 30000000;
int32_t tsStatusInterval = 1; // second int32_t tsStatusInterval = 1; // second
bool tsEnableTelemetryReporting = 0; bool tsEnableTelemetryReporting = false;
// common // common
int32_t tsRpcTimer = 300; int32_t tsRpcTimer = 300;
int32_t tsRpcMaxTime = 600; // seconds; int32_t tsRpcMaxTime = 600; // seconds;
bool tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default bool tsRpcForceTcp = true; // disable this, means query, show command use udp protocol as default
int32_t tsMaxShellConns = 50000; int32_t tsMaxShellConns = 50000;
int32_t tsMaxConnections = 50000; int32_t tsMaxConnections = 50000;
int32_t tsShellActivityTimer = 3; // second int32_t tsShellActivityTimer = 3; // second
float tsNumOfThreadsPerCore = 1.0f;
int32_t tsNumOfCommitThreads = 4;
float tsRatioOfQueryCores = 1.0f;
int32_t tsMaxBinaryDisplayWidth = 30; int32_t tsMaxBinaryDisplayWidth = 30;
bool tsEnableSlaveQuery = 1; bool tsEnableSlaveQuery = true;
bool tsPrintAuth = 0; bool tsPrintAuth = false;
bool tsMultiProcess = 0;
// multi process
bool tsMultiProcess = false;
int32_t tsMnodeShmSize = TSDB_MAX_WAL_SIZE * 2;
int32_t tsVnodeShmSize = TSDB_MAX_WAL_SIZE * 10;
int32_t tsQnodeShmSize = TSDB_MAX_WAL_SIZE * 4;
int32_t tsSnodeShmSize = TSDB_MAX_WAL_SIZE * 4;
int32_t tsBnodeShmSize = TSDB_MAX_WAL_SIZE * 4;
// queue & threads
int32_t tsNumOfRpcThreads = 1;
int32_t tsNumOfCommitThreads = 2;
int32_t tsNumOfTaskQueueThreads = 1;
int32_t tsNumOfMnodeQueryThreads = 1;
int32_t tsNumOfMnodeReadThreads = 1;
int32_t tsNumOfVnodeQueryThreads = 2;
int32_t tsNumOfVnodeFetchThreads = 2;
int32_t tsNumOfVnodeWriteThreads = 2;
int32_t tsNumOfVnodeSyncThreads = 2;
int32_t tsNumOfVnodeMergeThreads = 2;
int32_t tsNumOfQnodeQueryThreads = 2;
int32_t tsNumOfQnodeFetchThreads = 2;
int32_t tsNumOfSnodeSharedThreads = 2;
int32_t tsNumOfSnodeUniqueThreads = 2;
// monitor // monitor
bool tsEnableMonitor = 1; bool tsEnableMonitor = true;
int32_t tsMonitorInterval = 30; int32_t tsMonitorInterval = 30;
char tsMonitorFqdn[TSDB_FQDN_LEN] = {0}; char tsMonitorFqdn[TSDB_FQDN_LEN] = {0};
uint16_t tsMonitorPort = 6043; uint16_t tsMonitorPort = 6043;
@ -117,13 +137,13 @@ int32_t tsQueryBufferSize = -1;
int64_t tsQueryBufferSizeBytes = -1; int64_t tsQueryBufferSizeBytes = -1;
// in retrieve blocking model, the retrieve threads will wait for the completion of the query processing. // in retrieve blocking model, the retrieve threads will wait for the completion of the query processing.
bool tsRetrieveBlockingModel = 0; bool tsRetrieveBlockingModel = false;
// last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name // last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name
bool tsKeepOriginalColumnName = 0; bool tsKeepOriginalColumnName = false;
// kill long query // kill long query
bool tsDeadLockKillQuery = 0; bool tsDeadLockKillQuery = false;
// tsdb config // tsdb config
// For backward compatibility // For backward compatibility
@ -283,7 +303,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1; if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1;
if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1; if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1;
if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1; if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1;
if (cfgAddFloat(pCfg, "numOfThreadsPerCore", tsNumOfThreadsPerCore, 0, 10, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxTmrCtrl", tsMaxTmrCtrl, 8, 2048, 1) != 0) return -1; if (cfgAddInt32(pCfg, "maxTmrCtrl", tsMaxTmrCtrl, 8, 2048, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcTimer", tsRpcTimer, 100, 3000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "rpcTimer", tsRpcTimer, 100, 3000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcMaxTime", tsRpcMaxTime, 100, 7200, 1) != 0) return -1; if (cfgAddInt32(pCfg, "rpcMaxTime", tsRpcMaxTime, 100, 7200, 1) != 0) return -1;
@ -297,6 +316,11 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
return -1; return -1;
if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1; if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1) != 0) return -1; if (cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1) != 0) return -1;
tsNumOfTaskQueueThreads = tsNumOfCores / 4;
tsNumOfTaskQueueThreads = TRANGE(tsNumOfTaskQueueThreads, 1, 2);
if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 1, 1024, 0) != 0) return -1;
return 0; return 0;
} }
@ -329,8 +353,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536, 0) != 0) return -1; if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536, 0) != 0) return -1;
if (cfgAddDir(pCfg, "dataDir", tsDataDir, 0) != 0) return -1; if (cfgAddDir(pCfg, "dataDir", tsDataDir, 0) != 0) return -1;
if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0) != 0) return -1; if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 100, 0) != 0) return -1;
if (cfgAddFloat(pCfg, "ratioOfQueryCores", tsRatioOfQueryCores, 0, 2, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1;
if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelemetryReporting, 0) != 0) return -1; if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelemetryReporting, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "maxConnections", tsMaxConnections, 1, 100000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxConnections", tsMaxConnections, 1, 100000, 0) != 0) return -1;
@ -347,7 +369,65 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1; if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1;
if (cfgAddBool(pCfg, "slaveQuery", tsEnableSlaveQuery, 0) != 0) return -1; if (cfgAddBool(pCfg, "slaveQuery", tsEnableSlaveQuery, 0) != 0) return -1;
if (cfgAddBool(pCfg, "deadLockKillQuery", tsDeadLockKillQuery, 0) != 0) return -1; if (cfgAddBool(pCfg, "deadLockKillQuery", tsDeadLockKillQuery, 0) != 0) return -1;
if (cfgAddBool(pCfg, "multiProcess", tsMultiProcess, 0) != 0) return -1; if (cfgAddBool(pCfg, "multiProcess", tsMultiProcess, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "mnodeShmSize", tsMnodeShmSize, 4096, INT32_MAX, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "vnodeShmSize", tsVnodeShmSize, 4096, INT32_MAX, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "qnodeShmSize", tsQnodeShmSize, 4096, INT32_MAX, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "snodeShmSize", tsSnodeShmSize, 4096, INT32_MAX, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "bnodeShmSize", tsBnodeShmSize, 4096, INT32_MAX, 0) != 0) return -1;
tsNumOfRpcThreads = tsNumOfCores / 2;
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, 4);
if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, 0) != 0) return -1;
tsNumOfCommitThreads = tsNumOfCores / 2;
tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4);
if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, 0) != 0) return -1;
tsNumOfMnodeQueryThreads = tsNumOfCores / 8;
tsNumOfMnodeQueryThreads = TRANGE(tsNumOfMnodeQueryThreads, 1, 4);
if (cfgAddInt32(pCfg, "numOfMnodeQueryThreads", tsNumOfMnodeQueryThreads, 1, 1024, 0) != 0) return -1;
tsNumOfMnodeReadThreads = tsNumOfCores / 8;
tsNumOfMnodeReadThreads = TRANGE(tsNumOfMnodeReadThreads, 1, 4);
if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, 0) != 0) return -1;
tsNumOfVnodeQueryThreads = tsNumOfCores / 2;
tsNumOfVnodeQueryThreads = TMIN(tsNumOfVnodeQueryThreads, 1);
if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, 0) != 0) return -1;
tsNumOfVnodeFetchThreads = tsNumOfCores / 2;
tsNumOfVnodeFetchThreads = TRANGE(tsNumOfVnodeFetchThreads, 2, 4);
if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 1, 1024, 0) != 0) return -1;
tsNumOfVnodeWriteThreads = tsNumOfCores;
tsNumOfVnodeWriteThreads = TMIN(tsNumOfVnodeWriteThreads, 1);
if (cfgAddInt32(pCfg, "numOfVnodeWriteThreads", tsNumOfVnodeWriteThreads, 1, 1024, 0) != 0) return -1;
tsNumOfVnodeSyncThreads = tsNumOfCores / 2;
tsNumOfVnodeSyncThreads = TMIN(tsNumOfVnodeSyncThreads, 1);
if (cfgAddInt32(pCfg, "numOfVnodeSyncThreads", tsNumOfVnodeSyncThreads, 1, 1024, 0) != 0) return -1;
tsNumOfVnodeMergeThreads = tsNumOfCores / 8;
tsNumOfVnodeMergeThreads = TRANGE(tsNumOfVnodeMergeThreads, 1, 1);
if (cfgAddInt32(pCfg, "numOfVnodeMergeThreads", tsNumOfVnodeMergeThreads, 1, 1024, 0) != 0) return -1;
tsNumOfQnodeQueryThreads = tsNumOfCores / 2;
tsNumOfQnodeQueryThreads = TMIN(tsNumOfQnodeQueryThreads, 1);
if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, 0) != 0) return -1;
tsNumOfQnodeFetchThreads = tsNumOfCores / 2;
tsNumOfQnodeFetchThreads = TRANGE(tsNumOfQnodeFetchThreads, 2, 4);
if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1;
tsNumOfSnodeSharedThreads = tsNumOfCores / 4;
tsNumOfSnodeSharedThreads = TRANGE(tsNumOfSnodeSharedThreads, 2, 4);
if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeSharedThreads, 1, 1024, 0) != 0) return -1;
tsNumOfSnodeUniqueThreads = tsNumOfCores / 4;
tsNumOfSnodeUniqueThreads = TRANGE(tsNumOfSnodeUniqueThreads, 2, 4);
if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeUniqueThreads, 1, 1024, 0) != 0) return -1;
if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 360000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 360000, 0) != 0) return -1;
@ -411,7 +491,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
return -1; return -1;
} }
tsNumOfThreadsPerCore = cfgGetItem(pCfg, "numOfThreadsPerCore")->fval;
tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32; tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32;
tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32; tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32;
tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32; tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32;
@ -424,7 +503,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32; tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32;
tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32; tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32;
tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32;
return 0; return 0;
} }
@ -448,8 +527,6 @@ static void taosSetSystemCfg(SConfig *pCfg) {
static int32_t taosSetServerCfg(SConfig *pCfg) { static int32_t taosSetServerCfg(SConfig *pCfg) {
tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval;
tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32;
tsRatioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval;
tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32;
tsEnableTelemetryReporting = cfgGetItem(pCfg, "telemetryReporting")->bval; tsEnableTelemetryReporting = cfgGetItem(pCfg, "telemetryReporting")->bval;
tsMaxConnections = cfgGetItem(pCfg, "maxConnections")->i32; tsMaxConnections = cfgGetItem(pCfg, "maxConnections")->i32;
@ -466,7 +543,27 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval; tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval;
tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval; tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval;
tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->bval; tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->bval;
tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->bval; tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->bval;
tsMnodeShmSize = cfgGetItem(pCfg, "mnodeShmSize")->i32;
tsVnodeShmSize = cfgGetItem(pCfg, "vnodeShmSize")->i32;
tsQnodeShmSize = cfgGetItem(pCfg, "qnodeShmSize")->i32;
tsSnodeShmSize = cfgGetItem(pCfg, "snodeShmSize")->i32;
tsBnodeShmSize = cfgGetItem(pCfg, "bnodeShmSize")->i32;
tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32;
tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32;
tsNumOfMnodeQueryThreads = cfgGetItem(pCfg, "numOfMnodeQueryThreads")->i32;
tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32;
tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32;
tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32;
tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32;
tsNumOfVnodeMergeThreads = cfgGetItem(pCfg, "numOfVnodeMergeThreads")->i32;
tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32;
tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32;
tsNumOfSnodeUniqueThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32;

11
source/common/src/tvariant.c
View File

@ -512,6 +512,16 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result
setNull((char *)result, type, tDataTypes[type].bytes); setNull((char *)result, type, tDataTypes[type].bytes);
return 0; return 0;
} }
if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || (pVariant->nType == TSDB_DATA_TYPE_BOOL)) {
*result = pVariant->i;
} else if (IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) {
*result = pVariant->u;
} else if (IS_FLOAT_TYPE(pVariant->nType)) {
*result = (int64_t) pVariant->d;
} else {
//TODO: handling var types
}
#if 0 #if 0
errno = 0; errno = 0;
if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || (pVariant->nType == TSDB_DATA_TYPE_BOOL)) { if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || (pVariant->nType == TSDB_DATA_TYPE_BOOL)) {
@ -1038,4 +1048,3 @@ char * taosVariantGet(SVariant *pVar, int32_t type) {
return NULL; return NULL;
} }

4
source/dnode/mgmt/dm/dmHandle.c
View File

@ -118,7 +118,7 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
} }
static int32_t dmProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { static int32_t dmProcessCreateNodeMsg(SDnode *pDnode, EDndType ntype, SNodeMsg *pMsg) {
SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype);
if (pWrapper != NULL) { if (pWrapper != NULL) {
dndReleaseWrapper(pWrapper); dndReleaseWrapper(pWrapper);
@ -146,7 +146,7 @@ static int32_t dmProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg
return code; return code;
} }
static int32_t dmProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { static int32_t dmProcessDropNodeMsg(SDnode *pDnode, EDndType ntype, SNodeMsg *pMsg) {
SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype);
if (pWrapper == NULL) { if (pWrapper == NULL) {
terrno = TSDB_CODE_NODE_NOT_DEPLOYED; terrno = TSDB_CODE_NODE_NOT_DEPLOYED;

2
source/dnode/mgmt/exe/dndMain.c
View File

@ -26,7 +26,7 @@ static struct {
char apolloUrl[PATH_MAX]; char apolloUrl[PATH_MAX];
SArray *pArgs; // SConfigPair SArray *pArgs; // SConfigPair
SDnode *pDnode; SDnode *pDnode;
ENodeType ntype; EDndType ntype;
} global = {0}; } global = {0};
static void dndStopDnode(int signum, void *info, void *ctx) { static void dndStopDnode(int signum, void *info, void *ctx) {

12
source/dnode/mgmt/inc/dndInt.h
View File

@ -49,7 +49,7 @@ extern "C" {
#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }}
#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }}
typedef enum { DNODE, VNODES, QNODE, SNODE, MNODE, BNODE, NODE_MAX } ENodeType; typedef enum { DNODE, VNODES, QNODE, SNODE, MNODE, BNODE, NODE_MAX } EDndType;
typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EDndStatus; typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EDndStatus;
typedef enum { DND_ENV_INIT, DND_ENV_READY, DND_ENV_CLEANUP } EEnvStatus; typedef enum { DND_ENV_INIT, DND_ENV_READY, DND_ENV_CLEANUP } EEnvStatus;
typedef enum { PROC_SINGLE, PROC_CHILD, PROC_PARENT } EProcType; typedef enum { PROC_SINGLE, PROC_CHILD, PROC_PARENT } EProcType;
@ -92,7 +92,7 @@ typedef struct SMgmtWrapper {
char *path; char *path;
int32_t refCount; int32_t refCount;
SRWLatch latch; SRWLatch latch;
ENodeType ntype; EDndType ntype;
bool deployed; bool deployed;
bool required; bool required;
EProcType procType; EProcType procType;
@ -126,7 +126,7 @@ typedef struct SDnode {
int32_t numOfDisks; int32_t numOfDisks;
uint16_t serverPort; uint16_t serverPort;
bool dropped; bool dropped;
ENodeType ntype; EDndType ntype;
EDndStatus status; EDndStatus status;
EDndEvent event; EDndEvent event;
SStartupReq startup; SStartupReq startup;
@ -137,8 +137,8 @@ typedef struct SDnode {
// dndEnv.c // dndEnv.c
const char *dndStatStr(EDndStatus stat); const char *dndStatStr(EDndStatus stat);
const char *dndNodeLogStr(ENodeType ntype); const char *dndNodeLogStr(EDndType ntype);
const char *dndNodeProcStr(ENodeType ntype); const char *dndNodeProcStr(EDndType ntype);
const char *dndEventStr(EDndEvent ev); const char *dndEventStr(EDndEvent ev);
// dndExec.c // dndExec.c
@ -156,7 +156,7 @@ int32_t dndWriteShmFile(SDnode *pDnode);
EDndStatus dndGetStatus(SDnode *pDnode); EDndStatus dndGetStatus(SDnode *pDnode);
void dndSetStatus(SDnode *pDnode, EDndStatus stat); void dndSetStatus(SDnode *pDnode, EDndStatus stat);
void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId); void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId);
SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, EDndType nType);
int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); int32_t dndMarkWrapper(SMgmtWrapper *pWrapper);
void dndReleaseWrapper(SMgmtWrapper *pWrapper); void dndReleaseWrapper(SMgmtWrapper *pWrapper);
void dndHandleEvent(SDnode *pDnode, EDndEvent event); void dndHandleEvent(SDnode *pDnode, EDndEvent event);

4
source/dnode/mgmt/main/dndEnv.c
View File

@ -71,7 +71,7 @@ const char *dndStatStr(EDndStatus status) {
} }
} }
const char *dndNodeLogStr(ENodeType ntype) { const char *dndNodeLogStr(EDndType ntype) {
switch (ntype) { switch (ntype) {
case VNODES: case VNODES:
return "vnode"; return "vnode";
@ -88,7 +88,7 @@ const char *dndNodeLogStr(ENodeType ntype) {
} }
} }
const char *dndNodeProcStr(ENodeType ntype) { const char *dndNodeProcStr(EDndType ntype) {
switch (ntype) { switch (ntype) {
case VNODES: case VNODES:
return "taosv"; return "taosv";

32
source/dnode/mgmt/main/dndExec.c
View File

@ -66,7 +66,7 @@ void dndCloseNode(SMgmtWrapper *pWrapper) {
} }
static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) { static int32_t dndNewProc(SMgmtWrapper *pWrapper, EDndType n) {
char tstr[8] = {0}; char tstr[8] = {0};
char *args[6] = {0}; char *args[6] = {0};
snprintf(tstr, sizeof(tstr), "%d", n); snprintf(tstr, sizeof(tstr), "%d", n);
@ -89,6 +89,7 @@ static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) {
} }
static void dndProcessProcHandle(void *handle) { static void dndProcessProcHandle(void *handle) {
dWarn("handle:%p, the child process dies and send an offline rsp", handle);
SRpcMsg rpcMsg = {.handle = handle, .code = TSDB_CODE_DND_OFFLINE}; SRpcMsg rpcMsg = {.handle = handle, .code = TSDB_CODE_DND_OFFLINE};
rpcSendResponse(&rpcMsg); rpcSendResponse(&rpcMsg);
} }
@ -96,7 +97,7 @@ static void dndProcessProcHandle(void *handle) {
static int32_t dndRunInSingleProcess(SDnode *pDnode) { static int32_t dndRunInSingleProcess(SDnode *pDnode) {
dInfo("dnode run in single process"); dInfo("dnode run in single process");
for (ENodeType n = DNODE; n < NODE_MAX; ++n) { for (EDndType n = DNODE; n < NODE_MAX; ++n) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
pWrapper->required = dndRequireNode(pWrapper); pWrapper->required = dndRequireNode(pWrapper);
if (!pWrapper->required) continue; if (!pWrapper->required) continue;
@ -109,7 +110,7 @@ static int32_t dndRunInSingleProcess(SDnode *pDnode) {
dndSetStatus(pDnode, DND_STAT_RUNNING); dndSetStatus(pDnode, DND_STAT_RUNNING);
for (ENodeType n = 0; n < NODE_MAX; ++n) { for (EDndType n = 0; n < NODE_MAX; ++n) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
if (!pWrapper->required) continue; if (!pWrapper->required) continue;
if (pWrapper->fp.startFp == NULL) continue; if (pWrapper->fp.startFp == NULL) continue;
@ -141,12 +142,25 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
return -1; return -1;
} }
for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) { for (EDndType n = DNODE + 1; n < NODE_MAX; ++n) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
pWrapper->required = dndRequireNode(pWrapper); pWrapper->required = dndRequireNode(pWrapper);
if (!pWrapper->required) continue; if (!pWrapper->required) continue;
int32_t shmsize = 1024 * 1024 * 2; // size will be a configuration item int32_t shmsize = tsMnodeShmSize;
if (n == VNODES) {
shmsize = tsVnodeShmSize;
} else if (n == QNODE) {
shmsize = tsQnodeShmSize;
} else if (n == SNODE) {
shmsize = tsSnodeShmSize;
} else if (n == MNODE) {
shmsize = tsMnodeShmSize;
} else if (n == BNODE) {
shmsize = tsBnodeShmSize;
} else {
}
if (taosCreateShm(&pWrapper->shm, n, shmsize) != 0) { if (taosCreateShm(&pWrapper->shm, n, shmsize) != 0) {
terrno = TAOS_SYSTEM_ERROR(terrno); terrno = TAOS_SYSTEM_ERROR(terrno);
dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr()); dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr());
@ -169,7 +183,7 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
return -1; return -1;
} }
for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) { for (EDndType n = DNODE + 1; n < NODE_MAX; ++n) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
if (!pWrapper->required) continue; if (!pWrapper->required) continue;
@ -202,7 +216,7 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
dInfo("dnode is about to stop"); dInfo("dnode is about to stop");
dndSetStatus(pDnode, DND_STAT_STOPPED); dndSetStatus(pDnode, DND_STAT_STOPPED);
for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) { for (EDndType n = DNODE + 1; n < NODE_MAX; ++n) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
if (!pWrapper->required) continue; if (!pWrapper->required) continue;
if (pDnode->ntype == NODE_MAX) continue; if (pDnode->ntype == NODE_MAX) continue;
@ -217,13 +231,13 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
} }
break; break;
} else { } else {
for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) { for (EDndType n = DNODE + 1; n < NODE_MAX; ++n) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
if (!pWrapper->required) continue; if (!pWrapper->required) continue;
if (pDnode->ntype == NODE_MAX) continue; if (pDnode->ntype == NODE_MAX) continue;
if (pWrapper->procId <= 0 || !taosProcExist(pWrapper->procId)) { if (pWrapper->procId <= 0 || !taosProcExist(pWrapper->procId)) {
dInfo("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId); dWarn("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId);
taosProcCloseHandles(pWrapper->pProc, dndProcessProcHandle); taosProcCloseHandles(pWrapper->pProc, dndProcessProcHandle);
dndNewProc(pWrapper, n); dndNewProc(pWrapper, n);
} }

6
source/dnode/mgmt/main/dndFile.c
View File

@ -164,7 +164,7 @@ int32_t dndReadShmFile(SDnode *pDnode) {
goto _OVER; goto _OVER;
} }
for (ENodeType ntype = DNODE + 1; ntype < NODE_MAX; ++ntype) { for (EDndType ntype = DNODE + 1; ntype < NODE_MAX; ++ntype) {
snprintf(itemName, sizeof(itemName), "%s_shmid", dndNodeProcStr(ntype)); snprintf(itemName, sizeof(itemName), "%s_shmid", dndNodeProcStr(ntype));
cJSON *shmid = cJSON_GetObjectItem(root, itemName); cJSON *shmid = cJSON_GetObjectItem(root, itemName);
if (shmid && shmid->type == cJSON_Number) { if (shmid && shmid->type == cJSON_Number) {
@ -180,7 +180,7 @@ int32_t dndReadShmFile(SDnode *pDnode) {
} }
if (!tsMultiProcess || pDnode->ntype == DNODE || pDnode->ntype == NODE_MAX) { if (!tsMultiProcess || pDnode->ntype == DNODE || pDnode->ntype == NODE_MAX) {
for (ENodeType ntype = DNODE; ntype < NODE_MAX; ++ntype) { for (EDndType ntype = DNODE; ntype < NODE_MAX; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
if (pWrapper->shm.id >= 0) { if (pWrapper->shm.id >= 0) {
dDebug("shmid:%d, is closed, size:%d", pWrapper->shm.id, pWrapper->shm.size); dDebug("shmid:%d, is closed, size:%d", pWrapper->shm.id, pWrapper->shm.size);
@ -226,7 +226,7 @@ int32_t dndWriteShmFile(SDnode *pDnode) {
} }
len += snprintf(content + len, MAXLEN - len, "{\n"); len += snprintf(content + len, MAXLEN - len, "{\n");
for (ENodeType ntype = DNODE + 1; ntype < NODE_MAX; ++ntype) { for (EDndType ntype = DNODE + 1; ntype < NODE_MAX; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
len += snprintf(content + len, MAXLEN - len, " \"%s_shmid\":%d,\n", dndNodeProcStr(ntype), pWrapper->shm.id); len += snprintf(content + len, MAXLEN - len, " \"%s_shmid\":%d,\n", dndNodeProcStr(ntype), pWrapper->shm.id);
if (ntype == NODE_MAX - 1) { if (ntype == NODE_MAX - 1) {

10
source/dnode/mgmt/main/dndInt.c
View File

@ -46,7 +46,7 @@ static int32_t dndInitVars(SDnode *pDnode, const SDnodeOpt *pOption) {
} }
static void dndClearVars(SDnode *pDnode) { static void dndClearVars(SDnode *pDnode) {
for (ENodeType n = 0; n < NODE_MAX; ++n) { for (EDndType n = 0; n < NODE_MAX; ++n) {
SMgmtWrapper *pMgmt = &pDnode->wrappers[n]; SMgmtWrapper *pMgmt = &pDnode->wrappers[n];
taosMemoryFreeClear(pMgmt->path); taosMemoryFreeClear(pMgmt->path);
} }
@ -89,7 +89,7 @@ SDnode *dndCreate(const SDnodeOpt *pOption) {
smSetMgmtFp(&pDnode->wrappers[SNODE]); smSetMgmtFp(&pDnode->wrappers[SNODE]);
bmSetMgmtFp(&pDnode->wrappers[BNODE]); bmSetMgmtFp(&pDnode->wrappers[BNODE]);
for (ENodeType n = 0; n < NODE_MAX; ++n) { for (EDndType n = 0; n < NODE_MAX; ++n) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
snprintf(path, sizeof(path), "%s%s%s", pDnode->dataDir, TD_DIRSEP, pWrapper->name); snprintf(path, sizeof(path), "%s%s%s", pDnode->dataDir, TD_DIRSEP, pWrapper->name);
pWrapper->path = strdup(path); pWrapper->path = strdup(path);
@ -106,7 +106,7 @@ SDnode *dndCreate(const SDnodeOpt *pOption) {
} }
if (dndInitMsgHandle(pDnode) != 0) { if (dndInitMsgHandle(pDnode) != 0) {
dError("failed to msg handles since %s", terrstr()); dError("failed to init msg handles since %s", terrstr());
goto _OVER; goto _OVER;
} }
@ -134,7 +134,7 @@ _OVER:
void dndClose(SDnode *pDnode) { void dndClose(SDnode *pDnode) {
if (pDnode == NULL) return; if (pDnode == NULL) return;
for (ENodeType n = 0; n < NODE_MAX; ++n) { for (EDndType n = 0; n < NODE_MAX; ++n) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
dndCloseNode(pWrapper); dndCloseNode(pWrapper);
} }
@ -149,7 +149,7 @@ void dndHandleEvent(SDnode *pDnode, EDndEvent event) {
} }
} }
SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType ntype) { SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, EDndType ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
SMgmtWrapper *pRetWrapper = pWrapper; SMgmtWrapper *pRetWrapper = pWrapper;

9
source/dnode/mgmt/main/dndTransport.c
View File

@ -257,16 +257,11 @@ static int32_t dndRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *spi, ch
static int32_t dndInitServer(SDnode *pDnode) { static int32_t dndInitServer(SDnode *pDnode) {
STransMgmt *pMgmt = &pDnode->trans; STransMgmt *pMgmt = &pDnode->trans;
int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0);
if (numOfThreads < 1) {
numOfThreads = 1;
}
SRpcInit rpcInit; SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit)); memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localPort = pDnode->serverPort; rpcInit.localPort = pDnode->serverPort;
rpcInit.label = "DND"; rpcInit.label = "DND";
rpcInit.numOfThreads = numOfThreads; rpcInit.numOfThreads = tsNumOfRpcThreads;
rpcInit.cfp = (RpcCfp)dndProcessMsg; rpcInit.cfp = (RpcCfp)dndProcessMsg;
rpcInit.sessions = tsMaxShellConns; rpcInit.sessions = tsMaxShellConns;
rpcInit.connType = TAOS_CONN_SERVER; rpcInit.connType = TAOS_CONN_SERVER;
@ -307,7 +302,7 @@ void dndCleanupTrans(SDnode *pDnode) {
int32_t dndInitMsgHandle(SDnode *pDnode) { int32_t dndInitMsgHandle(SDnode *pDnode) {
STransMgmt *pMgmt = &pDnode->trans; STransMgmt *pMgmt = &pDnode->trans;
for (ENodeType n = 0; n < NODE_MAX; ++n) { for (EDndType n = 0; n < NODE_MAX; ++n) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) { for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) {

1
source/dnode/mgmt/mm/mmHandle.c
View File

@ -150,6 +150,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
// Requests handled by VNODE // Requests handled by VNODE
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);

16
source/dnode/mgmt/mm/mmWorker.c
View File

@ -129,25 +129,33 @@ int32_t mmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
} }
int32_t mmStartWorker(SMnodeMgmt *pMgmt) { int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
SSingleWorkerCfg qCfg = {.min = 0, .max = 1, .name = "mnode-query", .fp = (FItem)mmProcessQueryQueue, .param = pMgmt}; SSingleWorkerCfg qCfg = {.min = tsNumOfMnodeQueryThreads,
.max = tsNumOfMnodeQueryThreads,
.name = "mnode-query",
.fp = (FItem)mmProcessQueryQueue,
.param = pMgmt};
if (tSingleWorkerInit(&pMgmt->queryWorker, &qCfg) != 0) { if (tSingleWorkerInit(&pMgmt->queryWorker, &qCfg) != 0) {
dError("failed to start mnode-query worker since %s", terrstr()); dError("failed to start mnode-query worker since %s", terrstr());
return -1; return -1;
} }
SSingleWorkerCfg rCfg = {.min = 0, .max = 1, .name = "mnode-read", .fp = (FItem)mmProcessQueue, .param = pMgmt}; SSingleWorkerCfg rCfg = {.min = tsNumOfMnodeReadThreads,
.max = tsNumOfMnodeReadThreads,
.name = "mnode-read",
.fp = (FItem)mmProcessQueue,
.param = pMgmt};
if (tSingleWorkerInit(&pMgmt->readWorker, &rCfg) != 0) { if (tSingleWorkerInit(&pMgmt->readWorker, &rCfg) != 0) {
dError("failed to start mnode-read worker since %s", terrstr()); dError("failed to start mnode-read worker since %s", terrstr());
return -1; return -1;
} }
SSingleWorkerCfg wCfg = {.min = 0, .max = 1, .name = "mnode-write", .fp = (FItem)mmProcessQueue, .param = pMgmt}; SSingleWorkerCfg wCfg = {.min = 1, .max = 1, .name = "mnode-write", .fp = (FItem)mmProcessQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->writeWorker, &wCfg) != 0) { if (tSingleWorkerInit(&pMgmt->writeWorker, &wCfg) != 0) {
dError("failed to start mnode-write worker since %s", terrstr()); dError("failed to start mnode-write worker since %s", terrstr());
return -1; return -1;
} }
SSingleWorkerCfg sCfg = {.min = 0, .max = 1, .name = "mnode-sync", .fp = (FItem)mmProcessQueue, .param = pMgmt}; SSingleWorkerCfg sCfg = {.min = 1, .max = 1, .name = "mnode-sync", .fp = (FItem)mmProcessQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->syncWorker, &sCfg) != 0) { if (tSingleWorkerInit(&pMgmt->syncWorker, &sCfg) != 0) {
dError("failed to start mnode sync-worker since %s", terrstr()); dError("failed to start mnode sync-worker since %s", terrstr());
return -1; return -1;

13
source/dnode/mgmt/qm/qmWorker.c
View File

@ -106,13 +106,8 @@ int32_t qmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) {
} }
int32_t qmStartWorker(SQnodeMgmt *pMgmt) { int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
int32_t maxFetchThreads = 4; SSingleWorkerCfg queryCfg = {.min = tsNumOfVnodeQueryThreads,
int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores); .max = tsNumOfVnodeQueryThreads,
int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1);
int32_t maxQueryThreads = minQueryThreads;
SSingleWorkerCfg queryCfg = {.min = minQueryThreads,
.max = maxQueryThreads,
.name = "qnode-query", .name = "qnode-query",
.fp = (FItem)qmProcessQueryQueue, .fp = (FItem)qmProcessQueryQueue,
.param = pMgmt}; .param = pMgmt};
@ -122,8 +117,8 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
return -1; return -1;
} }
SSingleWorkerCfg fetchCfg = {.min = minFetchThreads, SSingleWorkerCfg fetchCfg = {.min = tsNumOfQnodeFetchThreads,
.max = maxFetchThreads, .max = tsNumOfQnodeFetchThreads,
.name = "qnode-fetch", .name = "qnode-fetch",
.fp = (FItem)qmProcessFetchQueue, .fp = (FItem)qmProcessFetchQueue,
.param = pMgmt}; .param = pMgmt};

8
source/dnode/mgmt/sm/smWorker.c
View File

@ -50,7 +50,7 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
return -1; return -1;
} }
for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) { for (int32_t i = 0; i < tsNumOfSnodeUniqueThreads; i++) {
SMultiWorker *pUniqueWorker = taosMemoryMalloc(sizeof(SMultiWorker)); SMultiWorker *pUniqueWorker = taosMemoryMalloc(sizeof(SMultiWorker));
if (pUniqueWorker == NULL) { if (pUniqueWorker == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -69,8 +69,8 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
} }
} }
SSingleWorkerCfg cfg = {.min = SND_SHARED_THREAD_NUM, SSingleWorkerCfg cfg = {.min = tsNumOfSnodeSharedThreads,
.max = SND_SHARED_THREAD_NUM, .max = tsNumOfSnodeSharedThreads,
.name = "snode-shared", .name = "snode-shared",
.fp = (FItem)smProcessSharedQueue, .fp = (FItem)smProcessSharedQueue,
.param = pMgmt}; .param = pMgmt};
@ -97,7 +97,7 @@ void smStopWorker(SSnodeMgmt *pMgmt) {
static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) { static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) {
SMsgHead *pHead = pMsg->pCont; SMsgHead *pHead = pMsg->pCont;
pHead->vgId = htonl(pHead->vgId); pHead->vgId = htonl(pHead->vgId);
return pHead->vgId % SND_UNIQUE_THREAD_NUM; return pHead->vgId % tsNumOfSnodeUniqueThreads;
} }
static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) { static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) {

2
source/dnode/mgmt/test/sut/inc/sut.h
View File

@ -40,8 +40,6 @@ class Testbase {
void ServerStart(); void ServerStart();
void ClientRestart(); void ClientRestart();
SRpcMsg* SendReq(tmsg_t msgType, void* pCont, int32_t contLen); SRpcMsg* SendReq(tmsg_t msgType, void* pCont, int32_t contLen);
private:
void InitLog(const char* path); void InitLog(const char* path);
private: private:

1
source/dnode/mgmt/vm/vmHandle.c
View File

@ -271,6 +271,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);

22
source/dnode/mgmt/vm/vmWorker.c
View File

@ -379,39 +379,31 @@ void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) {
} }
int32_t vmStartWorker(SVnodesMgmt *pMgmt) { int32_t vmStartWorker(SVnodesMgmt *pMgmt) {
int32_t maxFetchThreads = 4;
int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores);
int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1);
int32_t maxQueryThreads = minQueryThreads;
int32_t maxWriteThreads = TMAX(tsNumOfCores, 1);
int32_t maxSyncThreads = TMAX(tsNumOfCores / 2, 1);
int32_t maxMergeThreads = 1;
SQWorkerPool *pQPool = &pMgmt->queryPool; SQWorkerPool *pQPool = &pMgmt->queryPool;
pQPool->name = "vnode-query"; pQPool->name = "vnode-query";
pQPool->min = minQueryThreads; pQPool->min = tsNumOfVnodeQueryThreads;
pQPool->max = maxQueryThreads; pQPool->max = tsNumOfVnodeQueryThreads;
if (tQWorkerInit(pQPool) != 0) return -1; if (tQWorkerInit(pQPool) != 0) return -1;
SQWorkerPool *pFPool = &pMgmt->fetchPool; SQWorkerPool *pFPool = &pMgmt->fetchPool;
pFPool->name = "vnode-fetch"; pFPool->name = "vnode-fetch";
pFPool->min = minFetchThreads; pFPool->min = tsNumOfVnodeFetchThreads;
pFPool->max = maxFetchThreads; pFPool->max = tsNumOfVnodeFetchThreads;
if (tQWorkerInit(pFPool) != 0) return -1; if (tQWorkerInit(pFPool) != 0) return -1;
SWWorkerPool *pWPool = &pMgmt->writePool; SWWorkerPool *pWPool = &pMgmt->writePool;
pWPool->name = "vnode-write"; pWPool->name = "vnode-write";
pWPool->max = maxWriteThreads; pWPool->max = tsNumOfVnodeWriteThreads;
if (tWWorkerInit(pWPool) != 0) return -1; if (tWWorkerInit(pWPool) != 0) return -1;
pWPool = &pMgmt->syncPool; pWPool = &pMgmt->syncPool;
pWPool->name = "vnode-sync"; pWPool->name = "vnode-sync";
pWPool->max = maxSyncThreads; pWPool->max = tsNumOfVnodeSyncThreads;
if (tWWorkerInit(pWPool) != 0) return -1; if (tWWorkerInit(pWPool) != 0) return -1;
pWPool = &pMgmt->mergePool; pWPool = &pMgmt->mergePool;
pWPool->name = "vnode-merge"; pWPool->name = "vnode-merge";
pWPool->max = maxMergeThreads; pWPool->max = tsNumOfVnodeMergeThreads;
if (tWWorkerInit(pWPool) != 0) return -1; if (tWWorkerInit(pWPool) != 0) return -1;
SSingleWorkerCfg cfg = {.min = 1, .max = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt}; SSingleWorkerCfg cfg = {.min = 1, .max = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt};

3
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -413,6 +413,7 @@ typedef struct {
typedef struct { typedef struct {
int32_t vgId; // -1 for unassigned int32_t vgId; // -1 for unassigned
int32_t status; int32_t status;
int32_t epoch;
SEpSet epSet; SEpSet epSet;
int64_t oldConsumerId; int64_t oldConsumerId;
int64_t consumerId; // -1 for unassigned int64_t consumerId; // -1 for unassigned
@ -423,6 +424,7 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, const SMqConsumerEp
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeFixedI32(buf, pConsumerEp->vgId); tlen += taosEncodeFixedI32(buf, pConsumerEp->vgId);
tlen += taosEncodeFixedI32(buf, pConsumerEp->status); tlen += taosEncodeFixedI32(buf, pConsumerEp->status);
tlen += taosEncodeFixedI32(buf, pConsumerEp->epoch);
tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet); tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet);
tlen += taosEncodeFixedI64(buf, pConsumerEp->oldConsumerId); tlen += taosEncodeFixedI64(buf, pConsumerEp->oldConsumerId);
tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
@ -433,6 +435,7 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, const SMqConsumerEp
static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) {
buf = taosDecodeFixedI32(buf, &pConsumerEp->vgId); buf = taosDecodeFixedI32(buf, &pConsumerEp->vgId);
buf = taosDecodeFixedI32(buf, &pConsumerEp->status); buf = taosDecodeFixedI32(buf, &pConsumerEp->status);
buf = taosDecodeFixedI32(buf, &pConsumerEp->epoch);
buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet); buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet);
buf = taosDecodeFixedI64(buf, &pConsumerEp->oldConsumerId); buf = taosDecodeFixedI64(buf, &pConsumerEp->oldConsumerId);
buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);

27
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -61,6 +61,7 @@ static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqT
const SMqConsumerEp *pConsumerEp); const SMqConsumerEp *pConsumerEp);
static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp); static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp);
static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, const char* oldTopicName);
int32_t mndInitSubscribe(SMnode *pMnode) { int32_t mndInitSubscribe(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_SUBSCRIBE, SSdbTable table = {.sdbType = SDB_SUBSCRIBE,
@ -74,6 +75,7 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp); mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp);
mndSetMsgHandle(pMnode, TDMT_VND_MQ_REB_RSP, mndProcessSubscribeInternalRsp); mndSetMsgHandle(pMnode, TDMT_VND_MQ_REB_RSP, mndProcessSubscribeInternalRsp);
mndSetMsgHandle(pMnode, TDMT_VND_MQ_CANCEL_CONN_RSP, mndProcessSubscribeInternalRsp);
mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq);
mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessDoRebalanceMsg); mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessDoRebalanceMsg);
@ -154,11 +156,14 @@ static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqC
return 0; return 0;
} }
static int32_t mndBuildCancelConnReq(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp) { static int32_t mndBuildCancelConnReq(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp, const char* oldTopicName) {
SMqSetCVgReq req = {0}; SMqCancelConnReq req = {0};
req.consumerId = pConsumerEp->consumerId; req.consumerId = pConsumerEp->consumerId;
req.vgId = pConsumerEp->vgId;
req.epoch = pConsumerEp->epoch;
strcpy(req.topicName, oldTopicName);
int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); int32_t tlen = tEncodeSMqCancelConnReq(NULL, &req);
void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen);
if (buf == NULL) { if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -169,16 +174,16 @@ static int32_t mndBuildCancelConnReq(void **pBuf, int32_t *pLen, const SMqConsum
pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen); pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen);
pMsgHead->vgId = htonl(pConsumerEp->vgId); pMsgHead->vgId = htonl(pConsumerEp->vgId);
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tEncodeSMqSetCVgReq(&abuf, &req); tEncodeSMqCancelConnReq(&abuf, &req);
*pBuf = buf; *pBuf = buf;
*pLen = tlen; *pLen = tlen;
return 0; return 0;
} }
static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp) { static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, const char* oldTopicName) {
void *buf; void *buf;
int32_t tlen; int32_t tlen;
if (mndBuildCancelConnReq(&buf, &tlen, pConsumerEp) < 0) { if (mndBuildCancelConnReq(&buf, &tlen, pConsumerEp, oldTopicName) < 0) {
return -1; return -1;
} }
@ -189,7 +194,7 @@ static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMq
action.epSet = mndGetVgroupEpset(pMnode, pVgObj); action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
action.pCont = buf; action.pCont = buf;
action.contLen = sizeof(SMsgHead) + tlen; action.contLen = sizeof(SMsgHead) + tlen;
action.msgType = TDMT_VND_MQ_SET_CONN; action.msgType = TDMT_VND_MQ_CANCEL_CONN;
mndReleaseVgroup(pMnode, pVgObj); mndReleaseVgroup(pMnode, pVgObj);
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if (mndTransAppendRedoAction(pTrans, &action) != 0) {
@ -365,7 +370,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
if (status == MQ_CONSUMER_STATUS__MODIFY) { if (status == MQ_CONSUMER_STATUS__MODIFY) {
int32_t removeSz = taosArrayGetSize(pConsumer->recentRemovedTopics); int32_t removeSz = taosArrayGetSize(pConsumer->recentRemovedTopics);
for (int32_t i = 0; i < removeSz; i++) { for (int32_t i = 0; i < removeSz; i++) {
char *topicName = taosArrayGet(pConsumer->recentRemovedTopics, i); char *topicName = taosArrayGetP(pConsumer->recentRemovedTopics, i);
taosMemoryFree(topicName); taosMemoryFree(topicName);
} }
taosArrayClear(pConsumer->recentRemovedTopics); taosArrayClear(pConsumer->recentRemovedTopics);
@ -493,6 +498,8 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
pConsumerEp->oldConsumerId = pConsumerEp->consumerId; pConsumerEp->oldConsumerId = pConsumerEp->consumerId;
pConsumerEp->consumerId = pSubConsumer->consumerId; pConsumerEp->consumerId = pSubConsumer->consumerId;
//TODO
pConsumerEp->epoch = 0;
taosArrayPush(pSubConsumer->vgInfo, pConsumerEp); taosArrayPush(pSubConsumer->vgInfo, pConsumerEp);
if (pConsumerEp->oldConsumerId == -1) { if (pConsumerEp->oldConsumerId == -1) {
@ -795,7 +802,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo); int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo);
for (int32_t vgi = 0; vgi < vgsz; vgi++) { for (int32_t vgi = 0; vgi < vgsz; vgi++) {
SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, vgi); SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, vgi);
mndPersistCancelConnReq(pMnode, pTrans, pConsumerEp); mndPersistCancelConnReq(pMnode, pTrans, pConsumerEp, oldTopicName);
taosArrayPush(pSub->unassignedVg, pConsumerEp); taosArrayPush(pSub->unassignedVg, pConsumerEp);
} }
taosArrayRemove(pSub->consumers, ci); taosArrayRemove(pSub->consumers, ci);
@ -857,7 +864,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
} }
} }
if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void *))taosMemoryFree); /*if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void *))taosMemoryFree);*/
// persist consumerObj // persist consumerObj
SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer); SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);

2
source/dnode/vnode/CMakeLists.txt
View File

@ -1,5 +1,5 @@
set(META_DB_IMPL_LIST "BDB" "TDB") set(META_DB_IMPL_LIST "BDB" "TDB")
set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation") set(META_DB_IMPL "TDB" CACHE STRING "Use BDB as the default META implementation")
set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST}) set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST})
if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST) if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST)

10
source/dnode/vnode/inc/tsdb.h
View File

@ -88,6 +88,10 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp);
int tsdbPrepareCommit(STsdb *pTsdb); int tsdbPrepareCommit(STsdb *pTsdb);
int tsdbCommit(STsdb *pTsdb); int tsdbCommit(STsdb *pTsdb);
int32_t tsdbInitSma(STsdb *pTsdb);
int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg);
int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg);
/** /**
* @brief When submit msg received, update the relative expired window synchronously. * @brief When submit msg received, update the relative expired window synchronously.
* *
@ -95,7 +99,7 @@ int tsdbCommit(STsdb *pTsdb);
* @param msg * @param msg
* @return int32_t * @return int32_t
*/ */
int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, const char *msg); int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, SSubmitReq *pMsg);
/** /**
* @brief Insert tSma(Time-range-wise SMA) data from stream computing engine * @brief Insert tSma(Time-range-wise SMA) data from stream computing engine
@ -172,9 +176,9 @@ tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo
tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef);
int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* pTableBlockInfo); int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* pReader, STableBlockDistInfo* pTableBlockInfo);
bool isTsdbCacheLastRow(tsdbReaderT* pTsdbReadHandle); bool isTsdbCacheLastRow(tsdbReaderT* pReader);
/** /**
* *

4
source/dnode/vnode/src/inc/tqInt.h
View File

@ -67,7 +67,7 @@ extern "C" {
} \ } \
} }
#define TQ_BUFFER_SIZE 8 #define TQ_BUFFER_SIZE 4
#define TQ_BUCKET_MASK 0xFF #define TQ_BUCKET_MASK 0xFF
#define TQ_BUCKET_SIZE 256 #define TQ_BUCKET_SIZE 256
@ -206,7 +206,7 @@ typedef struct {
typedef struct { typedef struct {
int64_t consumerId; int64_t consumerId;
int64_t epoch; int32_t epoch;
char cgroup[TSDB_TOPIC_FNAME_LEN]; char cgroup[TSDB_TOPIC_FNAME_LEN];
SArray* topics; // SArray<STqTopic> SArray* topics; // SArray<STqTopic>
} STqConsumer; } STqConsumer;

16
source/dnode/vnode/src/inc/tsdbDef.h
View File

@ -56,17 +56,19 @@ struct STsdb {
STsdbFS * fs; STsdbFS * fs;
SMeta * pMeta; SMeta * pMeta;
STfs * pTfs; STfs * pTfs;
SSmaEnv * pTSmaEnv; SSmaEnvs smaEnvs;
SSmaEnv * pRSmaEnv;
}; };
#define REPO_ID(r) ((r)->vgId) #define REPO_ID(r) ((r)->vgId)
#define REPO_CFG(r) (&(r)->config) #define REPO_CFG(r) (&(r)->config)
#define REPO_FS(r) (r)->fs #define REPO_FS(r) ((r)->fs)
#define REPO_META(r) (r)->pMeta #define REPO_META(r) ((r)->pMeta)
#define REPO_TFS(r) (r)->pTfs #define REPO_TFS(r) ((r)->pTfs)
#define IS_REPO_LOCKED(r) (r)->repoLocked #define IS_REPO_LOCKED(r) ((r)->repoLocked)
#define REPO_SMA_ENV(r, t) ((TSDB_SMA_TYPE_ROLLUP == (t)) ? (r)->pRSmaEnv : (r)->pTSmaEnv) #define REPO_TSMA_NUM(r) ((r)->smaEnvs.nTSma)
#define REPO_RSMA_NUM(r) ((r)->smaEnvs.nRSma)
#define REPO_TSMA_ENV(r) ((r)->smaEnvs.pTSmaEnv)
#define REPO_RSMA_ENV(r) ((r)->smaEnvs.pRSmaEnv)
int tsdbLockRepo(STsdb *pTsdb); int tsdbLockRepo(STsdb *pTsdb);
int tsdbUnlockRepo(STsdb *pTsdb); int tsdbUnlockRepo(STsdb *pTsdb);

30
source/dnode/vnode/src/inc/tsdbReadImpl.h
View File

@ -17,12 +17,12 @@
#define _TD_TSDB_READ_IMPL_H_ #define _TD_TSDB_READ_IMPL_H_
#include "os.h" #include "os.h"
#include "tcommon.h"
#include "tfs.h" #include "tfs.h"
#include "tsdb.h" #include "tsdb.h"
#include "tsdbFile.h" #include "tsdbFile.h"
#include "tskiplist.h"
#include "tsdbMemory.h" #include "tsdbMemory.h"
#include "tcommon.h" #include "tskiplist.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -31,7 +31,6 @@ extern "C" {
typedef struct SReadH SReadH; typedef struct SReadH SReadH;
typedef struct { typedef struct {
int32_t tid;
uint32_t len; uint32_t len;
uint32_t offset; uint32_t offset;
uint32_t hasLast : 2; uint32_t hasLast : 2;
@ -165,19 +164,19 @@ typedef struct {
typedef void SAggrBlkData; // SBlockCol cols[]; typedef void SAggrBlkData; // SBlockCol cols[];
struct SReadH { struct SReadH {
STsdb * pRepo; STsdb *pRepo;
SDFileSet rSet; // FSET to read SDFileSet rSet; // FSET to read
SArray * aBlkIdx; // SBlockIdx array SArray *aBlkIdx; // SBlockIdx array
STable * pTable; // table to read STable *pTable; // table to read
SBlockIdx * pBlkIdx; // current reading table SBlockIdx SBlockIdx *pBlkIdx; // current reading table SBlockIdx
int cidx; int cidx;
SBlockInfo * pBlkInfo; SBlockInfo *pBlkInfo;
SBlockData * pBlkData; // Block info SBlockData *pBlkData; // Block info
SAggrBlkData *pAggrBlkData; // Aggregate Block info SAggrBlkData *pAggrBlkData; // Aggregate Block info
SDataCols * pDCols[2]; SDataCols *pDCols[2];
void * pBuf; // buffer void *pBuf; // buffer
void * pCBuf; // compression buffer void *pCBuf; // compression buffer
void * pExBuf; // extra buffer void *pExBuf; // extra buffer
}; };
#define TSDB_READ_REPO(rh) ((rh)->pRepo) #define TSDB_READ_REPO(rh) ((rh)->pRepo)
@ -222,14 +221,15 @@ int tsdbLoadBlockIdx(SReadH *pReadh);
int tsdbSetReadTable(SReadH *pReadh, STable *pTable); int tsdbSetReadTable(SReadH *pReadh, STable *pTable);
int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget); int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget);
int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo); int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo);
int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds); int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds,
int numOfColsIds);
int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock); int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);
int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx); int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx);
void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx); void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx);
void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock); void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock);
static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) { static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
void * pBuf = *ppBuf; void *pBuf = *ppBuf;
size_t tsize = taosTSizeof(pBuf); size_t tsize = taosTSizeof(pBuf);
if (tsize < size) { if (tsize < size) {

8
source/dnode/vnode/src/inc/tsdbSma.h
View File

@ -20,6 +20,7 @@
typedef struct SSmaStat SSmaStat; typedef struct SSmaStat SSmaStat;
typedef struct SSmaEnv SSmaEnv; typedef struct SSmaEnv SSmaEnv;
typedef struct SSmaEnvs SSmaEnvs;
struct SSmaEnv { struct SSmaEnv {
TdThreadRwlock lock; TdThreadRwlock lock;
@ -36,6 +37,13 @@ struct SSmaEnv {
#define SMA_ENV_STAT(env) ((env)->pStat) #define SMA_ENV_STAT(env) ((env)->pStat)
#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems) #define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems)
struct SSmaEnvs {
int16_t nTSma;
int16_t nRSma;
SSmaEnv *pTSmaEnv;
SSmaEnv *pRSmaEnv;
};
void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv); void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv);
void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv); void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv);
#if 0 #if 0

1
source/dnode/vnode/src/inc/vnd.h
View File

@ -197,6 +197,7 @@ int tqCommit(STQ*);
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
int32_t tqProcessSetConnReq(STQ* pTq, char* msg); int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
int32_t tqProcessRebReq(STQ* pTq, char* msg); int32_t tqProcessRebReq(STQ* pTq, char* msg);
int32_t tqProcessCancelConnReq(STQ* pTq, char* msg);
int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId); int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);

25
source/dnode/vnode/src/meta/metaBDBImpl.c
View File

@ -912,7 +912,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) {
pCur->uid = uid; pCur->uid = uid;
// TODO: lock? // TODO: lock?
ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &(pCur->pCur), 0); ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &(pCur->pCur), 0);
if (ret != 0) { if ((ret != 0) || (pCur->pCur == NULL)) {
taosMemoryFree(pCur); taosMemoryFree(pCur);
return NULL; return NULL;
} }
@ -996,32 +996,31 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
} }
SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
SArray * pUids = NULL; SArray *pUids = NULL;
SMetaDB *pDB = pMeta->pDB; SMetaDB *pDB = pMeta->pDB;
DBC * pCur = NULL; DBC *pCur = NULL;
DBT pkey = {0}, pval = {0}; DBT pkey = {0}, pval = {0};
uint32_t mode = isDup ? DB_NEXT_DUP : DB_NEXT_NODUP; uint32_t mode = isDup ? DB_NEXT_DUP : DB_NEXT_NODUP;
int ret; int ret;
pUids = taosArrayInit(16, sizeof(tb_uid_t));
if (!pUids) {
return NULL;
}
// TODO: lock? // TODO: lock?
ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &pCur, 0); ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &pCur, 0);
if (ret != 0) { if (ret != 0) {
taosArrayDestroy(pUids);
return NULL; return NULL;
} }
void *pBuf = NULL;
// TODO: lock? // TODO: lock?
while ((ret = pCur->get(pCur, &pkey, &pval, mode)) == 0) { while ((ret = pCur->get(pCur, &pkey, &pval, mode)) == 0) {
if (!pUids) {
pUids = taosArrayInit(16, sizeof(tb_uid_t));
if (!pUids) {
return NULL;
}
}
taosArrayPush(pUids, pkey.data); taosArrayPush(pUids, pkey.data);
} }
// TODO: lock?
if (pCur) { if (pCur) {
pCur->close(pCur); pCur->close(pCur);

21
source/dnode/vnode/src/meta/metaCommit.c
View File

@ -1,21 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "meta.h"
int metaCommit(SMeta *pMeta) {
// TODO
return 0;
}

135
source/dnode/vnode/src/meta/metaTDBImpl.c
View File

@ -16,8 +16,20 @@
#include "metaDef.h" #include "metaDef.h"
#include "tdbInt.h" #include "tdbInt.h"
typedef struct SPoolMem {
int64_t size;
struct SPoolMem *prev;
struct SPoolMem *next;
} SPoolMem;
static SPoolMem *openPool();
static void clearPool(SPoolMem *pPool);
static void closePool(SPoolMem *pPool);
static void *poolMalloc(void *arg, size_t size);
static void poolFree(void *arg, void *ptr);
struct SMetaDB { struct SMetaDB {
TXN txn;
TENV *pEnv; TENV *pEnv;
TDB *pTbDB; TDB *pTbDB;
TDB *pSchemaDB; TDB *pSchemaDB;
@ -25,6 +37,7 @@ struct SMetaDB {
TDB *pStbIdx; TDB *pStbIdx;
TDB *pNtbIdx; TDB *pNtbIdx;
TDB *pCtbIdx; TDB *pCtbIdx;
SPoolMem *pPool;
}; };
typedef struct __attribute__((__packed__)) { typedef struct __attribute__((__packed__)) {
@ -167,12 +180,19 @@ int metaOpenDB(SMeta *pMeta) {
return -1; return -1;
} }
pMetaDb->pPool = openPool();
tdbTxnOpen(&pMetaDb->txn, 0, poolMalloc, poolFree, pMetaDb->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
tdbBegin(pMetaDb->pEnv, NULL);
pMeta->pDB = pMetaDb; pMeta->pDB = pMetaDb;
return 0; return 0;
} }
void metaCloseDB(SMeta *pMeta) { void metaCloseDB(SMeta *pMeta) {
if (pMeta->pDB) { if (pMeta->pDB) {
tdbCommit(pMeta->pDB->pEnv, &pMeta->pDB->txn);
tdbTxnClose(&pMeta->pDB->txn);
clearPool(pMeta->pDB->pPool);
tdbDbClose(pMeta->pDB->pCtbIdx); tdbDbClose(pMeta->pDB->pCtbIdx);
tdbDbClose(pMeta->pDB->pNtbIdx); tdbDbClose(pMeta->pDB->pNtbIdx);
tdbDbClose(pMeta->pDB->pStbIdx); tdbDbClose(pMeta->pDB->pStbIdx);
@ -206,13 +226,21 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
uid = metaGenerateUid(pMeta); uid = metaGenerateUid(pMeta);
} }
// check name and uid unique
if (tdbDbGet(pMetaDb->pTbDB, &uid, sizeof(uid), NULL, NULL) == 0) {
return -1;
}
if (tdbDbGet(pMetaDb->pNameIdx, pTbCfg->name, strlen(pTbCfg->name) + 1, NULL, NULL) == 0) {
return -1;
}
// save to table.db // save to table.db
pKey = &uid; pKey = &uid;
kLen = sizeof(uid); kLen = sizeof(uid);
pVal = pBuf = buf; pVal = pBuf = buf;
metaEncodeTbInfo(&pBuf, pTbCfg); metaEncodeTbInfo(&pBuf, pTbCfg);
vLen = POINTER_DISTANCE(pBuf, buf); vLen = POINTER_DISTANCE(pBuf, buf);
ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen); ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, &pMetaDb->txn);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@ -226,15 +254,15 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
if (pTbCfg->type == META_SUPER_TABLE) { if (pTbCfg->type == META_SUPER_TABLE) {
schemaWrapper.nCols = pTbCfg->stbCfg.nCols; schemaWrapper.nCols = pTbCfg->stbCfg.nCols;
schemaWrapper.pSchema = pTbCfg->stbCfg.pSchema; schemaWrapper.pSchemaEx = pTbCfg->stbCfg.pSchema;
} else { } else {
schemaWrapper.nCols = pTbCfg->ntbCfg.nCols; schemaWrapper.nCols = pTbCfg->ntbCfg.nCols;
schemaWrapper.pSchema = pTbCfg->ntbCfg.pSchema; schemaWrapper.pSchemaEx = pTbCfg->ntbCfg.pSchema;
} }
pVal = pBuf = buf; pVal = pBuf = buf;
metaEncodeSchemaEx(&pBuf, &schemaWrapper); metaEncodeSchemaEx(&pBuf, &schemaWrapper);
vLen = POINTER_DISTANCE(pBuf, buf); vLen = POINTER_DISTANCE(pBuf, buf);
ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen); ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, &pMeta->pDB->txn);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@ -248,7 +276,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
kLen = nameLen + 1 + sizeof(uid); kLen = nameLen + 1 + sizeof(uid);
pVal = NULL; pVal = NULL;
vLen = 0; vLen = 0;
ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen); ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@ -259,7 +287,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
kLen = sizeof(uid); kLen = sizeof(uid);
pVal = NULL; pVal = NULL;
vLen = 0; vLen = 0;
ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen); ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@ -270,7 +298,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
kLen = sizeof(ctbIdxKey); kLen = sizeof(ctbIdxKey);
pVal = NULL; pVal = NULL;
vLen = 0; vLen = 0;
ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen); ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@ -279,12 +307,16 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
kLen = sizeof(uid); kLen = sizeof(uid);
pVal = NULL; pVal = NULL;
vLen = 0; vLen = 0;
ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen); ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
} }
if (pMeta->pDB->pPool->size > 0) {
metaCommit(pMeta);
}
return 0; return 0;
} }
@ -349,7 +381,7 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
} }
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
return *metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false); return metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false);
} }
static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx) { static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx) {
@ -523,7 +555,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
return 0; return 0;
} }
pCtbIdxKey = pCtbCur->pVal; pCtbIdxKey = pCtbCur->pKey;
return pCtbIdxKey->uid; return pCtbIdxKey->uid;
} }
@ -571,7 +603,7 @@ void metaCloseSmaCurosr(SMSmaCursor *pCur) {
SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
// TODO // TODO
ASSERT(0); // ASSERT(0); // comment this line to pass CI
return NULL; return NULL;
} }
@ -701,3 +733,84 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) {
} }
return buf; return buf;
} }
int metaCommit(SMeta *pMeta) {
TXN *pTxn = &pMeta->pDB->txn;
// Commit current txn
tdbCommit(pMeta->pDB->pEnv, pTxn);
tdbTxnClose(pTxn);
clearPool(pMeta->pDB->pPool);
// start a new txn
tdbTxnOpen(&pMeta->pDB->txn, 0, poolMalloc, poolFree, pMeta->pDB->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
tdbBegin(pMeta->pDB->pEnv, pTxn);
return 0;
}
static SPoolMem *openPool() {
SPoolMem *pPool = (SPoolMem *)tdbOsMalloc(sizeof(*pPool));
pPool->prev = pPool->next = pPool;
pPool->size = 0;
return pPool;
}
static void clearPool(SPoolMem *pPool) {
SPoolMem *pMem;
do {
pMem = pPool->next;
if (pMem == pPool) break;
pMem->next->prev = pMem->prev;
pMem->prev->next = pMem->next;
pPool->size -= pMem->size;
tdbOsFree(pMem);
} while (1);
assert(pPool->size == 0);
}
static void closePool(SPoolMem *pPool) {
clearPool(pPool);
tdbOsFree(pPool);
}
static void *poolMalloc(void *arg, size_t size) {
void *ptr = NULL;
SPoolMem *pPool = (SPoolMem *)arg;
SPoolMem *pMem;
pMem = (SPoolMem *)tdbOsMalloc(sizeof(*pMem) + size);
if (pMem == NULL) {
assert(0);
}
pMem->size = sizeof(*pMem) + size;
pMem->next = pPool->next;
pMem->prev = pPool;
pPool->next->prev = pMem;
pPool->next = pMem;
pPool->size += pMem->size;
ptr = (void *)(&pMem[1]);
return ptr;
}
static void poolFree(void *arg, void *ptr) {
SPoolMem *pPool = (SPoolMem *)arg;
SPoolMem *pMem;
pMem = &(((SPoolMem *)ptr)[-1]);
pMem->next->prev = pMem->prev;
pMem->prev->next = pMem->next;
pPool->size -= pMem->size;
tdbOsFree(pMem);
}

2
source/dnode/vnode/src/meta/metaTbUid.c
View File

@ -27,5 +27,5 @@ void metaCloseUidGnrt(SMeta *pMeta) { /* TODO */
tb_uid_t metaGenerateUid(SMeta *pMeta) { tb_uid_t metaGenerateUid(SMeta *pMeta) {
// Generate a new table UID // Generate a new table UID
return tGenIdPI32(); return tGenIdPI64();
} }

118
source/dnode/vnode/src/tq/tq.c
View File

@ -81,6 +81,13 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi
return -1; return -1;
} }
memcpy(data, msg, msgLen); memcpy(data, msg, msgLen);
if (msgType == TDMT_VND_SUBMIT) {
if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg) != 0) {
return -1;
}
}
SRpcMsg req = { SRpcMsg req = {
.msgType = TDMT_VND_STREAM_TRIGGER, .msgType = TDMT_VND_STREAM_TRIGGER,
.pCont = data, .pCont = data,
@ -167,7 +174,7 @@ static FORCE_INLINE int32_t tEncodeSTqConsumer(void** buf, const STqConsumer* pC
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); tlen += taosEncodeFixedI64(buf, pConsumer->consumerId);
tlen += taosEncodeFixedI64(buf, pConsumer->epoch); tlen += taosEncodeFixedI32(buf, pConsumer->epoch);
tlen += taosEncodeString(buf, pConsumer->cgroup); tlen += taosEncodeString(buf, pConsumer->cgroup);
sz = taosArrayGetSize(pConsumer->topics); sz = taosArrayGetSize(pConsumer->topics);
tlen += taosEncodeFixedI32(buf, sz); tlen += taosEncodeFixedI32(buf, sz);
@ -182,7 +189,7 @@ static FORCE_INLINE const void* tDecodeSTqConsumer(const void* buf, STqConsumer*
int32_t sz; int32_t sz;
buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); buf = taosDecodeFixedI64(buf, &pConsumer->consumerId);
buf = taosDecodeFixedI64(buf, &pConsumer->epoch); buf = taosDecodeFixedI32(buf, &pConsumer->epoch);
buf = taosDecodeStringTo(buf, pConsumer->cgroup); buf = taosDecodeStringTo(buf, pConsumer->cgroup);
buf = taosDecodeFixedI32(buf, &sz); buf = taosDecodeFixedI32(buf, &sz);
pConsumer->topics = taosArrayInit(sz, sizeof(STqTopic)); pConsumer->topics = taosArrayInit(sz, sizeof(STqTopic));
@ -255,6 +262,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
int64_t consumerId = pReq->consumerId; int64_t consumerId = pReq->consumerId;
int64_t fetchOffset; int64_t fetchOffset;
int64_t blockingTime = pReq->blockingTime; int64_t blockingTime = pReq->blockingTime;
int32_t reqEpoch = pReq->epoch;
if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
fetchOffset = 0; fetchOffset = 0;
@ -264,7 +272,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
fetchOffset = pReq->currentOffset + 1; fetchOffset = pReq->currentOffset + 1;
} }
/*printf("tmq poll vg %d req %ld %ld\n", pTq->pVnode->vgId, pReq->currentOffset, fetchOffset);*/ vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, pTq->pVnode->vgId, pReq->currentOffset, fetchOffset);
SMqPollRsp rsp = { SMqPollRsp rsp = {
/*.consumerId = consumerId,*/ /*.consumerId = consumerId,*/
@ -274,6 +282,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
if (pConsumer == NULL) { if (pConsumer == NULL) {
vWarn("tmq poll: consumer %ld (epoch %d) not found in vg %d", consumerId, pReq->epoch, pTq->pVnode->vgId);
pMsg->pCont = NULL; pMsg->pCont = NULL;
pMsg->contLen = 0; pMsg->contLen = 0;
pMsg->code = -1; pMsg->code = -1;
@ -281,30 +290,57 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
return 0; return 0;
} }
int32_t consumerEpoch = atomic_load_32(&pConsumer->epoch);
while (consumerEpoch < reqEpoch) {
consumerEpoch = atomic_val_compare_exchange_32(&pConsumer->epoch, consumerEpoch, reqEpoch);
}
STqTopic* pTopic = NULL;
int sz = taosArrayGetSize(pConsumer->topics); int sz = taosArrayGetSize(pConsumer->topics);
ASSERT(sz == 1); for (int32_t i = 0; i < sz; i++) {
STqTopic* pTopic = taosArrayGet(pConsumer->topics, 0); STqTopic* topic = taosArrayGet(pConsumer->topics, i);
ASSERT(strcmp(pTopic->topicName, pReq->topic) == 0); //TODO race condition
ASSERT(pConsumer->consumerId == consumerId); ASSERT(pConsumer->consumerId == consumerId);
if (strcmp(topic->topicName, pReq->topic) == 0) {
pTopic = topic;
break;
}
}
if (pTopic == NULL) {
vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic, pTq->pVnode->vgId);
pMsg->pCont = NULL;
pMsg->contLen = 0;
pMsg->code = -1;
tmsgSendRsp(pMsg);
return 0;
}
vDebug("poll topic %s from consumer %ld (epoch %d)", pTopic->topicName, consumerId, pReq->epoch);
rsp.reqOffset = pReq->currentOffset; rsp.reqOffset = pReq->currentOffset;
rsp.skipLogNum = 0; rsp.skipLogNum = 0;
while (1) { while (1) {
/*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/ /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/
//TODO
consumerEpoch = atomic_load_32(&pConsumer->epoch);
if (consumerEpoch > pReq->epoch) {
//TODO: return
break;
}
SWalReadHead* pHead; SWalReadHead* pHead;
if (walReadWithHandle_s(pTopic->pReadhandle, fetchOffset, &pHead) < 0) { if (walReadWithHandle_s(pTopic->pReadhandle, fetchOffset, &pHead) < 0) {
// TODO: no more log, set timer to wait blocking time // TODO: no more log, set timer to wait blocking time
// if data inserted during waiting, launch query and // if data inserted during waiting, launch query and
// response to user // response to user
vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, pTq->pVnode->vgId, fetchOffset);
break; break;
} }
/*printf("vg %d offset %ld msgType %d from epoch %d\n", pTq->pVnode->vgId, fetchOffset, pHead->msgType, pReq->epoch);*/ vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, pTq->pVnode->vgId, fetchOffset, pHead->msgType);
/*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/ /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/
/*pHead = pTopic->pReadhandle->pHead;*/ /*pHead = pTopic->pReadhandle->pHead;*/
if (pHead->msgType == TDMT_VND_SUBMIT) { if (pHead->msgType == TDMT_VND_SUBMIT) {
SSubmitReq* pCont = (SSubmitReq*)&pHead->body; SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
/*printf("from topic %s from consumer\n", pTopic->topicName, consumerId);*/
qTaskInfo_t task = pTopic->buffer.output[workerId].task; qTaskInfo_t task = pTopic->buffer.output[workerId].task;
ASSERT(task); ASSERT(task);
qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK); qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK);
@ -324,6 +360,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
} }
if (taosArrayGetSize(pRes) == 0) { if (taosArrayGetSize(pRes) == 0) {
vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId, pReq->epoch, pTq->pVnode->vgId, fetchOffset);
fetchOffset++; fetchOffset++;
rsp.skipLogNum++; rsp.skipLogNum++;
taosArrayDestroy(pRes); taosArrayDestroy(pRes);
@ -352,7 +389,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
pMsg->pCont = buf; pMsg->pCont = buf;
pMsg->contLen = tlen; pMsg->contLen = tlen;
pMsg->code = 0; pMsg->code = 0;
/*printf("vg %d offset %ld msgType %d from epoch %d actual rsp\n", pTq->pVnode->vgId, fetchOffset, pHead->msgType, pReq->epoch);*/ vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", pTq->pVnode->vgId, fetchOffset, pHead->msgType, consumerId, pReq->epoch);
tmsgSendRsp(pMsg); tmsgSendRsp(pMsg);
taosMemoryFree(pHead); taosMemoryFree(pHead);
return 0; return 0;
@ -383,7 +420,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
pMsg->contLen = tlen; pMsg->contLen = tlen;
pMsg->code = 0; pMsg->code = 0;
tmsgSendRsp(pMsg); tmsgSendRsp(pMsg);
/*printf("vg %d offset %ld from epoch %d not rsp\n", pTq->pVnode->vgId, fetchOffset, pReq->epoch);*/ vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", pTq->pVnode->vgId, fetchOffset, consumerId, pReq->epoch);
/*}*/ /*}*/
return 0; return 0;
@ -391,33 +428,77 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
int32_t tqProcessRebReq(STQ* pTq, char* msg) { int32_t tqProcessRebReq(STQ* pTq, char* msg) {
SMqMVRebReq req = {0}; SMqMVRebReq req = {0};
terrno = TSDB_CODE_SUCCESS;
tDecodeSMqMVRebReq(msg, &req); tDecodeSMqMVRebReq(msg, &req);
vDebug("vg %d set from consumer %ld to consumer %ld", req.vgId, req.oldConsumerId ,req.newConsumerId);
STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId); STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId);
ASSERT(pConsumer); ASSERT(pConsumer);
ASSERT(pConsumer->consumerId == req.oldConsumerId);
int32_t numOfTopics = taosArrayGetSize(pConsumer->topics);
if (numOfTopics == 1) {
STqTopic* pTopic = taosArrayGet(pConsumer->topics, 0);
ASSERT(strcmp(pTopic->topicName, req.topic) == 0);
STqConsumer* pNewConsumer = tqHandleGet(pTq->tqMeta, req.newConsumerId);
if (pNewConsumer == NULL) {
pConsumer->consumerId = req.newConsumerId; pConsumer->consumerId = req.newConsumerId;
tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer);
tqHandleCommit(pTq->tqMeta, req.newConsumerId); tqHandleCommit(pTq->tqMeta, req.newConsumerId);
tqHandlePurge(pTq->tqMeta, req.oldConsumerId); tqHandlePurge(pTq->tqMeta, req.oldConsumerId);
terrno = TSDB_CODE_SUCCESS; return 0;
} else {
taosArrayPush(pNewConsumer->topics, pTopic);
}
} else {
for (int32_t i = 0; i < numOfTopics; i++) {
STqTopic* pTopic = taosArrayGet(pConsumer->topics, i);
if (strcmp(pTopic->topicName, req.topic) == 0) {
STqConsumer* pNewConsumer = tqHandleGet(pTq->tqMeta, req.newConsumerId);
if (pNewConsumer == NULL) {
pNewConsumer = taosMemoryCalloc(1, sizeof(STqConsumer));
if (pNewConsumer == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1;
}
strcpy(pNewConsumer->cgroup, pConsumer->cgroup);
pNewConsumer->topics = taosArrayInit(0, sizeof(STqTopic));
pNewConsumer->consumerId = req.newConsumerId;
pNewConsumer->epoch = 0;
taosArrayPush(pNewConsumer->topics, pTopic);
tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer);
tqHandleCommit(pTq->tqMeta, req.newConsumerId);
return 0;
}
ASSERT(pNewConsumer->consumerId == req.newConsumerId);
taosArrayPush(pNewConsumer->topics, pTopic);
break;
}
}
//
}
return 0; return 0;
} }
int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
SMqSetCVgReq req = {0}; SMqSetCVgReq req = {0};
tDecodeSMqSetCVgReq(msg, &req); tDecodeSMqSetCVgReq(msg, &req);
bool create = false;
/*printf("vg %d set to consumer from %ld to %ld\n", req.vgId, req.oldConsumerId, req.newConsumerId);*/ vDebug("vg %d set to consumer %ld", req.vgId, req.consumerId);
STqConsumer* pConsumer = taosMemoryCalloc(1, sizeof(STqConsumer)); STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, req.consumerId);
if (pConsumer == NULL) {
pConsumer = taosMemoryCalloc(1, sizeof(STqConsumer));
if (pConsumer == NULL) { if (pConsumer == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1; return -1;
} }
strcpy(pConsumer->cgroup, req.cgroup); strcpy(pConsumer->cgroup, req.cgroup);
pConsumer->topics = taosArrayInit(0, sizeof(STqTopic)); pConsumer->topics = taosArrayInit(0, sizeof(STqTopic));
pConsumer->consumerId = req.consumerId; pConsumer->consumerId = req.consumerId;
pConsumer->epoch = 0; pConsumer->epoch = 0;
create = true;
}
STqTopic* pTopic = taosMemoryCalloc(1, sizeof(STqTopic)); STqTopic* pTopic = taosMemoryCalloc(1, sizeof(STqTopic));
if (pTopic == NULL) { if (pTopic == NULL) {
@ -450,10 +531,17 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle); pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle);
ASSERT(pTopic->buffer.output[i].task); ASSERT(pTopic->buffer.output[i].task);
} }
/*printf("set topic %s to consumer %ld on vg %d\n", pTopic->topicName, req.consumerId, pTq->pVnode->vgId);*/ vDebug("set topic %s to consumer %ld on vg %d", pTopic->topicName, req.consumerId, pTq->pVnode->vgId);
taosArrayPush(pConsumer->topics, pTopic); taosArrayPush(pConsumer->topics, pTopic);
if (create) {
tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer); tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer);
tqHandleCommit(pTq->tqMeta, req.consumerId); tqHandleCommit(pTq->tqMeta, req.consumerId);
}
terrno = TSDB_CODE_SUCCESS;
return 0;
}
int32_t tqProcessCancelConnReq(STQ* pTq, char* msg) {
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
return 0; return 0;
} }

1
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -701,7 +701,6 @@ int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray
// Set pIdx // Set pIdx
pBlock = taosArrayGetLast(pSupA); pBlock = taosArrayGetLast(pSupA);
pIdx->tid = TABLE_TID(pTable);
pIdx->uid = TABLE_UID(pTable); pIdx->uid = TABLE_UID(pTable);
pIdx->hasLast = pBlock->last ? 1 : 0; pIdx->hasLast = pBlock->last ? 1 : 0;
pIdx->maxKey = pBlock->keyLast; pIdx->maxKey = pBlock->keyLast;

10
source/dnode/vnode/src/tsdb/tsdbMain.c
View File

@ -80,9 +80,6 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg,
pTsdb->pmaf = pMAF; pTsdb->pmaf = pMAF;
pTsdb->pMeta = pMeta; pTsdb->pMeta = pMeta;
pTsdb->pTfs = pTfs; pTsdb->pTfs = pTfs;
pTsdb->pTSmaEnv = NULL;
pTsdb->pRSmaEnv = NULL;
pTsdb->fs = tsdbNewFS(pTsdbCfg); pTsdb->fs = tsdbNewFS(pTsdbCfg);
return pTsdb; return pTsdb;
@ -90,8 +87,8 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg,
static void tsdbFree(STsdb *pTsdb) { static void tsdbFree(STsdb *pTsdb) {
if (pTsdb) { if (pTsdb) {
tsdbFreeSmaEnv(pTsdb->pRSmaEnv); tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb));
tsdbFreeSmaEnv(pTsdb->pTSmaEnv); tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb));
tsdbFreeFS(pTsdb->fs); tsdbFreeFS(pTsdb->fs);
taosMemoryFreeClear(pTsdb->path); taosMemoryFreeClear(pTsdb->path);
taosMemoryFree(pTsdb); taosMemoryFree(pTsdb);
@ -100,7 +97,10 @@ static void tsdbFree(STsdb *pTsdb) {
static int tsdbOpenImpl(STsdb *pTsdb) { static int tsdbOpenImpl(STsdb *pTsdb) {
tsdbOpenFS(pTsdb); tsdbOpenFS(pTsdb);
tsdbInitSma(pTsdb);
// TODO // TODO
return 0; return 0;
} }

4
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -3046,8 +3046,8 @@ bool tsdbGetExternalRow(tsdbReaderT pHandle) {
// return code; // return code;
//} //}
bool isTsdbCacheLastRow(tsdbReaderT* pTsdbReadHandle) { bool isTsdbCacheLastRow(tsdbReaderT* pReader) {
return ((STsdbReadHandle *)pTsdbReadHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE; return ((STsdbReadHandle *)pReader)->cachelastrow > TSDB_CACHED_TYPE_NONE;
} }
int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo *groupList) { int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo *groupList) {

25
source/dnode/vnode/src/tsdb/tsdbReadImpl.c
View File

@ -98,7 +98,7 @@ int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet) {
void tsdbCloseAndUnsetFSet(SReadH *pReadh) { tsdbResetReadFile(pReadh); } void tsdbCloseAndUnsetFSet(SReadH *pReadh) { tsdbResetReadFile(pReadh); }
int tsdbLoadBlockIdx(SReadH *pReadh) { int tsdbLoadBlockIdx(SReadH *pReadh) {
SDFile * pHeadf = TSDB_READ_HEAD_FILE(pReadh); SDFile *pHeadf = TSDB_READ_HEAD_FILE(pReadh);
SBlockIdx blkIdx; SBlockIdx blkIdx;
ASSERT(taosArrayGetSize(pReadh->aBlkIdx) == 0); ASSERT(taosArrayGetSize(pReadh->aBlkIdx) == 0);
@ -149,8 +149,8 @@ int tsdbLoadBlockIdx(SReadH *pReadh) {
} }
tsize++; tsize++;
ASSERT(tsize == 1 || ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 2))->tid < // ASSERT(tsize == 1 || ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 2))->tid <
((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 1))->tid); // ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 1))->tid);
} }
return 0; return 0;
@ -180,7 +180,7 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) {
} }
SBlockIdx *pBlkIdx = taosArrayGet(pReadh->aBlkIdx, pReadh->cidx); SBlockIdx *pBlkIdx = taosArrayGet(pReadh->aBlkIdx, pReadh->cidx);
if (pBlkIdx->tid == TABLE_TID(pTable)) { if (pBlkIdx->uid == TABLE_TID(pTable)) {
if (pBlkIdx->uid == TABLE_UID(pTable)) { if (pBlkIdx->uid == TABLE_UID(pTable)) {
pReadh->pBlkIdx = pBlkIdx; pReadh->pBlkIdx = pBlkIdx;
} else { } else {
@ -188,7 +188,7 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) {
} }
pReadh->cidx++; pReadh->cidx++;
break; break;
} else if (pBlkIdx->tid > TABLE_TID(pTable)) { } else if (pBlkIdx->uid > TABLE_TID(pTable)) {
pReadh->pBlkIdx = NULL; pReadh->pBlkIdx = NULL;
break; break;
} else { } else {
@ -205,7 +205,7 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) {
int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) {
ASSERT(pReadh->pBlkIdx != NULL); ASSERT(pReadh->pBlkIdx != NULL);
SDFile * pHeadf = TSDB_READ_HEAD_FILE(pReadh); SDFile *pHeadf = TSDB_READ_HEAD_FILE(pReadh);
SBlockIdx *pBlkIdx = pReadh->pBlkIdx; SBlockIdx *pBlkIdx = pReadh->pBlkIdx;
if (tsdbSeekDFile(pHeadf, pBlkIdx->offset, SEEK_SET) < 0) { if (tsdbSeekDFile(pHeadf, pBlkIdx->offset, SEEK_SET) < 0) {
@ -237,7 +237,7 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) {
return -1; return -1;
} }
ASSERT(pBlkIdx->tid == pReadh->pBlkInfo->tid && pBlkIdx->uid == pReadh->pBlkInfo->uid); // ASSERT(pBlkIdx->tid == pReadh->pBlkInfo->tid && pBlkIdx->uid == pReadh->pBlkInfo->uid);
if (pTarget) { if (pTarget) {
memcpy(pTarget, (void *)(pReadh->pBlkInfo), pBlkIdx->len); memcpy(pTarget, (void *)(pReadh->pBlkInfo), pBlkIdx->len);
@ -275,7 +275,8 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
return 0; return 0;
} }
int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds) { int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds,
int numOfColsIds) {
ASSERT(pBlock->numOfSubBlocks > 0); ASSERT(pBlock->numOfSubBlocks > 0);
int8_t update = pReadh->pRepo->config.update; int8_t update = pReadh->pRepo->config.update;
@ -388,7 +389,7 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) {
int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx) { int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx) {
int tlen = 0; int tlen = 0;
tlen += taosEncodeVariantI32(buf, pIdx->tid); // tlen += taosEncodeVariantI32(buf, pIdx->tid);
tlen += taosEncodeVariantU32(buf, pIdx->len); tlen += taosEncodeVariantU32(buf, pIdx->len);
tlen += taosEncodeVariantU32(buf, pIdx->offset); tlen += taosEncodeVariantU32(buf, pIdx->offset);
tlen += taosEncodeFixedU8(buf, pIdx->hasLast); tlen += taosEncodeFixedU8(buf, pIdx->hasLast);
@ -404,7 +405,7 @@ void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) {
uint32_t numOfBlocks = 0; uint32_t numOfBlocks = 0;
uint64_t value = 0; uint64_t value = 0;
if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL; // if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL;
if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL; if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL;
if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL; if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL;
if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL; if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL;
@ -686,7 +687,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
ASSERT(colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID); ASSERT(colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID);
SDFile * pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
SBlockCol blockCol = {0}; SBlockCol blockCol = {0};
tdResetDataCols(pDataCols); tdResetDataCols(pDataCols);
@ -700,7 +701,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
int ccol = 0; int ccol = 0;
for (int i = 0; i < numOfColIds; i++) { for (int i = 0; i < numOfColIds; i++) {
int16_t colId = colIds[i]; int16_t colId = colIds[i];
SDataCol * pDataCol = NULL; SDataCol *pDataCol = NULL;
SBlockCol *pBlockCol = NULL; SBlockCol *pBlockCol = NULL;
while (true) { while (true) {

209
source/dnode/vnode/src/tsdb/tsdbSma.c
View File

@ -20,6 +20,7 @@ static const char *TSDB_SMA_DNAME[] = {
"tsma", // TSDB_SMA_TYPE_TIME_RANGE "tsma", // TSDB_SMA_TYPE_TIME_RANGE
"rsma", // TSDB_SMA_TYPE_ROLLUP "rsma", // TSDB_SMA_TYPE_ROLLUP
}; };
#undef _TEST_SMA_PRINT_DEBUG_LOG_ #undef _TEST_SMA_PRINT_DEBUG_LOG_
#define SMA_STORAGE_TSDB_DAYS 30 #define SMA_STORAGE_TSDB_DAYS 30
#define SMA_STORAGE_TSDB_TIMES 10 #define SMA_STORAGE_TSDB_TIMES 10
@ -81,7 +82,7 @@ struct SSmaStat {
// declaration of static functions // declaration of static functions
// expired window // expired window
static int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, const char *msg); static int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg);
static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey); static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey);
static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat); static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat);
static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem); static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem);
@ -117,6 +118,19 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg);
static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid);
// implementation // implementation
static FORCE_INLINE int16_t tsdbTSmaAdd(STsdb *pTsdb, int16_t n) { return atomic_add_fetch_16(&REPO_TSMA_NUM(pTsdb), n); }
static FORCE_INLINE int16_t tsdbTSmaSub(STsdb *pTsdb, int16_t n) { return atomic_sub_fetch_16(&REPO_TSMA_NUM(pTsdb), n); }
int32_t tsdbInitSma(STsdb *pTsdb) {
// tSma
int32_t numOfTSma = taosArrayGetSize(metaGetSmaTbUids(pTsdb->pMeta, false));
if (numOfTSma > 0) {
atomic_store_16(&REPO_TSMA_NUM(pTsdb), (int16_t)numOfTSma);
}
// TODO: rSma
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE int8_t tsdbSmaStat(SSmaStatItem *pStatItem) { static FORCE_INLINE int8_t tsdbSmaStat(SSmaStatItem *pStatItem) {
if (pStatItem) { if (pStatItem) {
return atomic_load_8(&pStatItem->state); return atomic_load_8(&pStatItem->state);
@ -246,8 +260,9 @@ void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv) {
static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat) { static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat) {
if (pStat == NULL) return 0; if (pStat == NULL) return 0;
int ref = T_REF_INC(pStat); int ref = T_REF_INC(pStat);
tsdbDebug("vgId:%d ref sma stat %p ref %d", REPO_ID(pTsdb), pStat, ref); tsdbDebug("vgId:%d ref sma stat:%p, val:%d", REPO_ID(pTsdb), pStat, ref);
return 0; return 0;
} }
@ -255,7 +270,7 @@ static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat) {
if (pStat == NULL) return 0; if (pStat == NULL) return 0;
int ref = T_REF_DEC(pStat); int ref = T_REF_DEC(pStat);
tsdbDebug("vgId:%d unref sma stat %p ref %d", REPO_ID(pTsdb), pStat, ref); tsdbDebug("vgId:%d unref sma stat:%p, val:%d", REPO_ID(pTsdb), pStat, ref);
return 0; return 0;
} }
@ -339,12 +354,12 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) {
// return if already init // return if already init
switch (smaType) { switch (smaType) {
case TSDB_SMA_TYPE_TIME_RANGE: case TSDB_SMA_TYPE_TIME_RANGE:
if ((pEnv = (SSmaEnv *)atomic_load_ptr(&pTsdb->pTSmaEnv)) != NULL) { if ((pEnv = (SSmaEnv *)atomic_load_ptr(&REPO_TSMA_ENV(pTsdb))) != NULL) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
break; break;
case TSDB_SMA_TYPE_ROLLUP: case TSDB_SMA_TYPE_ROLLUP:
if ((pEnv = (SSmaEnv *)atomic_load_ptr(&pTsdb->pRSmaEnv)) != NULL) { if ((pEnv = (SSmaEnv *)atomic_load_ptr(&REPO_RSMA_ENV(pTsdb))) != NULL) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
break; break;
@ -355,7 +370,8 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) {
// init sma env // init sma env
tsdbLockRepo(pTsdb); tsdbLockRepo(pTsdb);
pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&pTsdb->pTSmaEnv) : atomic_load_ptr(&pTsdb->pRSmaEnv); pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&REPO_TSMA_ENV(pTsdb))
: atomic_load_ptr(&REPO_RSMA_ENV(pTsdb));
if (pEnv == NULL) { if (pEnv == NULL) {
char rname[TSDB_FILENAME_LEN] = {0}; char rname[TSDB_FILENAME_LEN] = {0};
@ -377,8 +393,8 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
(smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&pTsdb->pTSmaEnv, pEnv) (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&REPO_TSMA_ENV(pTsdb), pEnv)
: atomic_store_ptr(&pTsdb->pRSmaEnv, pEnv); : atomic_store_ptr(&REPO_RSMA_ENV(pTsdb), pEnv);
} }
tsdbUnlockRepo(pTsdb); tsdbUnlockRepo(pTsdb);
@ -430,9 +446,12 @@ static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t
taosHashCleanup(pItem->expiredWindows); taosHashCleanup(pItem->expiredWindows);
taosMemoryFreeClear(pItem->pSma); taosMemoryFreeClear(pItem->pSma);
taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid));
tsdbWarn("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", REPO_ID(pTsdb), indexUid,
winSKey);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
tsdbDebug("vgId:%d smaIndex %" PRIi64 " tsKey %" PRIi64 " is put to hash", REPO_ID(pTsdb), indexUid, winSKey); tsdbDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", REPO_ID(pTsdb), indexUid,
winSKey);
} }
/** /**
@ -442,18 +461,21 @@ static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t
* @param msg SSubmitReq * @param msg SSubmitReq
* @return int32_t * @return int32_t
*/ */
int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, const char *msg) { int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg) {
const SSubmitReq *pMsg = (const SSubmitReq *)msg;
if (pMsg->length <= sizeof(SSubmitReq)) {
terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
return TSDB_CODE_FAILED;
}
if (!pTsdb->pMeta) { if (!pTsdb->pMeta) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
if (atomic_load_16(&REPO_TSMA_NUM(pTsdb)) <= 0) {
tsdbWarn("vgId:%d not update expire window since no tSma", REPO_ID(pTsdb));
return TSDB_CODE_SUCCESS;
}
if (tdScanAndConvertSubmitMsg(pMsg) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED;
}
// TODO: decode the msg from Stream Computing module => start // TODO: decode the msg from Stream Computing module => start
#ifdef TSDB_SMA_TESTx #ifdef TSDB_SMA_TESTx
int64_t indexUid = SMA_TEST_INDEX_UID; int64_t indexUid = SMA_TEST_INDEX_UID;
@ -480,7 +502,7 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, const char *msg) {
// Firstly, assume that tSma can only be created on super table/normal table. // Firstly, assume that tSma can only be created on super table/normal table.
// getActiveTimeWindow // getActiveTimeWindow
SSmaEnv *pEnv = REPO_SMA_ENV(pTsdb, TSDB_SMA_TYPE_TIME_RANGE); SSmaEnv *pEnv = REPO_TSMA_ENV(pTsdb);
SSmaStat *pStat = SMA_ENV_STAT(pEnv); SSmaStat *pStat = SMA_ENV_STAT(pEnv);
SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv);
@ -569,10 +591,12 @@ static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t ind
if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) {
// error handling // error handling
tsdbUnRefSmaStat(pTsdb, pStat); tsdbUnRefSmaStat(pTsdb, pStat);
tsdbWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " failed", REPO_ID(pTsdb), tsdbWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", REPO_ID(pTsdb),
skey, indexUid); skey, indexUid);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
tsdbDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", REPO_ID(pTsdb),
skey, indexUid);
// TODO: use a standalone interface to received state upate notification from stream computing module. // TODO: use a standalone interface to received state upate notification from stream computing module.
/** /**
* @brief state * @brief state
@ -651,13 +675,14 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
*/ */
static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) { static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) {
SDBFile *pDBFile = &pSmaH->dFile; SDBFile *pDBFile = &pSmaH->dFile;
printf("\nvgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 "\n",
REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
// TODO: insert sma data blocks into B+Tree(TDB) // TODO: insert sma data blocks into B+Tree(TDB)
if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) { if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) {
tsdbWarn("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
tsdbDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed",
REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ #ifdef _TEST_SMA_PRINT_DEBUG_LOG_
uint32_t valueSize = 0; uint32_t valueSize = 0;
@ -680,7 +705,6 @@ static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t k
* @return int64_t * @return int64_t
*/ */
static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) { static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) {
if (adjusted) { if (adjusted) {
return interval; return interval;
} }
@ -814,7 +838,7 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe
static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char *msg) { static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char *msg) {
STsdbCfg *pCfg = REPO_CFG(pTsdb); STsdbCfg *pCfg = REPO_CFG(pTsdb);
const SArray *pDataBlocks = (const SArray *)msg; const SArray *pDataBlocks = (const SArray *)msg;
SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); SSmaEnv *pEnv = atomic_load_ptr(&REPO_TSMA_ENV(pTsdb));
if (pEnv == NULL) { if (pEnv == NULL) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
@ -834,7 +858,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
SSmaStat *pStat = SMA_ENV_STAT(pTsdb->pTSmaEnv); SSmaStat *pStat = SMA_ENV_STAT(pEnv);
SSmaStatItem *pItem = NULL; SSmaStatItem *pItem = NULL;
tsdbRefSmaStat(pTsdb, pStat); tsdbRefSmaStat(pTsdb, pStat);
@ -874,27 +898,41 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
// key: skey + groupId // key: skey + groupId
char smaKey[SMA_KEY_LEN] = {0}; char smaKey[SMA_KEY_LEN] = {0};
char dataBuf[512] = {0}; char dataBuf[512] = {0};
void *pDataBuf = &dataBuf; void *pDataBuf = NULL;
int32_t sz = taosArrayGetSize(pDataBlocks); int32_t sz = taosArrayGetSize(pDataBlocks);
for (int32_t i = 0; i < sz; ++i) { for (int32_t i = 0; i < sz; ++i) {
SSDataBlock *pDataBlock = *(SSDataBlock **)taosArrayGet(pDataBlocks, i); SSDataBlock *pDataBlock = taosArrayGet(pDataBlocks, i);
int32_t colNum = pDataBlock->info.numOfCols; int32_t colNum = pDataBlock->info.numOfCols;
int32_t rows = pDataBlock->info.rows; int32_t rows = pDataBlock->info.rows;
int32_t rowSize = pDataBlock->info.rowSize; int32_t rowSize = pDataBlock->info.rowSize;
int64_t groupId = pDataBlock->info.groupId; int64_t groupId = pDataBlock->info.groupId;
for (int32_t j = 0; j < rows; ++j) { for (int32_t j = 0; j < rows; ++j) {
printf("|"); printf("|");
TSKEY skey = TSKEY_INITIAL_VAL; // the start key of TS window by interval TSKEY skey = 1649295200000; // TSKEY_INITIAL_VAL; // the start key of TS window by interval
void *pSmaKey = &smaKey; void *pSmaKey = &smaKey;
int32_t tlen = 0; bool isStartKey = false;
{
// just for debugging
isStartKey = true;
tsdbEncodeTSmaKey(groupId, skey, &pSmaKey);
}
int32_t tlen = 0; // reset the len
pDataBuf = &dataBuf; // reset the buf
for (int32_t k = 0; k < colNum; ++k) { for (int32_t k = 0; k < colNum; ++k) {
SColumnInfoData *pColInfoData = *(SColumnInfoData **)taosArrayGet(pDataBlock->pDataBlock, k); SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
void *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); void *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
switch (pColInfoData->info.type) { switch (pColInfoData->info.type) {
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
if (!isStartKey) {
isStartKey = true;
skey = *(TSKEY *)var; skey = *(TSKEY *)var;
printf("==> skey = %" PRIi64 " groupId = %" PRId64 "|", skey, groupId); printf("==> skey = %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId);
tsdbEncodeTSmaKey(groupId, skey, &pSmaKey); tsdbEncodeTSmaKey(groupId, skey, &pSmaKey);
} else {
printf(" %" PRIi64 " |", *(int64_t *)var);
tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var);
break;
}
break; break;
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_UTINYINT:
@ -918,6 +956,9 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var); tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var);
break; break;
case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_FLOAT:
printf(" %15f |", *(float *)var);
tlen += taosEncodeBinary(&pDataBuf, var, sizeof(float));
break;
case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UINT:
printf(" %15u |", *(uint32_t *)var); printf(" %15u |", *(uint32_t *)var);
tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var); tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var);
@ -927,6 +968,8 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var);
break; break;
case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_DOUBLE:
printf(" %15lf |", *(double *)var);
tlen += taosEncodeBinary(&pDataBuf, var, sizeof(double));
case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_UBIGINT:
printf(" %15lu |", *(uint64_t *)var); printf(" %15lu |", *(uint64_t *)var);
tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var); tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var);
@ -954,7 +997,8 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
break; break;
} }
} }
if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) { // if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) {
if (tlen > 0) {
int32_t fid = (int32_t)(TSDB_KEY_FID(skey, daysPerFile, pCfg->precision)); int32_t fid = (int32_t)(TSDB_KEY_FID(skey, daysPerFile, pCfg->precision));
// Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index
@ -966,7 +1010,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
tsdbCloseDBF(&tSmaH.dFile); tsdbCloseDBF(&tSmaH.dFile);
} }
tsdbSetTSmaDataFile(&tSmaH, indexUid, fid); tsdbSetTSmaDataFile(&tSmaH, indexUid, fid);
if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) { if (tsdbOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) {
tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb),
tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno));
tsdbDestroyTSmaWriteH(&tSmaH); tsdbDestroyTSmaWriteH(&tSmaH);
@ -975,21 +1019,20 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
} }
} }
if (tsdbInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, pDataBuf, tlen) != 0) { if (tsdbInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen) != 0) {
tsdbWarn("vgId:%d insert tSma data blocks failed for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 tsdbWarn("vgId:%d insert tSma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64
" since %s", " since %s",
REPO_ID(pTsdb), indexUid, skey, groupId, tstrerror(terrno)); REPO_ID(pTsdb), indexUid, skey, groupId, tstrerror(terrno));
tsdbDestroyTSmaWriteH(&tSmaH); tsdbDestroyTSmaWriteH(&tSmaH);
tsdbUnRefSmaStat(pTsdb, pStat); tsdbUnRefSmaStat(pTsdb, pStat);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} else {
tsdbWarn("vgId:%d insert tSma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64,
REPO_ID(pTsdb), indexUid, skey, groupId);
} }
tsdbDebug("vgId:%d insert tSma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64,
REPO_ID(pTsdb), indexUid, skey, groupId);
// TODO:tsdbEndTSmaCommit(); // TODO:tsdbEndTSmaCommit();
// Step 3: reset the SSmaStat // Step 3: reset the SSmaStat
tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), indexUid, skey); tsdbResetExpiredWindow(pTsdb, pStat, indexUid, skey);
} else { } else {
tsdbWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, tsdbWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64,
REPO_ID(pTsdb), skey, tlen, indexUid); REPO_ID(pTsdb), skey, tlen, indexUid);
@ -1012,7 +1055,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
* @return int32_t * @return int32_t
*/ */
static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid) { static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid) {
SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); SSmaEnv *pEnv = atomic_load_ptr(&REPO_TSMA_ENV(pTsdb));
// clear local cache // clear local cache
if (pEnv) { if (pEnv) {
@ -1035,12 +1078,17 @@ static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid) {
tsdbUnLockSma(pEnv); tsdbUnLockSma(pEnv);
int32_t nSleep = 0; int32_t nSleep = 0;
int32_t refVal = INT32_MAX;
while (true) { while (true) {
if (T_REF_VAL_GET(SMA_ENV_STAT(pEnv)) <= 0) { if ((refVal = T_REF_VAL_GET(SMA_ENV_STAT(pEnv))) <= 0) {
tsdbDebug("vgId:%d drop index %" PRIi64 " since refVal=%d", REPO_ID(pTsdb), indexUid, refVal);
break; break;
} }
tsdbDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", REPO_ID(pTsdb), indexUid, refVal);
taosSsleep(1); taosSsleep(1);
if (++nSleep > SMA_DROP_EXPIRED_TIME) { if (++nSleep > SMA_DROP_EXPIRED_TIME) {
tsdbDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", REPO_ID(pTsdb), indexUid, nSleep,
refVal);
break; break;
}; };
} }
@ -1066,7 +1114,7 @@ static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, int32_t fid) {
static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg) { static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg) {
STsdbCfg *pCfg = REPO_CFG(pTsdb); STsdbCfg *pCfg = REPO_CFG(pTsdb);
const SArray *pDataBlocks = (const SArray *)msg; const SArray *pDataBlocks = (const SArray *)msg;
SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pRSmaEnv); SSmaEnv *pEnv = atomic_load_ptr(&REPO_RSMA_ENV(pTsdb));
int64_t indexUid = SMA_TEST_INDEX_UID; int64_t indexUid = SMA_TEST_INDEX_UID;
if (pEnv == NULL) { if (pEnv == NULL) {
@ -1093,7 +1141,7 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
SSmaStat *pStat = SMA_ENV_STAT(pTsdb->pTSmaEnv); SSmaStat *pStat = SMA_ENV_STAT(pEnv);
SSmaStatItem *pItem = NULL; SSmaStatItem *pItem = NULL;
tsdbRefSmaStat(pTsdb, pStat); tsdbRefSmaStat(pTsdb, pStat);
@ -1241,7 +1289,8 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) {
* @return int32_t * @return int32_t
*/ */
static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) {
SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); SSmaEnv *pEnv = atomic_load_ptr(&REPO_TSMA_ENV(pTsdb));
SSmaStat *pStat = NULL;
if (!pEnv) { if (!pEnv) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
@ -1249,12 +1298,14 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid,
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
tsdbRefSmaStat(pTsdb, SMA_ENV_STAT(pEnv)); pStat = SMA_ENV_STAT(pEnv);
tsdbRefSmaStat(pTsdb, pStat);
SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid));
if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL)) { if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL)) {
// Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if
// it's NULL. // it's NULL.
tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); tsdbUnRefSmaStat(pTsdb, pStat);
terrno = TSDB_CODE_TDB_INVALID_ACTION; terrno = TSDB_CODE_TDB_INVALID_ACTION;
tsdbDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64, REPO_ID(pTsdb), indexUid); tsdbDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64, REPO_ID(pTsdb), indexUid);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
@ -1273,7 +1324,7 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid,
#if 1 #if 1
int8_t smaStat = 0; int8_t smaStat = 0;
if (!tsdbSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query if (!tsdbSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query
tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); tsdbUnRefSmaStat(pTsdb, pStat);
terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
tsdbWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, REPO_ID(pTsdb), indexUid, tsdbWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, REPO_ID(pTsdb), indexUid,
tstrerror(terrno), smaStat); tstrerror(terrno), smaStat);
@ -1291,18 +1342,16 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid,
STSma *pTSma = pItem->pSma; STSma *pTSma = pItem->pSma;
#endif #endif
STSmaReadH tReadH = {0}; STSmaReadH tReadH = {0};
tsdbInitTSmaReadH(&tReadH, pTsdb, pTSma->interval, pTSma->intervalUnit); tsdbInitTSmaReadH(&tReadH, pTsdb, pTSma->interval, pTSma->intervalUnit);
tsdbCloseDBF(&tReadH.dFile); tsdbCloseDBF(&tReadH.dFile);
tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); tsdbUnRefSmaStat(pTsdb, pStat);
tsdbInitTSmaFile(&tReadH, indexUid, querySKey); tsdbInitTSmaFile(&tReadH, indexUid, querySKey);
if (tsdbOpenDBF(SMA_ENV_ENV(pTsdb->pTSmaEnv), &tReadH.dFile) != 0) { if (tsdbOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) {
tsdbWarn("vgId:%d open DBF %s failed since %s", REPO_ID(pTsdb), tReadH.dFile.path, tstrerror(terrno)); tsdbWarn("vgId:%d open DBF %s failed since %s", REPO_ID(pTsdb), tReadH.dFile.path, tstrerror(terrno));
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -1359,6 +1408,60 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid,
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) {
SSmaCfg vCreateSmaReq = {0};
if (tDeserializeSVCreateTSmaReq(pMsg, &vCreateSmaReq) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tsdbWarn("vgId:%d TDMT_VND_CREATE_SMA received but deserialize failed since %s", REPO_ID(pTsdb), terrstr(terrno));
return -1;
}
tsdbDebug("vgId:%d TDMT_VND_CREATE_SMA msg received for %s:%" PRIi64, REPO_ID(pTsdb), vCreateSmaReq.tSma.indexName,
vCreateSmaReq.tSma.indexUid);
// record current timezone of server side
vCreateSmaReq.tSma.timezoneInt = tsTimezone;
if (metaCreateTSma(pTsdb->pMeta, &vCreateSmaReq) < 0) {
// TODO: handle error
tdDestroyTSma(&vCreateSmaReq.tSma);
return -1;
}
tsdbTSmaAdd(pTsdb, 1);
tdDestroyTSma(&vCreateSmaReq.tSma);
// TODO: return directly or go on follow steps?
}
int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) {
SVDropTSmaReq vDropSmaReq = {0};
if (tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
// TODO: send msg to stream computing to drop tSma
// if ((send msg to stream computing) < 0) {
// tdDestroyTSma(&vCreateSmaReq);
// return -1;
// }
//
if (metaDropTSma(pTsdb->pMeta, vDropSmaReq.indexUid) < 0) {
// TODO: handle error
return -1;
}
if (tsdbDropTSmaData(pTsdb, vDropSmaReq.indexUid) < 0) {
// TODO: handle error
return -1;
}
tsdbTSmaSub(pTsdb, 1);
// TODO: return directly or go on follow steps?
}
#if 0 #if 0
/** /**
* @brief Get the start TS key of the last data block of one interval/sliding. * @brief Get the start TS key of the last data block of one interval/sliding.
@ -1404,9 +1507,9 @@ int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg) {
return code; return code;
} }
int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, const char *msg) { int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, SSubmitReq *pMsg) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if ((code = tsdbUpdateExpiredWindowImpl(pTsdb, msg)) < 0) { if ((code = tsdbUpdateExpiredWindowImpl(pTsdb, pMsg)) < 0) {
tsdbWarn("vgId:%d update expired sma window failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); tsdbWarn("vgId:%d update expired sma window failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
} }
return code; return code;
@ -1420,7 +1523,7 @@ int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) {
return code; return code;
} }
int32_t tsdbGetTSmaData(STsdb *pTsdb, char*pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { int32_t tsdbGetTSmaData(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if ((code = tsdbGetTSmaDataImpl(pTsdb, pData, indexUid, querySKey, nMaxResult)) < 0) { if ((code = tsdbGetTSmaDataImpl(pTsdb, pData, indexUid, querySKey, nMaxResult)) < 0) {
tsdbWarn("vgId:%d get tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); tsdbWarn("vgId:%d get tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno));

2
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -47,7 +47,7 @@ int vnodeSyncCommit(SVnode *pVnode) {
static int vnodeCommit(void *arg) { static int vnodeCommit(void *arg) {
SVnode *pVnode = (SVnode *)arg; SVnode *pVnode = (SVnode *)arg;
metaCommit(pVnode->pMeta); // metaCommit(pVnode->pMeta);
tqCommit(pVnode->pTq); tqCommit(pVnode->pTq);
tsdbCommit(pVnode->pTsdb); tsdbCommit(pVnode->pTsdb);

27
source/dnode/vnode/src/vnd/vnodeWrite.c
View File

@ -191,6 +191,10 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
} }
} break; } break;
case TDMT_VND_MQ_CANCEL_CONN: {
if (tqProcessCancelConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
}
} break;
case TDMT_VND_TASK_DEPLOY: { case TDMT_VND_TASK_DEPLOY: {
if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
pMsg->contLen - sizeof(SMsgHead)) < 0) { pMsg->contLen - sizeof(SMsgHead)) < 0) {
@ -202,17 +206,17 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
} }
} break; } break;
case TDMT_VND_CREATE_SMA: { // timeRangeSMA case TDMT_VND_CREATE_SMA: { // timeRangeSMA
#if 1 #if 0
SSmaCfg vCreateSmaReq = {0}; SSmaCfg vCreateSmaReq = {0};
if (tDeserializeSVCreateTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateSmaReq) == NULL) { if (tDeserializeSVCreateTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateSmaReq) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
vWarn("vgId%d: TDMT_VND_CREATE_SMA received but deserialize failed since %s", pVnode->config.vgId, vWarn("vgId:%d TDMT_VND_CREATE_SMA received but deserialize failed since %s", pVnode->config.vgId,
terrstr(terrno)); terrstr(terrno));
return -1; return -1;
} }
vWarn("vgId%d: TDMT_VND_CREATE_SMA received for %s:%" PRIi64, pVnode->config.vgId, vCreateSmaReq.tSma.indexName, vDebug("vgId:%d TDMT_VND_CREATE_SMA msg received for %s:%" PRIi64, pVnode->config.vgId,
vCreateSmaReq.tSma.indexUid); vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid);
// record current timezone of server side // record current timezone of server side
vCreateSmaReq.tSma.timezoneInt = tsTimezone; vCreateSmaReq.tSma.timezoneInt = tsTimezone;
@ -222,19 +226,24 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
tdDestroyTSma(&vCreateSmaReq.tSma); tdDestroyTSma(&vCreateSmaReq.tSma);
return -1; return -1;
} }
// TODO: send msg to stream computing to create tSma
// if ((send msg to stream computing) < 0) { tsdbTSmaAdd(pVnode->pTsdb, 1);
// tdDestroyTSma(&vCreateSmaReq);
// return -1;
// }
tdDestroyTSma(&vCreateSmaReq.tSma); tdDestroyTSma(&vCreateSmaReq.tSma);
// TODO: return directly or go on follow steps? // TODO: return directly or go on follow steps?
#endif #endif
if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
// TODO
}
} break; } break;
case TDMT_VND_CANCEL_SMA: { // timeRangeSMA case TDMT_VND_CANCEL_SMA: { // timeRangeSMA
} break; } break;
case TDMT_VND_DROP_SMA: { // timeRangeSMA case TDMT_VND_DROP_SMA: { // timeRangeSMA
if (tsdbDropTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
// TODO
}
#if 0 #if 0
tsdbTSmaSub(pVnode->pTsdb, 1);
SVDropTSmaReq vDropSmaReq = {0}; SVDropTSmaReq vDropSmaReq = {0};
if (tDeserializeSVDropTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vDropSmaReq) == NULL) { if (tDeserializeSVDropTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vDropSmaReq) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;

2
source/dnode/vnode/test/tsdbSmaTest.cpp
View File

@ -408,7 +408,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
EXPECT_EQ(tdScanAndConvertSubmitMsg(pMsg), TSDB_CODE_SUCCESS); EXPECT_EQ(tdScanAndConvertSubmitMsg(pMsg), TSDB_CODE_SUCCESS);
EXPECT_EQ(tsdbUpdateSmaWindow(pTsdb, (const char *)pMsg), 0); EXPECT_EQ(tsdbUpdateSmaWindow(pTsdb, pMsg), 0);
// init // init
const int32_t tSmaGroupSize = 4; const int32_t tSmaGroupSize = 4;

32
source/libs/executor/inc/executorimpl.h
View File

@ -483,6 +483,7 @@ typedef struct STableIntervalOperatorInfo {
SOptrBasicInfo binfo; // basic info SOptrBasicInfo binfo; // basic info
SGroupResInfo groupResInfo; // multiple results build supporter SGroupResInfo groupResInfo; // multiple results build supporter
SInterval interval; // interval info SInterval interval; // interval info
int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator.
STimeWindow win; // query time range STimeWindow win; // query time range
bool timeWindowInterpo; // interpolation needed or not bool timeWindowInterpo; // interpolation needed or not
char **pRow; // previous row/tuple of already processed datablock char **pRow; // previous row/tuple of already processed datablock
@ -554,6 +555,7 @@ typedef struct SGroupbyOperatorInfo {
SOptrBasicInfo binfo; SOptrBasicInfo binfo;
SArray* pGroupCols; SArray* pGroupCols;
SArray* pGroupColVals; // current group column values, SArray<SGroupKeys> SArray* pGroupColVals; // current group column values, SArray<SGroupKeys>
SNode* pCondition;
bool isInit; // denote if current val is initialized or not bool isInit; // denote if current val is initialized or not
char* keyBuf; // group by keys for hash char* keyBuf; // group by keys for hash
int32_t groupKeyLen; // total group by column width int32_t groupKeyLen; // total group by column width
@ -623,23 +625,6 @@ typedef struct SSortOperatorInfo {
uint64_t totalElapsed; // total elapsed time uint64_t totalElapsed; // total elapsed time
} SSortOperatorInfo; } SSortOperatorInfo;
typedef struct SDistinctDataInfo {
int32_t index;
int32_t type;
int32_t bytes;
} SDistinctDataInfo;
typedef struct SDistinctOperatorInfo {
SHashObj* pSet;
SSDataBlock* pRes;
bool recordNullVal; // has already record the null value, no need to try again
int64_t threshold; // todo remove it
int64_t outputCapacity;// todo remove it
int32_t totalBytes; // todo remove it
char* buf;
SArray* pDistinctDataInfo;
} SDistinctOperatorInfo;
int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); int32_t operatorDummyOpenFn(SOperatorInfo* pOperator);
void operatorDummyCloseFn(void* param, int32_t numOfCols); void operatorDummyCloseFn(void* param, int32_t numOfCols);
int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num);
@ -654,6 +639,10 @@ void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput);
int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows,
char* pData, int32_t compLen, int32_t numOfOutput, int64_t startTs, char* pData, int32_t compLen, int32_t numOfOutput, int64_t startTs,
uint64_t* total, SArray* pColList); uint64_t* total, SArray* pColList);
void doSetOperatorCompleted(SOperatorInfo* pOperator);
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock);
SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity);
SSDataBlock* loadNextDataBlock(void* param);
SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfCols, int32_t repeatTime, SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfCols, int32_t repeatTime,
@ -666,19 +655,21 @@ SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR
SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName,
SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId); SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId);
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlot,
const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock,
SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SArray* pGroupColList, SNode* pCondition, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo); SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, SSDataBlock* pResBlock, SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, SSDataBlock* pResBlock,
int32_t fillType, char* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo); int32_t fillType, char* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createDistinctOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResultBlock, SArray* pSortInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
#if 0
SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,
SExprInfo* pExpr, int32_t numOfOutput); SExprInfo* pExpr, int32_t numOfOutput);
@ -700,6 +691,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI
SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema, SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema,
int32_t numOfOutput); int32_t numOfOutput);
#endif
void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order);

319
source/libs/executor/src/executorimpl.c
View File

@ -51,8 +51,6 @@
#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)
#define MULTI_KEY_DELIM "-"
enum { enum {
TS_JOIN_TS_EQUAL = 0, TS_JOIN_TS_EQUAL = 0,
TS_JOIN_TS_NOT_EQUALS = 1, TS_JOIN_TS_NOT_EQUALS = 1,
@ -221,12 +219,13 @@ static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput);
static void destroyOperatorInfo(SOperatorInfo* pOperator); static void destroyOperatorInfo(SOperatorInfo* pOperator);
static void destroySysTableScannerOperatorInfo(void* param, int32_t numOfOutput); static void destroySysTableScannerOperatorInfo(void* param, int32_t numOfOutput);
static void doSetOperatorCompleted(SOperatorInfo* pOperator) { void doSetOperatorCompleted(SOperatorInfo* pOperator) {
pOperator->status = OP_EXEC_DONE; pOperator->status = OP_EXEC_DONE;
if (pOperator->pTaskInfo != NULL) { if (pOperator->pTaskInfo != NULL) {
setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
} }
} }
#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED)
#define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED)
@ -298,6 +297,10 @@ SSDataBlock* createOutputBuf_rv1(SDataBlockDescNode* pNode) {
idata.info.slotId = pDescNode->slotId; idata.info.slotId = pDescNode->slotId;
idata.info.precision = pDescNode->dataType.precision; idata.info.precision = pDescNode->dataType.precision;
if (IS_VAR_DATA_TYPE(idata.info.type)) {
pBlock->info.hasVarCol = true;
}
taosArrayPush(pBlock->pDataBlock, &idata); taosArrayPush(pBlock->pDataBlock, &idata);
} }
@ -541,8 +544,9 @@ static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultR
existInCurrentResusltRowInfo = false; // this time window created by other timestamp that does not belongs to current table. existInCurrentResusltRowInfo = false; // this time window created by other timestamp that does not belongs to current table.
assert(pResultRowInfo->curPos == -1); assert(pResultRowInfo->curPos == -1);
} else if (pResultRowInfo->size == 1) { } else if (pResultRowInfo->size == 1) {
ASSERT(0); // ASSERT(0);
// existInCurrentResusltRowInfo = (pResultRowInfo->pResult[0] == (*p1)); SResultRowPosition* p = &pResultRowInfo->pPosition[0];
existInCurrentResusltRowInfo = (p->pageId == p1->pageId && p->offset == p1->offset);
} else { // check if current pResultRowInfo contains the existInCurrentResusltRowInfo pResultRow } else { // check if current pResultRowInfo contains the existInCurrentResusltRowInfo pResultRow
SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo); SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo);
int64_t* index = taosHashGet(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); int64_t* index = taosHashGet(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes));
@ -594,6 +598,8 @@ static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultR
int64_t index = pResultRowInfo->curPos; int64_t index = pResultRowInfo->curPos;
SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo); SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo);
taosHashPut(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); taosHashPut(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES);
} else {
pResult = getResultRowByPos(pResultBuf, p1);
} }
// too many time window in query // too many time window in query
@ -977,7 +983,7 @@ void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData*
for (int32_t k = 0; k < numOfOutput; ++k) { for (int32_t k = 0; k < numOfOutput; ++k) {
pCtx[k].startTs = pWin->skey; pCtx[k].startTs = pWin->skey;
// keep it temporarialy // keep it temporarily
bool hasAgg = pCtx[k].input.colDataAggIsSet; bool hasAgg = pCtx[k].input.colDataAggIsSet;
int32_t numOfRows = pCtx[k].input.numOfRows; int32_t numOfRows = pCtx[k].input.numOfRows;
int32_t startOffset = pCtx[k].input.startRowIndex; int32_t startOffset = pCtx[k].input.startRowIndex;
@ -1009,7 +1015,6 @@ void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData*
SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData}; SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};
pCtx[k].sfp.process(&tw, 1, &out); pCtx[k].sfp.process(&tw, 1, &out);
pEntryInfo->numOfRes = 1; pEntryInfo->numOfRes = 1;
pEntryInfo->hasResult = ',';
continue; continue;
} }
@ -1245,11 +1250,7 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunction
} }
static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, SArray* pPseudoList) { static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, SArray* pPseudoList) {
size_t num = 0; size_t num = (pPseudoList != NULL)? taosArrayGetSize(pPseudoList):0;
if (pPseudoList != NULL) {
num = taosArrayGetSize(pPseudoList);
}
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
pCtx[i].pOutput = taosArrayGet(pResult->pDataBlock, i); pCtx[i].pOutput = taosArrayGet(pResult->pDataBlock, i);
} }
@ -1265,6 +1266,12 @@ static void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSData
colDataAssign(pColInfoData, pCtx[k].input.pData[0], pCtx[k].input.numOfRows); colDataAssign(pColInfoData, pCtx[k].input.pData[0], pCtx[k].input.numOfRows);
pResult->info.rows = pCtx[0].input.numOfRows; pResult->info.rows = pCtx[0].input.numOfRows;
} else if (pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE) {
SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, k);
for (int32_t i = 0; i < pSrcBlock->info.rows; ++i) {
colDataAppend(pColInfoData, i, taosVariantGet(&pExpr[k].base.pParam[0].param, pExpr[k].base.pParam[0].param.nType), TSDB_DATA_TYPE_NULL == pExpr[k].base.pParam[0].param.nType);
}
pResult->info.rows = pSrcBlock->info.rows;
} else if (pExpr[k].pExpr->nodeType == QUERY_NODE_OPERATOR) { } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_OPERATOR) {
SArray* pBlockList = taosArrayInit(4, POINTER_BYTES); SArray* pBlockList = taosArrayInit(4, POINTER_BYTES);
taosArrayPush(pBlockList, &pSrcBlock); taosArrayPush(pBlockList, &pSrcBlock);
@ -1291,8 +1298,10 @@ static void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSData
pCtx[k].pOutput = taosArrayGet(pResult->pDataBlock, k); pCtx[k].pOutput = taosArrayGet(pResult->pDataBlock, k);
pCtx[k].offset = pResult->info.rows; // set the start offset pCtx[k].offset = pResult->info.rows; // set the start offset
if (taosArrayGetSize(pPseudoList) > 0) {
int32_t* outputColIndex = taosArrayGet(pPseudoList, 0); int32_t* outputColIndex = taosArrayGet(pPseudoList, 0);
pCtx[k].pTsOutput = (SColumnInfoData*)pCtx[*outputColIndex].pOutput; pCtx[k].pTsOutput = (SColumnInfoData*)pCtx[*outputColIndex].pOutput;
}
int32_t numOfRows = pCtx[k].fpSet.process(&pCtx[k]); int32_t numOfRows = pCtx[k].fpSet.process(&pCtx[k]);
pResult->info.rows += numOfRows; pResult->info.rows += numOfRows;
@ -1307,12 +1316,6 @@ static void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSData
pResult->info.rows = dest.numOfRows; pResult->info.rows = dest.numOfRows;
taosArrayDestroy(pBlockList); taosArrayDestroy(pBlockList);
} }
} else if (pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE) {
SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, k);
for (int32_t i = 0; i < pSrcBlock->info.rows; ++i) {
colDataAppend(pColInfoData, i, taosVariantGet(&pExpr[k].base.pParam[0].param, pExpr[k].base.pParam[0].type), TSDB_DATA_TYPE_NULL == pExpr[k].base.pParam[0].param.nType);
}
pResult->info.rows = pSrcBlock->info.rows;
} else { } else {
ASSERT(0); ASSERT(0);
} }
@ -1510,7 +1513,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
TSKEY* tsCols = NULL; TSKEY* tsCols = NULL;
if (pSDataBlock->pDataBlock != NULL) { if (pSDataBlock->pDataBlock != NULL) {
SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
tsCols = (int64_t*)pColDataInfo->pData; tsCols = (int64_t*)pColDataInfo->pData;
// assert(tsCols[0] == pSDataBlock->info.window.skey && tsCols[pSDataBlock->info.rows - 1] == // assert(tsCols[0] == pSDataBlock->info.window.skey && tsCols[pSDataBlock->info.rows - 1] ==
// pSDataBlock->info.window.ekey); // pSDataBlock->info.window.ekey);
@ -1612,9 +1615,10 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
// window start(end) key interpolation // window start(end) key interpolation
doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, pInfo->order, false); doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep,
pInfo->order, false);
updateTimeWindowInfo(&pInfo->timeWindowData, &win, true); updateTimeWindowInfo(&pInfo->timeWindowData, &nextWin, true);
doApplyFunctions(pInfo->binfo.pCtx, &nextWin, &pInfo->timeWindowData, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); doApplyFunctions(pInfo->binfo.pCtx, &nextWin, &pInfo->timeWindowData, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
} }
@ -3143,11 +3147,6 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD
} }
releaseBufPage(pBuf, bufPage); releaseBufPage(pBuf, bufPage);
/*
* set the number of output results for group by normal columns, the number of output rows usually is 1 except
* the top and bottom query
*/
// buf->numOfRows = (uint16_t)getNumOfResult(pCtx, numOfOutput);
} }
} }
@ -3278,6 +3277,51 @@ void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx,
} }
} }
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) {
if (pFilterNode == NULL) {
return;
}
SFilterInfo* filter = NULL;
// todo move to the initialization function
int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0);
SFilterColumnParam param1 = {.numOfCols = pBlock->info.numOfCols, .pDataBlock = pBlock->pDataBlock};
code = filterSetDataFromSlotId(filter, &param1);
int8_t* rowRes = NULL;
bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols);
SSDataBlock* px = createOneDataBlock(pBlock);
blockDataEnsureCapacity(px, pBlock->info.rows);
// todo extract method
int32_t numOfRow = 0;
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i);
SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i);
numOfRow = 0;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
if (rowRes[j] == 0) {
continue;
}
if (colDataIsNull_s(pSrc, j)) {
colDataAppendNULL(pDst, numOfRow);
} else {
colDataAppend(pDst, numOfRow, colDataGetData(pSrc, j), false);
}
numOfRow += 1;
}
*pSrc = *pDst;
}
pBlock->info.rows = numOfRow;
}
void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, int32_t tableGroupId, SExecTaskInfo* pTaskInfo) { void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, int32_t tableGroupId, SExecTaskInfo* pTaskInfo) {
// for simple group by query without interval, all the tables belong to one group result. // for simple group by query without interval, all the tables belong to one group result.
int64_t uid = 0; int64_t uid = 0;
@ -3486,7 +3530,7 @@ static int32_t doCopyToSDataBlock(SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResI
SResultRowEntryInfo* pEntryInfo = getResultCell(pRow, j, rowCellOffset); SResultRowEntryInfo* pEntryInfo = getResultCell(pRow, j, rowCellOffset);
char* in = GET_ROWCELL_INTERBUF(pEntryInfo); char* in = GET_ROWCELL_INTERBUF(pEntryInfo);
colDataAppend(pColInfoData, nrows, in, pEntryInfo->numOfRes == 0); colDataAppend(pColInfoData, nrows, in, pEntryInfo->isNullRes);
} }
releaseBufPage(pBuf, page); releaseBufPage(pBuf, page);
@ -4701,8 +4745,7 @@ static void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHan
pBlock->info.rows += 1; pBlock->info.rows += 1;
} }
static SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, bool hasVarCol, SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity) {
int32_t capacity) {
blockDataCleanup(pDataBlock); blockDataCleanup(pDataBlock);
while (1) { while (1) {
@ -4723,7 +4766,6 @@ static SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataB
SSDataBlock* loadNextDataBlock(void* param) { SSDataBlock* loadNextDataBlock(void* param) {
SOperatorInfo* pOperator = (SOperatorInfo*)param; SOperatorInfo* pOperator = (SOperatorInfo*)param;
bool newgroup = false; bool newgroup = false;
return pOperator->getNextFn(pOperator, &newgroup); return pOperator->getNextFn(pOperator, &newgroup);
} }
@ -4903,7 +4945,7 @@ static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator, bool* newgroup) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SSortedMergeOperatorInfo* pInfo = pOperator->info; SSortedMergeOperatorInfo* pInfo = pOperator->info;
if (pOperator->status == OP_RES_TO_RETURN) { if (pOperator->status == OP_RES_TO_RETURN) {
return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pInfo->hasVarCol, pInfo->binfo.capacity); return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pInfo->binfo.capacity);
} }
int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
@ -5048,15 +5090,14 @@ static SSDataBlock* doSort(SOperatorInfo* pOperator, bool* newgroup) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SSortOperatorInfo* pInfo = pOperator->info; SSortOperatorInfo* pInfo = pOperator->info;
bool hasVarCol = pInfo->pDataBlock->info.hasVarCol;
if (pOperator->status == OP_RES_TO_RETURN) { if (pOperator->status == OP_RES_TO_RETURN) {
return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, hasVarCol, pInfo->numOfRowsInRes); return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->numOfRowsInRes);
} }
int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, pInfo->bufPageSize, numOfBufPage, pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, pInfo->bufPageSize, numOfBufPage,
pInfo->pDataBlock, "GET_TASKID(pTaskInfo)"); pInfo->pDataBlock, pTaskInfo->id.str);
tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock);
@ -5070,27 +5111,23 @@ static SSDataBlock* doSort(SOperatorInfo* pOperator, bool* newgroup) {
} }
pOperator->status = OP_RES_TO_RETURN; pOperator->status = OP_RES_TO_RETURN;
return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, hasVarCol, pInfo->numOfRowsInRes); return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->numOfRowsInRes);
} }
SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExecTaskInfo* pTaskInfo) {
SExecTaskInfo* pTaskInfo) {
SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
taosMemoryFreeClear(pInfo); goto _error;
taosMemoryFreeClear(pOperator);
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
return NULL;
} }
pInfo->sortBufSize = 1024 * 16; // 1MB, TODO dynamic set the available sort buffer pInfo->sortBufSize = 1024 * 16; // TODO dynamic set the available sort buffer
pInfo->bufPageSize = 1024; pInfo->bufPageSize = 1024;
pInfo->numOfRowsInRes = 1024; pInfo->numOfRowsInRes = 1024;
pInfo->pDataBlock = pResBlock; pInfo->pDataBlock = pResBlock;
pInfo->pSortInfo = pSortInfo; pInfo->pSortInfo = pSortInfo;
pOperator->name = "Sort"; pOperator->name = "SortOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT;
pOperator->blockingOptr = true; pOperator->blockingOptr = true;
pOperator->status = OP_NOT_OPENED; pOperator->status = OP_NOT_OPENED;
@ -5102,6 +5139,12 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR
int32_t code = appendDownstream(pOperator, &downstream, 1); int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator; return pOperator;
_error:
pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pInfo);
taosMemoryFree(pOperator);
return NULL;
} }
static int32_t getTableScanOrder(STableScanInfo* pTableScanInfo) { return pTableScanInfo->order; } static int32_t getTableScanOrder(STableScanInfo* pTableScanInfo) { return pTableScanInfo->order; }
@ -6212,14 +6255,6 @@ static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) {
taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->pSortInfo);
} }
static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) {
SDistinctOperatorInfo* pInfo = (SDistinctOperatorInfo*)param;
taosHashCleanup(pInfo->pSet);
taosMemoryFreeClear(pInfo->buf);
taosArrayDestroy(pInfo->pDistinctDataInfo);
pInfo->pRes = blockDataDestroy(pInfo->pRes);
}
void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) { void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) {
SExchangeInfo* pExInfo = (SExchangeInfo*)param; SExchangeInfo* pExInfo = (SExchangeInfo*)param;
taosArrayDestroy(pExInfo->pSources); taosArrayDestroy(pExInfo->pSources);
@ -6324,7 +6359,7 @@ _error:
} }
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlot,
const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) {
STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
@ -6340,6 +6375,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
pInfo->win.skey = 0; pInfo->win.skey = 0;
pInfo->win.ekey = INT64_MAX; pInfo->win.ekey = INT64_MAX;
pInfo->primaryTsIndex = primaryTsSlot;
int32_t numOfRows = 4096; int32_t numOfRows = 4096;
int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, numOfRows, pResBlock, pTaskInfo->id.str); int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, numOfRows, pResBlock, pTaskInfo->id.str);
initExecTimeWindowInfo(&pInfo->timeWindowData, &pInfo->win); initExecTimeWindowInfo(&pInfo->timeWindowData, &pInfo->win);
@ -6791,146 +6828,7 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo
return pOperator; return pOperator;
} }
static bool initMultiDistinctInfo(SDistinctOperatorInfo* pInfo, SOperatorInfo* pOperator, SSDataBlock* pBlock) {
if (taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput) {
// distinct info already inited
return true;
}
for (int i = 0; i < pOperator->numOfOutput; i++) {
// pInfo->totalBytes += pOperator->pExpr[i].base.colBytes;
}
for (int i = 0; i < pOperator->numOfOutput; i++) {
int numOfBlock = (int)(taosArrayGetSize(pBlock->pDataBlock));
assert(i < numOfBlock);
for (int j = 0; j < numOfBlock; j++) {
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, j);
if (pColDataInfo->info.colId == pOperator->pExpr[i].base.resSchema.colId) {
SDistinctDataInfo item = {.index = j, .type = pColDataInfo->info.type, .bytes = pColDataInfo->info.bytes};
taosArrayInsert(pInfo->pDistinctDataInfo, i, &item);
}
}
}
pInfo->totalBytes += (int32_t)strlen(MULTI_KEY_DELIM) * (pOperator->numOfOutput);
pInfo->buf = taosMemoryCalloc(1, pInfo->totalBytes);
return taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput ? true : false;
}
static void buildMultiDistinctKey(SDistinctOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowId) {
char* p = pInfo->buf;
memset(p, 0, pInfo->totalBytes);
for (int i = 0; i < taosArrayGetSize(pInfo->pDistinctDataInfo); i++) {
SDistinctDataInfo* pDistDataInfo = (SDistinctDataInfo*)taosArrayGet(pInfo->pDistinctDataInfo, i);
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index);
char* val = ((char*)pColDataInfo->pData) + pColDataInfo->info.bytes * rowId;
if (isNull(val, pDistDataInfo->type)) {
p += pDistDataInfo->bytes;
continue;
}
if (IS_VAR_DATA_TYPE(pDistDataInfo->type)) {
memcpy(p, varDataVal(val), varDataLen(val));
p += varDataLen(val);
} else {
memcpy(p, val, pDistDataInfo->bytes);
p += pDistDataInfo->bytes;
}
memcpy(p, MULTI_KEY_DELIM, strlen(MULTI_KEY_DELIM));
p += strlen(MULTI_KEY_DELIM);
}
}
static SSDataBlock* hashDistinct(SOperatorInfo* pOperator, bool* newgroup) {
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
SDistinctOperatorInfo* pInfo = pOperator->info;
SSDataBlock* pRes = pInfo->pRes;
pRes->info.rows = 0;
SSDataBlock* pBlock = NULL;
SOperatorInfo* pDownstream = pOperator->pDownstream[0];
while (1) {
publishOperatorProfEvent(pDownstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
pBlock = pDownstream->getNextFn(pDownstream, newgroup);
publishOperatorProfEvent(pDownstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
doSetOperatorCompleted(pOperator);
break;
}
if (!initMultiDistinctInfo(pInfo, pOperator, pBlock)) {
doSetOperatorCompleted(pOperator);
break;
}
// ensure result output buf
if (pRes->info.rows + pBlock->info.rows > pInfo->outputCapacity) {
int32_t newSize = pRes->info.rows + pBlock->info.rows;
for (int i = 0; i < taosArrayGetSize(pRes->pDataBlock); i++) {
SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, i);
SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, i);
char* tmp = taosMemoryRealloc(pResultColInfoData->pData, newSize * pDistDataInfo->bytes);
if (tmp == NULL) {
return NULL;
} else {
pResultColInfoData->pData = tmp;
}
}
pInfo->outputCapacity = newSize;
}
for (int32_t i = 0; i < pBlock->info.rows; i++) {
buildMultiDistinctKey(pInfo, pBlock, i);
if (taosHashGet(pInfo->pSet, pInfo->buf, pInfo->totalBytes) == NULL) {
int32_t dummy;
taosHashPut(pInfo->pSet, pInfo->buf, pInfo->totalBytes, &dummy, sizeof(dummy));
for (int j = 0; j < taosArrayGetSize(pRes->pDataBlock); j++) {
SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, j); // distinct meta info
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); // src
SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, j); // dist
char* val = ((char*)pColInfoData->pData) + pDistDataInfo->bytes * i;
char* start = pResultColInfoData->pData + pDistDataInfo->bytes * pInfo->pRes->info.rows;
memcpy(start, val, pDistDataInfo->bytes);
}
pRes->info.rows += 1;
}
}
if (pRes->info.rows >= pInfo->threshold) {
break;
}
}
return (pInfo->pRes->info.rows > 0) ? pInfo->pRes : NULL;
}
SOperatorInfo* createDistinctOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo) {
SDistinctOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SDistinctOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
pOperator->resultInfo.capacity = 4096; // todo extract function.
pInfo->totalBytes = 0;
pInfo->buf = NULL;
pInfo->pDistinctDataInfo = taosArrayInit(numOfCols, sizeof(SDistinctDataInfo));
pInfo->pSet = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
pOperator->name = "DistinctOperator";
pOperator->blockingOptr = true;
pOperator->status = OP_NOT_OPENED;
// pOperator->operatorType = DISTINCT;
pOperator->pExpr = pExpr;
pOperator->numOfOutput = numOfCols;
pOperator->info = pInfo;
pOperator->getNextFn = hashDistinct;
pOperator->closeFn = destroyDistinctOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
}
static int32_t getColumnIndexInSource(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr, SColumnInfo* pTagCols) { static int32_t getColumnIndexInSource(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr, SColumnInfo* pTagCols) {
int32_t j = 0; int32_t j = 0;
@ -7066,6 +6964,17 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t*
pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pColNode->colName); pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pColNode->colName);
pExp->base.pParam[0].pCol = createColumn(pColNode->dataBlockId, pColNode->slotId, pType); pExp->base.pParam[0].pCol = createColumn(pColNode->dataBlockId, pColNode->slotId, pType);
pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN;
} else if (nodeType(pTargetNode->pExpr) == QUERY_NODE_VALUE) {
pExp->pExpr->nodeType = QUERY_NODE_VALUE;
SValueNode* pValNode = (SValueNode*)pTargetNode->pExpr;
pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam));
pExp->base.numOfParams = 1;
SDataType* pType = &pValNode->node.resType;
pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pValNode->node.aliasName);
pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE;
valueNodeToVariant(pValNode, &pExp->base.pParam[0].param);
} else if (nodeType(pTargetNode->pExpr) == QUERY_NODE_FUNCTION) { } else if (nodeType(pTargetNode->pExpr) == QUERY_NODE_FUNCTION) {
pExp->pExpr->nodeType = QUERY_NODE_FUNCTION; pExp->pExpr->nodeType = QUERY_NODE_FUNCTION;
SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr;
@ -7110,17 +7019,6 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t*
// pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; // pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN;
// pExp->base.pParam[0].pCol = createColumn(pTargetNode->dataBlockId, pTargetNode->slotId, pType); // pExp->base.pParam[0].pCol = createColumn(pTargetNode->dataBlockId, pTargetNode->slotId, pType);
} else if (nodeType(pTargetNode->pExpr) == QUERY_NODE_VALUE) {
pExp->pExpr->nodeType = QUERY_NODE_VALUE;
SValueNode* pValNode = (SValueNode*)pTargetNode->pExpr;
pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam));
pExp->base.numOfParams = 1;
SDataType* pType = &pValNode->node.resType;
pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pValNode->node.aliasName);
pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE;
valueNodeToVariant(pValNode, &pExp->base.pParam[0].param);
} else { } else {
ASSERT(0); ASSERT(0);
} }
@ -7164,7 +7062,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
int32_t numOfCols = 0; int32_t numOfCols = 0;
tsdbReaderT pDataReader = doCreateDataReader((STableScanPhysiNode*)pPhyNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); tsdbReaderT pDataReader = doCreateDataReader((STableScanPhysiNode*)pPhyNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId);
SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols);
return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count,
pScanPhyNode->reverse, pColList, pScanPhyNode->node.pConditions, pTaskInfo); pScanPhyNode->reverse, pColList, pScanPhyNode->node.pConditions, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pPhyNode)) { } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pPhyNode)) {
@ -7174,8 +7071,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == nodeType(pPhyNode)) { } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == nodeType(pPhyNode)) {
SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table.
int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId);
queryId, taskId);
SArray* tableIdList = extractTableIdList(pTableGroupInfo); SArray* tableIdList = extractTableIdList(pTableGroupInfo);
SSDataBlock* pResBlock = createOutputBuf_rv1(pScanPhyNode->node.pOutputDataBlockDesc); SSDataBlock* pResBlock = createOutputBuf_rv1(pScanPhyNode->node.pOutputDataBlockDesc);
@ -7232,7 +7128,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
if (pAggNode->pGroupKeys != NULL) { if (pAggNode->pGroupKeys != NULL) {
SArray* pColList = extractColumnInfo(pAggNode->pGroupKeys); SArray* pColList = extractColumnInfo(pAggNode->pGroupKeys);
return createGroupOperatorInfo(op, pExprInfo, num, pResBlock, pColList, pTaskInfo, NULL); return createGroupOperatorInfo(op, pExprInfo, num, pResBlock, pColList, pAggNode->node.pConditions, pTaskInfo, NULL);
} else { } else {
return createAggregateOperatorInfo(op, pExprInfo, num, pResBlock, pTaskInfo, pTableGroupInfo); return createAggregateOperatorInfo(op, pExprInfo, num, pResBlock, pTaskInfo, pTableGroupInfo);
} }
@ -7247,18 +7143,21 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
// todo: set the correct primary timestamp key column
int32_t num = 0; int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num);
SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc);
SInterval interval = {.interval = pIntervalPhyNode->interval, SInterval interval = {
.interval = pIntervalPhyNode->interval,
.sliding = pIntervalPhyNode->sliding, .sliding = pIntervalPhyNode->sliding,
.intervalUnit = pIntervalPhyNode->intervalUnit, .intervalUnit = pIntervalPhyNode->intervalUnit,
.slidingUnit = pIntervalPhyNode->slidingUnit, .slidingUnit = pIntervalPhyNode->slidingUnit,
.offset = pIntervalPhyNode->offset, .offset = pIntervalPhyNode->offset,
.precision = TSDB_TIME_PRECISION_MILLI}; .precision = pIntervalPhyNode->precision
return createIntervalOperatorInfo(op, pExprInfo, num, pResBlock, &interval, pTableGroupInfo, pTaskInfo); };
int32_t primaryTsSlotId = ((SColumnNode*) pIntervalPhyNode->pTspk)->slotId;
return createIntervalOperatorInfo(op, pExprInfo, num, pResBlock, &interval, primaryTsSlotId, pTableGroupInfo, pTaskInfo);
} }
} else if (QUERY_NODE_PHYSICAL_PLAN_SORT == nodeType(pPhyNode)) { } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == nodeType(pPhyNode)) {
size_t size = LIST_LENGTH(pPhyNode->pChildren); size_t size = LIST_LENGTH(pPhyNode->pChildren);

125
source/libs/executor/src/groupoperator.c
View File

@ -106,7 +106,7 @@ static bool groupKeyCompare(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, in
return true; return true;
} }
static void keepGroupKeys(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) { static void recordNewGroupKeys(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) {
SColumnDataAgg* pColAgg = NULL; SColumnDataAgg* pColAgg = NULL;
for (int32_t i = 0; i < numOfGroupCols; ++i) { for (int32_t i = 0; i < numOfGroupCols; ++i) {
@ -131,7 +131,7 @@ static void keepGroupKeys(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, int3
} }
} }
static int32_t generatedHashKey(void* pKey, int32_t* length, SArray* pGroupColVals) { static int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) {
ASSERT(pKey != NULL); ASSERT(pKey != NULL);
size_t numOfGroupCols = taosArrayGetSize(pGroupColVals); size_t numOfGroupCols = taosArrayGetSize(pGroupColVals);
@ -155,8 +155,7 @@ static int32_t generatedHashKey(void* pKey, int32_t* length, SArray* pGroupColVa
} }
} }
*length = (pStart - (char*)pKey); return (int32_t) (pStart - (char*)pKey);
return 0;
} }
// assign the group keys or user input constant values if required // assign the group keys or user input constant values if required
@ -170,11 +169,12 @@ static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t
char* dest = GET_ROWCELL_INTERBUF(pEntryInfo); char* dest = GET_ROWCELL_INTERBUF(pEntryInfo);
char* data = colDataGetData(pColInfoData, rowIndex); char* data = colDataGetData(pColInfoData, rowIndex);
// set result exists, todo refactor
memcpy(dest, data, pColInfoData->info.bytes); memcpy(dest, data, pColInfoData->info.bytes);
pEntryInfo->hasResult = DATA_SET_FLAG; } else { // it is a NULL value
pEntryInfo->numOfRes = 1; pEntryInfo->isNullRes = 1;
} }
pEntryInfo->numOfRes = 1;
} }
} }
} }
@ -197,7 +197,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
for (int32_t j = 0; j < pBlock->info.rows; ++j) { for (int32_t j = 0; j < pBlock->info.rows; ++j) {
// Compare with the previous row of this column, and do not set the output buffer again if they are identical. // Compare with the previous row of this column, and do not set the output buffer again if they are identical.
if (!pInfo->isInit) { if (!pInfo->isInit) {
keepGroupKeys(pInfo, pBlock, j, numOfGroupCols); recordNewGroupKeys(pInfo, pBlock, j, numOfGroupCols);
pInfo->isInit = true; pInfo->isInit = true;
num++; num++;
continue; continue;
@ -209,7 +209,14 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
continue; continue;
} }
/*int32_t ret = */ generatedHashKey(pInfo->keyBuf, &len, pInfo->pGroupColVals); // The first row of a new block does not belongs to the previous existed group
if (!equal && j == 0) {
num++;
recordNewGroupKeys(pInfo, pBlock, j, numOfGroupCols);
continue;
}
len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals);
int32_t ret = setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, 0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); int32_t ret = setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, 0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup);
if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code
longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
@ -220,12 +227,12 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
// assign the group keys or user input constant values if required // assign the group keys or user input constant values if required
doAssignGroupKeys(pCtx, pOperator->numOfOutput, pBlock->info.rows, rowIndex); doAssignGroupKeys(pCtx, pOperator->numOfOutput, pBlock->info.rows, rowIndex);
keepGroupKeys(pInfo, pBlock, j, numOfGroupCols); recordNewGroupKeys(pInfo, pBlock, j, numOfGroupCols);
num = 1; num = 1;
} }
if (num > 0) { if (num > 0) {
/*int32_t ret = */ generatedHashKey(pInfo->keyBuf, &len, pInfo->pGroupColVals); len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals);
int32_t ret = int32_t ret =
setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len,
0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); 0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup);
@ -245,13 +252,14 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou
} }
SGroupbyOperatorInfo* pInfo = pOperator->info; SGroupbyOperatorInfo* pInfo = pOperator->info;
SSDataBlock* pRes = pInfo->binfo.pRes;
if (pOperator->status == OP_RES_TO_RETURN) { if (pOperator->status == OP_RES_TO_RETURN) {
toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pInfo->binfo.pRes, pInfo->binfo.capacity, toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pRes, pInfo->binfo.capacity, pInfo->binfo.rowCellInfoOffset);
pInfo->binfo.rowCellInfoOffset); if (pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
pOperator->status = OP_EXEC_DONE; pOperator->status = OP_EXEC_DONE;
} }
return pInfo->binfo.pRes; return pRes;
} }
int32_t order = TSDB_ORDER_ASC; int32_t order = TSDB_ORDER_ASC;
@ -283,18 +291,28 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou
// pInfo->binfo.rowCellInfoOffset); // pInfo->binfo.rowCellInfoOffset);
// } // }
blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->binfo.capacity); blockDataEnsureCapacity(pRes, pInfo->binfo.capacity);
initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo);
toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pInfo->binfo.pRes, pInfo->binfo.capacity,
pInfo->binfo.rowCellInfoOffset); while(1) {
if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pRes, pInfo->binfo.capacity, pInfo->binfo.rowCellInfoOffset);
doFilter(pInfo->pCondition, pRes);
bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo);
if (!hasRemain) {
pOperator->status = OP_EXEC_DONE; pOperator->status = OP_EXEC_DONE;
break;
}
if (pRes->info.rows > 0) {
break;
}
} }
return pInfo->binfo.pRes; return pInfo->binfo.pRes;
} }
SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SExecTaskInfo* pTaskInfo, SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SNode* pCondition, SExecTaskInfo* pTaskInfo,
const STableGroupInfo* pTableGroupInfo) { const STableGroupInfo* pTableGroupInfo) {
SGroupbyOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupbyOperatorInfo)); SGroupbyOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupbyOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
@ -303,6 +321,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx
} }
pInfo->pGroupCols = pGroupColList; pInfo->pGroupCols = pGroupColList;
pInfo->pCondition = pCondition;
initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, 4096, pResultBlock, pTaskInfo->id.str); initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, 4096, pResultBlock, pTaskInfo->id.str);
initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8);
@ -326,7 +345,73 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx
return pOperator; return pOperator;
_error: _error:
pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFreeClear(pInfo); taosMemoryFreeClear(pInfo);
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
return NULL; return NULL;
} }
static SSDataBlock* doPartitionData(SOperatorInfo* pOperator, bool* newgroup) {
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SSortOperatorInfo* pInfo = pOperator->info;
bool hasVarCol = pInfo->pDataBlock->info.hasVarCol;
if (pOperator->status == OP_RES_TO_RETURN) {
return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->numOfRowsInRes);
}
int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, pInfo->bufPageSize, numOfBufPage,
pInfo->pDataBlock, pTaskInfo->id.str);
tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock);
SGenericSource* ps = taosMemoryCalloc(1, sizeof(SGenericSource));
ps->param = pOperator->pDownstream[0];
tsortAddSource(pInfo->pSortHandle, ps);
int32_t code = tsortOpen(pInfo->pSortHandle);
if (code != TSDB_CODE_SUCCESS) {
longjmp(pTaskInfo->env, terrno);
}
pOperator->status = OP_RES_TO_RETURN;
return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->numOfRowsInRes);
}
SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResultBlock, SArray* pSortInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) {
SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
pInfo->sortBufSize = 1024 * 16; // TODO dynamic set the available sort buffer
pInfo->bufPageSize = 1024;
pInfo->numOfRowsInRes = 1024;
pInfo->pDataBlock = pResultBlock;
pInfo->pSortInfo = pSortInfo;
pOperator->name = "PartitionOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION;
pOperator->blockingOptr = true;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->pTaskInfo = pTaskInfo;
pOperator->getNextFn = doPartitionData;
// pOperator->closeFn = destroyOrderOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
_error:
pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pInfo);
taosMemoryFree(pOperator);
return NULL;
}

35
source/libs/executor/src/scanoperator.c
View File

@ -34,6 +34,7 @@
#define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN)
#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC))
void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
for (int32_t i = 0; i < numOfOutput; ++i) { for (int32_t i = 0; i < numOfOutput; ++i) {
SWITCH_ORDER(pCtx[i].order); SWITCH_ORDER(pCtx[i].order);
@ -91,39 +92,7 @@ int32_t loadDataBlock(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo,
taosArraySet(pBlock->pDataBlock, pColMatchInfo->targetSlotId, p); taosArraySet(pBlock->pDataBlock, pColMatchInfo->targetSlotId, p);
} }
if (pTableScanInfo->pFilterNode != NULL) { doFilter(pTableScanInfo->pFilterNode, pBlock);
SFilterInfo* filter = NULL;
int32_t code = filterInitFromNode((SNode*)pTableScanInfo->pFilterNode, &filter, 0);
SFilterColumnParam param1 = {.numOfCols = pBlock->info.numOfCols, .pDataBlock = pBlock->pDataBlock};
code = filterSetDataFromSlotId(filter, &param1);
int8_t* rowRes = NULL;
bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols);
SSDataBlock* px = createOneDataBlock(pBlock);
blockDataEnsureCapacity(px, pBlock->info.rows);
int32_t numOfRow = 0;
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i);
SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i);
numOfRow = 0;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
if (rowRes[j] == 0) {
continue;
}
colDataAppend(pDst, numOfRow, colDataGetData(pSrc, j), false);
numOfRow += 1;
}
*pSrc = *pDst;
}
pBlock->info.rows = numOfRow;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

6
source/libs/executor/src/tsimplehash.c
View File

@ -37,7 +37,7 @@ typedef struct SHNode {
char data[]; char data[];
} SHNode; } SHNode;
typedef struct SSHashObj { struct SSHashObj {
SHNode **hashList; SHNode **hashList;
size_t capacity; // number of slots size_t capacity; // number of slots
int64_t size; // number of elements in hash table int64_t size; // number of elements in hash table
@ -45,7 +45,7 @@ typedef struct SSHashObj {
_equal_fn_t equalFp; // equal function _equal_fn_t equalFp; // equal function
int32_t keyLen; int32_t keyLen;
int32_t dataLen; int32_t dataLen;
} SSHashObj; };
static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
int32_t len = MIN(length, HASH_MAX_CAPACITY); int32_t len = MIN(length, HASH_MAX_CAPACITY);
@ -107,7 +107,7 @@ static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pDat
return pNewNode; return pNewNode;
} }
void taosHashTableResize(SSHashObj *pHashObj) { static void taosHashTableResize(SSHashObj *pHashObj) {
if (!HASH_NEED_RESIZE(pHashObj)) { if (!HASH_NEED_RESIZE(pHashObj)) {
return; return;
} }

18
source/libs/function/src/builtinsimpl.c
View File

@ -25,7 +25,6 @@
break; \ break; \
} \ } \
(_info)->numOfRes = (res); \ (_info)->numOfRes = (res); \
(_info)->hasResult = DATA_SET_FLAG; \
} while (0) } while (0)
typedef struct SSumRes { typedef struct SSumRes {
@ -49,11 +48,11 @@ bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
return true; return true;
} }
static void doFinalizer(SResultRowEntryInfo* pResInfo) { cleanupResultRowEntry(pResInfo); }
void functionFinalize(SqlFunctionCtx *pCtx) { void functionFinalize(SqlFunctionCtx *pCtx) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
doFinalizer(pResInfo);
cleanupResultRowEntry(pResInfo);
pResInfo->isNullRes = (pResInfo->numOfRes == 0)? 1:0;
} }
bool getCountFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { bool getCountFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
@ -715,7 +714,6 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) {
} }
SET_VAL(pResInfo, notNullElems, 1); SET_VAL(pResInfo, notNullElems, 1);
pResInfo->hasResult = DATA_SET_FLAG;
} }
// TODO set the correct parameter. // TODO set the correct parameter.
@ -775,9 +773,7 @@ int32_t firstFunction(SqlFunctionCtx *pCtx) {
// DO_UPDATE_TAG_COLUMNS(pCtx, k); // DO_UPDATE_TAG_COLUMNS(pCtx, k);
// } // }
pResInfo->hasResult = DATA_SET_FLAG;
pResInfo->complete = true; pResInfo->complete = true;
numOfElems++; numOfElems++;
break; break;
} }
@ -815,8 +811,6 @@ int32_t lastFunction(SqlFunctionCtx *pCtx) {
// TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; // TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0;
// DO_UPDATE_TAG_COLUMNS(pCtx, ts); // DO_UPDATE_TAG_COLUMNS(pCtx, ts);
pResInfo->hasResult = DATA_SET_FLAG;
pResInfo->complete = true; // set query completed on this column pResInfo->complete = true; // set query completed on this column
numOfElems++; numOfElems++;
break; break;
@ -830,10 +824,8 @@ int32_t lastFunction(SqlFunctionCtx *pCtx) {
char* data = colDataGetData(pInputCol, i); char* data = colDataGetData(pInputCol, i);
TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0;
if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { if (pResInfo->numOfRes == 0 || (*(TSKEY*)buf) < ts) {
pResInfo->hasResult = DATA_SET_FLAG;
memcpy(buf, data, pCtx->inputBytes); memcpy(buf, data, pCtx->inputBytes);
*(TSKEY*)buf = ts; *(TSKEY*)buf = ts;
// DO_UPDATE_TAG_COLUMNS(pCtx, ts); // DO_UPDATE_TAG_COLUMNS(pCtx, ts);
} }
@ -917,7 +909,7 @@ int32_t diffFunction(SqlFunctionCtx *pCtx) {
colDataAppendInt32(pOutput, pos, &delta); colDataAppendInt32(pOutput, pos, &delta);
} }
if (tsList != NULL) { if (pTsOutput != NULL) {
colDataAppendInt64(pTsOutput, pos, &tsList[i]); colDataAppendInt64(pTsOutput, pos, &tsList[i]);
} }
} }

2
source/libs/function/src/taggfunction.c
View File

@ -208,7 +208,7 @@ int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock
SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i);
SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[i]); SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[i]);
if (!pResInfo->hasResult) { if (pResInfo->numOfRes == 0) {
for(int32_t j = 0; j < pResInfo->numOfRes; ++j) { for(int32_t j = 0; j < pResInfo->numOfRes; ++j) {
colDataAppend(pCol, j, NULL, true); // TODO add set null data api colDataAppend(pCol, j, NULL, true); // TODO add set null data api
} }

8
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -283,6 +283,12 @@ static SNode* logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) {
return (SNode*)pDst; return (SNode*)pDst;
} }
static SNode* logicPartitionCopy(const SPartitionLogicNode* pSrc, SPartitionLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
CLONE_NODE_LIST_FIELD(pPartitionKeys);
return (SNode*)pDst;
}
static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) { static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) {
CLONE_NODE_FIELD(pNode); CLONE_NODE_FIELD(pNode);
COPY_SCALAR_FIELD(subplanType); COPY_SCALAR_FIELD(subplanType);
@ -367,6 +373,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) {
return logicWindowCopy((const SWindowLogicNode*)pNode, (SWindowLogicNode*)pDst); return logicWindowCopy((const SWindowLogicNode*)pNode, (SWindowLogicNode*)pDst);
case QUERY_NODE_LOGIC_PLAN_SORT: case QUERY_NODE_LOGIC_PLAN_SORT:
return logicSortCopy((const SSortLogicNode*)pNode, (SSortLogicNode*)pDst); return logicSortCopy((const SSortLogicNode*)pNode, (SSortLogicNode*)pDst);
case QUERY_NODE_LOGIC_PLAN_PARTITION:
return logicPartitionCopy((const SPartitionLogicNode*)pNode, (SPartitionLogicNode*)pDst);
case QUERY_NODE_LOGIC_SUBPLAN: case QUERY_NODE_LOGIC_SUBPLAN:
return logicSubplanCopy((const SLogicSubplan*)pNode, (SLogicSubplan*)pDst); return logicSubplanCopy((const SLogicSubplan*)pNode, (SLogicSubplan*)pDst);
default: default:

81
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -198,6 +198,10 @@ const char* nodesNodeName(ENodeType type) {
return "PhysiInterval"; return "PhysiInterval";
case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:
return "PhysiSessionWindow"; return "PhysiSessionWindow";
case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW:
return "PhysiStateWindow";
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
return "PhysiPartition";
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return "PhysiDispatch"; return "PhysiDispatch";
case QUERY_NODE_PHYSICAL_PLAN_INSERT: case QUERY_NODE_PHYSICAL_PLAN_INSERT:
@ -990,6 +994,7 @@ static int32_t jsonToPhysiExchangeNode(const SJson* pJson, void* pObj) {
static const char* jkSortPhysiPlanExprs = "Exprs"; static const char* jkSortPhysiPlanExprs = "Exprs";
static const char* jkSortPhysiPlanSortKeys = "SortKeys"; static const char* jkSortPhysiPlanSortKeys = "SortKeys";
static const char* jkSortPhysiPlanTargets = "Targets";
static int32_t physiSortNodeToJson(const void* pObj, SJson* pJson) { static int32_t physiSortNodeToJson(const void* pObj, SJson* pJson) {
const SSortPhysiNode* pNode = (const SSortPhysiNode*)pObj; const SSortPhysiNode* pNode = (const SSortPhysiNode*)pObj;
@ -1001,6 +1006,9 @@ static int32_t physiSortNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkSortPhysiPlanSortKeys, pNode->pSortKeys); code = nodeListToJson(pJson, jkSortPhysiPlanSortKeys, pNode->pSortKeys);
} }
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkSortPhysiPlanTargets, pNode->pTargets);
}
return code; return code;
} }
@ -1015,6 +1023,9 @@ static int32_t jsonToPhysiSortNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSortPhysiPlanSortKeys, &pNode->pSortKeys); code = jsonToNodeList(pJson, jkSortPhysiPlanSortKeys, &pNode->pSortKeys);
} }
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSortPhysiPlanTargets, &pNode->pTargets);
}
return code; return code;
} }
@ -1147,6 +1158,68 @@ static int32_t jsonToPhysiSessionWindowNode(const SJson* pJson, void* pObj) {
return code; return code;
} }
static const char* jkStateWindowPhysiPlanStateKey = "StateKey";
static int32_t physiStateWindowNodeToJson(const void* pObj, SJson* pJson) {
const SStateWinodwPhysiNode* pNode = (const SStateWinodwPhysiNode*)pObj;
int32_t code = physiWindowNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkStateWindowPhysiPlanStateKey, nodeToJson, pNode->pStateKey);
}
return code;
}
static int32_t jsonToPhysiStateWindowNode(const SJson* pJson, void* pObj) {
SStateWinodwPhysiNode* pNode = (SStateWinodwPhysiNode*)pObj;
int32_t code = jsonToPhysiWindowNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkStateWindowPhysiPlanStateKey, &pNode->pStateKey);
}
return code;
}
static const char* jkPartitionPhysiPlanExprs = "Exprs";
static const char* jkPartitionPhysiPlanPartitionKeys = "PartitionKeys";
static const char* jkPartitionPhysiPlanTargets = "Targets";
static int32_t physiPartitionNodeToJson(const void* pObj, SJson* pJson) {
const SPartitionPhysiNode* pNode = (const SPartitionPhysiNode*)pObj;
int32_t code = physicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkPartitionPhysiPlanExprs, pNode->pExprs);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkPartitionPhysiPlanPartitionKeys, pNode->pPartitionKeys);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkPartitionPhysiPlanTargets, pNode->pTargets);
}
return code;
}
static int32_t jsonToPhysiPartitionNode(const SJson* pJson, void* pObj) {
SPartitionPhysiNode* pNode = (SPartitionPhysiNode*)pObj;
int32_t code = jsonToPhysicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkPartitionPhysiPlanExprs, &pNode->pExprs);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkPartitionPhysiPlanPartitionKeys, &pNode->pPartitionKeys);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkPartitionPhysiPlanTargets, &pNode->pTargets);
}
return code;
}
static const char* jkDataSinkInputDataBlockDesc = "InputDataBlockDesc"; static const char* jkDataSinkInputDataBlockDesc = "InputDataBlockDesc";
static int32_t physicDataSinkNodeToJson(const void* pObj, SJson* pJson) { static int32_t physicDataSinkNodeToJson(const void* pObj, SJson* pJson) {
@ -2420,6 +2493,10 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return physiIntervalNodeToJson(pObj, pJson); return physiIntervalNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:
return physiSessionWindowNodeToJson(pObj, pJson); return physiSessionWindowNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW:
return physiStateWindowNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
return physiPartitionNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return physiDispatchNodeToJson(pObj, pJson); return physiDispatchNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_INSERT: case QUERY_NODE_PHYSICAL_PLAN_INSERT:
@ -2512,6 +2589,10 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToPhysiIntervalNode(pJson, pObj); return jsonToPhysiIntervalNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:
return jsonToPhysiSessionWindowNode(pJson, pObj); return jsonToPhysiSessionWindowNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW:
return jsonToPhysiStateWindowNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
return jsonToPhysiPartitionNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return jsonToPhysiDispatchNode(pJson, pObj); return jsonToPhysiDispatchNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_SUBPLAN: case QUERY_NODE_PHYSICAL_SUBPLAN:

11
source/libs/nodes/src/nodesTraverseFuncs.c
View File

@ -78,7 +78,7 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker
res = walkNode(((SOrderByExprNode*)pNode)->pExpr, order, walker, pContext); res = walkNode(((SOrderByExprNode*)pNode)->pExpr, order, walker, pContext);
break; break;
case QUERY_NODE_STATE_WINDOW: case QUERY_NODE_STATE_WINDOW:
res = walkNode(((SStateWindowNode*)pNode)->pCol, order, walker, pContext); res = walkNode(((SStateWindowNode*)pNode)->pExpr, order, walker, pContext);
break; break;
case QUERY_NODE_SESSION_WINDOW: { case QUERY_NODE_SESSION_WINDOW: {
SSessionWindowNode* pSession = (SSessionWindowNode*)pNode; SSessionWindowNode* pSession = (SSessionWindowNode*)pNode;
@ -212,7 +212,7 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit
res = rewriteNode(&(((SOrderByExprNode*)pNode)->pExpr), order, rewriter, pContext); res = rewriteNode(&(((SOrderByExprNode*)pNode)->pExpr), order, rewriter, pContext);
break; break;
case QUERY_NODE_STATE_WINDOW: case QUERY_NODE_STATE_WINDOW:
res = rewriteNode(&(((SStateWindowNode*)pNode)->pCol), order, rewriter, pContext); res = rewriteNode(&(((SStateWindowNode*)pNode)->pExpr), order, rewriter, pContext);
break; break;
case QUERY_NODE_SESSION_WINDOW: case QUERY_NODE_SESSION_WINDOW:
res = rewriteNode(&(((SSessionWindowNode*)pNode)->pCol), order, rewriter, pContext); res = rewriteNode(&(((SSessionWindowNode*)pNode)->pCol), order, rewriter, pContext);
@ -301,10 +301,10 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa
case SQL_CLAUSE_GROUP_BY: case SQL_CLAUSE_GROUP_BY:
nodesWalkExpr(pSelect->pHaving, walker, pContext); nodesWalkExpr(pSelect->pHaving, walker, pContext);
case SQL_CLAUSE_HAVING: case SQL_CLAUSE_HAVING:
case SQL_CLAUSE_DISTINCT:
nodesWalkExprs(pSelect->pOrderByList, walker, pContext); nodesWalkExprs(pSelect->pOrderByList, walker, pContext);
case SQL_CLAUSE_ORDER_BY: case SQL_CLAUSE_ORDER_BY:
nodesWalkExprs(pSelect->pProjectionList, walker, pContext); nodesWalkExprs(pSelect->pProjectionList, walker, pContext);
case SQL_CLAUSE_SELECT:
default: default:
break; break;
} }
@ -330,9 +330,10 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit
case SQL_CLAUSE_GROUP_BY: case SQL_CLAUSE_GROUP_BY:
nodesRewriteExpr(&(pSelect->pHaving), rewriter, pContext); nodesRewriteExpr(&(pSelect->pHaving), rewriter, pContext);
case SQL_CLAUSE_HAVING: case SQL_CLAUSE_HAVING:
nodesRewriteExprs(pSelect->pProjectionList, rewriter, pContext); case SQL_CLAUSE_DISTINCT:
case SQL_CLAUSE_SELECT:
nodesRewriteExprs(pSelect->pOrderByList, rewriter, pContext); nodesRewriteExprs(pSelect->pOrderByList, rewriter, pContext);
case SQL_CLAUSE_ORDER_BY:
nodesRewriteExprs(pSelect->pProjectionList, rewriter, pContext);
default: default:
break; break;
} }

8
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -169,6 +169,8 @@ SNodeptr nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SWindowLogicNode)); return makeNode(type, sizeof(SWindowLogicNode));
case QUERY_NODE_LOGIC_PLAN_SORT: case QUERY_NODE_LOGIC_PLAN_SORT:
return makeNode(type, sizeof(SSortLogicNode)); return makeNode(type, sizeof(SSortLogicNode));
case QUERY_NODE_LOGIC_PLAN_PARTITION:
return makeNode(type, sizeof(SPartitionLogicNode));
case QUERY_NODE_LOGIC_SUBPLAN: case QUERY_NODE_LOGIC_SUBPLAN:
return makeNode(type, sizeof(SLogicSubplan)); return makeNode(type, sizeof(SLogicSubplan));
case QUERY_NODE_LOGIC_PLAN: case QUERY_NODE_LOGIC_PLAN:
@ -197,6 +199,10 @@ SNodeptr nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SIntervalPhysiNode)); return makeNode(type, sizeof(SIntervalPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:
return makeNode(type, sizeof(SSessionWinodwPhysiNode)); return makeNode(type, sizeof(SSessionWinodwPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW:
return makeNode(type, sizeof(SStateWinodwPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
return makeNode(type, sizeof(SPartitionPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return makeNode(type, sizeof(SDataDispatcherNode)); return makeNode(type, sizeof(SDataDispatcherNode));
case QUERY_NODE_PHYSICAL_PLAN_INSERT: case QUERY_NODE_PHYSICAL_PLAN_INSERT:
@ -302,7 +308,7 @@ void nodesDestroyNode(SNodeptr pNode) {
case QUERY_NODE_LIMIT: // no pointer field case QUERY_NODE_LIMIT: // no pointer field
break; break;
case QUERY_NODE_STATE_WINDOW: case QUERY_NODE_STATE_WINDOW:
nodesDestroyNode(((SStateWindowNode*)pNode)->pCol); nodesDestroyNode(((SStateWindowNode*)pNode)->pExpr);
break; break;
case QUERY_NODE_SESSION_WINDOW: { case QUERY_NODE_SESSION_WINDOW: {
SSessionWindowNode* pSession = (SSessionWindowNode*)pNode; SSessionWindowNode* pSession = (SSessionWindowNode*)pNode;

31
source/libs/parser/inc/parAst.h
View File

@ -35,7 +35,7 @@ typedef struct SAstCreateContext {
} SAstCreateContext; } SAstCreateContext;
typedef enum EDatabaseOptionType { typedef enum EDatabaseOptionType {
DB_OPTION_BLOCKS = 0, DB_OPTION_BLOCKS = 1,
DB_OPTION_CACHE, DB_OPTION_CACHE,
DB_OPTION_CACHELAST, DB_OPTION_CACHELAST,
DB_OPTION_COMP, DB_OPTION_COMP,
@ -52,26 +52,22 @@ typedef enum EDatabaseOptionType {
DB_OPTION_VGROUPS, DB_OPTION_VGROUPS,
DB_OPTION_SINGLE_STABLE, DB_OPTION_SINGLE_STABLE,
DB_OPTION_STREAM_MODE, DB_OPTION_STREAM_MODE,
DB_OPTION_RETENTIONS, DB_OPTION_RETENTIONS
DB_OPTION_MAX
} EDatabaseOptionType; } EDatabaseOptionType;
typedef enum ETableOptionType { typedef enum ETableOptionType {
TABLE_OPTION_KEEP = 0, TABLE_OPTION_KEEP = 1,
TABLE_OPTION_TTL, TABLE_OPTION_TTL,
TABLE_OPTION_COMMENT, TABLE_OPTION_COMMENT,
TABLE_OPTION_SMA, TABLE_OPTION_SMA,
TABLE_OPTION_FILE_FACTOR, TABLE_OPTION_FILE_FACTOR,
TABLE_OPTION_DELAY, TABLE_OPTION_DELAY
TABLE_OPTION_MAX
} ETableOptionType; } ETableOptionType;
typedef struct SAlterOption { typedef struct SAlterOption {
int32_t type; int32_t type;
SToken val; SValueNode* pVal;
SNodeList* pKeep; SNodeList* pList;
} SAlterOption; } SAlterOption;
extern SToken nil_token; extern SToken nil_token;
@ -97,13 +93,14 @@ SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNo
SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList); SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList);
SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList); SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList);
SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2);
SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias); SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias);
SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias); SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias);
SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond); SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond);
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset);
SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder);
SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap); SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap);
SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol); SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr);
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill); SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill);
SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode); SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
@ -119,20 +116,12 @@ SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit);
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable); SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable);
SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight); SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight);
SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt); SNode* createDatabaseOptions(SAstCreateContext* pCxt);
SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt);
SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal);
SNode* setDatabaseKeepOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pKeep);
SNode* setDatabaseAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption); SNode* setDatabaseAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption);
SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions); SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions);
SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pDbName); SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pDbName);
SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions); SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions);
SNode* createDefaultTableOptions(SAstCreateContext* pCxt); SNode* createTableOptions(SAstCreateContext* pCxt);
SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt);
SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal);
SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pSma);
SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs);
SNode* setTableKeepOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pKeep);
SNode* setTableAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption); SNode* setTableAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption);
SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment); SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment);
SDataType createDataType(uint8_t type); SDataType createDataType(uint8_t type);

97
source/libs/parser/inc/sql.y
View File

@ -128,44 +128,59 @@ not_exists_opt(A) ::= .
exists_opt(A) ::= IF EXISTS. { A = true; } exists_opt(A) ::= IF EXISTS. { A = true; }
exists_opt(A) ::= . { A = false; } exists_opt(A) ::= . { A = false; }
db_options(A) ::= . { A = createDefaultDatabaseOptions(pCxt); } db_options(A) ::= . { A = createDatabaseOptions(pCxt); }
db_options(A) ::= db_options(B) BLOCKS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_BLOCKS, &C); } db_options(A) ::= db_options(B) BLOCKS NK_INTEGER(C). { ((SDatabaseOptions*)B)->pNumOfBlocks = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
db_options(A) ::= db_options(B) CACHE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHE, &C); } db_options(A) ::= db_options(B) CACHE NK_INTEGER(C). { ((SDatabaseOptions*)B)->pCacheBlockSize = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
db_options(A) ::= db_options(B) CACHELAST NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELAST, &C); } db_options(A) ::= db_options(B) CACHELAST NK_INTEGER(C). { ((SDatabaseOptions*)B)->pCachelast = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
db_options(A) ::= db_options(B) COMP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMP, &C); } db_options(A) ::= db_options(B) COMP NK_INTEGER(C). { ((SDatabaseOptions*)B)->pCompressionLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
db_options(A) ::= db_options(B) DAYS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); } db_options(A) ::= db_options(B) DAYS NK_INTEGER(C). { ((SDatabaseOptions*)B)->pDaysPerFile = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
db_options(A) ::= db_options(B) FSYNC NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_FSYNC, &C); } db_options(A) ::= db_options(B) DAYS NK_VARIABLE(C). { ((SDatabaseOptions*)B)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, &C); A = B; }
db_options(A) ::= db_options(B) MAXROWS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_MAXROWS, &C); } db_options(A) ::= db_options(B) FSYNC NK_INTEGER(C). { ((SDatabaseOptions*)B)->pFsyncPeriod = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
db_options(A) ::= db_options(B) MINROWS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_MINROWS, &C); } db_options(A) ::= db_options(B) MAXROWS NK_INTEGER(C). { ((SDatabaseOptions*)B)->pMaxRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
db_options(A) ::= db_options(B) KEEP integer_list(C). { A = setDatabaseKeepOption(pCxt, B, C); } db_options(A) ::= db_options(B) MINROWS NK_INTEGER(C). { ((SDatabaseOptions*)B)->pMinRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
db_options(A) ::= db_options(B) PRECISION NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PRECISION, &C); } db_options(A) ::= db_options(B) KEEP integer_list(C). { ((SDatabaseOptions*)B)->pKeep = C; A = B; }
db_options(A) ::= db_options(B) QUORUM NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_QUORUM, &C); } db_options(A) ::= db_options(B) KEEP variable_list(C). { ((SDatabaseOptions*)B)->pKeep = C; A = B; }
db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_REPLICA, &C); } db_options(A) ::= db_options(B) PRECISION NK_STRING(C). { ((SDatabaseOptions*)B)->pPrecision = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &C); A = B; }
db_options(A) ::= db_options(B) TTL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TTL, &C); } db_options(A) ::= db_options(B) QUORUM NK_INTEGER(C). { ((SDatabaseOptions*)B)->pQuorum = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL, &C); } db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). { ((SDatabaseOptions*)B)->pReplica = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); } db_options(A) ::= db_options(B) TTL NK_INTEGER(C). { ((SDatabaseOptions*)B)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLE_STABLE, &C); } db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { ((SDatabaseOptions*)B)->pWalLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAM_MODE, &C); } db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { ((SDatabaseOptions*)B)->pNumOfVgroups = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
db_options(A) ::= db_options(B) RETENTIONS NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_RETENTIONS, &C); } db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { ((SDatabaseOptions*)B)->pSingleStable = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { ((SDatabaseOptions*)B)->pStreamMode = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
db_options(A) ::= db_options(B) RETENTIONS retention_list(C). { ((SDatabaseOptions*)B)->pRetentions = C; A = B; }
alter_db_options(A) ::= alter_db_option(B). { A = createDefaultAlterDatabaseOptions(pCxt); A = setDatabaseAlterOption(pCxt, A, &B); } alter_db_options(A) ::= alter_db_option(B). { A = createDatabaseOptions(pCxt); A = setDatabaseAlterOption(pCxt, A, &B); }
alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setDatabaseAlterOption(pCxt, B, &C); } alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setDatabaseAlterOption(pCxt, B, &C); }
%type alter_db_option { SAlterOption } %type alter_db_option { SAlterOption }
%destructor alter_db_option { } %destructor alter_db_option { }
alter_db_option(A) ::= BLOCKS NK_INTEGER(B). { A.type = DB_OPTION_BLOCKS; A.val = B; } alter_db_option(A) ::= BLOCKS NK_INTEGER(B). { A.type = DB_OPTION_BLOCKS; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); }
alter_db_option(A) ::= FSYNC NK_INTEGER(B). { A.type = DB_OPTION_FSYNC; A.val = B; } alter_db_option(A) ::= FSYNC NK_INTEGER(B). { A.type = DB_OPTION_FSYNC; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); }
alter_db_option(A) ::= KEEP integer_list(B). { A.type = DB_OPTION_KEEP; A.pKeep = B; } alter_db_option(A) ::= KEEP integer_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; }
alter_db_option(A) ::= WAL NK_INTEGER(B). { A.type = DB_OPTION_WAL; A.val = B; } alter_db_option(A) ::= KEEP variable_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; }
alter_db_option(A) ::= QUORUM NK_INTEGER(B). { A.type = DB_OPTION_QUORUM; A.val = B; } alter_db_option(A) ::= WAL NK_INTEGER(B). { A.type = DB_OPTION_WAL; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); }
alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.val = B; } alter_db_option(A) ::= QUORUM NK_INTEGER(B). { A.type = DB_OPTION_QUORUM; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); }
alter_db_option(A) ::= REPLICA NK_INTEGER(B). { A.type = DB_OPTION_REPLICA; A.val = B; } alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); }
alter_db_option(A) ::= REPLICA NK_INTEGER(B). { A.type = DB_OPTION_REPLICA; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); }
%type integer_list { SNodeList* } %type integer_list { SNodeList* }
%destructor integer_list { nodesDestroyList($$); } %destructor integer_list { nodesDestroyList($$); }
integer_list(A) ::= NK_INTEGER(B). { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); } integer_list(A) ::= NK_INTEGER(B). { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); }
integer_list(A) ::= integer_list(B) NK_COMMA NK_INTEGER(C). { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); } integer_list(A) ::= integer_list(B) NK_COMMA NK_INTEGER(C). { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); }
%type variable_list { SNodeList* }
%destructor variable_list { nodesDestroyList($$); }
variable_list(A) ::= NK_VARIABLE(B). { A = createNodeList(pCxt, createDurationValueNode(pCxt, &B)); }
variable_list(A) ::= variable_list(B) NK_COMMA NK_VARIABLE(C). { A = addNodeToList(pCxt, B, createDurationValueNode(pCxt, &C)); }
%type retention_list { SNodeList* }
%destructor retention_list { nodesDestroyList($$); }
retention_list(A) ::= retention(B). { A = createNodeList(pCxt, B); }
retention_list(A) ::= retention_list(B) NK_COMMA retention(C). { A = addNodeToList(pCxt, B, C); }
retention(A) ::= NK_VARIABLE(B) NK_COLON NK_VARIABLE(C). { A = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &B), createDurationValueNode(pCxt, &C)); }
/************************************************ create/drop table/stable ********************************************/ /************************************************ create/drop table/stable ********************************************/
cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B) cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B)
NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); } NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); }
@ -263,23 +278,23 @@ tags_def_opt(A) ::= tags_def(B).
%destructor tags_def { nodesDestroyList($$); } %destructor tags_def { nodesDestroyList($$); }
tags_def(A) ::= TAGS NK_LP column_def_list(B) NK_RP. { A = B; } tags_def(A) ::= TAGS NK_LP column_def_list(B) NK_RP. { A = B; }
table_options(A) ::= . { A = createDefaultTableOptions(pCxt); } table_options(A) ::= . { A = createTableOptions(pCxt); }
table_options(A) ::= table_options(B) COMMENT NK_STRING(C). { A = setTableOption(pCxt, B, TABLE_OPTION_COMMENT, &C); } table_options(A) ::= table_options(B) COMMENT NK_STRING(C). { ((STableOptions*)B)->pComments = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &C); A = B; }
table_options(A) ::= table_options(B) KEEP integer_list(C). { A = setTableKeepOption(pCxt, B, C); } table_options(A) ::= table_options(B) KEEP integer_list(C). { ((STableOptions*)B)->pKeep = C; A = B; }
table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); } table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { ((STableOptions*)B)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableSmaOption(pCxt, B, C); } table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { ((STableOptions*)B)->pSma = C; A = B; }
table_options(A) ::= table_options(B) ROLLUP NK_LP func_name_list(C) NK_RP. { A = setTableRollupOption(pCxt, B, C); } table_options(A) ::= table_options(B) ROLLUP NK_LP func_name_list(C) NK_RP. { ((STableOptions*)B)->pFuncs = C; A = B; }
table_options(A) ::= table_options(B) FILE_FACTOR NK_FLOAT(C). { A = setTableOption(pCxt, B, TABLE_OPTION_FILE_FACTOR, &C); } table_options(A) ::= table_options(B) FILE_FACTOR NK_FLOAT(C). { ((STableOptions*)B)->pFilesFactor = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &C); A = B; }
table_options(A) ::= table_options(B) DELAY NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_DELAY, &C); } table_options(A) ::= table_options(B) DELAY NK_INTEGER(C). { ((STableOptions*)B)->pDelay = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C); A = B; }
alter_table_options(A) ::= alter_table_option(B). { A = createDefaultAlterTableOptions(pCxt); A = setTableAlterOption(pCxt, A, &B); } alter_table_options(A) ::= alter_table_option(B). { A = createTableOptions(pCxt); A = setTableAlterOption(pCxt, A, &B); }
alter_table_options(A) ::= alter_table_options(B) alter_table_option(C). { A = setTableAlterOption(pCxt, B, &C); } alter_table_options(A) ::= alter_table_options(B) alter_table_option(C). { A = setTableAlterOption(pCxt, B, &C); }
%type alter_table_option { SAlterOption } %type alter_table_option { SAlterOption }
%destructor alter_table_option { } %destructor alter_table_option { }
alter_table_option(A) ::= COMMENT NK_STRING(B). { A.type = TABLE_OPTION_COMMENT; A.val = B; } alter_table_option(A) ::= COMMENT NK_STRING(B). { A.type = TABLE_OPTION_COMMENT; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); }
alter_table_option(A) ::= KEEP integer_list(B). { A.type = TABLE_OPTION_KEEP; A.pKeep = B; } alter_table_option(A) ::= KEEP integer_list(B). { A.type = TABLE_OPTION_KEEP; A.pList = B; }
alter_table_option(A) ::= TTL NK_INTEGER(B). { A.type = TABLE_OPTION_TTL; A.val = B; } alter_table_option(A) ::= TTL NK_INTEGER(B). { A.type = TABLE_OPTION_TTL; A.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); }
%type col_name_list { SNodeList* } %type col_name_list { SNodeList* }
%destructor col_name_list { nodesDestroyList($$); } %destructor col_name_list { nodesDestroyList($$); }
@ -471,6 +486,8 @@ column_name(A) ::= NK_ID(B).
%type function_name { SToken } %type function_name { SToken }
%destructor function_name { } %destructor function_name { }
function_name(A) ::= NK_ID(B). { A = B; } function_name(A) ::= NK_ID(B). { A = B; }
function_name(A) ::= FIRST(B). { A = B; }
function_name(A) ::= LAST(B). { A = B; }
%type table_alias { SToken } %type table_alias { SToken }
%destructor table_alias { } %destructor table_alias { }
@ -716,7 +733,7 @@ partition_by_clause_opt(A) ::= PARTITION BY expression_list(B).
twindow_clause_opt(A) ::= . { A = NULL; } twindow_clause_opt(A) ::= . { A = NULL; }
twindow_clause_opt(A) ::= twindow_clause_opt(A) ::=
SESSION NK_LP column_reference(B) NK_COMMA duration_literal(C) NK_RP. { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)); } SESSION NK_LP column_reference(B) NK_COMMA duration_literal(C) NK_RP. { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)); }
twindow_clause_opt(A) ::= STATE_WINDOW NK_LP column_reference(B) NK_RP. { A = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, B)); } twindow_clause_opt(A) ::= STATE_WINDOW NK_LP expression(B) NK_RP. { A = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, B)); }
twindow_clause_opt(A) ::= twindow_clause_opt(A) ::=
INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), NULL, C, D); } INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), NULL, C, D); }
twindow_clause_opt(A) ::= twindow_clause_opt(A) ::=

515
source/libs/parser/src/parAstCreater.c
View File

@ -16,6 +16,7 @@
#include "parAst.h" #include "parAst.h"
#include "parUtil.h" #include "parUtil.h"
#include "ttime.h"
#define CHECK_OUT_OF_MEM(p) \ #define CHECK_OUT_OF_MEM(p) \
do { \ do { \
@ -36,314 +37,6 @@
SToken nil_token = { .type = TK_NK_NIL, .n = 0, .z = NULL }; SToken nil_token = { .type = TK_NK_NIL, .n = 0, .z = NULL };
typedef SDatabaseOptions* (*FSetDatabaseOption)(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal);
static FSetDatabaseOption setDbOptionFuncs[DB_OPTION_MAX];
typedef STableOptions* (*FSetTableOption)(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal);
static FSetTableOption setTableOptionFuncs[TABLE_OPTION_MAX];
static SDatabaseOptions* setDbBlocks(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_TOTAL_BLOCKS || val > TSDB_MAX_TOTAL_BLOCKS) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option totalBlocks: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS);
pCxt->valid = false;
return pOptions;
}
pOptions->numOfBlocks = val;
return pOptions;
}
static SDatabaseOptions* setDbCache(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option cacheBlockSize: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE);
pCxt->valid = false;
return pOptions;
}
pOptions->cacheBlockSize = val;
return pOptions;
}
static SDatabaseOptions* setDbCacheLast(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DB_CACHE_LAST_ROW || val > TSDB_MAX_DB_CACHE_LAST_ROW) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option cacheLast: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_DB_CACHE_LAST_ROW, TSDB_MAX_DB_CACHE_LAST_ROW);
pCxt->valid = false;
return pOptions;
}
pOptions->cachelast = val;
return pOptions;
}
static SDatabaseOptions* setDbComp(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_COMP_LEVEL || val > TSDB_MAX_COMP_LEVEL) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option compression: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL);
pCxt->valid = false;
return pOptions;
}
pOptions->compressionLevel = val;
return pOptions;
}
static SDatabaseOptions* setDbDays(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option daysPerFile: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE);
pCxt->valid = false;
return pOptions;
}
pOptions->daysPerFile = val;
return pOptions;
}
static SDatabaseOptions* setDbFsync(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option fsyncPeriod: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD);
pCxt->valid = false;
return pOptions;
}
pOptions->fsyncPeriod = val;
return pOptions;
}
static SDatabaseOptions* setDbMaxRows(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_MAX_ROW_FBLOCK || val > TSDB_MAX_MAX_ROW_FBLOCK) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option maxRowsPerBlock: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_MAX_ROW_FBLOCK, TSDB_MAX_MAX_ROW_FBLOCK);
pCxt->valid = false;
return pOptions;
}
pOptions->maxRowsPerBlock = val;
return pOptions;
}
static SDatabaseOptions* setDbMinRows(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_MIN_ROW_FBLOCK || val > TSDB_MAX_MIN_ROW_FBLOCK) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option minRowsPerBlock: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK);
pCxt->valid = false;
return pOptions;
}
pOptions->minRowsPerBlock = val;
return pOptions;
}
static SDatabaseOptions* setDbPrecision(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
char val[10] = {0};
trimString(pVal->z, pVal->n, val, sizeof(val));
if (0 == strcmp(val, TSDB_TIME_PRECISION_MILLI_STR)) {
pOptions->precision = TSDB_TIME_PRECISION_MILLI;
} else if (0 == strcmp(val, TSDB_TIME_PRECISION_MICRO_STR)) {
pOptions->precision = TSDB_TIME_PRECISION_MICRO;
} else if (0 == strcmp(val, TSDB_TIME_PRECISION_NANO_STR)) {
pOptions->precision = TSDB_TIME_PRECISION_NANO;
} else {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option precision: %s", val);
pCxt->valid = false;
}
return pOptions;
}
static SDatabaseOptions* setDbQuorum(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DB_QUORUM_OPTION || val > TSDB_MAX_DB_QUORUM_OPTION) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option quorum: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION);
pCxt->valid = false;
return pOptions;
}
pOptions->quorum = val;
return pOptions;
}
static SDatabaseOptions* setDbReplica(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (!(val == TSDB_MIN_DB_REPLICA_OPTION || val == TSDB_MAX_DB_REPLICA_OPTION)) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option replications: %"PRId64", only 1, 3 allowed", val);
pCxt->valid = false;
return pOptions;
}
pOptions->replica = val;
return pOptions;
}
static SDatabaseOptions* setDbTtl(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DB_TTL_OPTION) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option ttl: %"PRId64", should be greater than or equal to %d", val, TSDB_MIN_DB_TTL_OPTION);
pCxt->valid = false;
return pOptions;
}
pOptions->ttl = val;
return pOptions;
}
static SDatabaseOptions* setDbWal(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_WAL_LEVEL || val > TSDB_MAX_WAL_LEVEL) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option walLevel: %"PRId64", only 1-2 allowed", val);
pCxt->valid = false;
return pOptions;
}
pOptions->walLevel = val;
return pOptions;
}
static SDatabaseOptions* setDbVgroups(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_VNODES_PER_DB || val > TSDB_MAX_VNODES_PER_DB) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid db option vgroups: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB);
pCxt->valid = false;
return pOptions;
}
pOptions->numOfVgroups = val;
return pOptions;
}
static SDatabaseOptions* setDbSingleStable(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DB_SINGLE_STABLE_OPTION || val > TSDB_MAX_DB_SINGLE_STABLE_OPTION) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option singleStable: %"PRId64", only 0-1 allowed", val);
pCxt->valid = false;
return pOptions;
}
pOptions->singleStable = val;
return pOptions;
}
static SDatabaseOptions* setDbStreamMode(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DB_STREAM_MODE_OPTION || val > TSDB_MAX_DB_STREAM_MODE_OPTION) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option streamMode: %"PRId64", only 0-1 allowed", val);
pCxt->valid = false;
return pOptions;
}
pOptions->streamMode = val;
return pOptions;
}
static SDatabaseOptions* setDbRetentions(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
pOptions->pRetentions = nodesMakeList();
if (NULL == pOptions->pRetentions) {
pCxt->valid = false;
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory");
return pOptions;
}
if (pVal->n > 2) {
char* pStart = pVal->z + 1;
char* pEnd = pVal->z + pVal->n - 1;
int32_t sepOrder = 1;
while (1) {
char* pPos = strchr(pStart, (0 == (sepOrder++) % 2) ? ',' : ':');
SToken t = { .type = TK_NK_VARIABLE, .z = pStart, .n = (NULL == pPos ? pEnd - pStart : pPos - pStart)};
if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pOptions->pRetentions, createDurationValueNode(pCxt, &t))) {
pCxt->valid = false;
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory");
return pOptions;
}
if (NULL == pPos) {
break;
}
pStart = pPos + 1;
}
}
if (LIST_LENGTH(pOptions->pRetentions) < 2 || LIST_LENGTH(pOptions->pRetentions) % 2 != 0) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option retentions: %s", pVal->z);
pCxt->valid = false;
}
return pOptions;
}
static void initSetDatabaseOptionFp() {
setDbOptionFuncs[DB_OPTION_BLOCKS] = setDbBlocks;
setDbOptionFuncs[DB_OPTION_CACHE] = setDbCache;
setDbOptionFuncs[DB_OPTION_CACHELAST] = setDbCacheLast;
setDbOptionFuncs[DB_OPTION_COMP] = setDbComp;
setDbOptionFuncs[DB_OPTION_DAYS] = setDbDays;
setDbOptionFuncs[DB_OPTION_FSYNC] = setDbFsync;
setDbOptionFuncs[DB_OPTION_MAXROWS] = setDbMaxRows;
setDbOptionFuncs[DB_OPTION_MINROWS] = setDbMinRows;
setDbOptionFuncs[DB_OPTION_PRECISION] = setDbPrecision;
setDbOptionFuncs[DB_OPTION_QUORUM] = setDbQuorum;
setDbOptionFuncs[DB_OPTION_REPLICA] = setDbReplica;
setDbOptionFuncs[DB_OPTION_TTL] = setDbTtl;
setDbOptionFuncs[DB_OPTION_WAL] = setDbWal;
setDbOptionFuncs[DB_OPTION_VGROUPS] = setDbVgroups;
setDbOptionFuncs[DB_OPTION_SINGLE_STABLE] = setDbSingleStable;
setDbOptionFuncs[DB_OPTION_STREAM_MODE] = setDbStreamMode;
setDbOptionFuncs[DB_OPTION_RETENTIONS] = setDbRetentions;
}
static STableOptions* setTableTtl(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DB_TTL_OPTION) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid table option ttl: %"PRId64", should be greater than or equal to %d", val, TSDB_MIN_DB_TTL_OPTION);
pCxt->valid = false;
return pOptions;
}
pOptions->ttl = val;
return pOptions;
}
static STableOptions* setTableComment(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) {
if (pVal->n >= sizeof(pOptions->comments)) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid table option comment, length cannot exceed %d", (int32_t)(sizeof(pOptions->comments) - 1));
pCxt->valid = false;
return pOptions;
}
trimString(pVal->z, pVal->n, pOptions->comments, sizeof(pOptions->comments));
return pOptions;
}
static STableOptions* setTableFileFactor(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) {
double val = strtod(pVal->z, NULL);
if (val < TSDB_MIN_DB_FILE_FACTOR || val > TSDB_MAX_DB_FILE_FACTOR) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid table option file_factor: %f valid range: [%d, %d]", val, TSDB_MIN_DB_FILE_FACTOR, TSDB_MAX_DB_FILE_FACTOR);
pCxt->valid = false;
return pOptions;
}
pOptions->filesFactor = val;
return pOptions;
}
static STableOptions* setTableDelay(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DB_DELAY || val > TSDB_MAX_DB_DELAY) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid table option delay: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_DB_DELAY, TSDB_MAX_DB_DELAY);
pCxt->valid = false;
return pOptions;
}
pOptions->delay = val;
return pOptions;
}
static void initSetTableOptionFp() {
setTableOptionFuncs[TABLE_OPTION_TTL] = setTableTtl;
setTableOptionFuncs[TABLE_OPTION_COMMENT] = setTableComment;
setTableOptionFuncs[TABLE_OPTION_FILE_FACTOR] = setTableFileFactor;
setTableOptionFuncs[TABLE_OPTION_DELAY] = setTableDelay;
}
void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) { void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) {
pCxt->pQueryCxt = pParseCxt; pCxt->pQueryCxt = pParseCxt;
pCxt->msgBuf.buf = pParseCxt->pMsg; pCxt->msgBuf.buf = pParseCxt->pMsg;
@ -351,8 +44,6 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) {
pCxt->notSupport = false; pCxt->notSupport = false;
pCxt->valid = true; pCxt->valid = true;
pCxt->pRootNode = NULL; pCxt->pRootNode = NULL;
initSetDatabaseOptionFp();
initSetTableOptionFp();
} }
static void trimEscape(SToken* pName) { static void trimEscape(SToken* pName) {
@ -652,6 +343,16 @@ SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList) {
return (SNode*)list; return (SNode*)list;
} }
SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2) {
SNodeListNode* list = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST);
CHECK_OUT_OF_MEM(list);
list->pNodeList = nodesMakeList();
CHECK_OUT_OF_MEM(list->pNodeList);
nodesListAppend(list->pNodeList, p1);
nodesListAppend(list->pNodeList, p2);
return (SNode*)list;
}
SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias) { SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias) {
if (!checkDbName(pCxt, pDbName, true) || !checkTableName(pCxt, pTableName) || !checkTableName(pCxt, pTableAlias)) { if (!checkDbName(pCxt, pDbName, true) || !checkTableName(pCxt, pTableName) || !checkTableName(pCxt, pTableAlias)) {
return NULL; return NULL;
@ -730,10 +431,10 @@ SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap
return (SNode*)session; return (SNode*)session;
} }
SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol) { SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr) {
SStateWindowNode* state = (SStateWindowNode*)nodesMakeNode(QUERY_NODE_STATE_WINDOW); SStateWindowNode* state = (SStateWindowNode*)nodesMakeNode(QUERY_NODE_STATE_WINDOW);
CHECK_OUT_OF_MEM(state); CHECK_OUT_OF_MEM(state);
state->pCol = pCol; state->pExpr = pExpr;
return (SNode*)state; return (SNode*)state;
} }
@ -855,60 +556,12 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode*
return (SNode*)setOp; return (SNode*)setOp;
} }
SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { SNode* createDatabaseOptions(SAstCreateContext* pCxt) {
SDatabaseOptions* pOptions = nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS); SDatabaseOptions* pOptions = nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions); CHECK_OUT_OF_MEM(pOptions);
pOptions->numOfBlocks = TSDB_DEFAULT_TOTAL_BLOCKS;
pOptions->cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
pOptions->cachelast = TSDB_DEFAULT_CACHE_LAST_ROW;
pOptions->compressionLevel = TSDB_DEFAULT_COMP_LEVEL;
pOptions->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
pOptions->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
pOptions->maxRowsPerBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK;
pOptions->minRowsPerBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK;
pOptions->keep0 = TSDB_DEFAULT_KEEP;
pOptions->keep1 = TSDB_DEFAULT_KEEP;
pOptions->keep2 = TSDB_DEFAULT_KEEP;
pOptions->precision = TSDB_TIME_PRECISION_MILLI;
pOptions->quorum = TSDB_DEFAULT_DB_QUORUM_OPTION;
pOptions->replica = TSDB_DEFAULT_DB_REPLICA_OPTION;
pOptions->ttl = TSDB_DEFAULT_DB_TTL_OPTION;
pOptions->walLevel = TSDB_DEFAULT_WAL_LEVEL;
pOptions->numOfVgroups = TSDB_DEFAULT_VN_PER_DB;
pOptions->singleStable = TSDB_DEFAULT_DB_SINGLE_STABLE_OPTION;
pOptions->streamMode = TSDB_DEFAULT_DB_STREAM_MODE_OPTION;
return (SNode*)pOptions; return (SNode*)pOptions;
} }
SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt) {
SDatabaseOptions* pOptions = nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions);
pOptions->numOfBlocks = -1;
pOptions->cacheBlockSize = -1;
pOptions->cachelast = -1;
pOptions->compressionLevel = -1;
pOptions->daysPerFile = -1;
pOptions->fsyncPeriod = -1;
pOptions->maxRowsPerBlock = -1;
pOptions->minRowsPerBlock = -1;
pOptions->keep0 = -1;
pOptions->keep1 = -1;
pOptions->keep2= -1;
pOptions->precision = -1;
pOptions->quorum = -1;
pOptions->replica = -1;
pOptions->ttl = -1;
pOptions->walLevel = -1;
pOptions->numOfVgroups = -1;
pOptions->singleStable = -1;
pOptions->streamMode = -1;
return (SNode*)pOptions;
}
SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal) {
return (SNode*)setDbOptionFuncs[type](pCxt, (SDatabaseOptions*)pOptions, pVal);
}
static bool checkAndSetKeepOption(SAstCreateContext* pCxt, SNodeList* pKeep, int32_t* pKeep0, int32_t* pKeep1, int32_t* pKeep2) { static bool checkAndSetKeepOption(SAstCreateContext* pCxt, SNodeList* pKeep, int32_t* pKeep0, int32_t* pKeep1, int32_t* pKeep2) {
int32_t numOfKeep = LIST_LENGTH(pKeep); int32_t numOfKeep = LIST_LENGTH(pKeep);
if (numOfKeep > 3 || numOfKeep < 1) { if (numOfKeep > 3 || numOfKeep < 1) {
@ -937,18 +590,66 @@ static bool checkAndSetKeepOption(SAstCreateContext* pCxt, SNodeList* pKeep, int
return true; return true;
} }
SNode* setDatabaseKeepOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pKeep) {
SDatabaseOptions* pOp = (SDatabaseOptions*)pOptions;
pCxt->valid = checkAndSetKeepOption(pCxt, pKeep, &pOp->keep0, &pOp->keep1, &pOp->keep2);
return pOptions;
}
SNode* setDatabaseAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption) { SNode* setDatabaseAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption) {
if (DB_OPTION_KEEP == pAlterOption->type) { switch (pAlterOption->type) {
return setDatabaseKeepOption(pCxt, pOptions, pAlterOption->pKeep); case DB_OPTION_BLOCKS:
} else { ((SDatabaseOptions*)pOptions)->pNumOfBlocks = pAlterOption->pVal;
return setDatabaseOption(pCxt, pOptions, pAlterOption->type, &pAlterOption->val); break;
case DB_OPTION_CACHE:
((SDatabaseOptions*)pOptions)->pCacheBlockSize = pAlterOption->pVal;
break;
case DB_OPTION_CACHELAST:
((SDatabaseOptions*)pOptions)->pCachelast = pAlterOption->pVal;
break;
case DB_OPTION_COMP:
((SDatabaseOptions*)pOptions)->pCompressionLevel = pAlterOption->pVal;
break;
case DB_OPTION_DAYS:
((SDatabaseOptions*)pOptions)->pDaysPerFile = pAlterOption->pVal;
break;
case DB_OPTION_FSYNC:
((SDatabaseOptions*)pOptions)->pFsyncPeriod = pAlterOption->pVal;
break;
case DB_OPTION_MAXROWS:
((SDatabaseOptions*)pOptions)->pMaxRowsPerBlock = pAlterOption->pVal;
break;
case DB_OPTION_MINROWS:
((SDatabaseOptions*)pOptions)->pMinRowsPerBlock = pAlterOption->pVal;
break;
case DB_OPTION_KEEP:
((SDatabaseOptions*)pOptions)->pKeep = pAlterOption->pList;
break;
case DB_OPTION_PRECISION:
((SDatabaseOptions*)pOptions)->pPrecision = pAlterOption->pVal;
break;
case DB_OPTION_QUORUM:
((SDatabaseOptions*)pOptions)->pQuorum = pAlterOption->pVal;
break;
case DB_OPTION_REPLICA:
((SDatabaseOptions*)pOptions)->pReplica = pAlterOption->pVal;
break;
case DB_OPTION_TTL:
((SDatabaseOptions*)pOptions)->pTtl = pAlterOption->pVal;
break;
case DB_OPTION_WAL:
((SDatabaseOptions*)pOptions)->pWalLevel = pAlterOption->pVal;
break;
case DB_OPTION_VGROUPS:
((SDatabaseOptions*)pOptions)->pNumOfVgroups = pAlterOption->pVal;
break;
case DB_OPTION_SINGLE_STABLE:
((SDatabaseOptions*)pOptions)->pSingleStable = pAlterOption->pVal;
break;
case DB_OPTION_STREAM_MODE:
((SDatabaseOptions*)pOptions)->pStreamMode = pAlterOption->pVal;
break;
case DB_OPTION_RETENTIONS:
((SDatabaseOptions*)pOptions)->pRetentions = pAlterOption->pList;
break;
default:
break;
} }
return pOptions;
} }
SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions) { SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions) {
@ -985,61 +686,36 @@ SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode*
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createDefaultTableOptions(SAstCreateContext* pCxt) { SNode* createTableOptions(SAstCreateContext* pCxt) {
STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS); STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions); CHECK_OUT_OF_MEM(pOptions);
pOptions->keep0 = TSDB_DEFAULT_KEEP;
pOptions->keep1 = TSDB_DEFAULT_KEEP;
pOptions->keep2 = TSDB_DEFAULT_KEEP;
pOptions->ttl = TSDB_DEFAULT_DB_TTL_OPTION;
pOptions->filesFactor = TSDB_DEFAULT_DB_FILE_FACTOR;
pOptions->delay = TSDB_DEFAULT_DB_DELAY;
return (SNode*)pOptions; return (SNode*)pOptions;
} }
SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt) {
STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions);
pOptions->keep0 = -1;
pOptions->keep1 = -1;
pOptions->keep2 = -1;
pOptions->ttl = -1;
pOptions->filesFactor = -1;
pOptions->delay = -1;
return (SNode*)pOptions;
}
SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal) {
return (SNode*)setTableOptionFuncs[type](pCxt, (STableOptions*)pOptions, pVal);
}
SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pSma) {
((STableOptions*)pOptions)->pSma = pSma;
return pOptions;
}
SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs) {
if (1 != LIST_LENGTH(pFuncs)) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table option rollup: only one function is allowed");
pCxt->valid = false;
return pOptions;
}
((STableOptions*)pOptions)->pFuncs = pFuncs;
return pOptions;
}
SNode* setTableKeepOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pKeep) {
STableOptions* pOp = (STableOptions*)pOptions;
pCxt->valid = checkAndSetKeepOption(pCxt, pKeep, &pOp->keep0, &pOp->keep1, &pOp->keep2);
return pOptions;
}
SNode* setTableAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption) { SNode* setTableAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption) {
if (TABLE_OPTION_KEEP == pAlterOption->type) { switch (pAlterOption->type) {
return setTableKeepOption(pCxt, pOptions, pAlterOption->pKeep); case TABLE_OPTION_KEEP:
} else { ((STableOptions*)pOptions)->pKeep = pAlterOption->pList;
return setTableOption(pCxt, pOptions, pAlterOption->type, &pAlterOption->val); break;
case TABLE_OPTION_TTL:
((STableOptions*)pOptions)->pTtl = pAlterOption->pVal;
break;
case TABLE_OPTION_COMMENT:
((STableOptions*)pOptions)->pComments = pAlterOption->pVal;
break;
case TABLE_OPTION_SMA:
((STableOptions*)pOptions)->pSma = pAlterOption->pList;
break;
case TABLE_OPTION_FILE_FACTOR:
((STableOptions*)pOptions)->pFilesFactor = pAlterOption->pVal;
break;
case TABLE_OPTION_DELAY:
((STableOptions*)pOptions)->pDelay = pAlterOption->pVal;
break;
default:
break;
} }
return pOptions;
} }
SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment) { SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment) {
@ -1050,6 +726,7 @@ SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDat
if (NULL != pComment) { if (NULL != pComment) {
trimString(pComment->z, pComment->n, pCol->comments, sizeof(pCol->comments)); trimString(pComment->z, pComment->n, pCol->comments, sizeof(pCol->comments));
} }
pCol->sma = true;
return (SNode*)pCol; return (SNode*)pCol;
} }

83
source/libs/parser/src/parCalcConst.c
View File

@ -90,14 +90,91 @@ static EDealRes calcConst(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
static bool isCondition(const SNode* pNode) {
if (QUERY_NODE_OPERATOR == nodeType(pNode)) {
return nodesIsComparisonOp((const SOperatorNode*)pNode);
}
return (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode));
}
static int32_t rewriteIsTrue(SNode* pSrc, SNode** pIsTrue) {
SOperatorNode* pOp = nodesMakeNode(QUERY_NODE_OPERATOR);
if (NULL == pOp) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pOp->opType = OP_TYPE_IS_TRUE;
pOp->pLeft = pSrc;
*pIsTrue = (SNode*)pOp;
return TSDB_CODE_SUCCESS;
}
static EDealRes doRewriteCondition(SNode** pNode, void* pContext) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pNode)) {
SNode* pParam = NULL;
FOREACH(pParam, ((SLogicConditionNode*)*pNode)->pParameterList) {
if (!isCondition(pParam)) {
SNode* pIsTrue = NULL;
if (TSDB_CODE_SUCCESS != rewriteIsTrue(pParam, &pIsTrue)) {
((SCalcConstContext*)pContext)->code = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}
REPLACE_NODE(pIsTrue);
}
}
}
return DEAL_RES_CONTINUE;
}
static int32_t rewriteCondition(SCalcConstContext* pCxt, SNode** pNode) {
if (!isCondition(*pNode)) {
return rewriteIsTrue(*pNode, pNode);
}
nodesRewriteExprPostOrder(pNode, doRewriteCondition, pCxt);
return pCxt->code;
}
static int32_t rewriteConditionForFromTable(SCalcConstContext* pCxt, SNode* pTable) {
if (QUERY_NODE_JOIN_TABLE == nodeType(pTable)) {
SJoinTableNode* pJoin = (SJoinTableNode*)pTable;
pCxt->code = rewriteConditionForFromTable(pCxt, pJoin->pLeft);
if (TSDB_CODE_SUCCESS == pCxt->code) {
pCxt->code = rewriteConditionForFromTable(pCxt, pJoin->pRight);
}
if (TSDB_CODE_SUCCESS == pCxt->code) {
pCxt->code = rewriteCondition(pCxt, &pJoin->pOnCond);
}
}
return pCxt->code;
}
static int32_t calcConstFromTable(SCalcConstContext* pCxt, SSelectStmt* pSelect) {
nodesRewriteExprPostOrder(&pSelect->pFromTable, calcConst, pCxt);
if (TSDB_CODE_SUCCESS == pCxt->code) {
pCxt->code = rewriteConditionForFromTable(pCxt, pSelect->pFromTable);
}
return pCxt->code;
}
static int32_t calcConstCondition(SCalcConstContext* pCxt, SNode** pCond) {
if (NULL == *pCond) {
return TSDB_CODE_SUCCESS;
}
nodesRewriteExprPostOrder(pCond, calcConst, pCxt);
if (TSDB_CODE_SUCCESS == pCxt->code) {
pCxt->code = rewriteCondition(pCxt, pCond);
}
return pCxt->code;
}
static int32_t calcConstSelect(SSelectStmt* pSelect) { static int32_t calcConstSelect(SSelectStmt* pSelect) {
SCalcConstContext cxt = { .code = TSDB_CODE_SUCCESS }; SCalcConstContext cxt = { .code = TSDB_CODE_SUCCESS };
nodesRewriteExprsPostOrder(pSelect->pProjectionList, calcConst, &cxt); nodesRewriteExprsPostOrder(pSelect->pProjectionList, calcConst, &cxt);
if (TSDB_CODE_SUCCESS == cxt.code) { if (TSDB_CODE_SUCCESS == cxt.code) {
nodesRewriteExprPostOrder(&pSelect->pFromTable, calcConst, &cxt); cxt.code = calcConstFromTable(&cxt, pSelect);
} }
if (TSDB_CODE_SUCCESS == cxt.code) { if (TSDB_CODE_SUCCESS == cxt.code) {
nodesRewriteExprPostOrder(&pSelect->pWhere, calcConst, &cxt); cxt.code = calcConstCondition(&cxt, &pSelect->pWhere);
} }
if (TSDB_CODE_SUCCESS == cxt.code) { if (TSDB_CODE_SUCCESS == cxt.code) {
nodesRewriteExprsPostOrder(pSelect->pPartitionByList, calcConst, &cxt); nodesRewriteExprsPostOrder(pSelect->pPartitionByList, calcConst, &cxt);
@ -109,7 +186,7 @@ static int32_t calcConstSelect(SSelectStmt* pSelect) {
nodesRewriteExprsPostOrder(pSelect->pGroupByList, calcConst, &cxt); nodesRewriteExprsPostOrder(pSelect->pGroupByList, calcConst, &cxt);
} }
if (TSDB_CODE_SUCCESS == cxt.code) { if (TSDB_CODE_SUCCESS == cxt.code) {
nodesRewriteExprPostOrder(&pSelect->pHaving, calcConst, &cxt); cxt.code = calcConstCondition(&cxt, &pSelect->pHaving);
} }
if (TSDB_CODE_SUCCESS == cxt.code) { if (TSDB_CODE_SUCCESS == cxt.code) {
nodesRewriteExprsPostOrder(pSelect->pOrderByList, calcConst, &cxt); nodesRewriteExprsPostOrder(pSelect->pOrderByList, calcConst, &cxt);

3
source/libs/parser/src/parTokenizer.c
View File

@ -72,6 +72,7 @@ static SKeyword keywordTable[] = {
{"EXPLAIN", TK_EXPLAIN}, {"EXPLAIN", TK_EXPLAIN},
{"FILE_FACTOR", TK_FILE_FACTOR}, {"FILE_FACTOR", TK_FILE_FACTOR},
{"FILL", TK_FILL}, {"FILL", TK_FILL},
{"FIRST", TK_FIRST},
{"FLOAT", TK_FLOAT}, {"FLOAT", TK_FLOAT},
{"FROM", TK_FROM}, {"FROM", TK_FROM},
{"FSYNC", TK_FSYNC}, {"FSYNC", TK_FSYNC},
@ -95,6 +96,7 @@ static SKeyword keywordTable[] = {
{"JSON", TK_JSON}, {"JSON", TK_JSON},
{"KEEP", TK_KEEP}, {"KEEP", TK_KEEP},
{"KILL", TK_KILL}, {"KILL", TK_KILL},
{"LAST", TK_LAST},
{"LICENCE", TK_LICENCE}, {"LICENCE", TK_LICENCE},
{"LIKE", TK_LIKE}, {"LIKE", TK_LIKE},
{"LIMIT", TK_LIMIT}, {"LIMIT", TK_LIMIT},
@ -113,6 +115,7 @@ static SKeyword keywordTable[] = {
{"NOT", TK_NOT}, {"NOT", TK_NOT},
{"NOW", TK_NOW}, {"NOW", TK_NOW},
{"NULL", TK_NULL}, {"NULL", TK_NULL},
{"NULLS", TK_NULLS},
{"OFFSET", TK_OFFSET}, {"OFFSET", TK_OFFSET},
{"ON", TK_ON}, {"ON", TK_ON},
{"OR", TK_OR}, {"OR", TK_OR},

399
source/libs/parser/src/parTranslater.c
View File

@ -21,6 +21,8 @@
#include "parUtil.h" #include "parUtil.h"
#include "ttime.h" #include "ttime.h"
#define GET_OPTION_VAL(pVal, defaultVal) (NULL == (pVal) ? (defaultVal) : ((SValueNode*)(pVal))->datum.i)
typedef struct STranslateContext { typedef struct STranslateContext {
SParseContext* pParseCxt; SParseContext* pParseCxt;
int32_t errCode; int32_t errCode;
@ -934,7 +936,7 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) {
static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbReq* pReq) { static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbReq* pReq) {
if (NULL != pRetentions) { if (NULL != pRetentions) {
pReq->pRetensions = taosArrayInit(LIST_LENGTH(pRetentions) / 2, sizeof(SRetention)); pReq->pRetensions = taosArrayInit(LIST_LENGTH(pRetentions), sizeof(SRetention));
if (NULL == pReq->pRetensions) { if (NULL == pReq->pRetensions) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -943,10 +945,8 @@ static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbRe
SNode* pNode = NULL; SNode* pNode = NULL;
int32_t index = 0; int32_t index = 0;
FOREACH(pNode, pRetentions) { FOREACH(pNode, pRetentions) {
if (0 == ((index++) & 1)) { pFreq = (SValueNode*)nodesListGetNode(((SNodeListNode*)pNode)->pNodeList, 0);
pFreq = (SValueNode*)pNode; pKeep = (SValueNode*)nodesListGetNode(((SNodeListNode*)pNode)->pNodeList, 1);
} else {
pKeep = (SValueNode*)pNode;
SRetention retention = { SRetention retention = {
.freq = pFreq->datum.i, .freq = pFreq->datum.i,
.freqUnit = pFreq->unit, .freqUnit = pFreq->unit,
@ -955,7 +955,6 @@ static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbRe
}; };
taosArrayPush(pReq->pRetensions, &retention); taosArrayPush(pReq->pRetensions, &retention);
} }
}
pReq->numOfRetensions = taosArrayGetSize(pReq->pRetensions); pReq->numOfRetensions = taosArrayGetSize(pReq->pRetensions);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -965,41 +964,220 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS
SName name = {0}; SName name = {0};
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
tNameGetFullDbName(&name, pReq->db); tNameGetFullDbName(&name, pReq->db);
pReq->numOfVgroups = pStmt->pOptions->numOfVgroups; pReq->numOfVgroups = GET_OPTION_VAL(pStmt->pOptions->pNumOfVgroups, TSDB_DEFAULT_VN_PER_DB);
pReq->cacheBlockSize = pStmt->pOptions->cacheBlockSize; pReq->cacheBlockSize = GET_OPTION_VAL(pStmt->pOptions->pCacheBlockSize, TSDB_DEFAULT_CACHE_BLOCK_SIZE);
pReq->totalBlocks = pStmt->pOptions->numOfBlocks; pReq->totalBlocks = GET_OPTION_VAL(pStmt->pOptions->pNumOfBlocks, TSDB_DEFAULT_TOTAL_BLOCKS);
pReq->daysPerFile = pStmt->pOptions->daysPerFile; pReq->daysPerFile = GET_OPTION_VAL(pStmt->pOptions->pDaysPerFile, TSDB_DEFAULT_DAYS_PER_FILE);
pReq->daysToKeep0 = pStmt->pOptions->keep0; pReq->daysToKeep0 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 0), TSDB_DEFAULT_KEEP);
pReq->daysToKeep1 = pStmt->pOptions->keep1; pReq->daysToKeep1 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 1), TSDB_DEFAULT_KEEP);
pReq->daysToKeep2 = pStmt->pOptions->keep2; pReq->daysToKeep2 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 2), TSDB_DEFAULT_KEEP);
pReq->minRows = pStmt->pOptions->minRowsPerBlock; pReq->minRows = GET_OPTION_VAL(pStmt->pOptions->pMinRowsPerBlock, TSDB_DEFAULT_MIN_ROW_FBLOCK);
pReq->maxRows = pStmt->pOptions->maxRowsPerBlock; pReq->maxRows = GET_OPTION_VAL(pStmt->pOptions->pMaxRowsPerBlock, TSDB_DEFAULT_MAX_ROW_FBLOCK);
pReq->commitTime = -1; pReq->commitTime = -1;
pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod; pReq->fsyncPeriod = GET_OPTION_VAL(pStmt->pOptions->pFsyncPeriod, TSDB_DEFAULT_FSYNC_PERIOD);
pReq->walLevel = pStmt->pOptions->walLevel; pReq->walLevel = GET_OPTION_VAL(pStmt->pOptions->pWalLevel, TSDB_DEFAULT_WAL_LEVEL);
pReq->precision = pStmt->pOptions->precision; pReq->precision = GET_OPTION_VAL(pStmt->pOptions->pPrecision, TSDB_TIME_PRECISION_MILLI);
pReq->compression = pStmt->pOptions->compressionLevel; pReq->compression = GET_OPTION_VAL(pStmt->pOptions->pCompressionLevel, TSDB_DEFAULT_COMP_LEVEL);
pReq->replications = pStmt->pOptions->replica; pReq->replications = GET_OPTION_VAL(pStmt->pOptions->pReplica, TSDB_DEFAULT_DB_REPLICA_OPTION);
pReq->quorum = pStmt->pOptions->quorum; pReq->quorum = GET_OPTION_VAL(pStmt->pOptions->pQuorum, TSDB_DEFAULT_DB_QUORUM_OPTION);
pReq->update = -1; pReq->update = -1;
pReq->cacheLastRow = pStmt->pOptions->cachelast; pReq->cacheLastRow = GET_OPTION_VAL(pStmt->pOptions->pCachelast, TSDB_DEFAULT_CACHE_LAST_ROW);
pReq->ignoreExist = pStmt->ignoreExists; pReq->ignoreExist = pStmt->ignoreExists;
pReq->streamMode = pStmt->pOptions->streamMode; pReq->streamMode = GET_OPTION_VAL(pStmt->pOptions->pStreamMode, TSDB_DEFAULT_DB_STREAM_MODE_OPTION);
return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq); return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq);
} }
static int32_t checkCreateDatabase(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt) { static int32_t checkRangeOption(STranslateContext* pCxt, const char* pName, SValueNode* pVal, int32_t minVal, int32_t maxVal) {
if (NULL != pStmt->pOptions->pRetentions) { if (NULL != pVal) {
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode;
}
int64_t val = pVal->datum.i;
if (val < minVal || val > maxVal) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_RANGE_OPTION, pName, val, minVal, maxVal);
}
}
return TSDB_CODE_SUCCESS;
}
static void convertValueFromStrToInt(SValueNode* pVal, int64_t val) {
taosMemoryFreeClear(pVal->datum.p);
pVal->datum.i = val;
pVal->node.resType.type = TSDB_DATA_TYPE_BIGINT;
pVal->node.resType.bytes = tDataTypes[pVal->node.resType.type].bytes;
}
static int32_t checkDbPrecisionOption(STranslateContext* pCxt, SValueNode* pVal) {
if (NULL != pVal) {
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode;
}
char* pRrecision = varDataVal(pVal->datum.p);
if (0 == strcmp(pRrecision, TSDB_TIME_PRECISION_MILLI_STR)) {
convertValueFromStrToInt(pVal, TSDB_TIME_PRECISION_MILLI);
} else if (0 == strcmp(pRrecision, TSDB_TIME_PRECISION_MICRO_STR)) {
convertValueFromStrToInt(pVal, TSDB_TIME_PRECISION_MICRO);
} else if (0 == strcmp(pRrecision, TSDB_TIME_PRECISION_NANO_STR)) {
convertValueFromStrToInt(pVal, TSDB_TIME_PRECISION_NANO);
} else {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STR_OPTION, "precision", pVal->datum.p);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, SValueNode* pVal, int32_t v1, int32_t v2) {
if (NULL != pVal) {
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode;
}
int64_t val = pVal->datum.i;
if (val != v1 && val != v2) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_ENUM_OPTION, pName, val, v1, v2);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkTtlOption(STranslateContext* pCxt, SValueNode* pVal) {
if (NULL != pVal) {
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode;
}
int64_t val = pVal->datum.i;
if (val < TSDB_MIN_DB_TTL_OPTION) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_TTL_OPTION, val, TSDB_MIN_DB_TTL_OPTION);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkKeepOption(STranslateContext* pCxt, SNodeList* pKeep) {
if (NULL == pKeep) {
return TSDB_CODE_SUCCESS;
}
int32_t numOfKeep = LIST_LENGTH(pKeep);
if (numOfKeep > 3 || numOfKeep < 1) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_KEEP_NUM);
}
SNode* pNode = NULL; SNode* pNode = NULL;
FOREACH(pNode, pStmt->pOptions->pRetentions) { FOREACH(pNode, pKeep) {
if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pNode)) { if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pNode)) {
return pCxt->errCode; return pCxt->errCode;
} }
} }
if (1 == numOfKeep) {
if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pKeep, nodesCloneNode(nodesListGetNode(pKeep, 0)))) {
return TSDB_CODE_OUT_OF_MEMORY;
} }
++numOfKeep;
}
if (2 == numOfKeep) {
if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pKeep, nodesCloneNode(nodesListGetNode(pKeep, 1)))) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
int32_t daysToKeep0 = ((SValueNode*)nodesListGetNode(pKeep, 0))->datum.i;
int32_t daysToKeep1 = ((SValueNode*)nodesListGetNode(pKeep, 1))->datum.i;
int32_t daysToKeep2 = ((SValueNode*)nodesListGetNode(pKeep, 2))->datum.i;
if (daysToKeep0 < TSDB_MIN_KEEP || daysToKeep1 < TSDB_MIN_KEEP || daysToKeep2 < TSDB_MIN_KEEP ||
daysToKeep0 > TSDB_MAX_KEEP || daysToKeep1 > TSDB_MAX_KEEP || daysToKeep2 > TSDB_MAX_KEEP) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_KEEP_VALUE, daysToKeep0, daysToKeep1, daysToKeep2, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
}
if (!((daysToKeep0 <= daysToKeep1) && (daysToKeep1 <= daysToKeep2))) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_KEEP_ORDER);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRetentions) {
if (NULL == pRetentions) {
return TSDB_CODE_SUCCESS;
}
if (LIST_LENGTH(pRetentions) > 3) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION);
}
SNode* pNode = NULL;
FOREACH(pNode, pRetentions) {
SNode* pVal = NULL;
FOREACH(pVal, ((SNodeListNode*)pNode)->pNodeList) {
if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pVal)) {
return pCxt->errCode;
}
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkDatabaseOptions(STranslateContext* pCxt, SDatabaseOptions* pOptions) {
int32_t code = checkRangeOption(pCxt, "totalBlocks", pOptions->pNumOfBlocks, TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS);
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "cacheBlockSize", pOptions->pCacheBlockSize, TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "cacheLast", pOptions->pCachelast, TSDB_MIN_DB_CACHE_LAST_ROW, TSDB_MAX_DB_CACHE_LAST_ROW);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "compression", pOptions->pCompressionLevel, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "daysPerFile", pOptions->pDaysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "fsyncPeriod", pOptions->pFsyncPeriod, TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "maxRowsPerBlock", pOptions->pMaxRowsPerBlock, TSDB_MIN_MAX_ROW_FBLOCK, TSDB_MAX_MAX_ROW_FBLOCK);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "minRowsPerBlock", pOptions->pMinRowsPerBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkKeepOption(pCxt, pOptions->pKeep);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbPrecisionOption(pCxt, pOptions->pPrecision);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "quorum", pOptions->pQuorum, TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbEnumOption(pCxt, "replications", pOptions->pReplica, TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkTtlOption(pCxt, pOptions->pTtl);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbEnumOption(pCxt, "walLevel", pOptions->pWalLevel, TSDB_MIN_WAL_LEVEL, TSDB_MAX_WAL_LEVEL);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "vgroups", pOptions->pNumOfVgroups, TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbEnumOption(pCxt, "singleStable", pOptions->pSingleStable, TSDB_MIN_DB_SINGLE_STABLE_OPTION, TSDB_MAX_DB_SINGLE_STABLE_OPTION);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbEnumOption(pCxt, "streamMode", pOptions->pStreamMode, TSDB_MIN_DB_STREAM_MODE_OPTION, TSDB_MAX_DB_STREAM_MODE_OPTION);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbRetentionsOption(pCxt, pOptions->pRetentions);
}
return code;
}
static int32_t checkCreateDatabase(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt) {
return checkDatabaseOptions(pCxt, pStmt->pOptions);
}
static int32_t translateCreateDatabase(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt) { static int32_t translateCreateDatabase(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt) {
SCreateDbReq createReq = {0}; SCreateDbReq createReq = {0};
@ -1053,19 +1231,24 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt,
SName name = {0}; SName name = {0};
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
tNameGetFullDbName(&name, pReq->db); tNameGetFullDbName(&name, pReq->db);
pReq->totalBlocks = pStmt->pOptions->numOfBlocks; pReq->totalBlocks = GET_OPTION_VAL(pStmt->pOptions->pNumOfBlocks, -1);
pReq->daysToKeep0 = pStmt->pOptions->keep0; pReq->daysToKeep0 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 0), -1);
pReq->daysToKeep1 = pStmt->pOptions->keep1; pReq->daysToKeep1 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 1), -1);
pReq->daysToKeep2 = pStmt->pOptions->keep2; pReq->daysToKeep2 = GET_OPTION_VAL(nodesListGetNode(pStmt->pOptions->pKeep, 2), -1);
pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod; pReq->fsyncPeriod = GET_OPTION_VAL(pStmt->pOptions->pFsyncPeriod, -1);
pReq->walLevel = pStmt->pOptions->walLevel; pReq->walLevel = GET_OPTION_VAL(pStmt->pOptions->pWalLevel, -1);
pReq->quorum = pStmt->pOptions->quorum; pReq->quorum = GET_OPTION_VAL(pStmt->pOptions->pQuorum, -1);
pReq->cacheLastRow = pStmt->pOptions->cachelast; pReq->cacheLastRow = GET_OPTION_VAL(pStmt->pOptions->pCachelast, -1);
pReq->replications = pStmt->pOptions->replica; pReq->replications = GET_OPTION_VAL(pStmt->pOptions->pReplica, -1);
return; return;
} }
static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt) { static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt) {
int32_t code = checkDatabaseOptions(pCxt, pStmt->pOptions);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
SAlterDbReq alterReq = {0}; SAlterDbReq alterReq = {0};
buildAlterDbReq(pCxt, pStmt, &alterReq); buildAlterDbReq(pCxt, pStmt, &alterReq);
@ -1119,7 +1302,7 @@ static int32_t columnNodeToField(SNodeList* pList, SArray** pArray) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static const SColumnDefNode* findColDef(const SNodeList* pCols, const SColumnNode* pCol) { static SColumnDefNode* findColDef(SNodeList* pCols, const SColumnNode* pCol) {
SNode* pColDef = NULL; SNode* pColDef = NULL;
FOREACH(pColDef, pCols) { FOREACH(pColDef, pCols) {
if (0 == strcmp(pCol->colName, ((SColumnDefNode*)pColDef)->colName)) { if (0 == strcmp(pCol->colName, ((SColumnDefNode*)pColDef)->colName)) {
@ -1129,27 +1312,87 @@ static const SColumnDefNode* findColDef(const SNodeList* pCols, const SColumnNod
return NULL; return NULL;
} }
static int32_t checkCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) { static int32_t checkTableCommentOption(STranslateContext* pCxt, SValueNode* pVal) {
if (NULL != pVal) {
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode;
}
if (pVal->node.resType.bytes >= TSDB_STB_COMMENT_LEN) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COMMENT_OPTION, TSDB_STB_COMMENT_LEN - 1);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t checTableFactorOption(STranslateContext* pCxt, SValueNode* pVal) {
if (NULL != pVal) {
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode;
}
if (pVal->datum.d < TSDB_MIN_DB_FILE_FACTOR || pVal->datum.d > TSDB_MAX_DB_FILE_FACTOR) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_F_RANGE_OPTION, "file_factor", pVal->datum.d, TSDB_MIN_DB_FILE_FACTOR, TSDB_MAX_DB_FILE_FACTOR);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
if (NULL != pStmt->pOptions->pSma) { if (NULL != pStmt->pOptions->pSma) {
SNode* pNode = NULL; SNode* pNode = NULL;
FOREACH(pNode, pStmt->pCols) {
((SColumnDefNode*)pNode)->sma = false;
}
FOREACH(pNode, pStmt->pOptions->pSma) { FOREACH(pNode, pStmt->pOptions->pSma) {
SColumnNode* pSmaCol = (SColumnNode*)pNode; SColumnNode* pSmaCol = (SColumnNode*)pNode;
const SColumnDefNode* pColDef = findColDef(pStmt->pCols, pSmaCol); SColumnDefNode* pColDef = findColDef(pStmt->pCols, pSmaCol);
if (NULL == pColDef) { if (NULL == pColDef) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pSmaCol->colName); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pSmaCol->colName);
} }
pSmaCol->node.resType = pColDef->dataType; pSmaCol->node.resType = pColDef->dataType;
} pColDef->sma = true;
}
if (NULL != pStmt->pOptions->pFuncs) {
SFunctionNode* pFunc = nodesListGetNode(pStmt->pOptions->pFuncs, 0);
if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName);
} }
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs) {
if (NULL == pFuncs) {
return TSDB_CODE_SUCCESS;
}
if (1 != LIST_LENGTH(pFuncs)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_ROLLUP_OPTION);
}
SFunctionNode* pFunc = nodesListGetNode(pFuncs, 0);
if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName);
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
int32_t code = checkKeepOption(pCxt, pStmt->pOptions->pKeep);
if (TSDB_CODE_SUCCESS == code) {
code = checkTtlOption(pCxt, pStmt->pOptions->pTtl);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkTableCommentOption(pCxt, pStmt->pOptions->pComments);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkTableSmaOption(pCxt, pStmt);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkTableRollupOption(pCxt, pStmt->pOptions->pFuncs);
}
if (TSDB_CODE_SUCCESS == code) {
code = checTableFactorOption(pCxt, pStmt->pOptions->pFilesFactor);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "delay", pStmt->pOptions->pDelay, TSDB_MIN_DB_DELAY, TSDB_MAX_DB_DELAY);
}
return code;
}
static int32_t getAggregationMethod(SNodeList* pFuncs) { static int32_t getAggregationMethod(SNodeList* pFuncs) {
if (NULL == pFuncs) { if (NULL == pFuncs) {
return -1; return -1;
@ -1158,7 +1401,7 @@ static int32_t getAggregationMethod(SNodeList* pFuncs) {
} }
static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) { static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
int32_t code = checkCreateSuperTable(pCxt, pStmt); int32_t code = checkCreateTable(pCxt, pStmt);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return code; return code;
} }
@ -1166,8 +1409,8 @@ static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableSt
SMCreateStbReq createReq = {0}; SMCreateStbReq createReq = {0};
createReq.igExists = pStmt->ignoreExists; createReq.igExists = pStmt->ignoreExists;
createReq.aggregationMethod = getAggregationMethod(pStmt->pOptions->pFuncs); createReq.aggregationMethod = getAggregationMethod(pStmt->pOptions->pFuncs);
createReq.xFilesFactor = pStmt->pOptions->filesFactor; createReq.xFilesFactor = GET_OPTION_VAL(pStmt->pOptions->pFilesFactor, TSDB_DEFAULT_DB_FILE_FACTOR);
createReq.delay = pStmt->pOptions->delay; createReq.delay = GET_OPTION_VAL(pStmt->pOptions->pDelay, TSDB_DEFAULT_DB_DELAY);
columnDefNodeToField(pStmt->pCols, &createReq.pColumns); columnDefNodeToField(pStmt->pCols, &createReq.pColumns);
columnDefNodeToField(pStmt->pTags, &createReq.pTags); columnDefNodeToField(pStmt->pTags, &createReq.pTags);
createReq.numOfColumns = LIST_LENGTH(pStmt->pCols); createReq.numOfColumns = LIST_LENGTH(pStmt->pCols);
@ -1489,7 +1732,7 @@ static int32_t nodeTypeToShowType(ENodeType nt) {
case QUERY_NODE_SHOW_CONNECTIONS_STMT: case QUERY_NODE_SHOW_CONNECTIONS_STMT:
return TSDB_MGMT_TABLE_CONNS; return TSDB_MGMT_TABLE_CONNS;
case QUERY_NODE_SHOW_LICENCE_STMT: case QUERY_NODE_SHOW_LICENCE_STMT:
return 0; // todo return TSDB_MGMT_TABLE_GRANTS;
case QUERY_NODE_SHOW_QUERIES_STMT: case QUERY_NODE_SHOW_QUERIES_STMT:
return TSDB_MGMT_TABLE_QUERIES; return TSDB_MGMT_TABLE_QUERIES;
case QUERY_NODE_SHOW_SCORES_STMT: case QUERY_NODE_SHOW_SCORES_STMT:
@ -2161,11 +2404,11 @@ typedef struct SVgroupTablesBatch {
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
} SVgroupTablesBatch; } SVgroupTablesBatch;
static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchemaEx* pSchema) { static void toSchemaEx(const SColumnDefNode* pCol, col_id_t colId, SSchemaEx* pSchema) {
pSchema->colId = colId; pSchema->colId = colId;
pSchema->type = pCol->dataType.type; pSchema->type = pCol->dataType.type;
pSchema->bytes = calcTypeBytes(pCol->dataType); pSchema->bytes = calcTypeBytes(pCol->dataType);
pSchema->sma = TSDB_BSMA_TYPE_LATEST; // TODO: use default value currently, and use the real value later. pSchema->sma = pCol->sma ? TSDB_BSMA_TYPE_LATEST : TSDB_BSMA_TYPE_NONE;
strcpy(pSchema->name, pCol->colName); strcpy(pSchema->name, pCol->colName);
} }
@ -2175,33 +2418,60 @@ static void destroyCreateTbReq(SVCreateTbReq* pReq) {
taosMemoryFreeClear(pReq->ntbCfg.pSchema); taosMemoryFreeClear(pReq->ntbCfg.pSchema);
} }
static int32_t buildNormalTableBatchReq(int32_t acctId, const char* pDbName, const char* pTableName, static int32_t buildSmaParam(STableOptions* pOptions, SVCreateTbReq* pReq) {
const SNodeList* pColumns, const SVgroupInfo* pVgroupInfo, SVgroupTablesBatch* pBatch) { if (0 == LIST_LENGTH(pOptions->pFuncs)) {
return TSDB_CODE_SUCCESS;
}
pReq->ntbCfg.pRSmaParam = taosMemoryCalloc(1, sizeof(SRSmaParam));
if (NULL == pReq->ntbCfg.pRSmaParam) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pReq->ntbCfg.pRSmaParam->delay = GET_OPTION_VAL(pOptions->pDelay, TSDB_DEFAULT_DB_DELAY);
pReq->ntbCfg.pRSmaParam->xFilesFactor = GET_OPTION_VAL(pOptions->pFilesFactor, TSDB_DEFAULT_DB_FILE_FACTOR);
pReq->ntbCfg.pRSmaParam->nFuncIds = LIST_LENGTH(pOptions->pFuncs);
pReq->ntbCfg.pRSmaParam->pFuncIds = taosMemoryCalloc(pReq->ntbCfg.pRSmaParam->nFuncIds, sizeof(func_id_t));
if (NULL == pReq->ntbCfg.pRSmaParam->pFuncIds) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t index = 0;
SNode* pFunc = NULL;
FOREACH(pFunc, pOptions->pFuncs) {
pReq->ntbCfg.pRSmaParam->pFuncIds[index++] = ((SFunctionNode*)pFunc)->funcId;
}
return TSDB_CODE_SUCCESS;
}
static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* pStmt, const SVgroupInfo* pVgroupInfo, SVgroupTablesBatch* pBatch) {
char dbFName[TSDB_DB_FNAME_LEN] = {0}; char dbFName[TSDB_DB_FNAME_LEN] = {0};
SName name = { .type = TSDB_DB_NAME_T, .acctId = acctId }; SName name = { .type = TSDB_DB_NAME_T, .acctId = acctId };
strcpy(name.dbname, pDbName); strcpy(name.dbname, pStmt->dbName);
tNameGetFullDbName(&name, dbFName); tNameGetFullDbName(&name, dbFName);
SVCreateTbReq req = {0}; SVCreateTbReq req = {0};
req.type = TD_NORMAL_TABLE; req.type = TD_NORMAL_TABLE;
req.dbFName = strdup(dbFName); req.dbFName = strdup(dbFName);
req.name = strdup(pTableName); req.name = strdup(pStmt->tableName);
req.ntbCfg.nCols = LIST_LENGTH(pColumns); req.ntbCfg.nCols = LIST_LENGTH(pStmt->pCols);
req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchemaEx)); req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchemaEx));
if (NULL == req.name || NULL == req.ntbCfg.pSchema) { if (NULL == req.name || NULL == req.ntbCfg.pSchema) {
destroyCreateTbReq(&req); destroyCreateTbReq(&req);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
SNode* pCol; SNode* pCol;
int32_t index = 0; col_id_t index = 0;
FOREACH(pCol, pColumns) { FOREACH(pCol, pStmt->pCols) {
toSchema((SColumnDefNode*)pCol, index + 1, req.ntbCfg.pSchema + index); toSchemaEx((SColumnDefNode*)pCol, index + 1, req.ntbCfg.pSchema + index);
++index; ++index;
} }
// TODO: use the real sma for normal table. if (TSDB_CODE_SUCCESS != buildSmaParam(pStmt->pOptions, &req)) {
destroyCreateTbReq(&req);
return TSDB_CODE_OUT_OF_MEMORY;
}
pBatch->info = *pVgroupInfo; pBatch->info = *pVgroupInfo;
strcpy(pBatch->dbName, pDbName); strcpy(pBatch->dbName, pStmt->dbName);
pBatch->req.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq)); pBatch->req.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq));
if (NULL == pBatch->req.pArray) { if (NULL == pBatch->req.pArray) {
destroyCreateTbReq(&req); destroyCreateTbReq(&req);
@ -2282,7 +2552,7 @@ static int32_t buildCreateTableDataBlock(int32_t acctId, const SCreateTableStmt*
} }
SVgroupTablesBatch tbatch = {0}; SVgroupTablesBatch tbatch = {0};
int32_t code = buildNormalTableBatchReq(acctId, pStmt->dbName, pStmt->tableName, pStmt->pCols, pInfo, &tbatch); int32_t code = buildNormalTableBatchReq(acctId, pStmt, pInfo, &tbatch);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = serializeVgroupTablesBatch(&tbatch, *pBufArray); code = serializeVgroupTablesBatch(&tbatch, *pBufArray);
} }
@ -2297,8 +2567,11 @@ static int32_t buildCreateTableDataBlock(int32_t acctId, const SCreateTableStmt*
static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) {
SCreateTableStmt* pStmt = (SCreateTableStmt*)pQuery->pRoot; SCreateTableStmt* pStmt = (SCreateTableStmt*)pQuery->pRoot;
int32_t code = checkCreateTable(pCxt, pStmt);
SVgroupInfo info = {0}; SVgroupInfo info = {0};
int32_t code = getTableHashVgroup(pCxt, pStmt->dbName, pStmt->tableName, &info); if (TSDB_CODE_SUCCESS == code) {
code = getTableHashVgroup(pCxt, pStmt->dbName, pStmt->tableName, &info);
}
SArray* pBufArray = NULL; SArray* pBufArray = NULL;
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = buildCreateTableDataBlock(pCxt->pParseCxt->acctId, pStmt, &info, &pBufArray); code = buildCreateTableDataBlock(pCxt->pParseCxt->acctId, pStmt, &info, &pBufArray);
@ -2405,7 +2678,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau
static int32_t checkCreateSubTable(STranslateContext* pCxt, SCreateSubTableClause* pStmt) { static int32_t checkCreateSubTable(STranslateContext* pCxt, SCreateSubTableClause* pStmt) {
if (0 != strcmp(pStmt->dbName, pStmt->useDbName)) { if (0 != strcmp(pStmt->dbName, pStmt->useDbName)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR);; return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

22
source/libs/parser/src/parUtil.c
View File

@ -67,6 +67,28 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "Invalid identifier name : %s"; return "Invalid identifier name : %s";
case TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR: case TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR:
return "corresponding super table not in this db"; return "corresponding super table not in this db";
case TSDB_CODE_PAR_INVALID_RANGE_OPTION:
return "invalid option %s: %"PRId64" valid range: [%d, %d]";
case TSDB_CODE_PAR_INVALID_STR_OPTION:
return "invalid option %s: %s";
case TSDB_CODE_PAR_INVALID_ENUM_OPTION:
return "invalid option %s: %"PRId64", only %d, %d allowed";
case TSDB_CODE_PAR_INVALID_TTL_OPTION:
return "invalid option ttl: %"PRId64", should be greater than or equal to %d";
case TSDB_CODE_PAR_INVALID_KEEP_NUM:
return "invalid number of keep options";
case TSDB_CODE_PAR_INVALID_KEEP_ORDER:
return "invalid keep value, should be keep0 <= keep1 <= keep2";
case TSDB_CODE_PAR_INVALID_KEEP_VALUE:
return "invalid option keep: %d, %d, %d valid range: [%d, %d]";
case TSDB_CODE_PAR_INVALID_COMMENT_OPTION:
return "invalid option comment, length cannot exceed %d";
case TSDB_CODE_PAR_INVALID_F_RANGE_OPTION:
return "invalid option %s: %f valid range: [%d, %d]";
case TSDB_CODE_PAR_INVALID_ROLLUP_OPTION:
return "invalid option rollup: only one function is allowed";
case TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION:
return "invalid option retentions";
case TSDB_CODE_OUT_OF_MEMORY: case TSDB_CODE_OUT_OF_MEMORY:
return "Out of memory"; return "Out of memory";
default: default:

4683
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

28
source/libs/parser/test/parserAstTest.cpp
View File

@ -44,7 +44,7 @@ protected:
query_ = nullptr; query_ = nullptr;
bool res = runImpl(parseCode, translateCode); bool res = runImpl(parseCode, translateCode);
qDestroyQuery(query_); qDestroyQuery(query_);
if (!res) { if (1/*!res*/) {
dump(); dump();
} }
return res; return res;
@ -69,6 +69,12 @@ private:
return (terrno == translateCode); return (terrno == translateCode);
} }
translatedAstStr_ = toString(query_->pRoot); translatedAstStr_ = toString(query_->pRoot);
code = calculateConstant(&cxt_, query_);
if (code != TSDB_CODE_SUCCESS) {
calcConstErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_;
return false;
}
calcConstAstStr_ = toString(query_->pRoot);
return (TSDB_CODE_SUCCESS == translateCode); return (TSDB_CODE_SUCCESS == translateCode);
} }
@ -88,6 +94,13 @@ private:
cout << "translate output: " << endl; cout << "translate output: " << endl;
cout << translatedAstStr_ << endl; cout << translatedAstStr_ << endl;
} }
if (!calcConstErrStr_.empty()) {
cout << "calculateConstant error: " << calcConstErrStr_ << endl;
}
if (!calcConstAstStr_.empty()) {
cout << "calculateConstant output: " << endl;
cout << calcConstAstStr_ << endl;
}
} }
string toString(const SNode* pRoot, bool format = false) { string toString(const SNode* pRoot, bool format = false) {
@ -112,6 +125,8 @@ private:
parsedAstStr_.clear(); parsedAstStr_.clear();
translateErrStr_.clear(); translateErrStr_.clear();
translatedAstStr_.clear(); translatedAstStr_.clear();
calcConstErrStr_.clear();
calcConstAstStr_.clear();
} }
string acctId_; string acctId_;
@ -124,6 +139,8 @@ private:
string parsedAstStr_; string parsedAstStr_;
string translateErrStr_; string translateErrStr_;
string translatedAstStr_; string translatedAstStr_;
string calcConstErrStr_;
string calcConstAstStr_;
}; };
TEST_F(ParserTest, createAccount) { TEST_F(ParserTest, createAccount) {
@ -191,6 +208,9 @@ TEST_F(ParserTest, selectConstant) {
bind("SELECT 1234567890123456789012345678901234567890, 20.1234567890123456789012345678901234567890, 'abc', \"wxy\", TIMESTAMP '2022-02-09 17:30:20', true, false, 15s FROM t1"); bind("SELECT 1234567890123456789012345678901234567890, 20.1234567890123456789012345678901234567890, 'abc', \"wxy\", TIMESTAMP '2022-02-09 17:30:20', true, false, 15s FROM t1");
ASSERT_TRUE(run()); ASSERT_TRUE(run());
bind("SELECT 123 + 45 FROM t1 where 2 - 1");
ASSERT_TRUE(run());
} }
TEST_F(ParserTest, selectExpression) { TEST_F(ParserTest, selectExpression) {
@ -426,6 +446,12 @@ TEST_F(ParserTest, createDatabase) {
"RETENTIONS '15s:7d,1m:21d,15m:5y'" "RETENTIONS '15s:7d,1m:21d,15m:5y'"
); );
ASSERT_TRUE(run()); ASSERT_TRUE(run());
bind("create database if not exists wxy_db "
"DAYS 100m "
"KEEP 200m,300h,400d "
);
ASSERT_TRUE(run());
} }
TEST_F(ParserTest, alterDatabase) { TEST_F(ParserTest, alterDatabase) {

67
source/libs/planner/src/planLogicCreater.c
View File

@ -477,6 +477,18 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm
return code; return code;
} }
static int32_t createWindowLogicNodeByState(SLogicPlanContext* pCxt, SStateWindowNode* pState, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW);
if (NULL == pWindow) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pWindow->winType = WINDOW_TYPE_STATE;
pWindow->pStateExpr = nodesCloneNode(pState->pExpr);
return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode);
}
static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionWindowNode* pSession, SSelectStmt* pSelect, SLogicNode** pLogicNode) { static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionWindowNode* pSession, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW); SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW);
if (NULL == pWindow) { if (NULL == pWindow) {
@ -525,6 +537,8 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele
} }
switch (nodeType(pSelect->pWindow)) { switch (nodeType(pSelect->pWindow)) {
case QUERY_NODE_STATE_WINDOW:
return createWindowLogicNodeByState(pCxt, (SStateWindowNode*)pSelect->pWindow, pSelect, pLogicNode);
case QUERY_NODE_SESSION_WINDOW: case QUERY_NODE_SESSION_WINDOW:
return createWindowLogicNodeBySession(pCxt, (SSessionWindowNode*)pSelect->pWindow, pSelect, pLogicNode); return createWindowLogicNodeBySession(pCxt, (SSessionWindowNode*)pSelect->pWindow, pSelect, pLogicNode);
case QUERY_NODE_INTERVAL_WINDOW: case QUERY_NODE_INTERVAL_WINDOW:
@ -642,6 +656,29 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
SNodeList* pCols = NULL;
int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_PARTITION_BY, NULL, &pCols);
if (TSDB_CODE_SUCCESS == code && NULL != pCols) {
pPartition->node.pTargets = nodesCloneList(pCols);
if (NULL == pPartition->node.pTargets) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code) {
pPartition->pPartitionKeys = nodesCloneList(pSelect->pPartitionByList);
if (NULL == pPartition->pPartitionKeys) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pPartition;
} else {
nodesDestroyNode(pPartition);
}
return code;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -650,7 +687,35 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
return TSDB_CODE_SUCCESS; SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG);
if (NULL == pAgg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = TSDB_CODE_SUCCESS;
// set grouyp keys, agg funcs and having conditions
pAgg->pGroupKeys = nodesCloneList(pSelect->pProjectionList);
if (NULL == pAgg->pGroupKeys) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
// rewrite the expression in subsequent clauses
if (TSDB_CODE_SUCCESS == code) {
code = rewriteExpr(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_DISTINCT);
}
// set the output
if (TSDB_CODE_SUCCESS == code) {
code = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys, &pAgg->node.pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pAgg;
} else {
nodesDestroyNode(pAgg);
}
return code;
} }
static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {

124
source/libs/planner/src/planPhysiCreater.c
View File

@ -176,6 +176,10 @@ static int16_t getUnsetSlotId(const SArray* pSlotIdsInfo) {
} }
static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc, const char* pStmtName, bool output) { static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc, const char* pStmtName, bool output) {
if (NULL == pList) {
return TSDB_CODE_SUCCESS;
}
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SHashObj* pHash = taosArrayGetP(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId); SHashObj* pHash = taosArrayGetP(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId);
int16_t nextSlotId = taosHashGetSize(pHash), slotId = 0; int16_t nextSlotId = taosHashGetSize(pHash), slotId = 0;
@ -219,6 +223,23 @@ static int32_t addDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDat
return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, NULL, false); return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, NULL, false);
} }
static int32_t addDataBlockSlot(SPhysiPlanContext* pCxt, SNode** pNode, SDataBlockDescNode* pDataBlockDesc) {
if (NULL == pNode || NULL == *pNode) {
return TSDB_CODE_SUCCESS;
}
SNodeList* pList = NULL;
int32_t code = nodesListMakeAppend(&pList, *pNode);
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, pList, pDataBlockDesc);
}
if (TSDB_CODE_SUCCESS == code) {
*pNode = nodesListGetNode(pList, 0);
}
nodesClearList(pList);
return code;
}
static int32_t addDataBlockSlotsForProject(SPhysiPlanContext* pCxt, const char* pStmtName, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { static int32_t addDataBlockSlotsForProject(SPhysiPlanContext* pCxt, const char* pStmtName, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) {
return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, true); return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, true);
} }
@ -244,6 +265,7 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) {
} }
// pIndex is definitely not NULL, otherwise it is a bug // pIndex is definitely not NULL, otherwise it is a bug
if (NULL == pIndex) { if (NULL == pIndex) {
pCxt->errCode = TSDB_CODE_PLAN_INTERNAL_ERROR;
return DEAL_RES_ERROR; return DEAL_RES_ERROR;
} }
((SColumnNode*)pNode)->dataBlockId = pIndex->dataBlockId; ((SColumnNode*)pNode)->dataBlockId = pIndex->dataBlockId;
@ -614,6 +636,25 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN
return cxt.errCode; return cxt.errCode;
} }
static int32_t rewritePrecalcExpr(SPhysiPlanContext* pCxt, SNode* pNode, SNodeList** pPrecalcExprs, SNode** pRewritten) {
if (NULL == pNode) {
return TSDB_CODE_SUCCESS;
}
SNodeList* pList = NULL;
int32_t code = nodesListMakeAppend(&pList, pNode);
SNodeList* pRewrittenList = NULL;
if (TSDB_CODE_SUCCESS == code) {
code = rewritePrecalcExprs(pCxt, pList, pPrecalcExprs, &pRewrittenList);
}
if (TSDB_CODE_SUCCESS == code) {
*pRewritten = nodesListGetNode(pRewrittenList, 0);
}
nodesClearList(pList);
nodesClearList(pRewrittenList);
return code;
}
static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode, SPhysiNode** pPhyNode) { static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode, SPhysiNode** pPhyNode) {
SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pAggLogicNode, QUERY_NODE_PHYSICAL_PLAN_AGG); SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pAggLogicNode, QUERY_NODE_PHYSICAL_PLAN_AGG);
if (NULL == pAgg) { if (NULL == pAgg) {
@ -818,6 +859,40 @@ static int32_t createSessionWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList*
return createWindowPhysiNodeFinalize(pCxt, pChildren, &pSession->window, pWindowLogicNode, pPhyNode); return createWindowPhysiNodeFinalize(pCxt, pChildren, &pSession->window, pWindowLogicNode, pPhyNode);
} }
static int32_t createStateWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) {
SStateWinodwPhysiNode* pState = (SStateWinodwPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pWindowLogicNode, QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW);
if (NULL == pState) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SNodeList* pPrecalcExprs = NULL;
SNode* pStateKey = NULL;
int32_t code = rewritePrecalcExpr(pCxt, pWindowLogicNode->pStateExpr, &pPrecalcExprs, &pStateKey);
SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc);
// push down expression to pOutputDataBlockDesc of child node
if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) {
code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pState->window.pExprs);
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, pState->window.pExprs, pChildTupe);
}
}
if (TSDB_CODE_SUCCESS == code) {
code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pStateKey, &pState->pStateKey);
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlot(pCxt, &pState->pStateKey, pState->window.node.pOutputDataBlockDesc);
}
}
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode(pState);
return code;
}
return createWindowPhysiNodeFinalize(pCxt, pChildren, &pState->window, pWindowLogicNode, pPhyNode);
}
static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) {
switch (pWindowLogicNode->winType) { switch (pWindowLogicNode->winType) {
case WINDOW_TYPE_INTERVAL: case WINDOW_TYPE_INTERVAL:
@ -825,7 +900,7 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr
case WINDOW_TYPE_SESSION: case WINDOW_TYPE_SESSION:
return createSessionWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); return createSessionWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode);
case WINDOW_TYPE_STATE: case WINDOW_TYPE_STATE:
break; return createStateWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode);
default: default:
break; break;
} }
@ -853,8 +928,12 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pSortKeys, &pSort->pSortKeys); code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pSortKeys, &pSort->pSortKeys);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, pSort->pSortKeys, pSort->node.pOutputDataBlockDesc); code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pSortLogicNode->node.pTargets, &pSort->pTargets);
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, pSort->pTargets, pSort->node.pOutputDataBlockDesc);
} }
} }
@ -867,6 +946,45 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
return code; return code;
} }
static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SPartitionLogicNode* pPartLogicNode, SPhysiNode** pPhyNode) {
SPartitionPhysiNode* pPart = (SPartitionPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pPartLogicNode, QUERY_NODE_PHYSICAL_PLAN_PARTITION);
if (NULL == pPart) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SNodeList* pPrecalcExprs = NULL;
SNodeList* pPartitionKeys = NULL;
int32_t code = rewritePrecalcExprs(pCxt, pPartLogicNode->pPartitionKeys, &pPrecalcExprs, &pPartitionKeys);
SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc);
// push down expression to pOutputDataBlockDesc of child node
if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) {
code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pPart->pExprs);
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, pPart->pExprs, pChildTupe);
}
}
if (TSDB_CODE_SUCCESS == code) {
code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPartitionKeys, &pPart->pPartitionKeys);
}
if (TSDB_CODE_SUCCESS == code) {
code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPartLogicNode->node.pTargets, &pPart->pTargets);
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlots(pCxt, pPart->pTargets, pPart->node.pOutputDataBlockDesc);
}
}
if (TSDB_CODE_SUCCESS == code) {
*pPhyNode = (SPhysiNode*)pPart;
} else {
nodesDestroyNode(pPart);
}
return code;
}
static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubplan* pSubplan, SNodeList* pChildren, SPhysiNode** pPhyNode) { static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubplan* pSubplan, SNodeList* pChildren, SPhysiNode** pPhyNode) {
switch (nodeType(pLogicNode)) { switch (nodeType(pLogicNode)) {
case QUERY_NODE_LOGIC_PLAN_SCAN: case QUERY_NODE_LOGIC_PLAN_SCAN:
@ -883,6 +1001,8 @@ static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode
return createWindowPhysiNode(pCxt, pChildren, (SWindowLogicNode*)pLogicNode, pPhyNode); return createWindowPhysiNode(pCxt, pChildren, (SWindowLogicNode*)pLogicNode, pPhyNode);
case QUERY_NODE_LOGIC_PLAN_SORT: case QUERY_NODE_LOGIC_PLAN_SORT:
return createSortPhysiNode(pCxt, pChildren, (SSortLogicNode*)pLogicNode, pPhyNode); return createSortPhysiNode(pCxt, pChildren, (SSortLogicNode*)pLogicNode, pPhyNode);
case QUERY_NODE_LOGIC_PLAN_PARTITION:
return createPartitionPhysiNode(pCxt, pChildren, (SPartitionLogicNode*)pLogicNode, pPhyNode);
default: default:
break; break;
} }

42
source/libs/planner/test/plannerTest.cpp
View File

@ -217,6 +217,32 @@ TEST_F(PlannerTest, sessionWindow) {
ASSERT_TRUE(run()); ASSERT_TRUE(run());
} }
TEST_F(PlannerTest, stateWindow) {
setDatabase("root", "test");
bind("SELECT count(*) FROM t1 state_window(c1)");
ASSERT_TRUE(run());
bind("SELECT count(*) FROM t1 state_window(c1 + 10)");
ASSERT_TRUE(run());
}
TEST_F(PlannerTest, partitionBy) {
setDatabase("root", "test");
bind("SELECT * FROM t1 partition by c1");
ASSERT_TRUE(run());
bind("SELECT count(*) FROM t1 partition by c1");
ASSERT_TRUE(run());
bind("SELECT count(*) FROM t1 partition by c1 group by c2");
ASSERT_TRUE(run());
bind("SELECT count(*) FROM st1 partition by tag1, tag2 interval(10s)");
ASSERT_TRUE(run());
}
TEST_F(PlannerTest, orderBy) { TEST_F(PlannerTest, orderBy) {
setDatabase("root", "test"); setDatabase("root", "test");
@ -228,6 +254,22 @@ TEST_F(PlannerTest, orderBy) {
bind("SELECT * FROM t1 order by c1 + 10, c2"); bind("SELECT * FROM t1 order by c1 + 10, c2");
ASSERT_TRUE(run()); ASSERT_TRUE(run());
bind("SELECT * FROM t1 order by c1 desc nulls first");
ASSERT_TRUE(run());
}
TEST_F(PlannerTest, distinct) {
setDatabase("root", "test");
bind("SELECT distinct c1 FROM t1");
ASSERT_TRUE(run());
bind("SELECT distinct c1, c2 + 10 FROM t1");
ASSERT_TRUE(run());
bind("SELECT distinct c1 + 10 a FROM t1 order by a");
ASSERT_TRUE(run());
} }
TEST_F(PlannerTest, limit) { TEST_F(PlannerTest, limit) {

8
source/libs/qcom/src/queryUtil.c
View File

@ -97,18 +97,14 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTag
static void* pTaskQueue = NULL; static void* pTaskQueue = NULL;
int32_t initTaskQueue() { int32_t initTaskQueue() {
double factor = 4.0;
int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2);
int32_t queueSize = tsMaxConnections * 2; int32_t queueSize = tsMaxConnections * 2;
pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); pTaskQueue = taosInitScheduler(queueSize, tsNumOfTaskQueueThreads, "tsc");
if (NULL == pTaskQueue) { if (NULL == pTaskQueue) {
qError("failed to init task queue"); qError("failed to init task queue");
return -1; return -1;
} }
qDebug("task queue is initialized, numOfThreads: %d", numOfThreads); qDebug("task queue is initialized, numOfThreads: %d", tsNumOfTaskQueueThreads);
return 0; return 0;
} }

14
source/libs/tdb/src/btree/tdbBtreeBalance.c
View File

@ -1,14 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

14
source/libs/tdb/src/btree/tdbBtreeCommon.c
View File

@ -1,14 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

14
source/libs/tdb/src/btree/tdbBtreeDelete.c
View File

@ -1,14 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

14
source/libs/tdb/src/btree/tdbBtreeInsert.c
View File

@ -1,14 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

27
source/libs/tdb/src/btree/tdbBtreeInt.h
View File

@ -1,27 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TDB_BTREE_INT_H_
#define _TDB_BTREE_INT_H_
#ifdef __cplusplus
extern "C" {
#endif
#ifdef __cplusplus
}
#endif
#endif /*_TDB_BTREE_INT_H_*/

14
source/libs/tdb/src/btree/tdbBtreeOpen.c
View File

@ -1,14 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

52
source/libs/tdb/src/db/tdbBtree.c
View File

@ -127,7 +127,7 @@ int tdbBtreeClose(SBTree *pBt) {
return 0; return 0;
} }
int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen) { int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) {
SBTC btc; SBTC btc;
SCell *pCell; SCell *pCell;
void *pBuf; void *pBuf;
@ -137,7 +137,7 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in
int idx; int idx;
int c; int c;
tdbBtcOpen(&btc, pBt); tdbBtcOpen(&btc, pBt, pTxn);
// move to the position to insert // move to the position to insert
ret = tdbBtcMoveTo(&btc, pKey, kLen, &c); ret = tdbBtcMoveTo(&btc, pKey, kLen, &c);
@ -225,7 +225,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL
void *pTVal = NULL; void *pTVal = NULL;
SCellDecoder cd; SCellDecoder cd;
tdbBtcOpen(&btc, pBt); tdbBtcOpen(&btc, pBt, NULL);
ret = tdbBtcMoveTo(&btc, pKey, kLen, &cret); ret = tdbBtcMoveTo(&btc, pKey, kLen, &cret);
if (ret < 0) { if (ret < 0) {
@ -233,7 +233,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL
ASSERT(0); ASSERT(0);
} }
if (cret) { if (btc.idx < 0 || cret) {
tdbBtcClose(&btc); tdbBtcClose(&btc);
return -1; return -1;
} }
@ -253,6 +253,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL
memcpy(*ppKey, cd.pKey, cd.kLen); memcpy(*ppKey, cd.pKey, cd.kLen);
} }
if (ppVal) {
pTVal = TDB_REALLOC(*ppVal, cd.vLen); pTVal = TDB_REALLOC(*ppVal, cd.vLen);
if (pTVal == NULL) { if (pTVal == NULL) {
tdbBtcClose(&btc); tdbBtcClose(&btc);
@ -262,6 +263,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL
*ppVal = pTVal; *ppVal = pTVal;
*vLen = cd.vLen; *vLen = cd.vLen;
memcpy(*ppVal, cd.pVal, cd.vLen); memcpy(*ppVal, cd.pVal, cd.vLen);
}
tdbBtcClose(&btc); tdbBtcClose(&btc);
@ -297,7 +299,8 @@ static int tdbBtreeOpenImpl(SBTree *pBt) {
{ {
// 1. TODO: Search the main DB to check if the DB exists // 1. TODO: Search the main DB to check if the DB exists
pgno = 0; ret = tdbPagerOpenDB(pBt->pPager, &pgno, true);
ASSERT(ret == 0);
} }
if (pgno != 0) { if (pgno != 0) {
@ -307,13 +310,13 @@ static int tdbBtreeOpenImpl(SBTree *pBt) {
// Try to create a new database // Try to create a new database
SBtreeInitPageArg zArg = {.flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF, .pBt = pBt}; SBtreeInitPageArg zArg = {.flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF, .pBt = pBt};
ret = tdbPagerNewPage(pBt->pPager, &pgno, &pPage, tdbBtreeZeroPage, &zArg); ret = tdbPagerNewPage(pBt->pPager, &pgno, &pPage, tdbBtreeZeroPage, &zArg, NULL);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
// TODO: here still has problem // TODO: here still has problem
tdbPagerReturnPage(pBt->pPager, pPage); tdbPagerReturnPage(pBt->pPager, pPage, NULL);
ASSERT(pgno != 0); ASSERT(pgno != 0);
pBt->root = pgno; pBt->root = pgno;
@ -385,7 +388,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) {
} }
// TDB_BTREE_BALANCE ===================== // TDB_BTREE_BALANCE =====================
static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN *pTxn) {
SPager *pPager; SPager *pPager;
SPage *pChild; SPage *pChild;
SPgno pgnoChild; SPgno pgnoChild;
@ -402,7 +405,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) {
// Allocate a new child page // Allocate a new child page
zArg.flags = TDB_FLAG_REMOVE(flags, TDB_BTREE_ROOT); zArg.flags = TDB_FLAG_REMOVE(flags, TDB_BTREE_ROOT);
zArg.pBt = pBt; zArg.pBt = pBt;
ret = tdbPagerNewPage(pPager, &pgnoChild, &pChild, tdbBtreeZeroPage, &zArg); ret = tdbPagerNewPage(pPager, &pgnoChild, &pChild, tdbBtreeZeroPage, &zArg, pTxn);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@ -436,7 +439,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) {
return 0; return 0;
} }
static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTxn) {
int ret; int ret;
int nOlds; int nOlds;
@ -477,7 +480,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
pgno = *(SPgno *)pCell; pgno = *(SPgno *)pCell;
} }
ret = tdbPagerFetchPage(pBt->pPager, pgno, pOlds + i, tdbBtreeInitPage, pBt); ret = tdbPagerFetchPage(pBt->pPager, pgno, pOlds + i, tdbBtreeInitPage, pBt, pTxn);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
@ -640,7 +643,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
} else { } else {
iarg.pBt = pBt; iarg.pBt = pBt;
iarg.flags = flags; iarg.flags = flags;
ret = tdbPagerNewPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeZeroPage, &iarg); ret = tdbPagerNewPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeZeroPage, &iarg, pTxn);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
} }
@ -767,9 +770,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
// TODO: here is not corrent for drop case // TODO: here is not corrent for drop case
for (int i = 0; i < nNews; i++) { for (int i = 0; i < nNews; i++) {
if (i < nOlds) { if (i < nOlds) {
tdbPagerReturnPage(pBt->pPager, pOlds[i]); tdbPagerReturnPage(pBt->pPager, pOlds[i], pTxn);
} else { } else {
tdbPagerReturnPage(pBt->pPager, pNews[i]); tdbPagerReturnPage(pBt->pPager, pNews[i], pTxn);
} }
} }
@ -805,7 +808,7 @@ static int tdbBtreeBalance(SBTC *pBtc) {
// ignore the case of empty // ignore the case of empty
if (pPage->nOverflow == 0) break; if (pPage->nOverflow == 0) break;
ret = tdbBtreeBalanceDeeper(pBtc->pBt, pPage, &(pBtc->pgStack[1])); ret = tdbBtreeBalanceDeeper(pBtc->pBt, pPage, &(pBtc->pgStack[1]), pBtc->pTxn);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@ -819,12 +822,12 @@ static int tdbBtreeBalance(SBTC *pBtc) {
// Generalized balance step // Generalized balance step
pParent = pBtc->pgStack[iPage - 1]; pParent = pBtc->pgStack[iPage - 1];
ret = tdbBtreeBalanceNonRoot(pBtc->pBt, pParent, pBtc->idxStack[pBtc->iPage - 1]); ret = tdbBtreeBalanceNonRoot(pBtc->pBt, pParent, pBtc->idxStack[pBtc->iPage - 1], pBtc->pTxn);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage); tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage, pBtc->pTxn);
pBtc->iPage--; pBtc->iPage--;
pBtc->pPage = pBtc->pgStack[pBtc->iPage]; pBtc->pPage = pBtc->pgStack[pBtc->iPage];
@ -1024,11 +1027,12 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) {
// TDB_BTREE_CELL // TDB_BTREE_CELL
// TDB_BTREE_CURSOR ===================== // TDB_BTREE_CURSOR =====================
int tdbBtcOpen(SBTC *pBtc, SBTree *pBt) { int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn) {
pBtc->pBt = pBt; pBtc->pBt = pBt;
pBtc->iPage = -1; pBtc->iPage = -1;
pBtc->pPage = NULL; pBtc->pPage = NULL;
pBtc->idx = -1; pBtc->idx = -1;
pBtc->pTxn = pTxn;
return 0; return 0;
} }
@ -1045,7 +1049,7 @@ int tdbBtcMoveToFirst(SBTC *pBtc) {
if (pBtc->iPage < 0) { if (pBtc->iPage < 0) {
// move a clean cursor // move a clean cursor
ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt); ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt, pBtc->pTxn);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
@ -1110,7 +1114,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) {
if (pBtc->iPage < 0) { if (pBtc->iPage < 0) {
// move a clean cursor // move a clean cursor
ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt); ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt, pBtc->pTxn);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
@ -1284,7 +1288,7 @@ static int tdbBtcMoveDownward(SBTC *pBtc) {
pBtc->pPage = NULL; pBtc->pPage = NULL;
pBtc->idx = -1; pBtc->idx = -1;
ret = tdbPagerFetchPage(pBtc->pBt->pPager, pgno, &pBtc->pPage, tdbBtreeInitPage, pBtc->pBt); ret = tdbPagerFetchPage(pBtc->pBt->pPager, pgno, &pBtc->pPage, tdbBtreeInitPage, pBtc->pBt, pBtc->pTxn);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
@ -1296,7 +1300,7 @@ static int tdbBtcMoveDownward(SBTC *pBtc) {
static int tdbBtcMoveUpward(SBTC *pBtc) { static int tdbBtcMoveUpward(SBTC *pBtc) {
if (pBtc->iPage == 0) return -1; if (pBtc->iPage == 0) return -1;
tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage); tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage, pBtc->pTxn);
pBtc->iPage--; pBtc->iPage--;
pBtc->pPage = pBtc->pgStack[pBtc->iPage]; pBtc->pPage = pBtc->pgStack[pBtc->iPage];
@ -1319,7 +1323,7 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) {
if (pBtc->iPage < 0) { if (pBtc->iPage < 0) {
// move from a clear cursor // move from a clear cursor
ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt); ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt, pBtc->pTxn);
if (ret < 0) { if (ret < 0) {
// TODO // TODO
ASSERT(0); ASSERT(0);
@ -1456,7 +1460,7 @@ int tdbBtcClose(SBTC *pBtc) {
for (;;) { for (;;) {
ASSERT(pBtc->pPage); ASSERT(pBtc->pPage);
tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage); tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage, pBtc->pTxn);
pBtc->iPage--; pBtc->iPage--;
if (pBtc->iPage < 0) break; if (pBtc->iPage < 0) break;

6
source/libs/tdb/src/db/tdbDb.c
View File

@ -75,8 +75,8 @@ int tdbDbDrop(TDB *pDb) {
return 0; return 0;
} }
int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) { int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) {
return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen); return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen, pTxn);
} }
int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) {
@ -97,7 +97,7 @@ int tdbDbcOpen(TDB *pDb, TDBC **ppDbc) {
return -1; return -1;
} }
tdbBtcOpen(&pDbc->btc, pDb->pBt); tdbBtcOpen(&pDbc->btc, pDb->pBt, NULL);
// TODO: move to first now, we can move to any key-value // TODO: move to first now, we can move to any key-value
// and in any direction, design new APIs. // and in any direction, design new APIs.

10
source/libs/tdb/src/db/tdbEnv.c
View File

@ -73,12 +73,12 @@ int tdbEnvClose(TENV *pEnv) {
return 0; return 0;
} }
int tdbBegin(TENV *pEnv) { int tdbBegin(TENV *pEnv, TXN *pTxn) {
SPager *pPager; SPager *pPager;
int ret; int ret;
for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) { for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) {
ret = tdbPagerBegin(pPager); ret = tdbPagerBegin(pPager, pTxn);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
@ -88,12 +88,12 @@ int tdbBegin(TENV *pEnv) {
return 0; return 0;
} }
int tdbCommit(TENV *pEnv) { int tdbCommit(TENV *pEnv, TXN *pTxn) {
SPager *pPager; SPager *pPager;
int ret; int ret;
for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) { for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) {
ret = tdbPagerCommit(pPager); ret = tdbPagerCommit(pPager, pTxn);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
@ -103,7 +103,7 @@ int tdbCommit(TENV *pEnv) {
return 0; return 0;
} }
int tdbRollback(TENV *pEnv) { int tdbRollback(TENV *pEnv, TXN *pTxn) {
ASSERT(0); ASSERT(0);
return 0; return 0;
} }

106
source/libs/tdb/src/db/tdbPCache.c
View File

@ -18,6 +18,7 @@ struct SPCache {
int pageSize; int pageSize;
int cacheSize; int cacheSize;
tdb_mutex_t mutex; tdb_mutex_t mutex;
SPage *pList;
int nFree; int nFree;
SPage *pFree; SPage *pFree;
int nPage; int nPage;
@ -35,16 +36,17 @@ struct SPCache {
#define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL) #define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL)
static int tdbPCacheOpenImpl(SPCache *pCache); static int tdbPCacheOpenImpl(SPCache *pCache);
static void tdbPCacheInitLock(SPCache *pCache); static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn);
static void tdbPCacheClearLock(SPCache *pCache);
static void tdbPCacheLock(SPCache *pCache);
static void tdbPCacheUnlock(SPCache *pCache);
static bool tdbPCacheLocked(SPCache *pCache);
static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage);
static void tdbPCachePinPage(SPCache *pCache, SPage *pPage); static void tdbPCachePinPage(SPCache *pCache, SPage *pPage);
static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage); static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage);
static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage); static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage);
static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage); static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage);
static int tdbPCacheCloseImpl(SPCache *pCache);
static void tdbPCacheInitLock(SPCache *pCache) { tdbMutexInit(&(pCache->mutex), NULL); }
static void tdbPCacheDestroyLock(SPCache *pCache) { tdbMutexDestroy(&(pCache->mutex)); }
static void tdbPCacheLock(SPCache *pCache) { tdbMutexLock(&(pCache->mutex)); }
static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexUnlock(&(pCache->mutex)); }
int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
SPCache *pCache; SPCache *pCache;
@ -69,16 +71,19 @@ int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
} }
int tdbPCacheClose(SPCache *pCache) { int tdbPCacheClose(SPCache *pCache) {
/* TODO */ if (pCache) {
tdbPCacheCloseImpl(pCache);
tdbOsFree(pCache);
}
return 0; return 0;
} }
SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) { SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
SPage *pPage; SPage *pPage;
tdbPCacheLock(pCache); tdbPCacheLock(pCache);
pPage = tdbPCacheFetchImpl(pCache, pPgid, alcNewPage); pPage = tdbPCacheFetchImpl(pCache, pPgid, pTxn);
if (pPage) { if (pPage) {
TDB_REF_PAGE(pPage); TDB_REF_PAGE(pPage);
} }
@ -88,32 +93,40 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) {
return pPage; return pPage;
} }
void tdbPCacheRelease(SPCache *pCache, SPage *pPage) { void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
i32 nRef; i32 nRef;
nRef = TDB_UNREF_PAGE(pPage); nRef = TDB_UNREF_PAGE(pPage);
ASSERT(nRef >= 0); ASSERT(nRef >= 0);
if (nRef == 0) { if (nRef == 0) {
tdbPCacheLock(pCache);
// test the nRef again to make sure
// it is safe th handle the page
nRef = TDB_GET_PAGE_REF(pPage);
if (nRef == 0) {
if (pPage->isLocal) {
tdbPCacheUnpinPage(pCache, pPage); tdbPCacheUnpinPage(pCache, pPage);
} else {
// remove from hash
tdbPCacheRemovePageFromHash(pCache, pPage);
// free the page
if (pTxn && pTxn->xFree) {
tdbPageDestroy(pPage, pTxn->xFree, pTxn->xArg);
}
}
}
tdbPCacheUnlock(pCache);
} }
} }
static void tdbPCacheInitLock(SPCache *pCache) { tdbMutexInit(&(pCache->mutex), NULL); } int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; }
static void tdbPCacheClearLock(SPCache *pCache) { tdbMutexDestroy(&(pCache->mutex)); } static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
int ret;
static void tdbPCacheLock(SPCache *pCache) { tdbMutexLock(&(pCache->mutex)); }
static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexUnlock(&(pCache->mutex)); }
static bool tdbPCacheLocked(SPCache *pCache) {
assert(0);
// TODO
return true;
}
static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) {
SPage *pPage; SPage *pPage;
// 1. Search the hash table // 1. Search the hash table
@ -123,10 +136,10 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe
pPage = pPage->pHashNext; pPage = pPage->pHashNext;
} }
if (pPage || !alcNewPage) {
if (pPage) { if (pPage) {
// TODO: the page need to be copied and
// replaced the page in hash table
tdbPCachePinPage(pCache, pPage); tdbPCachePinPage(pCache, pPage);
}
return pPage; return pPage;
} }
@ -145,7 +158,20 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe
tdbPCachePinPage(pCache, pPage); tdbPCachePinPage(pCache, pPage);
} }
// 4. Try a stress allocation (TODO) // 4. Try a create new page
if (!pPage && pTxn && pTxn->xMalloc) {
ret = tdbPageCreate(pCache->pageSize, &pPage, pTxn->xMalloc, pTxn->xArg);
if (ret < 0) {
// TODO
ASSERT(0);
return NULL;
}
// init the page fields
pPage->isAnchor = 0;
pPage->isLocal = 0;
TDB_INIT_PAGE_REF(pPage);
}
// 5. Page here are just created from a free list // 5. Page here are just created from a free list
// or by recycling or allocated streesly, // or by recycling or allocated streesly,
@ -154,6 +180,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe
memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid)); memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid));
pPage->pLruNext = NULL; pPage->pLruNext = NULL;
pPage->pPager = NULL; pPage->pPager = NULL;
// TODO: allocated page may not add to hash
tdbPCacheAddPageToHash(pCache, pPage); tdbPCacheAddPageToHash(pCache, pPage);
} }
@ -173,25 +201,17 @@ static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) {
static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) { static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) {
i32 nRef; i32 nRef;
tdbPCacheLock(pCache);
ASSERT(!pPage->isDirty); ASSERT(!pPage->isDirty);
ASSERT(TDB_GET_PAGE_REF(pPage) == 0);
nRef = TDB_GET_PAGE_REF(pPage);
ASSERT(nRef >= 0);
if (nRef == 0) {
// Add the page to LRU list
ASSERT(pPage->pLruNext == NULL); ASSERT(pPage->pLruNext == NULL);
pPage->pLruPrev = &(pCache->lru); pPage->pLruPrev = &(pCache->lru);
pPage->pLruNext = pCache->lru.pLruNext; pPage->pLruNext = pCache->lru.pLruNext;
pCache->lru.pLruNext->pLruPrev = pPage; pCache->lru.pLruNext->pLruPrev = pPage;
pCache->lru.pLruNext = pPage; pCache->lru.pLruNext = pPage;
}
pCache->nRecyclable++; pCache->nRecyclable++;
tdbPCacheUnlock(pCache);
} }
static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) {
@ -238,13 +258,14 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
// pPage->pgid = 0; // pPage->pgid = 0;
pPage->isAnchor = 0; pPage->isAnchor = 0;
pPage->isLocalPage = 1; pPage->isLocal = 1;
TDB_INIT_PAGE_REF(pPage); TDB_INIT_PAGE_REF(pPage);
pPage->pHashNext = NULL; pPage->pHashNext = NULL;
pPage->pLruNext = NULL; pPage->pLruNext = NULL;
pPage->pLruPrev = NULL; pPage->pLruPrev = NULL;
pPage->pDirtyNext = NULL; pPage->pDirtyNext = NULL;
// add page to free list
pPage->pFreeNext = pCache->pFree; pPage->pFreeNext = pCache->pFree;
pCache->pFree = pPage; pCache->pFree = pPage;
pCache->nFree++; pCache->nFree++;
@ -268,4 +289,13 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
return 0; return 0;
} }
int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; } static int tdbPCacheCloseImpl(SPCache *pCache) {
SPage *pPage;
for (pPage = pCache->pList; pPage; pPage = pCache->pList) {
pCache->pList = pPage->pCacheNext;
tdbPageDestroy(pPage, NULL, NULL);
}
tdbPCacheDestroyLock(pCache);
}

2
source/libs/tdb/src/db/tdbPage.c
View File

@ -278,7 +278,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) {
// 2. Try to allocate from the page free list // 2. Try to allocate from the page free list
cellFree = TDB_PAGE_FCELL(pPage); cellFree = TDB_PAGE_FCELL(pPage);
ASSERT(cellFree == 0 || cellFree > pPage->pFreeEnd - pPage->pData); ASSERT(cellFree == 0 || cellFree >= pPage->pFreeEnd - pPage->pData);
if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) { if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) {
SCell *pPrevFreeCell = NULL; SCell *pPrevFreeCell = NULL;
int szPrevFreeCell; int szPrevFreeCell;

34
source/libs/tdb/src/db/tdbPager.c
View File

@ -27,7 +27,6 @@ TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct")
#define TDB_PAGE_INITIALIZED(pPage) ((pPage)->pPager != NULL) #define TDB_PAGE_INITIALIZED(pPage) ((pPage)->pPager != NULL)
static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg, u8 loadPage); static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg, u8 loadPage);
static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage); static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage);
static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage); static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage);
@ -77,6 +76,8 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
// pPager->jfd = -1; // pPager->jfd = -1;
pPager->pageSize = tdbPCacheGetPageSize(pCache); pPager->pageSize = tdbPCacheGetPageSize(pCache);
// pPager->dbOrigSize
ret = tdbGetFileSize(pPager->fd, pPager->pageSize, &(pPager->dbOrigSize));
*ppPager = pPager; *ppPager = pPager;
return 0; return 0;
@ -92,9 +93,15 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate) {
SPage *pPage; SPage *pPage;
int ret; int ret;
if (pPager->dbOrigSize > 0) {
pgno = 1;
} else {
pgno = 0;
}
{ {
// TODO: try to search the main DB to get the page number // TODO: try to search the main DB to get the page number
pgno = 0; // pgno = 0;
} }
// if (pgno == 0 && toCreate) { // if (pgno == 0 && toCreate) {
@ -157,7 +164,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
return 0; return 0;
} }
int tdbPagerBegin(SPager *pPager) { int tdbPagerBegin(SPager *pPager, TXN *pTxn) {
if (pPager->inTran) { if (pPager->inTran) {
return 0; return 0;
} }
@ -175,7 +182,7 @@ int tdbPagerBegin(SPager *pPager) {
return 0; return 0;
} }
int tdbPagerCommit(SPager *pPager) { int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
SPage *pPage; SPage *pPage;
int ret; int ret;
@ -204,7 +211,7 @@ int tdbPagerCommit(SPager *pPager) {
pPage->isDirty = 0; pPage->isDirty = 0;
tdbPCacheRelease(pPager->pCache, pPage); tdbPCacheRelease(pPager->pCache, pPage, pTxn);
} }
// sync the db file // sync the db file
@ -219,7 +226,8 @@ int tdbPagerCommit(SPager *pPager) {
return 0; return 0;
} }
int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg) { int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg,
TXN *pTxn) {
SPage *pPage; SPage *pPage;
SPgid pgid; SPgid pgid;
int ret; int ret;
@ -227,7 +235,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage
// Fetch a page container from the page cache // Fetch a page container from the page cache
memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN); memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN);
pgid.pgno = pgno; pgid.pgno = pgno;
pPage = tdbPCacheFetch(pPager->pCache, &pgid, 1); pPage = tdbPCacheFetch(pPager->pCache, &pgid, pTxn);
if (pPage == NULL) { if (pPage == NULL) {
return -1; return -1;
} }
@ -247,7 +255,8 @@ int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage
return 0; return 0;
} }
int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg) { int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg,
TXN *pTxn) {
int ret; int ret;
SPage *pPage; SPage *pPage;
SPgid pgid; SPgid pgid;
@ -255,6 +264,7 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage
// Allocate a page number // Allocate a page number
ret = tdbPagerAllocPage(pPager, ppgno); ret = tdbPagerAllocPage(pPager, ppgno);
if (ret < 0) { if (ret < 0) {
ASSERT(0);
return -1; return -1;
} }
@ -263,8 +273,9 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage
// Fetch a page container from the page cache // Fetch a page container from the page cache
memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN); memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN);
pgid.pgno = *ppgno; pgid.pgno = *ppgno;
pPage = tdbPCacheFetch(pPager->pCache, &pgid, 1); pPage = tdbPCacheFetch(pPager->pCache, &pgid, pTxn);
if (pPage == NULL) { if (pPage == NULL) {
ASSERT(0);
return -1; return -1;
} }
@ -273,6 +284,7 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage
// Initialize the page if need // Initialize the page if need
ret = tdbPagerInitPage(pPager, pPage, initPage, arg, 0); ret = tdbPagerInitPage(pPager, pPage, initPage, arg, 0);
if (ret < 0) { if (ret < 0) {
ASSERT(0);
return -1; return -1;
} }
@ -283,7 +295,7 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage
return 0; return 0;
} }
void tdbPagerReturnPage(SPager *pPager, SPage *pPage) { tdbPCacheRelease(pPager->pCache, pPage); } void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn) { tdbPCacheRelease(pPager->pCache, pPage, pTxn); }
static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) { static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) {
// TODO: Allocate a page from the free list // TODO: Allocate a page from the free list
@ -295,7 +307,7 @@ static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) {
return 0; return 0;
} }
static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) { int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) {
int ret; int ret;
*ppgno = 0; *ppgno = 0;

36
source/libs/tdb/src/db/tdbTxn.c
View File

@ -15,29 +15,17 @@
#include "tdbInt.h" #include "tdbInt.h"
// int tdbTxnBegin(TENV *pEnv) { int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg,
// // TODO int flags) {
// return 0; // not support read-committed version at the moment
// } ASSERT(flags == 0 || flags == TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
// int tdbTxnCommit(TENV *pEnv) { pTxn->flags = flags;
// SPager *pPager = NULL; pTxn->txnId = txnid;
// int ret; pTxn->xMalloc = xMalloc;
pTxn->xFree = xFree;
pTxn->xArg = xArg;
return 0;
}
// for (;;) { int tdbTxnClose(TXN *pTxn) { return 0; }
// break;
// ret = tdbPagerCommit(pPager);
// if (ret < 0) {
// ASSERT(0);
// return -1;
// }
// }
// // TODO
// return 0;
// }
// int tdbTxnRollback(TENV *pEnv) {
// // TODO
// return 0;
// }

13
source/libs/tdb/src/db/tdbUtil.c
View File

@ -32,3 +32,16 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) {
return 0; return 0;
} }
int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size) {
int ret;
int64_t szBytes;
ret = tdbOsFileSize(fd, &szBytes);
if (ret < 0) {
return -1;
}
*size = szBytes / szPage;
return 0;
}

5
source/libs/tdb/src/inc/tdbBtree.h
View File

@ -35,17 +35,18 @@ struct SBTC {
int idx; int idx;
int idxStack[BTREE_MAX_DEPTH + 1]; int idxStack[BTREE_MAX_DEPTH + 1];
SPage *pgStack[BTREE_MAX_DEPTH + 1]; SPage *pgStack[BTREE_MAX_DEPTH + 1];
TXN *pTxn;
}; };
// SBTree // SBTree
int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt); int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt);
int tdbBtreeClose(SBTree *pBt); int tdbBtreeClose(SBTree *pBt);
int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen); int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen);
int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
// SBTC // SBTC
int tdbBtcOpen(SBTC *pCur, SBTree *pBt); int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn);
int tdbBtcMoveToFirst(SBTC *pBtc); int tdbBtcMoveToFirst(SBTC *pBtc);
int tdbBtcMoveToLast(SBTC *pBtc); int tdbBtcMoveToLast(SBTC *pBtc);
int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen);

Some files were not shown because too many files have changed in this diff Show More