xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/3.0_debug_wxy' into feature/insertselect

This commit is contained in:
dapan1121 2022-07-08 10:28:53 +08:00
parent b849252eed 80172d5c61
commit 0925064d24
106 changed files with 5440 additions and 4368 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmake/cmake.options
View File

@ -51,6 +51,13 @@ IF(${TD_WINDOWS})
"If build unit tests using googletest"
ON
)
ELSEIF (TD_DARWIN_64)
add_definitions(-DCOMPILER_SUPPORTS_CXX13)
option(
BUILD_TEST
"If build unit tests using googletest"
ON
)
ELSE ()
include(CheckCXXCompilerFlag)
CHECK_CXX_COMPILER_FLAG("-std=c++13" COMPILER_SUPPORTS_CXX13)

4
examples/c/tmq.c
View File

@ -241,7 +241,7 @@ int32_t create_topic() {
taos_free_result(pRes);
pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");
// pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");
/*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/
if (taos_errno(pRes) != 0) {
printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
return -1;
@ -302,7 +302,7 @@ tmq_t* build_consumer() {
tmq_conf_set(conf, "msg.with.table.name", "true");
tmq_conf_set(conf, "enable.auto.commit", "true");
tmq_conf_set(conf, "experimental.snapshot.enable", "false");
/*tmq_conf_set(conf, "experimental.snapshot.enable", "true");*/
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);

26
include/common/tcommon.h
View File

@ -32,6 +32,18 @@ enum {
TMQ_CONF__RESET_OFFSET__LATEST = -1,
};
// clang-format off
#define IS_META_MSG(x) ( \
x == TDMT_VND_CREATE_STB \
|| x == TDMT_VND_ALTER_STB \
|| x == TDMT_VND_DROP_STB \
|| x == TDMT_VND_CREATE_TABLE \
|| x == TDMT_VND_ALTER_TABLE \
|| x == TDMT_VND_DROP_TABLE \
|| x == TDMT_VND_DROP_TTL_TABLE \
)
// clang-format on
enum {
TMQ_MSG_TYPE__DUMMY = 0,
TMQ_MSG_TYPE__POLL_RSP,
@ -105,6 +117,20 @@ typedef struct SSDataBlock {
SDataBlockInfo info;
} SSDataBlock;
enum {
FETCH_TYPE__DATA = 1,
FETCH_TYPE__META,
FETCH_TYPE__NONE,
};
typedef struct {
int8_t fetchType;
union {
SSDataBlock data;
void* meta;
};
} SFetchRet;
typedef struct SVarColAttr {
int32_t* offset; // start position for each entry in the list
uint32_t length; // used buffer size that contain the valid data

8
include/common/tmsg.h
View File

@ -2828,8 +2828,8 @@ typedef struct {
static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
// tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
// tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
tlen += taosEncodeFixedI16(buf, pRsp->resMsgType);
tlen += taosEncodeFixedI32(buf, pRsp->metaRspLen);
tlen += taosEncodeBinary(buf, pRsp->metaRsp, pRsp->metaRspLen);
@ -2837,8 +2837,8 @@ static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp
}
static FORCE_INLINE void* tDecodeSMqMetaRsp(const void* buf, SMqMetaRsp* pRsp) {
buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
// buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
// buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
buf = taosDecodeFixedI16(buf, &pRsp->resMsgType);
buf = taosDecodeFixedI32(buf, &pRsp->metaRspLen);
buf = taosDecodeBinary(buf, &pRsp->metaRsp, pRsp->metaRspLen);

379
include/common/ttokendef.h
View File

@ -79,195 +79,196 @@
#define TK_EXISTS 61
#define TK_BUFFER 62
#define TK_CACHELAST 63
#define TK_COMP 64
#define TK_DURATION 65
#define TK_NK_VARIABLE 66
#define TK_FSYNC 67
#define TK_MAXROWS 68
#define TK_MINROWS 69
#define TK_KEEP 70
#define TK_PAGES 71
#define TK_PAGESIZE 72
#define TK_PRECISION 73
#define TK_REPLICA 74
#define TK_STRICT 75
#define TK_WAL 76
#define TK_VGROUPS 77
#define TK_SINGLE_STABLE 78
#define TK_RETENTIONS 79
#define TK_SCHEMALESS 80
#define TK_NK_COLON 81
#define TK_TABLE 82
#define TK_NK_LP 83
#define TK_NK_RP 84
#define TK_STABLE 85
#define TK_ADD 86
#define TK_COLUMN 87
#define TK_MODIFY 88
#define TK_RENAME 89
#define TK_TAG 90
#define TK_SET 91
#define TK_NK_EQ 92
#define TK_USING 93
#define TK_TAGS 94
#define TK_COMMENT 95
#define TK_BOOL 96
#define TK_TINYINT 97
#define TK_SMALLINT 98
#define TK_INT 99
#define TK_INTEGER 100
#define TK_BIGINT 101
#define TK_FLOAT 102
#define TK_DOUBLE 103
#define TK_BINARY 104
#define TK_TIMESTAMP 105
#define TK_NCHAR 106
#define TK_UNSIGNED 107
#define TK_JSON 108
#define TK_VARCHAR 109
#define TK_MEDIUMBLOB 110
#define TK_BLOB 111
#define TK_VARBINARY 112
#define TK_DECIMAL 113
#define TK_MAX_DELAY 114
#define TK_WATERMARK 115
#define TK_ROLLUP 116
#define TK_TTL 117
#define TK_SMA 118
#define TK_FIRST 119
#define TK_LAST 120
#define TK_SHOW 121
#define TK_DATABASES 122
#define TK_TABLES 123
#define TK_STABLES 124
#define TK_MNODES 125
#define TK_MODULES 126
#define TK_QNODES 127
#define TK_FUNCTIONS 128
#define TK_INDEXES 129
#define TK_ACCOUNTS 130
#define TK_APPS 131
#define TK_CONNECTIONS 132
#define TK_LICENCE 133
#define TK_GRANTS 134
#define TK_QUERIES 135
#define TK_SCORES 136
#define TK_TOPICS 137
#define TK_VARIABLES 138
#define TK_BNODES 139
#define TK_SNODES 140
#define TK_CLUSTER 141
#define TK_TRANSACTIONS 142
#define TK_DISTRIBUTED 143
#define TK_CONSUMERS 144
#define TK_SUBSCRIPTIONS 145
#define TK_LIKE 146
#define TK_INDEX 147
#define TK_FUNCTION 148
#define TK_INTERVAL 149
#define TK_TOPIC 150
#define TK_AS 151
#define TK_WITH 152
#define TK_META 153
#define TK_CONSUMER 154
#define TK_GROUP 155
#define TK_DESC 156
#define TK_DESCRIBE 157
#define TK_RESET 158
#define TK_QUERY 159
#define TK_CACHE 160
#define TK_EXPLAIN 161
#define TK_ANALYZE 162
#define TK_VERBOSE 163
#define TK_NK_BOOL 164
#define TK_RATIO 165
#define TK_NK_FLOAT 166
#define TK_COMPACT 167
#define TK_VNODES 168
#define TK_IN 169
#define TK_OUTPUTTYPE 170
#define TK_AGGREGATE 171
#define TK_BUFSIZE 172
#define TK_STREAM 173
#define TK_INTO 174
#define TK_TRIGGER 175
#define TK_AT_ONCE 176
#define TK_WINDOW_CLOSE 177
#define TK_IGNORE 178
#define TK_EXPIRED 179
#define TK_KILL 180
#define TK_CONNECTION 181
#define TK_TRANSACTION 182
#define TK_BALANCE 183
#define TK_VGROUP 184
#define TK_MERGE 185
#define TK_REDISTRIBUTE 186
#define TK_SPLIT 187
#define TK_SYNCDB 188
#define TK_DELETE 189
#define TK_INSERT 190
#define TK_NULL 191
#define TK_NK_QUESTION 192
#define TK_NK_ARROW 193
#define TK_ROWTS 194
#define TK_TBNAME 195
#define TK_QSTARTTS 196
#define TK_QENDTS 197
#define TK_WSTARTTS 198
#define TK_WENDTS 199
#define TK_WDURATION 200
#define TK_CAST 201
#define TK_NOW 202
#define TK_TODAY 203
#define TK_TIMEZONE 204
#define TK_CLIENT_VERSION 205
#define TK_SERVER_VERSION 206
#define TK_SERVER_STATUS 207
#define TK_CURRENT_USER 208
#define TK_COUNT 209
#define TK_LAST_ROW 210
#define TK_BETWEEN 211
#define TK_IS 212
#define TK_NK_LT 213
#define TK_NK_GT 214
#define TK_NK_LE 215
#define TK_NK_GE 216
#define TK_NK_NE 217
#define TK_MATCH 218
#define TK_NMATCH 219
#define TK_CONTAINS 220
#define TK_JOIN 221
#define TK_INNER 222
#define TK_SELECT 223
#define TK_DISTINCT 224
#define TK_WHERE 225
#define TK_PARTITION 226
#define TK_BY 227
#define TK_SESSION 228
#define TK_STATE_WINDOW 229
#define TK_SLIDING 230
#define TK_FILL 231
#define TK_VALUE 232
#define TK_NONE 233
#define TK_PREV 234
#define TK_LINEAR 235
#define TK_NEXT 236
#define TK_HAVING 237
#define TK_RANGE 238
#define TK_EVERY 239
#define TK_ORDER 240
#define TK_SLIMIT 241
#define TK_SOFFSET 242
#define TK_LIMIT 243
#define TK_OFFSET 244
#define TK_ASC 245
#define TK_NULLS 246
#define TK_ID 247
#define TK_NK_BITNOT 248
#define TK_VALUES 249
#define TK_IMPORT 250
#define TK_NK_SEMI 251
#define TK_FILE 252
#define TK_CACHELASTSIZE 64
#define TK_COMP 65
#define TK_DURATION 66
#define TK_NK_VARIABLE 67
#define TK_FSYNC 68
#define TK_MAXROWS 69
#define TK_MINROWS 70
#define TK_KEEP 71
#define TK_PAGES 72
#define TK_PAGESIZE 73
#define TK_PRECISION 74
#define TK_REPLICA 75
#define TK_STRICT 76
#define TK_WAL 77
#define TK_VGROUPS 78
#define TK_SINGLE_STABLE 79
#define TK_RETENTIONS 80
#define TK_SCHEMALESS 81
#define TK_NK_COLON 82
#define TK_TABLE 83
#define TK_NK_LP 84
#define TK_NK_RP 85
#define TK_STABLE 86
#define TK_ADD 87
#define TK_COLUMN 88
#define TK_MODIFY 89
#define TK_RENAME 90
#define TK_TAG 91
#define TK_SET 92
#define TK_NK_EQ 93
#define TK_USING 94
#define TK_TAGS 95
#define TK_COMMENT 96
#define TK_BOOL 97
#define TK_TINYINT 98
#define TK_SMALLINT 99
#define TK_INT 100
#define TK_INTEGER 101
#define TK_BIGINT 102
#define TK_FLOAT 103
#define TK_DOUBLE 104
#define TK_BINARY 105
#define TK_TIMESTAMP 106
#define TK_NCHAR 107
#define TK_UNSIGNED 108
#define TK_JSON 109
#define TK_VARCHAR 110
#define TK_MEDIUMBLOB 111
#define TK_BLOB 112
#define TK_VARBINARY 113
#define TK_DECIMAL 114
#define TK_MAX_DELAY 115
#define TK_WATERMARK 116
#define TK_ROLLUP 117
#define TK_TTL 118
#define TK_SMA 119
#define TK_FIRST 120
#define TK_LAST 121
#define TK_SHOW 122
#define TK_DATABASES 123
#define TK_TABLES 124
#define TK_STABLES 125
#define TK_MNODES 126
#define TK_MODULES 127
#define TK_QNODES 128
#define TK_FUNCTIONS 129
#define TK_INDEXES 130
#define TK_ACCOUNTS 131
#define TK_APPS 132
#define TK_CONNECTIONS 133
#define TK_LICENCE 134
#define TK_GRANTS 135
#define TK_QUERIES 136
#define TK_SCORES 137
#define TK_TOPICS 138
#define TK_VARIABLES 139
#define TK_BNODES 140
#define TK_SNODES 141
#define TK_CLUSTER 142
#define TK_TRANSACTIONS 143
#define TK_DISTRIBUTED 144
#define TK_CONSUMERS 145
#define TK_SUBSCRIPTIONS 146
#define TK_LIKE 147
#define TK_INDEX 148
#define TK_FUNCTION 149
#define TK_INTERVAL 150
#define TK_TOPIC 151
#define TK_AS 152
#define TK_WITH 153
#define TK_META 154
#define TK_CONSUMER 155
#define TK_GROUP 156
#define TK_DESC 157
#define TK_DESCRIBE 158
#define TK_RESET 159
#define TK_QUERY 160
#define TK_CACHE 161
#define TK_EXPLAIN 162
#define TK_ANALYZE 163
#define TK_VERBOSE 164
#define TK_NK_BOOL 165
#define TK_RATIO 166
#define TK_NK_FLOAT 167
#define TK_COMPACT 168
#define TK_VNODES 169
#define TK_IN 170
#define TK_OUTPUTTYPE 171
#define TK_AGGREGATE 172
#define TK_BUFSIZE 173
#define TK_STREAM 174
#define TK_INTO 175
#define TK_TRIGGER 176
#define TK_AT_ONCE 177
#define TK_WINDOW_CLOSE 178
#define TK_IGNORE 179
#define TK_EXPIRED 180
#define TK_KILL 181
#define TK_CONNECTION 182
#define TK_TRANSACTION 183
#define TK_BALANCE 184
#define TK_VGROUP 185
#define TK_MERGE 186
#define TK_REDISTRIBUTE 187
#define TK_SPLIT 188
#define TK_SYNCDB 189
#define TK_DELETE 190
#define TK_INSERT 191
#define TK_NULL 192
#define TK_NK_QUESTION 193
#define TK_NK_ARROW 194
#define TK_ROWTS 195
#define TK_TBNAME 196
#define TK_QSTARTTS 197
#define TK_QENDTS 198
#define TK_WSTARTTS 199
#define TK_WENDTS 200
#define TK_WDURATION 201
#define TK_CAST 202
#define TK_NOW 203
#define TK_TODAY 204
#define TK_TIMEZONE 205
#define TK_CLIENT_VERSION 206
#define TK_SERVER_VERSION 207
#define TK_SERVER_STATUS 208
#define TK_CURRENT_USER 209
#define TK_COUNT 210
#define TK_LAST_ROW 211
#define TK_BETWEEN 212
#define TK_IS 213
#define TK_NK_LT 214
#define TK_NK_GT 215
#define TK_NK_LE 216
#define TK_NK_GE 217
#define TK_NK_NE 218
#define TK_MATCH 219
#define TK_NMATCH 220
#define TK_CONTAINS 221
#define TK_JOIN 222
#define TK_INNER 223
#define TK_SELECT 224
#define TK_DISTINCT 225
#define TK_WHERE 226
#define TK_PARTITION 227
#define TK_BY 228
#define TK_SESSION 229
#define TK_STATE_WINDOW 230
#define TK_SLIDING 231
#define TK_FILL 232
#define TK_VALUE 233
#define TK_NONE 234
#define TK_PREV 235
#define TK_LINEAR 236
#define TK_NEXT 237
#define TK_HAVING 238
#define TK_RANGE 239
#define TK_EVERY 240
#define TK_ORDER 241
#define TK_SLIMIT 242
#define TK_SOFFSET 243
#define TK_LIMIT 244
#define TK_OFFSET 245
#define TK_ASC 246
#define TK_NULLS 247
#define TK_ID 248
#define TK_NK_BITNOT 249
#define TK_VALUES 250
#define TK_IMPORT 251
#define TK_NK_SEMI 252
#define TK_FILE 253
#define TK_NK_SPACE 300
#define TK_NK_COMMENT 301

13
include/libs/executor/executor.h
View File

@ -30,15 +30,15 @@ struct SRpcMsg;
struct SSubplan;
typedef struct SReadHandle {
void* reader;
void* tqReader;
void* meta;
void* config;
void* vnode;
void* mnd;
SMsgCb* pMsgCb;
// int8_t initTsdbReader;
bool tqReader;
bool initMetaReader;
bool initTableReader;
bool initTqReader;
} SReadHandle;
typedef enum {
@ -52,7 +52,7 @@ typedef enum {
* @param streamReadHandle
* @return
*/
qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle);
qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers);
/**
* Switch the stream scan to snapshot mode
@ -176,6 +176,9 @@ int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts);
int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts);
void* qExtractReaderFromStreamScanner(void* scanner);
int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner);
#ifdef __cplusplus
}
#endif

23
include/libs/index/index.h
View File

@ -127,7 +127,7 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result);
* @parma opt (input, rebuild index opts)
* @return error code
*/
int indexRebuild(SIndex* index, SIndexOpts* opt);
// int indexRebuild(SIndex* index, SIndexOpts* opt);
/*
* open index
@ -185,6 +185,25 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t c
int32_t nColName, const char* colVal, int32_t nColVal);
void indexTermDestroy(SIndexTerm* p);
/*
* rebuild index
*/
void indexRebuild(SIndexJson* idx, void* iter);
/*
* check index json status
**/
bool indexIsRebuild(SIndex* idx);
/*
* rebuild index json
*/
void indexJsonRebuild(SIndexJson* idx, void* iter);
/*
* check index json status
**/
bool indexJsonIsRebuild(SIndexJson* idx);
/*
* init index env
*
@ -203,7 +222,7 @@ typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltS
SIdxFltStatus idxGetFltStatus(SNode* pFilterNode);
int32_t doFilterTag(const SNode* pFilterNode, SIndexMetaArg* metaArg, SArray* result);
int32_t doFilterTag(SNode* pFilterNode, SIndexMetaArg* metaArg, SArray* result, SIdxFltStatus* status);
/*
* destory index env
*

3
include/libs/nodes/cmdnodes.h
View File

@ -51,7 +51,8 @@ extern "C" {
typedef struct SDatabaseOptions {
ENodeType type;
int32_t buffer;
int8_t cachelast;
int8_t cacheLast;
int32_t cacheLastSize;
int8_t compressionLevel;
int32_t daysPerFile;
SValueNode* pDaysPerFile;

8
include/libs/stream/tstream.h
View File

@ -223,7 +223,7 @@ typedef struct {
SEpSet epSet;
} SStreamChildEpInfo;
struct SStreamTask {
typedef struct SStreamTask {
int64_t streamId;
int32_t taskId;
int8_t isDataScan;
@ -277,7 +277,7 @@ struct SStreamTask {
// msg handle
SMsgCb* pMsgCb;
};
} SStreamTask;
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo);
@ -288,6 +288,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
void tFreeSStreamTask(SStreamTask* pTask);
static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) {
#if 0
while (1) {
int8_t inputStatus =
atomic_val_compare_exchange_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL, TASK_INPUT_STATUS__PROCESSING);
@ -296,6 +297,7 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem
}
ASSERT(0);
}
#endif
if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem);
@ -316,8 +318,10 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem
atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE, TASK_TRIGGER_STATUS__ACTIVE);
}
#if 0
// TODO: back pressure
atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL);
#endif
return 0;
}

48
include/libs/wal/wal.h
View File

@ -88,7 +88,7 @@ typedef struct {
EWalType level; // wal level
} SWalCfg;
typedef struct SWalVer {
typedef struct {
int64_t firstVer;
int64_t verInSnapshotting;
int64_t snapshotVer;
@ -149,17 +149,22 @@ typedef struct SWal {
SWalCkHead writeHead;
} SWal; // WAL HANDLE
typedef struct SWalReadHandle {
SWal *pWal;
TdFilePtr pReadLogTFile;
TdFilePtr pReadIdxTFile;
int64_t curFileFirstVer;
int64_t curVersion;
int64_t capacity;
int64_t status; // if cursor valid
TdThreadMutex mutex;
SWalCkHead *pHead;
} SWalReadHandle;
typedef struct {
int8_t scanUncommited;
int8_t scanMeta;
} SWalFilterCond;
typedef struct {
SWal *pWal;
TdFilePtr pLogFile;
TdFilePtr pIdxFile;
int64_t curFileFirstVer;
int64_t curVersion;
int64_t capacity;
TdThreadMutex mutex;
SWalFilterCond cond;
SWalCkHead *pHead;
} SWalReader;
// module initialization
int32_t walInit();
@ -178,7 +183,6 @@ void walFsync(SWal *, bool force);
// apis for lifecycle management
int32_t walCommit(SWal *, int64_t ver);
// truncate after
int32_t walRollback(SWal *, int64_t ver);
// notify that previous logs can be pruned safely
int32_t walBeginSnapshot(SWal *, int64_t ver);
@ -187,15 +191,16 @@ int32_t walRestoreFromSnapshot(SWal *, int64_t ver);
// int32_t walDataCorrupted(SWal*);
// read
SWalReadHandle *walOpenReadHandle(SWal *);
void walCloseReadHandle(SWalReadHandle *);
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond);
void walCloseReader(SWalReader *pRead);
int32_t walReadVer(SWalReader *pRead, int64_t ver);
int32_t walNextValidMsg(SWalReader *pRead);
// only for tq usage
void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity);
int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalCkHead *pHead);
int32_t walFetchBody(SWalReadHandle *pRead, SWalCkHead **ppHead);
int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalCkHead *pHead);
void walSetReaderCapacity(SWalReader *pRead, int32_t capacity);
int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead);
int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead);
int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead);
typedef struct {
int64_t refId;
@ -207,10 +212,11 @@ void walCloseRef(SWalRef *);
int32_t walRefVer(SWalRef *, int64_t ver);
int32_t walUnrefVer(SWal *);
// help function for raft
bool walLogExist(SWal *, int64_t ver);
bool walIsEmpty(SWal *);
// lifecycle check
bool walIsEmpty(SWal *);
int64_t walGetFirstVer(SWal *);
int64_t walGetSnapshotVer(SWal *);
int64_t walGetLastVer(SWal *);

1
include/util/tutil.h
View File

@ -45,6 +45,7 @@ void taosIp2String(uint32_t ip, char *str);
void taosIpPort2String(uint32_t ip, uint16_t port, char *str);
void *tmemmem(const char *haystack, int hlen, const char *needle, int nlen);
char *strDupUnquo(const char *src);
static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) {
T_MD5_CTX context;

2
source/client/CMakeLists.txt
View File

@ -26,6 +26,8 @@ if(TD_WINDOWS)
/DEF:${CMAKE_CURRENT_SOURCE_DIR}/src/taos.def
)
INCLUDE_DIRECTORIES(jni/windows)
INCLUDE_DIRECTORIES(jni/windows/win32)
INCLUDE_DIRECTORIES(jni/windows/win32/bridge)
else()
INCLUDE_DIRECTORIES(jni/linux)
endif()

8
source/client/src/clientImpl.c
View File

@ -1298,16 +1298,16 @@ void doProcessMsgFromServer(SSchedMsg* schedMsg) {
pSendInfo->fp(pSendInfo->param, &buf, pMsg->code);
rpcFreeCont(pMsg->pCont);
destroySendMsgInfo(pSendInfo);
taosMemoryFree(arg);
}
void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
SSchedMsg schedMsg = {0};
SEpSet* tEpSet = pEpSet != NULL ? taosMemoryCalloc(1, sizeof(SEpSet)) : NULL;
if (tEpSet != NULL) {
*tEpSet = *pEpSet;
SEpSet* tEpSet = NULL;
if (pEpSet != NULL) {
tEpSet = taosMemoryCalloc(1, sizeof(SEpSet));
memcpy((void*)tEpSet, (void*)pEpSet, sizeof(SEpSet));
}
SchedArg* arg = taosMemoryCalloc(1, sizeof(SchedArg));

27
source/client/src/tmq.c
View File

@ -50,19 +50,18 @@ struct tmq_list_t {
};
struct tmq_conf_t {
char clientId[256];
char groupId[TSDB_CGROUP_LEN];
int8_t autoCommit;
int8_t resetOffset;
int8_t withTbName;
int8_t spEnable;
int32_t spBatchSize;
uint16_t port;
int32_t autoCommitInterval;
char* ip;
char* user;
char* pass;
/*char* db;*/
char clientId[256];
char groupId[TSDB_CGROUP_LEN];
int8_t autoCommit;
int8_t resetOffset;
int8_t withTbName;
int8_t spEnable;
int32_t spBatchSize;
uint16_t port;
int32_t autoCommitInterval;
char* ip;
char* user;
char* pass;
tmq_commit_cb* commitCb;
void* commitCbUserParam;
};
@ -338,7 +337,7 @@ tmq_list_t* tmq_list_new() {
int32_t tmq_list_append(tmq_list_t* list, const char* src) {
SArray* container = &list->container;
char* topic = strdup(src);
char* topic = strDupUnquo(src);
if (taosArrayPush(container, &topic) == NULL) return -1;
return 0;
}

13
source/common/src/tdatablock.c
View File

@ -1737,41 +1737,54 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
int32_t len = 0;
len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id %lu|\n", flag,
(int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId);
if (len >= size -1) return dumpBuf;
for (int32_t j = 0; j < rows; j++) {
len += snprintf(dumpBuf + len, size - len, "%s |", flag);
if (len >= size -1) return dumpBuf;
for (int32_t k = 0; k < colNum; k++) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
if (colDataIsNull(pColInfoData, rows, j, NULL)) {
len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL");
if (len >= size -1) return dumpBuf;
continue;
}
switch (pColInfoData->info.type) {
case TSDB_DATA_TYPE_TIMESTAMP:
formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI);
len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf);
if (len >= size -1) return dumpBuf;
break;
case TSDB_DATA_TYPE_INT:
len += snprintf(dumpBuf + len, size - len, " %15d |", *(int32_t*)var);
if (len >= size -1) return dumpBuf;
break;
case TSDB_DATA_TYPE_UINT:
len += snprintf(dumpBuf + len, size - len, " %15u |", *(uint32_t*)var);
if (len >= size -1) return dumpBuf;
break;
case TSDB_DATA_TYPE_BIGINT:
len += snprintf(dumpBuf + len, size - len, " %15ld |", *(int64_t*)var);
if (len >= size -1) return dumpBuf;
break;
case TSDB_DATA_TYPE_UBIGINT:
len += snprintf(dumpBuf + len, size - len, " %15lu |", *(uint64_t*)var);
if (len >= size -1) return dumpBuf;
break;
case TSDB_DATA_TYPE_FLOAT:
len += snprintf(dumpBuf + len, size - len, " %15f |", *(float*)var);
if (len >= size -1) return dumpBuf;
break;
case TSDB_DATA_TYPE_DOUBLE:
len += snprintf(dumpBuf + len, size - len, " %15lf |", *(double*)var);
if (len >= size -1) return dumpBuf;
break;
}
}
len += snprintf(dumpBuf + len, size - len, "\n");
if (len >= size -1) return dumpBuf;
}
len += snprintf(dumpBuf + len, size - len, "%s |end\n", flag);
return dumpBuf;

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -238,10 +238,12 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;

2
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -377,10 +377,12 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SET_VNODE_STANDBY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;

3
source/dnode/mgmt/test/sut/src/sut.cpp
View File

@ -43,6 +43,9 @@ if (taosInitLog("taosdlog", 1) != 0) {
}
void Testbase::Init(const char* path, int16_t port) {
#ifdef _TD_DARWIN_64
osDefaultInit();
#endif
tsServerPort = port;
strcpy(tsLocalFqdn, "localhost");
snprintf(tsLocalEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort);

4
source/dnode/mnode/impl/src/mndStb.c
View File

@ -270,13 +270,12 @@ static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) {
static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) {
mTrace("stb:%s, perform delete action, row:%p", pStb->name, pStb);
taosArrayDestroy(pStb->pFuncs);
taosMemoryFreeClear(pStb->pColumns);
taosMemoryFreeClear(pStb->pTags);
taosMemoryFreeClear(pStb->comment);
taosMemoryFreeClear(pStb->pFuncs);
taosMemoryFreeClear(pStb->pAst1);
taosMemoryFreeClear(pStb->pAst2);
taosArrayDestroy(pStb->pFuncs);
return 0;
}
@ -798,6 +797,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea
_OVER:
mndTransDrop(pTrans);
mndStbActionDelete(pMnode->pSdb, &stbObj);
return code;
}

40
source/dnode/vnode/inc/vnode.h
View File

@ -136,6 +136,8 @@ SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdLis
int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond, int32_t tWinIdx);
int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
void *tsdbGetIdx(SMeta *pMeta);
void *tsdbGetIvtIdx(SMeta *pMeta);
int32_t tsdbLastRowReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t *colId, int32_t numOfCols,
void **pReader);
@ -145,19 +147,37 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6
// tq
typedef struct STqReadHandle SStreamReader;
typedef struct STqReader {
int64_t ver;
const SSubmitReq *pMsg;
SSubmitBlk *pBlock;
SSubmitMsgIter msgIter;
SSubmitBlkIter blkIter;
SStreamReader *tqInitSubmitMsgScanner(SMeta *pMeta);
SWalReader *pWalReader;
void tqReadHandleSetColIdList(SStreamReader *pReadHandle, SArray *pColIdList);
int32_t tqReadHandleSetTbUidList(SStreamReader *pHandle, const SArray *tbUidList);
int32_t tqReadHandleAddTbUidList(SStreamReader *pHandle, const SArray *tbUidList);
int32_t tqReadHandleRemoveTbUidList(SStreamReader *pHandle, const SArray *tbUidList);
SMeta *pVnodeMeta;
SHashObj *tbIdHash;
SArray *pColIdList; // SArray<int16_t>
int32_t tqReadHandleSetMsg(SStreamReader *pHandle, SSubmitReq *pMsg, int64_t ver);
bool tqNextDataBlock(SStreamReader *pHandle);
bool tqNextDataBlockFilterOut(SStreamReader *pHandle, SHashObj *filterOutUids);
int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, SStreamReader *pHandle);
int32_t cachedSchemaVer;
int64_t cachedSchemaSuid;
SSchemaWrapper *pSchemaWrapper;
STSchema *pSchema;
} STqReader;
STqReader *tqOpenReader(SVnode *pVnode);
void tqCloseReader(STqReader *);
void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList);
int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList);
int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList);
int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
int32_t tqReaderSetDataMsg(STqReader *pReader, SSubmitReq *pMsg, int64_t ver);
bool tqNextDataBlock(STqReader *pReader);
bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader);
// sma
int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);

34
source/dnode/vnode/src/inc/tq.h
View File

@ -40,38 +40,10 @@ extern "C" {
#define tqDebug(...) do { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0)
#define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
#define IS_META_MSG(x) ( \
x == TDMT_VND_CREATE_STB \
|| x == TDMT_VND_ALTER_STB \
|| x == TDMT_VND_DROP_STB \
|| x == TDMT_VND_CREATE_TABLE \
|| x == TDMT_VND_ALTER_TABLE \
|| x == TDMT_VND_DROP_TABLE \
|| x == TDMT_VND_DROP_TTL_TABLE \
)
// clang-format on
typedef struct STqOffsetStore STqOffsetStore;
// tqRead
struct STqReadHandle {
int64_t ver;
const SSubmitReq* pMsg;
SSubmitBlk* pBlock;
SSubmitMsgIter msgIter;
SSubmitBlkIter blkIter;
SMeta* pVnodeMeta;
SHashObj* tbIdHash;
SArray* pColIdList; // SArray<int16_t>
int32_t cachedSchemaVer;
int64_t cachedSchemaSuid;
SSchemaWrapper* pSchemaWrapper;
STSchema* pSchema;
};
// tqPush
typedef struct {
@ -111,7 +83,7 @@ typedef struct {
typedef struct {
int8_t subType;
SStreamReader* pExecReader[5];
STqReader* pExecReader[5];
union {
STqExecCol execCol;
STqExecTb execTb;
@ -127,8 +99,8 @@ typedef struct {
int32_t epoch;
int8_t fetchMeta;
// reader
SWalReadHandle* pWalReader;
// TODO remove
SWalReader* pWalReader;
// push
STqPushHandle pushHandle;

1
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -439,6 +439,7 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv
tDecoderInit(&dc, pData, nData);
tDecodeSSchemaWrapper(&dc, pSchemaWrapper);
tDecoderClear(&dc);
tdbFree(pData);
// convert
STSchemaBuilder sb = {0};

141
source/dnode/vnode/src/meta/metaTable.c
View File

@ -16,6 +16,7 @@
#include "meta.h"
static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME);
static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME);
@ -74,7 +75,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
SIndexTerm *term = NULL;
if (type == TSDB_DATA_TYPE_NULL) {
// handle null value
term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0);
} else if (type == TSDB_DATA_TYPE_NCHAR) {
if (pTagVal->nData > 0) {
char * val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
@ -83,18 +84,74 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
type = TSDB_DATA_TYPE_VARCHAR;
term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, val, len);
} else if (pTagVal->nData == 0) {
char * val = NULL;
int32_t len = 0;
// handle NULL key
term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, pTagVal->pData, 0);
}
} else if (type == TSDB_DATA_TYPE_DOUBLE) {
double val = *(double *)(&pTagVal->i64);
int len = 0;
int len = sizeof(val);
term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len);
} else if (type == TSDB_DATA_TYPE_BOOL) {
int val = *(int *)(&pTagVal->i64);
int len = 0;
term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_INT, key, nKey, (const char *)&val, len);
int len = sizeof(val);
term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BOOL, key, nKey, (const char *)&val, len);
}
if (term != NULL) {
indexMultiTermAdd(terms, term);
}
}
indexJsonPut(pMeta->pTagIvtIdx, terms, tuid);
indexMultiTermDestroy(terms);
#endif
return 0;
}
int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) {
#ifdef USE_INVERTED_INDEX
if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
return -1;
}
void * data = pCtbEntry->ctbEntry.pTags;
const char *tagName = pSchema->name;
tb_uid_t suid = pCtbEntry->ctbEntry.suid;
tb_uid_t tuid = pCtbEntry->uid;
const void *pTagData = pCtbEntry->ctbEntry.pTags;
int32_t nTagData = 0;
SArray *pTagVals = NULL;
if (tTagToValArray((const STag *)data, &pTagVals) != 0) {
return -1;
}
SIndexMultiTerm *terms = indexMultiTermCreate();
int16_t nCols = taosArrayGetSize(pTagVals);
for (int i = 0; i < nCols; i++) {
STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
char type = pTagVal->type;
char * key = pTagVal->pKey;
int32_t nKey = strlen(key);
SIndexTerm *term = NULL;
if (type == TSDB_DATA_TYPE_NULL) {
term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0);
} else if (type == TSDB_DATA_TYPE_NCHAR) {
if (pTagVal->nData > 0) {
char * val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
type = TSDB_DATA_TYPE_VARCHAR;
term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, val, len);
} else if (pTagVal->nData == 0) {
term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, pTagVal->pData, 0);
}
} else if (type == TSDB_DATA_TYPE_DOUBLE) {
double val = *(double *)(&pTagVal->i64);
int len = sizeof(val);
term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, (const char *)&val, len);
} else if (type == TSDB_DATA_TYPE_BOOL) {
int val = *(int *)(&pTagVal->i64);
int len = sizeof(val);
term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_BOOL, key, nKey, (const char *)&val, len);
}
if (term != NULL) {
indexMultiTermAdd(terms, term);
@ -380,22 +437,22 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) {
metaWLock(pMeta);
int ret = metaTtlSmaller(pMeta, ttl, tbUids);
if(ret != 0){
if (ret != 0) {
metaULock(pMeta);
return ret;
}
for (int i = 0; i < taosArrayGetSize(tbUids); ++i) {
tb_uid_t *uid = (tb_uid_t *)taosArrayGet(tbUids, i);
metaDropTableByUid(pMeta, *uid, NULL);
metaDebug("ttl drop table:%"PRId64, *uid);
metaDebug("ttl drop table:%" PRId64, *uid);
}
metaULock(pMeta);
return 0;
}
static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME){
int64_t ttlDays;
int64_t ctime;
static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME) {
int64_t ttlDays;
int64_t ctime;
if (pME->type == TSDB_CHILD_TABLE) {
ctime = pME->ctbEntry.ctime;
ttlDays = pME->ctbEntry.ttlDays;
@ -415,11 +472,10 @@ static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME){
static int metaDeleteTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
STtlIdxKey ttlKey = {0};
metaBuildTtlIdxKey(&ttlKey, pME);
if(ttlKey.dtime == 0) return 0;
if (ttlKey.dtime == 0) return 0;
return tdbTbDelete(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), &pMeta->txn);
}
static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
void * pData = NULL;
int nData = 0;
@ -437,11 +493,34 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
if (type) *type = e.type;
if (e.type == TSDB_CHILD_TABLE) {
void *tData = NULL;
int tLen = 0;
if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) {
version = *(int64_t *)tData;
STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = version};
if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) {
SDecoder tdc = {0};
SMetaEntry stbEntry = {0};
tDecoderInit(&tdc, tData, tLen);
metaDecodeEntry(&tdc, &stbEntry);
const SSchema *pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0];
if (pTagColumn->type == TSDB_DATA_TYPE_JSON) {
metaDelJsonVarFromIdx(pMeta, &e, pTagColumn);
}
tDecoderClear(&tdc);
}
tdbFree(tData);
}
}
tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pMeta->txn);
tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, &pMeta->txn);
tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), &pMeta->txn);
if(e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e);
if (e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e);
if (e.type == TSDB_CHILD_TABLE) {
tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), &pMeta->txn);
@ -765,15 +844,15 @@ _err:
}
static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
void * pVal = NULL;
int nVal = 0;
const void * pData = NULL;
int nData = 0;
int ret = 0;
tb_uid_t uid;
int64_t oversion;
SMetaEntry entry = {0};
int c = 0;
void * pVal = NULL;
int nVal = 0;
const void *pData = NULL;
int nData = 0;
int ret = 0;
tb_uid_t uid;
int64_t oversion;
SMetaEntry entry = {0};
int c = 0;
// search name index
ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal);
@ -816,22 +895,22 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p
metaWLock(pMeta);
// build SMetaEntry
if (entry.type == TSDB_CHILD_TABLE) {
if(pAlterTbReq->updateTTL) {
if (pAlterTbReq->updateTTL) {
metaDeleteTtlIdx(pMeta, &entry);
entry.ctbEntry.ttlDays = pAlterTbReq->newTTL;
metaUpdateTtlIdx(pMeta, &entry);
}
if(pAlterTbReq->newCommentLen >= 0) {
if (pAlterTbReq->newCommentLen >= 0) {
entry.ctbEntry.commentLen = pAlterTbReq->newCommentLen;
entry.ctbEntry.comment = pAlterTbReq->newComment;
}
} else {
if(pAlterTbReq->updateTTL) {
if (pAlterTbReq->updateTTL) {
metaDeleteTtlIdx(pMeta, &entry);
entry.ntbEntry.ttlDays = pAlterTbReq->newTTL;
metaUpdateTtlIdx(pMeta, &entry);
}
if(pAlterTbReq->newCommentLen >= 0) {
if (pAlterTbReq->newCommentLen >= 0) {
entry.ntbEntry.commentLen = pAlterTbReq->newCommentLen;
entry.ntbEntry.comment = pAlterTbReq->newComment;
}
@ -930,7 +1009,7 @@ static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
STtlIdxKey ttlKey = {0};
metaBuildTtlIdxKey(&ttlKey, pME);
if(ttlKey.dtime == 0) return 0;
if (ttlKey.dtime == 0) return 0;
return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn);
}
@ -988,7 +1067,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
SDecoder dc = {0};
// get super table
if(tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0){
if (tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0) {
return -1;
}
tbDbKey.uid = pCtbEntry->ctbEntry.suid;
@ -1096,7 +1175,7 @@ static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
if (pME->type == TSDB_SUPER_TABLE) {
if (metaUpdateSuidIdx(pMeta, pME) < 0) goto _err;
}
}
}
if (pME->type != TSDB_SUPER_TABLE) {

10
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -277,6 +277,8 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo
pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY;
}
pItem->level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2);
smaInfo("vgId:%d table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 ", finally maxdelay:%"PRIi32, SMA_VID(pSma),
pRSmaInfo->suid, idx + 1, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay);
}
return TSDB_CODE_SUCCESS;
_err:
@ -325,14 +327,14 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
return TSDB_CODE_FAILED;
}
SStreamReader *pReadHandle = tqInitSubmitMsgScanner(pMeta);
if (!pReadHandle) {
STqReader *pReader = tqOpenReader(pVnode);
if (!pReader) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
SReadHandle handle = {
.reader = pReadHandle,
.tqReader = pReader,
.meta = pMeta,
.pMsgCb = pMsgCb,
.vnode = pVnode,
@ -364,7 +366,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
return TSDB_CODE_SUCCESS;
_err:
tdFreeRSmaInfo(pRSmaInfo);
taosMemoryFree(pReadHandle);
taosMemoryFree(pReader);
return TSDB_CODE_FAILED;
}

52
source/dnode/vnode/src/tq/tq.c
View File

@ -28,8 +28,12 @@ int32_t tqInit() {
atomic_store_8(&tqMgmt.inited, 0);
return -1;
}
if (streamInit() < 0) {
return -1;
}
atomic_store_8(&tqMgmt.inited, 1);
}
return 0;
}
@ -42,6 +46,7 @@ void tqCleanUp() {
if (old == 1) {
taosTmrCleanUp(tqMgmt.timer);
streamCleanUp();
atomic_store_8(&tqMgmt.inited, 0);
}
}
@ -144,7 +149,6 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
SEncoder encoder;
tEncoderInit(&encoder, abuf, len);
tEncodeSMqDataRsp(&encoder, pRsp);
/*tEncodeSMqDataBlkRsp(&abuf, pRsp);*/
SRpcMsg rsp = {
.info = pMsg->info,
@ -361,8 +365,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
ASSERT(IS_META_MSG(pHead->msgType));
tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType);
SMqMetaRsp metaRsp = {0};
metaRsp.reqOffset = pReq->reqOffset.version;
metaRsp.rspOffset = fetchVer;
/*metaRsp.reqOffset = pReq->reqOffset.version;*/
/*metaRsp.rspOffset = fetchVer;*/
/*metaRsp.rspOffsetNew.version = fetchVer;*/
tqOffsetResetToLog(&metaRsp.reqOffsetNew, pReq->reqOffset.version);
tqOffsetResetToLog(&metaRsp.rspOffsetNew, fetchVer);
metaRsp.resMsgType = pHead->msgType;
metaRsp.metaRspLen = pHead->bodyLen;
metaRsp.metaRsp = pHead->body;
@ -439,27 +446,39 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
pHandle->execHandle.subType = req.subType;
pHandle->fetchMeta = req.withMeta;
pHandle->pWalReader = walOpenReadHandle(pTq->pVnode->pWal);
for (int32_t i = 0; i < 5; i++) {
pHandle->execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
}
pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
/*for (int32_t i = 0; i < 5; i++) {*/
/*pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);*/
/*}*/
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
pHandle->execHandle.execCol.qmsg = req.qmsg;
req.qmsg = NULL;
for (int32_t i = 0; i < 5; i++) {
SReadHandle handle = {
.reader = pHandle->execHandle.pExecReader[i],
.tqReader = pHandle->execHandle.pExecReader[i],
.meta = pTq->pVnode->pMeta,
.vnode = pTq->pVnode,
.tqReader = true,
.initTableReader = true,
.initTqReader = true,
};
pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle);
ASSERT(pHandle->execHandle.execCol.task[i]);
void* scanner = NULL;
qExtractStreamScanner(pHandle->execHandle.execCol.task[i], &scanner);
ASSERT(scanner);
pHandle->execHandle.pExecReader[i] = qExtractReaderFromStreamScanner(scanner);
ASSERT(pHandle->execHandle.pExecReader[i]);
}
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) {
for (int32_t i = 0; i < 5; i++) {
pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);
}
pHandle->execHandle.execDb.pFilterOutTbUid =
taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
for (int32_t i = 0; i < 5; i++) {
pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);
}
pHandle->execHandle.execTb.suid = req.suid;
SArray* tbUidList = taosArrayInit(0, sizeof(int64_t));
vnodeGetCtbIdList(pTq->pVnode, req.suid, tbUidList);
@ -469,7 +488,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
tqDebug("vg %d, idx %d, uid: %ld", TD_VID(pTq->pVnode), i, tbUid);
}
for (int32_t i = 0; i < 5; i++) {
tqReadHandleSetTbUidList(pHandle->execHandle.pExecReader[i], tbUidList);
tqReaderSetTbUidList(pHandle->execHandle.pExecReader[i], tbUidList);
}
taosArrayDestroy(tbUidList);
}
@ -522,19 +541,16 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
if (pTask->execType != TASK_EXEC__NONE) {
// expand runners
if (pTask->isDataScan) {
SStreamReader* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
SReadHandle handle = {
.reader = pStreamReader,
.meta = pTq->pVnode->pMeta,
.vnode = pTq->pVnode,
SReadHandle handle = {
.meta = pTq->pVnode->pMeta,
.vnode = pTq->pVnode,
.initTqReader = 1,
};
/*pTask->exec.inputHandle = pStreamReader;*/
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
ASSERT(pTask->exec.executor);
} else {
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL);
ASSERT(pTask->exec.executor);
}
ASSERT(pTask->exec.executor);
}
// sink

8
source/dnode/vnode/src/tq/tqExec.c
View File

@ -135,8 +135,8 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR
}
} else if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
pRsp->withSchema = 1;
SStreamReader* pReader = pExec->pExecReader[workerId];
tqReadHandleSetMsg(pReader, pReq, 0);
STqReader* pReader = pExec->pExecReader[workerId];
tqReaderSetDataMsg(pReader, pReq, 0);
while (tqNextDataBlock(pReader)) {
SSDataBlock block = {0};
if (tqRetrieveDataBlock(&block, pReader) < 0) {
@ -153,8 +153,8 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR
}
} else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
pRsp->withSchema = 1;
SStreamReader* pReader = pExec->pExecReader[workerId];
tqReadHandleSetMsg(pReader, pReq, 0);
STqReader* pReader = pExec->pExecReader[workerId];
tqReaderSetDataMsg(pReader, pReq, 0);
while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) {
SSDataBlock block = {0};
if (tqRetrieveDataBlock(&block, pReader) < 0) {

6
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -77,14 +77,14 @@ int32_t tqMetaOpen(STQ* pTq) {
STqHandle handle;
tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
tDecodeSTqHandle(&decoder, &handle);
handle.pWalReader = walOpenReadHandle(pTq->pVnode->pWal);
handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
for (int32_t i = 0; i < 5; i++) {
handle.execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
handle.execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);
}
if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
for (int32_t i = 0; i < 5; i++) {
SReadHandle reader = {
.reader = handle.execHandle.pExecReader[i],
.tqReader = handle.execHandle.pExecReader[i],
.meta = pTq->pVnode->pMeta,
.pMsgCb = &pTq->pVnode->msgCb,
.vnode = pTq->pVnode,

172
source/dnode/vnode/src/tq/tqRead.c
View File

@ -15,6 +15,11 @@
#include "tq.h"
int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset) {
/*if ()*/
return 0;
}
int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) {
int32_t code = 0;
taosThreadMutexLock(&pHandle->pWalReader->mutex);
@ -73,53 +78,107 @@ END:
return code;
}
SStreamReader* tqInitSubmitMsgScanner(SMeta* pMeta) {
SStreamReader* pReadHandle = taosMemoryMalloc(sizeof(SStreamReader));
if (pReadHandle == NULL) {
STqReader* tqOpenReader(SVnode* pVnode) {
STqReader* pReader = taosMemoryMalloc(sizeof(STqReader));
if (pReader == NULL) {
return NULL;
}
pReadHandle->pVnodeMeta = pMeta;
pReadHandle->pMsg = NULL;
pReadHandle->ver = -1;
pReadHandle->pColIdList = NULL;
pReadHandle->cachedSchemaVer = 0;
pReadHandle->cachedSchemaSuid = 0;
pReadHandle->pSchema = NULL;
pReadHandle->pSchemaWrapper = NULL;
pReadHandle->tbIdHash = NULL;
return pReadHandle;
// TODO open
/*pReader->pWalReader = walOpenReader(pVnode->pWal, NULL);*/
pReader->pVnodeMeta = pVnode->pMeta;
pReader->pMsg = NULL;
pReader->ver = -1;
pReader->pColIdList = NULL;
pReader->cachedSchemaVer = 0;
pReader->cachedSchemaSuid = 0;
pReader->pSchema = NULL;
pReader->pSchemaWrapper = NULL;
pReader->tbIdHash = NULL;
return pReader;
}
int32_t tqReadHandleSetMsg(SStreamReader* pReadHandle, SSubmitReq* pMsg, int64_t ver) {
pReadHandle->pMsg = pMsg;
void tqCloseReader(STqReader* pReader) {
// close wal reader
// free cached schema
// free hash
taosMemoryFree(pReader);
}
if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1;
int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
bool fromProcessedMsg = pReader->pMsg != NULL;
while (1) {
if (!fromProcessedMsg) {
if (walNextValidMsg(pReader->pWalReader) < 0) {
ret->fetchType = FETCH_TYPE__NONE;
return -1;
}
void* body = pReader->pWalReader->pHead->head.body;
if (pReader->pWalReader->pHead->head.msgType != TDMT_VND_SUBMIT) {
// TODO do filter
ret->fetchType = FETCH_TYPE__META;
ret->meta = pReader->pWalReader->pHead->head.body;
return 0;
} else {
tqReaderSetDataMsg(pReader, body, pReader->pWalReader->pHead->head.version);
}
}
while (tqNextDataBlock(pReader)) {
memset(&ret->data, 0, sizeof(SSDataBlock));
int32_t code = tqRetrieveDataBlock(&ret->data, pReader);
if (code != 0 || ret->data.info.rows == 0) {
if (fromProcessedMsg) {
ret->fetchType = FETCH_TYPE__NONE;
return 0;
} else {
break;
}
}
ret->fetchType = FETCH_TYPE__DATA;
return 0;
}
if (fromProcessedMsg) {
ret->fetchType = FETCH_TYPE__NONE;
return 0;
}
}
}
int32_t tqReaderSetDataMsg(STqReader* pReader, SSubmitReq* pMsg, int64_t ver) {
pReader->pMsg = pMsg;
if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1;
while (true) {
if (tGetSubmitMsgNext(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1;
if (pReadHandle->pBlock == NULL) break;
if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) return -1;
if (pReader->pBlock == NULL) break;
}
if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1;
pReadHandle->ver = ver;
memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter));
if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1;
pReader->ver = ver;
memset(&pReader->blkIter, 0, sizeof(SSubmitBlkIter));
return 0;
}
bool tqNextDataBlock(SStreamReader* pHandle) {
if (pHandle->pMsg == NULL) return false;
bool tqNextDataBlock(STqReader* pReader) {
if (pReader->pMsg == NULL) return false;
while (1) {
if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) {
return false;
}
if (pHandle->pBlock == NULL) {
pHandle->pMsg = NULL;
if (pReader->pBlock == NULL) {
pReader->pMsg = NULL;
return false;
}
if (pHandle->tbIdHash == NULL) {
if (pReader->tbIdHash == NULL) {
return true;
}
void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->msgIter.uid, sizeof(int64_t));
void* ret = taosHashGet(pReader->tbIdHash, &pReader->msgIter.uid, sizeof(int64_t));
/*tqDebug("search uid %ld", pHandle->msgIter.uid);*/
if (ret != NULL) {
/*tqDebug("find uid %ld", pHandle->msgIter.uid);*/
@ -129,7 +188,7 @@ bool tqNextDataBlock(SStreamReader* pHandle) {
return false;
}
bool tqNextDataBlockFilterOut(SStreamReader* pHandle, SHashObj* filterOutUids) {
bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) {
while (1) {
if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
return false;
@ -145,38 +204,38 @@ bool tqNextDataBlockFilterOut(SStreamReader* pHandle, SHashObj* filterOutUids) {
return false;
}
int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) {
int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) {
// TODO: cache multiple schema
int32_t sversion = htonl(pHandle->pBlock->sversion);
if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion ||
pHandle->cachedSchemaSuid != pHandle->msgIter.suid) {
if (pHandle->pSchema) taosMemoryFree(pHandle->pSchema);
pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion);
if (pHandle->pSchema == NULL) {
int32_t sversion = htonl(pReader->pBlock->sversion);
if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion ||
pReader->cachedSchemaSuid != pReader->msgIter.suid) {
if (pReader->pSchema) taosMemoryFree(pReader->pSchema);
pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion);
if (pReader->pSchema == NULL) {
tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table",
pHandle->msgIter.uid, pHandle->msgIter.suid, pHandle->cachedSchemaVer);
pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer);
/*ASSERT(0);*/
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1;
}
if (pHandle->pSchemaWrapper) tDeleteSSchemaWrapper(pHandle->pSchemaWrapper);
pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion, true);
if (pHandle->pSchemaWrapper == NULL) {
if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, true);
if (pReader->pSchemaWrapper == NULL) {
tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table",
pHandle->msgIter.uid, pHandle->cachedSchemaVer);
pReader->msgIter.uid, pReader->cachedSchemaVer);
/*ASSERT(0);*/
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1;
}
pHandle->cachedSchemaVer = sversion;
pHandle->cachedSchemaSuid = pHandle->msgIter.suid;
pReader->cachedSchemaVer = sversion;
pReader->cachedSchemaSuid = pReader->msgIter.suid;
}
STSchema* pTschema = pHandle->pSchema;
SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper;
STSchema* pTschema = pReader->pSchema;
SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList);
if (colNumNeed == 0) {
int32_t colMeta = 0;
@ -199,7 +258,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) {
while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) {
SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta];
col_id_t colIdSchema = pColSchema->colId;
col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pHandle->pColIdList, colNeed);
col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed);
if (colIdSchema < colIdNeed) {
colMeta++;
} else if (colIdSchema > colIdNeed) {
@ -216,7 +275,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) {
}
}
if (blockDataEnsureCapacity(pBlock, pHandle->msgIter.numOfRows) < 0) {
if (blockDataEnsureCapacity(pBlock, pReader->msgIter.numOfRows) < 0) {
goto FAIL;
}
@ -227,13 +286,12 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) {
STSRow* row;
int32_t curRow = 0;
tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter);
tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter);
pBlock->info.groupId = 0;
pBlock->info.uid = pHandle->msgIter.uid;
pBlock->info.rows = pHandle->msgIter.numOfRows;
pBlock->info.uid = pReader->msgIter.uid;
pBlock->info.rows = pReader->msgIter.numOfRows;
while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) {
tdSTSRowIterReset(&iter, row);
// get all wanted col of that block
for (int32_t i = 0; i < colActual; i++) {
@ -255,9 +313,9 @@ FAIL:
return -1;
}
void tqReadHandleSetColIdList(SStreamReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; }
void tqReaderSetColIdList(STqReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; }
int tqReadHandleSetTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
int tqReaderSetTbUidList(STqReader* pHandle, const SArray* tbUidList) {
if (pHandle->tbIdHash) {
taosHashClear(pHandle->tbIdHash);
}
@ -276,7 +334,7 @@ int tqReadHandleSetTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
return 0;
}
int tqReadHandleAddTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
int tqReaderAddTbUidList(STqReader* pHandle, const SArray* tbUidList) {
if (pHandle->tbIdHash == NULL) {
pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (pHandle->tbIdHash == NULL) {
@ -293,7 +351,7 @@ int tqReadHandleAddTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
return 0;
}
int tqReadHandleRemoveTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
int tqReaderRemoveTbUidList(STqReader* pHandle, const SArray* tbUidList) {
ASSERT(pHandle->tbIdHash != NULL);
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {

4
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -309,9 +309,11 @@ static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t
if (code) goto _err;
// code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx);
pDelIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ);
SDelIdx *pIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ);
if (code) goto _err;
*pDelIdx = *pIdx;
if (pDelIdxArray) {
taosArrayDestroy(pDelIdxArray);
}

8
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -168,8 +168,6 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel
tb_uid_t suid;
tb_uid_t uid;
taosArrayClear(pCommitter->aDelData);
if (pTbData) {
suid = pTbData->suid;
uid = pTbData->uid;
@ -185,6 +183,8 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel
code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, NULL);
if (code) goto _err;
} else {
taosArrayClear(pCommitter->aDelData);
}
if (pTbData == NULL && pDelIdx == NULL) goto _exit;
@ -205,7 +205,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel
if (code) goto _err;
// put delIdx
if (taosArrayPush(pCommitter->aDelIdx, &delIdx) == NULL) {
if (taosArrayPush(pCommitter->aDelIdxN, &delIdx) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
@ -854,7 +854,7 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) {
if (pCommitter->pReader) {
code = tsdbDataFReaderClose(&pCommitter->pReader);
goto _err;
if (code) goto _err;
}
_exit:

2
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -520,7 +520,7 @@ static int32_t tsdbScanAndTryFixFS(STsdbFS *pFS, int8_t deepScan) {
return code;
_err:
tsdbError("vgId:%d tsdb can and try fix fs failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
tsdbError("vgId:%d tsdb scan and try fix fs failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code;
}

4
source/dnode/vnode/src/tsdb/tsdbMemTable.c
View File

@ -105,7 +105,7 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
// check if table exists (todo: refact)
SMetaReader mr = {0};
SMetaEntry me = {0};
// SMetaEntry me = {0};
metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0);
if (metaGetTableEntryByUid(&mr, pMsgIter->uid) < 0) {
metaReaderClear(&mr);
@ -117,6 +117,8 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
if (mr.me.type == TSDB_NORMAL_TABLE) {
sverNew = mr.me.ntbEntry.schemaRow.version;
} else {
tDecoderClear(&mr.coder);
metaGetTableEntryByUid(&mr, mr.me.ctbEntry.suid);
sverNew = mr.me.stbEntry.schemaRow.version;
}

185
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -17,26 +17,26 @@
#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC)
typedef struct {
STbDataIter *iter;
STbDataIter* iter;
int32_t index;
bool hasVal;
} SIterInfo;
typedef struct STableBlockScanInfo {
uint64_t uid;
TSKEY lastKey;
SBlockIdx blockIdx;
SArray* pBlockList; // block data index list
SIterInfo iter; // mem buffer skip list iterator
SIterInfo iiter; // imem buffer skip list iterator
SArray* delSkyline; // delete info for this table
int32_t fileDelIndex;
bool iterInit; // whether to initialize the in-memory skip list iterator or not
uint64_t uid;
TSKEY lastKey;
SBlockIdx blockIdx;
SArray* pBlockList; // block data index list
SIterInfo iter; // mem buffer skip list iterator
SIterInfo iiter; // imem buffer skip list iterator
SArray* delSkyline; // delete info for this table
int32_t fileDelIndex;
bool iterInit; // whether to initialize the in-memory skip list iterator or not
} STableBlockScanInfo;
typedef struct SBlockOrderWrapper {
int64_t uid;
SBlock* pBlock;
int64_t uid;
SBlock* pBlock;
} SBlockOrderWrapper;
typedef struct SBlockOrderSupporter {
@ -70,40 +70,41 @@ typedef struct SFilesetIter {
} SFilesetIter;
typedef struct SFileDataBlockInfo {
int32_t tbBlockIdx; // index position in STableBlockScanInfo in order to check whether neighbor block overlaps with it
uint64_t uid;
int32_t
tbBlockIdx; // index position in STableBlockScanInfo in order to check whether neighbor block overlaps with it
uint64_t uid;
} SFileDataBlockInfo;
typedef struct SDataBlockIter {
int32_t numOfBlocks;
int32_t index;
SArray* blockList; // SArray<SFileDataBlockInfo>
int32_t order;
int32_t numOfBlocks;
int32_t index;
SArray* blockList; // SArray<SFileDataBlockInfo>
int32_t order;
} SDataBlockIter;
typedef struct SFileBlockDumpInfo {
int32_t totalRows;
int32_t rowIndex;
int64_t lastKey;
bool allDumped;
int32_t totalRows;
int32_t rowIndex;
int64_t lastKey;
bool allDumped;
} SFileBlockDumpInfo;
typedef struct SVersionRange {
uint64_t minVer;
uint64_t maxVer;
uint64_t minVer;
uint64_t maxVer;
} SVersionRange;
typedef struct SReaderStatus {
bool loadFromFile; // check file stage
SHashObj* pTableMap; // SHash<STableBlockScanInfo>
bool loadFromFile; // check file stage
SHashObj* pTableMap; // SHash<STableBlockScanInfo>
STableBlockScanInfo* pTableIter; // table iterator used in building in-memory buffer data blocks.
SFileBlockDumpInfo fBlockDumpInfo;
SDFileSet* pCurrentFileset; // current opened file set
SBlockData fileBlockData;
SFilesetIter fileIter;
SDataBlockIter blockIter;
bool composedDataBlock;// the returned data block is a composed block or not
SDFileSet* pCurrentFileset; // current opened file set
SBlockData fileBlockData;
SFilesetIter fileIter;
SDataBlockIter blockIter;
bool composedDataBlock; // the returned data block is a composed block or not
} SReaderStatus;
struct STsdbReader {
@ -118,10 +119,10 @@ struct STsdbReader {
int32_t type; // query type: 1. retrieve all data blocks, 2. retrieve direct prev|next rows
SBlockLoadSuppInfo suppInfo;
SIOCostSummary cost;
STSchema* pSchema;
SDataFReader* pFileReader;
SVersionRange verRange;
SIOCostSummary cost;
STSchema* pSchema;
SDataFReader* pFileReader;
SVersionRange verRange;
#if 0
SArray* prev; // previous row which is before than time window
SArray* next; // next row which is after the query time window
@ -144,17 +145,21 @@ static int buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, i
static TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader);
static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader,
SRowMerger* pMerger);
static int32_t doMergeRowsInBuf(SIterInfo *pIter, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader);
static int32_t doMergeRowsInBuf(SIterInfo* pIter, int64_t ts, SArray* pDelList, SRowMerger* pMerger,
STsdbReader* pReader);
static int32_t doAppendOneRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow);
static void setComposedBlockFlag(STsdbReader* pReader, bool composed);
static void updateSchema(TSDBROW* pRow, uint64_t uid, STsdbReader* pReader);
static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey);
static void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo *pIter, SArray* pDelList, STSRow** pTSRow, STsdbReader* pReader);
static void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow,
STsdbReader* pReader);
static void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader,
STSRow** pTSRow);
static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData, STbData* piMemTbData);
static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr, int8_t *pLevel);
static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData,
STbData* piMemTbData);
static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr,
int8_t* pLevel);
static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level);
static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) {
@ -215,10 +220,10 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK
static void resetDataBlockScanInfo(SHashObj* pTableMap) {
STableBlockScanInfo* p = NULL;
while((p = taosHashIterate(pTableMap, p)) != NULL) {
while ((p = taosHashIterate(pTableMap, p)) != NULL) {
p->iterInit = false;
p->iiter.hasVal = false;
if (p->iter.iter != NULL) {
if (p->iter.iter != NULL) {
tsdbTbDataIterDestroy(p->iter.iter);
}
@ -234,12 +239,12 @@ static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) {
// Update the query time window according to the data time to live(TTL) information, in order to avoid to return
// the expired data to client, even it is queried already.
static STimeWindow updateQueryTimeWindow(STsdb* pTsdb, STimeWindow* pWindow) {
STsdbKeepCfg* pCfg = &pTsdb->keepCfg;
STsdbKeepCfg* pCfg = &pTsdb->keepCfg;
int64_t now = taosGetTimestamp(pCfg->precision);
int64_t earilyTs = now - (tsTickPerMin[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick
int64_t now = taosGetTimestamp(pCfg->precision);
int64_t earilyTs = now - (tsTickPerMin[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick
STimeWindow win = *pWindow;
STimeWindow win = *pWindow;
if (win.skey < earilyTs) {
win.skey = earilyTs;
}
@ -352,8 +357,8 @@ static void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order) {
}
static void initReaderStatus(SReaderStatus* pStatus) {
pStatus->pTableIter = NULL;
pStatus->loadFromFile = true;
pStatus->pTableIter = NULL;
pStatus->loadFromFile = true;
}
static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity) {
@ -390,14 +395,15 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
initReaderStatus(&pReader->status);
pReader->pTsdb = getTsdbByRetentions(pVnode, pCond->twindows[0].skey, pVnode->config.tsdbCfg.retentions, idstr, &level);
pReader->suid = pCond->suid;
pReader->order = pCond->order;
pReader->capacity = 4096;
pReader->idStr = (idstr != NULL)? strdup(idstr):NULL;
pReader->verRange = getQueryVerRange(pVnode, pCond, level);
pReader->pTsdb =
getTsdbByRetentions(pVnode, pCond->twindows[0].skey, pVnode->config.tsdbCfg.retentions, idstr, &level);
pReader->suid = pCond->suid;
pReader->order = pCond->order;
pReader->capacity = 4096;
pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL;
pReader->verRange = getQueryVerRange(pVnode, pCond, level);
pReader->type = pCond->type;
pReader->window = updateQueryTimeWindow(pVnode->pTsdb, pCond->twindows);
pReader->window = updateQueryTimeWindow(pVnode->pTsdb, pCond->twindows);
// todo remove this
setQueryTimewindow(pReader, pCond, 0);
@ -660,7 +666,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, uint32_
*numOfValidTables = 0;
STableBlockScanInfo* px = NULL;
while(1) {
while (1) {
px = taosHashIterate(pReader->status.pTableMap, px);
if (px == NULL) {
break;
@ -669,7 +675,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, uint32_
taosArrayClear(px->pBlockList);
}
for(int32_t i = 0; i < numOfTables; ++i) {
for (int32_t i = 0; i < numOfTables; ++i) {
SBlockIdx* pBlockIdx = taosArrayGet(pIndexList, i);
SMapData mapData = {0};
@ -830,7 +836,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
uint8_t *pb = NULL, *pb1 = NULL;
int32_t code = tsdbReadColData(pReader->pFileReader, &pBlockScanInfo->blockIdx, pBlock, pSupInfo->colIds, numOfCols,
pBlockData, &pb, &pb1);
pBlockData, &pb, &pb1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -1812,7 +1818,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo*
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
SBlockData* pBlockData = &pReader->status.fileBlockData;
SArray* pDelList = pBlockScanInfo->delSkyline;
SArray* pDelList = pBlockScanInfo->delSkyline;
TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, pDelList, pReader);
TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, pDelList, pReader);
@ -1936,8 +1942,8 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo*
ASSERT(0);
}
static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, STableBlockScanInfo* pBlockScanInfo,
STsdbReader* pReader) {
static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo,
STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
// check for version and time range
int64_t ver = pBlockData->aVersion[pDumpInfo->rowIndex];
if (ver > pReader->verRange.maxVer || ver < pReader->verRange.minVer) {
@ -1980,7 +1986,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI
// mem + file
if (pBlockScanInfo->iter.hasVal) {
return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, pTSRow, &pBlockScanInfo->iter,key);
return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, pTSRow, &pBlockScanInfo->iter, key);
}
// imem & mem are all empty, only file exist
@ -2113,7 +2119,8 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea
return TSDB_CODE_SUCCESS;
}
int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData, STbData* piMemTbData) {
int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData,
STbData* piMemTbData) {
if (pBlockScanInfo->delSkyline != NULL) {
return TSDB_CODE_SUCCESS;
}
@ -2123,7 +2130,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader*
SArray* pDelData = taosArrayInit(4, sizeof(SDelData));
SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->cState);
SDelFile* pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->cState);
if (pDelFile) {
SDelFReader* pDelFReader = NULL;
code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL);
@ -2173,7 +2180,8 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader*
}
taosArrayDestroy(pDelData);
pBlockScanInfo->iter.index = ASCENDING_TRAVERSE(pReader->order)? 0:taosArrayGetSize(pBlockScanInfo->delSkyline) - 1;
pBlockScanInfo->iter.index =
ASCENDING_TRAVERSE(pReader->order) ? 0 : taosArrayGetSize(pBlockScanInfo->delSkyline) - 1;
pBlockScanInfo->iiter.index = pBlockScanInfo->iter.index;
pBlockScanInfo->fileDelIndex = pBlockScanInfo->iter.index;
return code;
@ -2488,7 +2496,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey) {
if (pKey->ts > last->ts) {
return false;
} else if (pKey->ts == last->ts) {
size_t size = taosArrayGetSize(pDelList);
size_t size = taosArrayGetSize(pDelList);
TSDBKEY* prev = taosArrayGet(pDelList, size - 2);
if (prev->version >= pKey->version) {
return true;
@ -2527,7 +2535,8 @@ TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pRea
}
// it is a valid data version
if ((key.version <= pReader->verRange.maxVer && key.version >= pReader->verRange.minVer) && (!hasBeenDropped(pDelList, &pIter->index, &key))) {
if ((key.version <= pReader->verRange.maxVer && key.version >= pReader->verRange.minVer) &&
(!hasBeenDropped(pDelList, &pIter->index, &key))) {
return pRow;
}
@ -2545,13 +2554,14 @@ TSDBROW* getValidRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pRea
return NULL;
}
if (key.version <= pReader->verRange.maxVer && key.version >= pReader->verRange.minVer && (!hasBeenDropped(pDelList, &pIter->index, &key))) {
if (key.version <= pReader->verRange.maxVer && key.version >= pReader->verRange.minVer &&
(!hasBeenDropped(pDelList, &pIter->index, &key))) {
return pRow;
}
}
}
int32_t doMergeRowsInBuf(SIterInfo *pIter, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader) {
int32_t doMergeRowsInBuf(SIterInfo* pIter, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader) {
while (1) {
pIter->hasVal = tsdbTbDataIterNext(pIter->iter);
if (!pIter->hasVal) {
@ -2685,7 +2695,8 @@ void updateSchema(TSDBROW* pRow, uint64_t uid, STsdbReader* pReader) {
}
}
void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo *pIter, SArray* pDelList, STSRow** pTSRow, STsdbReader* pReader) {
void doMergeMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow,
STsdbReader* pReader) {
SRowMerger merge = {0};
TSDBKEY k = TSDBROW_KEY(pRow);
@ -2728,7 +2739,7 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
int64_t endKey) {
TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader);
TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader);
SArray* pDelList = pBlockScanInfo->delSkyline;
SArray* pDelList = pBlockScanInfo->delSkyline;
// todo refactor
bool asc = ASCENDING_TRAVERSE(pReader->order);
@ -2855,6 +2866,20 @@ int32_t tsdbSetTableId(STsdbReader* pReader, int64_t uid) {
return TDB_CODE_SUCCESS;
}
void* tsdbGetIdx(SMeta* pMeta) {
if (pMeta == NULL) {
return NULL;
}
return metaGetIdx(pMeta);
}
void* tsdbGetIvtIdx(SMeta* pMeta) {
if (pMeta == NULL) {
return NULL;
}
return metaGetIvtIdx(pMeta);
}
/**
* @brief Get all suids since suid
*
@ -3105,11 +3130,11 @@ void tsdbReaderClose(STsdbReader* pReader) {
taosMemoryFreeClear(pReader->suppInfo.plist);
taosMemoryFree(pReader->suppInfo.slotIds);
if (!isEmptyQueryTimeWindow(&pReader->window)) {
// tsdbMayUnTakeMemSnapshot(pTsdbReadHandle);
} else {
ASSERT(pReader->status.pTableMap == NULL);
}
if (!isEmptyQueryTimeWindow(&pReader->window)) {
// tsdbMayUnTakeMemSnapshot(pTsdbReadHandle);
} else {
ASSERT(pReader->status.pTableMap == NULL);
}
#if 0
// if (pReader->status.pTableScanInfo != NULL) {
// pReader->status.pTableScanInfo = destroyTableCheckInfo(pReader->status.pTableScanInfo);
@ -3288,10 +3313,10 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_
setQueryTimewindow(pReader, pCond, tWinIdx);
pReader->order = pCond->order;
pReader->type = BLOCK_LOAD_OFFSET_ORDER;
pReader->order = pCond->order;
pReader->type = BLOCK_LOAD_OFFSET_ORDER;
pReader->status.loadFromFile = true;
pReader->status.pTableIter = NULL;
pReader->status.pTableIter = NULL;
pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows[tWinIdx]);
@ -3319,8 +3344,8 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond, int32_
}
}
tsdbDebug("%p reset reader, suid:%"PRIu64", numOfTables:%d, query range:%"PRId64" - %"PRId64" in query %s", pReader, pReader->suid,
numOfTables, pReader->window.skey, pReader->window.ekey, pReader->idStr);
tsdbDebug("%p reset reader, suid:%" PRIu64 ", numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s",
pReader, pReader->suid, numOfTables, pReader->window.skey, pReader->window.ekey, pReader->idStr);
return code;
}

2
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -49,7 +49,7 @@ int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb
}
pDelFWriter->fDel.size = TSDB_FHDR_SIZE;
pDelFWriter->fDel.size = 0;
pDelFWriter->fDel.offset = 0;
*ppWriter = pDelFWriter;
return code;

16
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -281,8 +281,9 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
vTrace("message in fetch queue is processing");
if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG)
&& !vnodeIsLeader(pVnode)) {
if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META ||
pMsg->msgType == TDMT_VND_TABLE_CFG) &&
!vnodeIsLeader(pVnode)) {
vnodeRedirectRpcMsg(pVnode, pMsg);
return 0;
}
@ -348,7 +349,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p
if (tbUids == NULL) return TSDB_CODE_OUT_OF_MEMORY;
int32_t t = ntohl(*(int32_t *)pReq);
vError("rec ttl time:%d", t);
vDebug("rec ttl time:%d", t);
int32_t ret = metaTtlDropTable(pVnode->pMeta, t, tbUids);
if (ret != 0) {
goto end;
@ -389,10 +390,14 @@ static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *p
goto _err;
}
taosMemoryFree(req.schemaRow.pSchema);
taosMemoryFree(req.schemaTag.pSchema);
tDecoderClear(&coder);
return 0;
_err:
taosMemoryFree(req.schemaRow.pSchema);
taosMemoryFree(req.schemaTag.pSchema);
tDecoderClear(&coder);
return -1;
}
@ -811,7 +816,8 @@ _exit:
taosArrayDestroy(submitRsp.pArray);
// TODO: the partial success scenario and the error case
// => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level 1/level 2.
// => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level
// 1/level 2.
// TODO: refactor
if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) {
tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT);
@ -915,4 +921,4 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq
_err:
return code;
}
}

23
source/libs/executor/inc/executorimpl.h
View File

@ -348,7 +348,7 @@ typedef struct SessionWindowSupporter {
uint8_t parentType;
} SessionWindowSupporter;
typedef struct SStreamBlockScanInfo {
typedef struct SStreamScanInfo {
uint64_t tableUid; // queried super table uid
SExprInfo* pPseudoExpr;
int32_t numOfPseudoExpr;
@ -365,7 +365,7 @@ typedef struct SStreamBlockScanInfo {
int32_t blockType; // current block type
int32_t validBlockIndex; // Is current data has returned?
uint64_t numOfExec; // execution times
void* streamBlockReader;// stream block reader handle
STqReader* tqReader;
int32_t tsArrayIndex;
SArray* tsArray;
@ -374,7 +374,7 @@ typedef struct SStreamBlockScanInfo {
EStreamScanMode scanMode;
SOperatorInfo* pStreamScanOp;
SOperatorInfo* pSnapshotReadOp;
SOperatorInfo* pTableScanOp;
SArray* childIds;
SessionWindowSupporter sessionSup;
bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA.
@ -383,7 +383,12 @@ typedef struct SStreamBlockScanInfo {
SSDataBlock* pPullDataRes; // pull data SSDataBlock
SSDataBlock* pDeleteDataRes; // delete data SSDataBlock
int32_t deleteDataIndex;
} SStreamBlockScanInfo;
// status for tmq
//SSchemaWrapper schema;
STqOffset offset;
} SStreamScanInfo;
typedef struct SSysTableScanInfo {
SRetrieveMetaTableRsp* pRsp;
@ -518,6 +523,7 @@ typedef struct SIndefOperatorInfo {
SAggSupporter aggSup;
SArray* pPseudoColInfo;
SExprSupp scalarSup;
SNode* pCondition;
} SIndefOperatorInfo;
typedef struct SFillOperatorInfo {
@ -527,6 +533,8 @@ typedef struct SFillOperatorInfo {
void** p;
SSDataBlock* existNewGroupBlock;
bool multigroupResult;
STimeWindow win;
SNode* pCondition;
} SFillOperatorInfo;
typedef struct SGroupbyOperatorInfo {
@ -587,6 +595,7 @@ typedef struct SSessionAggOperatorInfo {
int64_t gap; // session window gap
int32_t tsSlotId; // primary timestamp slot id
STimeWindowAggSupp twAggSup;
SNode *pCondition;
} SSessionAggOperatorInfo;
typedef struct SResultWindowInfo {
@ -648,6 +657,7 @@ typedef struct SStateWindowOperatorInfo {
int32_t tsSlotId; // primary timestamp column slot id
STimeWindowAggSupp twAggSup;
// bool reptScan;
const SNode *pCondition;
} SStateWindowOperatorInfo;
typedef struct SStreamStateAggOperatorInfo {
@ -805,7 +815,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExpr
STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp,
SExecTaskInfo* pTaskInfo);
SNode* pCondition, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResultBlock, SArray* pGroupColList, SNode* pCondition,
SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo);
@ -819,7 +829,8 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols,
SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSupp, int32_t tsSlotId, SColumn* pStateKeyCol, SExecTaskInfo* pTaskInfo);
SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSupp, int32_t tsSlotId,
SColumn* pStateKeyCol, SNode* pCondition, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo);

18
source/libs/executor/src/executil.c
View File

@ -305,9 +305,21 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
SNode* pTagIndexCond = (SNode*)pListInfo->pTagIndexCond;
if (pScanNode->tableType == TSDB_SUPER_TABLE) {
if (pTagIndexCond) {
SArray* res = taosArrayInit(8, sizeof(uint64_t));
// code = doFilterTag(pTagIndexCond, &metaArg, res);
code = TSDB_CODE_INDEX_REBUILDING;
///<<<<<<< HEAD
SIndexMetaArg metaArg = {
.metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid};
SArray* res = taosArrayInit(8, sizeof(uint64_t));
SIdxFltStatus status = SFLT_NOT_INDEX;
code = doFilterTag(pTagIndexCond, &metaArg, res, &status);
if (code != 0 || status == SFLT_NOT_INDEX) {
code = TSDB_CODE_INDEX_REBUILDING;
}
//=======
// SArray* res = taosArrayInit(8, sizeof(uint64_t));
// // code = doFilterTag(pTagIndexCond, &metaArg, res);
// code = TSDB_CODE_INDEX_REBUILDING;
//>>>>>>> dvv
if (code == TSDB_CODE_INDEX_REBUILDING) {
code = vnodeGetAllTableList(pVnode, tableUid, pListInfo->pTableList);
} else if (code != TSDB_CODE_SUCCESS) {

18
source/libs/executor/src/executor.c
View File

@ -37,7 +37,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
} else {
pOperator->status = OP_NOT_OPENED;
SStreamBlockScanInfo* pInfo = pOperator->info;
SStreamScanInfo* pInfo = pOperator->info;
pInfo->assignBlockUid = assignUid;
// TODO: if a block was set but not consumed,
@ -45,7 +45,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
pInfo->blockType = type;
if (type == STREAM_INPUT__DATA_SUBMIT) {
if (tqReadHandleSetMsg(pInfo->streamBlockReader, input, 0) < 0) {
if (tqReaderSetDataMsg(pInfo->tqReader, input, 0) < 0) {
qError("submit msg messed up when initing stream block, %s" PRIx64, id);
return TSDB_CODE_QRY_APP_ERROR;
}
@ -105,7 +105,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO
return code;
}
qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) {
qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) {
if (msg == NULL) {
return NULL;
}
@ -120,7 +120,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) {
}
qTaskInfo_t pTaskInfo = NULL;
code = qCreateExecTask(streamReadHandle, 0, 0, plan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM);
code = qCreateExecTask(readers, 0, 0, plan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM);
if (code != TSDB_CODE_SUCCESS) {
// TODO: destroy SSubplan & pTaskInfo
terrno = code;
@ -130,7 +130,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) {
return pTaskInfo;
}
static SArray* filterQualifiedChildTables(const SStreamBlockScanInfo* pScanInfo, const SArray* tableIdList) {
static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList) {
SArray* qa = taosArrayInit(4, sizeof(tb_uid_t));
// let's discard the tables those are not created according to the queried super table.
@ -168,17 +168,17 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
pInfo = pInfo->pDownstream[0];
}
int32_t code = 0;
SStreamBlockScanInfo* pScanInfo = pInfo->info;
int32_t code = 0;
SStreamScanInfo* pScanInfo = pInfo->info;
if (isAdd) { // add new table id
SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList);
qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa));
code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa);
code = tqReaderAddTbUidList(pScanInfo->tqReader, qa);
taosArrayDestroy(qa);
} else { // remove the table id in current list
qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList));
code = tqReadHandleRemoveTbUidList(pScanInfo->streamBlockReader, tableIdList);
code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList);
}
return code;

31
source/libs/executor/src/executorMain.c
View File

@ -236,6 +236,37 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le
return decodeOperator(pTaskInfo->pRoot, pInput, len);
}
int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
SOperatorInfo* pOperator = pTaskInfo->pRoot;
while (1) {
uint8_t type = pOperator->operatorType;
if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
*scanner = pOperator->info;
return 0;
} else {
ASSERT(pOperator->numOfDownstream == 1);
pOperator = pOperator->pDownstream[0];
}
}
}
void* qExtractReaderFromStreamScanner(void* scanner) {
SStreamScanInfo* pInfo = scanner;
return (void*)pInfo->tqReader;
}
const SSchemaWrapper* qExtractSchemaFromStreamScanner(void* scanner) {
SStreamScanInfo* pInfo = scanner;
return pInfo->tqReader->pSchemaWrapper;
}
const STqOffset* qExtractStatusFromStreamScanner(void* scanner) {
SStreamScanInfo* pInfo = scanner;
return &pInfo->offset;
}
int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;

93
source/libs/executor/src/executorimpl.c
View File

@ -1340,6 +1340,8 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) {
extractQualifiedTupleByFilterResult(pBlock, rowRes, keep);
blockDataUpdateTsWindow(pBlock, 0);
taosMemoryFree(rowRes);
}
void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep) {
@ -2842,17 +2844,17 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
}
int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) {
int32_t type = pOperator->operatorType;
uint8_t type = pOperator->operatorType;
pOperator->status = OP_OPENED;
if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
SStreamBlockScanInfo* pScanInfo = pOperator->info;
SStreamScanInfo* pScanInfo = pOperator->info;
pScanInfo->blockType = STREAM_INPUT__DATA_SCAN;
pScanInfo->pSnapshotReadOp->status = OP_OPENED;
pScanInfo->pTableScanOp->status = OP_OPENED;
STableScanInfo* pInfo = pScanInfo->pSnapshotReadOp->info;
STableScanInfo* pInfo = pScanInfo->pTableScanOp->info;
ASSERT(pInfo->scanMode == TABLE_SCAN__TABLE_ORDER);
if (uid == 0) {
@ -2912,8 +2914,8 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) {
int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts) {
int32_t type = pOperator->operatorType;
if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
SStreamBlockScanInfo* pScanInfo = pOperator->info;
STableScanInfo* pSnapShotScanInfo = pScanInfo->pSnapshotReadOp->info;
SStreamScanInfo* pScanInfo = pOperator->info;
STableScanInfo* pSnapShotScanInfo = pScanInfo->pTableScanOp->info;
*uid = pSnapShotScanInfo->lastStatus.uid;
*ts = pSnapShotScanInfo->lastStatus.ts;
} else {
@ -3334,7 +3336,7 @@ static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo* pInfo, SResult
SExecTaskInfo* pTaskInfo) {
pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows;
int64_t ekey = Q_STATUS_EQUAL(pTaskInfo->status, TASK_COMPLETED) ? pTaskInfo->window.ekey
int64_t ekey = Q_STATUS_EQUAL(pTaskInfo->status, TASK_COMPLETED) ? pInfo->win.ekey
: pInfo->existNewGroupBlock->info.window.ekey;
taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo));
@ -3362,7 +3364,7 @@ static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo* pInfo, SResultInf
}
}
static SSDataBlock* doFill(SOperatorInfo* pOperator) {
static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
SFillOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -3370,9 +3372,6 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
SSDataBlock* pResBlock = pInfo->pRes;
blockDataCleanup(pResBlock);
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
// todo handle different group data interpolation
bool n = false;
@ -3395,7 +3394,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
// Fill the previous group data block, before handle the data block of new group.
// Close the fill operation for previous group data block
taosFillSetStartInfo(pInfo->pFillInfo, 0, pTaskInfo->window.ekey);
taosFillSetStartInfo(pInfo->pFillInfo, 0, pInfo->win.ekey);
} else {
if (pBlock == NULL) {
if (pInfo->totalInputRows == 0) {
@ -3403,7 +3402,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
return NULL;
}
taosFillSetStartInfo(pInfo->pFillInfo, 0, pTaskInfo->window.ekey);
taosFillSetStartInfo(pInfo->pFillInfo, 0, pInfo->win.ekey);
} else {
pInfo->totalInputRows += pBlock->info.rows;
taosFillSetStartInfo(pInfo->pFillInfo, pBlock->info.rows, pBlock->info.window.ekey);
@ -3438,6 +3437,39 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
}
}
static SSDataBlock* doFill(SOperatorInfo* pOperator) {
SFillOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
SSDataBlock* fillResult = NULL;
while (true) {
fillResult = doFillImpl(pOperator);
if (fillResult != NULL) {
doFilter(pInfo->pCondition, fillResult);
}
if (fillResult == NULL) {
doSetOperatorCompleted(pOperator);
break;
}
if (fillResult->info.rows > 0) {
break;
}
}
if (fillResult != NULL) {
size_t rows = fillResult->info.rows;
pOperator->resultInfo.totalRows += rows;
}
return fillResult;
}
static void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) {
for (int32_t i = 0; i < numOfExprs; ++i) {
SExprInfo* pExprInfo = &pExpr[i];
@ -3830,6 +3862,8 @@ static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) {
}
}
doFilter(pIndefInfo->pCondition, pInfo->pRes);
size_t rows = pInfo->pRes->info.rows;
pOperator->resultInfo.totalRows += rows;
@ -3883,6 +3917,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
pInfo->binfo.pRes = pResBlock;
pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr);
pInfo->pCondition = pPhyNode->node.pConditions;
pOperator->name = "IndefinitOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT;
@ -3920,7 +3955,8 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t
int32_t order = TSDB_ORDER_ASC;
pInfo->pFillInfo = taosCreateFillInfo(order, w.skey, 0, capacity, numOfCols, pInterval, fillType, pColInfo, id);
pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES);
pInfo->win = win;
pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES);
if (pInfo->pFillInfo == NULL || pInfo->p == NULL) {
taosMemoryFree(pInfo->pFillInfo);
taosMemoryFree(pInfo->p);
@ -3955,6 +3991,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
pInfo->pRes = pResBlock;
pInfo->multigroupResult = multigroupResult;
pInfo->pCondition = pPhyFillNode->node.pConditions;
pOperator->name = "FillOperator";
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
@ -4309,11 +4346,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
} else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) {
SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode;
// int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
// if (code) {
// pTaskInfo->code = code;
// return NULL;
// }
// int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
// if (code) {
// pTaskInfo->code = code;
// return NULL;
// }
int32_t code = extractTableSchemaVersion(pHandle, pScanNode->uid, pTaskInfo);
if (code != TSDB_CODE_SUCCESS) {
@ -4370,8 +4407,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pOptr = createGroupOperatorInfo(ops[0], pExprInfo, num, pResBlock, pColList, pAggNode->node.pConditions,
pScalarExprInfo, numOfScalarExpr, pTaskInfo);
} else {
pOptr =
createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions, pScalarExprInfo, numOfScalarExpr, pTaskInfo);
pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions,
pScalarExprInfo, numOfScalarExpr, pTaskInfo);
}
} else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) {
SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
@ -4452,7 +4489,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;
pOptr =
createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, pTaskInfo);
createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, pPhyNode->pConditions, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION == type) {
pOptr = createStreamSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION == type) {
@ -4474,7 +4511,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
SColumn col = extractColumnFromColumnNode(pColNode);
pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, &col, pTaskInfo);
pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, &col, pPhyNode->pConditions, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE == type) {
pOptr = createStreamStateAggOperatorInfo(ops[0], pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN == type) {
@ -4538,7 +4575,8 @@ SArray* extractColumnInfo(SNodeList* pNodeList) {
return pList;
}
STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableListInfo, const char* idstr) {
STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
STableListInfo* pTableListInfo, const char* idstr) {
int32_t code = getTableList(pHandle->meta, pHandle->vnode, &pTableScanNode->scan, pTableListInfo);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
@ -4584,9 +4622,9 @@ static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanI
}
return extractTbscanInStreamOpTree(pOperator->pDownstream[0], ppInfo);
} else {
SStreamBlockScanInfo* pInfo = pOperator->info;
ASSERT(pInfo->pSnapshotReadOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
*ppInfo = pInfo->pSnapshotReadOp->info;
SStreamScanInfo* pInfo = pOperator->info;
ASSERT(pInfo->pTableScanOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
*ppInfo = pInfo->pTableScanOp->info;
return 0;
}
}
@ -4792,7 +4830,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
(*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId,
&(*pTaskInfo)->tableqinfoList, pPlan->user);
if (NULL == (*pTaskInfo)->pRoot) {
code = (*pTaskInfo)->code;
goto _complete;

19
source/libs/executor/src/groupoperator.c
View File

@ -302,14 +302,21 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
SSDataBlock* pRes = pInfo->binfo.pRes;
if (pOperator->status == OP_RES_TO_RETURN) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
while(1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pInfo->pCondition, pRes);
size_t rows = pRes->info.rows;
if (rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
doSetOperatorCompleted(pOperator);
bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
if (!hasRemain) {
doSetOperatorCompleted(pOperator);
break;
}
if (pRes->info.rows > 0) {
break;
}
}
pOperator->resultInfo.totalRows += rows;
pOperator->resultInfo.totalRows += pRes->info.rows;
return (pRes->info.rows == 0)? NULL:pRes;
}

253
source/libs/executor/src/scanoperator.c
View File

@ -780,7 +780,7 @@ _error:
return NULL;
}
static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) {
static void doClearBufferedBlocks(SStreamScanInfo* pInfo) {
size_t total = taosArrayGetSize(pInfo->pBlockLists);
pInfo->validBlockIndex = 0;
@ -791,21 +791,20 @@ static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) {
taosArrayClear(pInfo->pBlockLists);
}
static bool isSessionWindow(SStreamBlockScanInfo* pInfo) {
static bool isSessionWindow(SStreamScanInfo* pInfo) {
return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
}
static bool isStateWindow(SStreamBlockScanInfo* pInfo) {
static bool isStateWindow(SStreamScanInfo* pInfo) {
return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
}
static bool isIntervalWindow(SStreamBlockScanInfo* pInfo) {
static bool isIntervalWindow(SStreamScanInfo* pInfo) {
return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL ||
pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL;
pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL;
}
static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) {
uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t));
if (groupId) {
@ -827,19 +826,17 @@ static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) {
*/
}
static void setGroupId(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) {
static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) {
ASSERT(rowIndex < pBlock->info.rows);
switch (pBlock->info.type)
{
case STREAM_DELETE_DATA:
case STREAM_RETRIEVE: {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex);
uint64_t* groupCol = (uint64_t*)pColInfo->pData;
pInfo->groupId = groupCol[rowIndex];
}
break;
default:
break;
switch (pBlock->info.type) {
case STREAM_DELETE_DATA:
case STREAM_RETRIEVE: {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex);
uint64_t* groupCol = (uint64_t*)pColInfo->pData;
pInfo->groupId = groupCol[rowIndex];
} break;
default:
break;
}
}
@ -854,17 +851,17 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) {
pTableScanInfo->currentGroupId = -1;
}
static bool prepareRangeScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) {
static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) {
if ((*pRowIndex) == pBlock->info.rows) {
return false;
}
ASSERT(taosArrayGetSize(pBlock->pDataBlock) >= 3);
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* startData = (TSKEY*)pStartTsCol->pData;
TSKEY* startData = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
TSKEY* endData = (TSKEY*)pEndTsCol->pData;
STimeWindow win = {.skey = startData[*pRowIndex], .ekey = endData[*pRowIndex]};
TSKEY* endData = (TSKEY*)pEndTsCol->pData;
STimeWindow win = {.skey = startData[*pRowIndex], .ekey = endData[*pRowIndex]};
setGroupId(pInfo, pBlock, GROUPID_COLUMN_INDEX, *pRowIndex);
(*pRowIndex)++;
@ -877,16 +874,16 @@ static bool prepareRangeScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, i
win.skey = TMIN(win.skey, startData[*pRowIndex]);
continue;
}
ASSERT( (win.skey > startData[*pRowIndex] && win.ekey < endData[*pRowIndex]) ||
( isInTimeWindow(&win, startData[*pRowIndex], 0) || isInTimeWindow(&win, endData[*pRowIndex], 0) ) );
ASSERT((win.skey > startData[*pRowIndex] && win.ekey < endData[*pRowIndex]) ||
(isInTimeWindow(&win, startData[*pRowIndex], 0) || isInTimeWindow(&win, endData[*pRowIndex], 0)));
break;
}
resetTableScanInfo(pInfo->pSnapshotReadOp->info, &win);
resetTableScanInfo(pInfo->pTableScanOp->info, &win);
return true;
}
static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
static bool prepareDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
STimeWindow win = {
.skey = INT64_MIN,
.ekey = INT64_MAX,
@ -907,11 +904,10 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3
setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex);
(*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, *pRowIndex, gap, NULL);
} else {
win =
getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, pInfo->interval.precision, NULL);
win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, pInfo->interval.precision, NULL);
setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex);
(*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, *pRowIndex, win.ekey, binarySearchForKey, NULL,
TSDB_ORDER_ASC);
(*pRowIndex) +=
getNumOfRowsInTimeWindow(&pSDB->info, tsCols, *pRowIndex, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
}
needRead = true;
} else if (isStateWindow(pInfo)) {
@ -929,7 +925,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3
if (!needRead) {
return false;
}
resetTableScanInfo(pInfo->pSnapshotReadOp->info, &win);
resetTableScanInfo(pInfo->pTableScanOp->info, &win);
return true;
}
@ -946,13 +942,13 @@ static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRow
dest->info.rows++;
}
static SSDataBlock* doRangeScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
while (1) {
SSDataBlock* pResult = NULL;
pResult = doTableScan(pInfo->pSnapshotReadOp);
pResult = doTableScan(pInfo->pTableScanOp);
if (!pResult && prepareRangeScan(pInfo, pSDB, pRowIndex)) {
// scan next window data
pResult = doTableScan(pInfo->pSnapshotReadOp);
pResult = doTableScan(pInfo->pTableScanOp);
}
if (!pResult) {
blockDataCleanup(pSDB);
@ -966,14 +962,14 @@ static SSDataBlock* doRangeScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB,
}
}
static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
static SSDataBlock* doDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
while (1) {
SSDataBlock* pResult = NULL;
pResult = doTableScan(pInfo->pSnapshotReadOp);
pResult = doTableScan(pInfo->pTableScanOp);
if (pResult == NULL) {
if (prepareDataScan(pInfo, pSDB, tsColIndex, pRowIndex)) {
// scan next window data
pResult = doTableScan(pInfo->pSnapshotReadOp);
pResult = doTableScan(pInfo->pTableScanOp);
}
}
if (!pResult) {
@ -997,8 +993,8 @@ static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, i
return pResult;
*/
}
static void generateIntervalTs(SStreamBlockScanInfo* pInfo, SSDataBlock* pDelBlock, SOperatorInfo* pOperator, SSDataBlock* pUpdateRes) {
static void generateIntervalTs(SStreamScanInfo* pInfo, SSDataBlock* pDelBlock, SOperatorInfo* pOperator,
SSDataBlock* pUpdateRes) {
if (pDelBlock->info.rows == 0) {
return;
}
@ -1006,18 +1002,20 @@ static void generateIntervalTs(SStreamBlockScanInfo* pInfo, SSDataBlock* pDelBlo
blockDataEnsureCapacity(pUpdateRes, 64);
ASSERT(taosArrayGetSize(pDelBlock->pDataBlock) >= 3);
SColumnInfoData* pStartTsCol = taosArrayGet(pDelBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* startData = (TSKEY*)pStartTsCol->pData;
TSKEY* startData = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pEndTsCol = taosArrayGet(pDelBlock->pDataBlock, END_TS_COLUMN_INDEX);
TSKEY* endData = (TSKEY*)pEndTsCol->pData;
TSKEY* endData = (TSKEY*)pEndTsCol->pData;
SColumnInfoData* pGpCol = taosArrayGet(pDelBlock->pDataBlock, UID_COLUMN_INDEX);
uint64_t* uidCol = (uint64_t*)pGpCol->pData;
uint64_t* uidCol = (uint64_t*)pGpCol->pData;
SColumnInfoData* pDestTsCol = taosArrayGet(pUpdateRes->pDataBlock, START_TS_COLUMN_INDEX);
SColumnInfoData* pDestGpCol = taosArrayGet(pUpdateRes->pDataBlock, GROUPID_COLUMN_INDEX);
for (int32_t i = pInfo->deleteDataIndex ; i < pDelBlock->info.rows &&
i < pDelBlock->info.capacity - (endData[i] - startData[i])/pInfo->interval.interval - 1; i++) {
for (int32_t i = pInfo->deleteDataIndex;
i < pDelBlock->info.rows &&
i < pDelBlock->info.capacity - (endData[i] - startData[i]) / pInfo->interval.interval - 1;
i++) {
uint64_t groupId = getGroupId(pOperator, uidCol[i]);
for (TSKEY startTs = startData[i]; startTs <= endData[i]; ) {
for (TSKEY startTs = startData[i]; startTs <= endData[i];) {
colDataAppend(pDestTsCol, pUpdateRes->info.rows, (const char*)&startTs, false);
colDataAppend(pDestGpCol, pUpdateRes->info.rows, (const char*)&groupId, false);
pUpdateRes->info.rows++;
@ -1032,33 +1030,36 @@ static void generateIntervalTs(SStreamBlockScanInfo* pInfo, SSDataBlock* pDelBlo
}
}
static void generateScanRange(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SOperatorInfo* pOperator, SSDataBlock* pUpdateRes) {
if (pBlock->info.rows == 0) {
static void generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SOperatorInfo* pOperator,
SSDataBlock* pUpdateRes) {
if (pBlock->info.rows == 0) {
return;
}
blockDataCleanup(pUpdateRes);
blockDataEnsureCapacity(pUpdateRes, pBlock->info.rows);
ASSERT(taosArrayGetSize(pBlock->pDataBlock) >= 3);
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* startData = (TSKEY*)pStartTsCol->pData;
TSKEY* startData = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
TSKEY* endData = (TSKEY*)pEndTsCol->pData;
TSKEY* endData = (TSKEY*)pEndTsCol->pData;
SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
uint64_t* uidCol = (uint64_t*)pGpCol->pData;
uint64_t* uidCol = (uint64_t*)pGpCol->pData;
SColumnInfoData* pDestStartCol = taosArrayGet(pUpdateRes->pDataBlock, START_TS_COLUMN_INDEX);
SColumnInfoData* pDestEndCol = taosArrayGet(pUpdateRes->pDataBlock, END_TS_COLUMN_INDEX);
SColumnInfoData* pDestGpCol = taosArrayGet(pUpdateRes->pDataBlock, GROUPID_COLUMN_INDEX);
int32_t dummy = 0;
for (int32_t i = 0 ; i < pBlock->info.rows; i++) {
int32_t dummy = 0;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
uint64_t groupId = getGroupId(pOperator, uidCol[i]);
//gap must be 0.
SResultWindowInfo* pStartWin = getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, startData[i], endData[i], groupId, 0, &dummy);
// gap must be 0.
SResultWindowInfo* pStartWin =
getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, startData[i], endData[i], groupId, 0, &dummy);
if (!pStartWin) {
// window has been closed.
continue;
}
SResultWindowInfo* pEndWin = getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, endData[i], endData[i], groupId, 0, &dummy);
SResultWindowInfo* pEndWin =
getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, endData[i], endData[i], groupId, 0, &dummy);
ASSERT(pEndWin);
colDataAppend(pDestStartCol, i, (const char*)&pStartWin->win.skey, false);
colDataAppend(pDestEndCol, i, (const char*)&pEndWin->win.ekey, false);
@ -1066,7 +1067,7 @@ static void generateScanRange(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock,
pUpdateRes->info.rows++;
}
}
static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlock* pUpdateBlock) {
static void setUpdateData(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlock* pUpdateBlock) {
blockDataCleanup(pUpdateBlock);
int32_t size = taosArrayGetSize(pInfo->tsArray);
if (pInfo->tsArrayIndex < size) {
@ -1075,11 +1076,11 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa
blockDataEnsureCapacity(pUpdateBlock, size);
int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex);
pInfo->groupId = getGroupId(pInfo->pSnapshotReadOp, pBlock->info.uid);
pInfo->groupId = getGroupId(pInfo->pTableScanOp, pBlock->info.uid);
int32_t i = 0;
for (; i < size; i++) {
rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, i + pInfo->tsArrayIndex);
uint64_t id = getGroupId(pInfo->pSnapshotReadOp, pBlock->info.uid);
uint64_t id = getGroupId(pInfo->pTableScanOp, pBlock->info.uid);
if (pInfo->groupId != id) {
break;
}
@ -1098,12 +1099,11 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa
}
if (size == 0) {
generateIntervalTs(pInfo, pInfo->pDeleteDataRes, pInfo->pSnapshotReadOp, pUpdateBlock);
generateIntervalTs(pInfo, pInfo->pDeleteDataRes, pInfo->pTableScanOp, pUpdateBlock);
}
}
static void checkUpdateData(SStreamBlockScanInfo* pInfo, bool invertible, SSDataBlock* pBlock,
bool out) {
static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) {
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP);
TSKEY* ts = (TSKEY*)pColDataInfo->pData;
@ -1119,15 +1119,15 @@ static void setBlockGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, uidColIndex);
uint64_t* uidCol = (uint64_t*)pColDataInfo->pData;
ASSERT(pBlock->info.rows > 0);
for (int32_t i = 0 ; i < pBlock->info.rows; i++) {
for (int32_t i = 0; i < pBlock->info.rows; i++) {
uidCol[i] = getGroupId(pOperator, uidCol[i]);
}
}
static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
// NOTE: this operator does never check if current status is done or not
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamBlockScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStreamScanInfo* pInfo = pOperator->info;
pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
if (pTaskInfo->code != TSDB_CODE_SUCCESS || pOperator->status == OP_EXEC_DONE) {
@ -1145,36 +1145,35 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
int32_t current = pInfo->validBlockIndex++;
SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current);
// TODO move into scan
blockDataUpdateTsWindow(pBlock, 0);
switch (pBlock->info.type) {
case STREAM_RETRIEVE:{
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE;
copyDataBlock(pInfo->pPullDataRes, pBlock);
pInfo->pullDataResIndex = 0;
prepareDataScan(pInfo, pInfo->pPullDataRes, START_TS_COLUMN_INDEX, &pInfo->pullDataResIndex);
updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo);
}
break;
case STREAM_DELETE_DATA: {
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
pInfo->updateResIndex = 0;
if (isIntervalWindow(pInfo)) {
copyDataBlock(pInfo->pDeleteDataRes, pBlock);
generateIntervalTs(pInfo, pInfo->pDeleteDataRes, pInfo->pSnapshotReadOp, pInfo->pUpdateRes);
prepareDataScan(pInfo, pInfo->pUpdateRes, START_TS_COLUMN_INDEX, &pInfo->updateResIndex);
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
} else {
generateScanRange(pInfo, pBlock, pInfo->pSnapshotReadOp, pInfo->pUpdateRes);
prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
}
pInfo->pUpdateRes->info.type = STREAM_DELETE_DATA;
return pInfo->pUpdateRes;
}
break;
default:
break;
case STREAM_RETRIEVE: {
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE;
copyDataBlock(pInfo->pPullDataRes, pBlock);
pInfo->pullDataResIndex = 0;
prepareDataScan(pInfo, pInfo->pPullDataRes, START_TS_COLUMN_INDEX, &pInfo->pullDataResIndex);
updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo);
} break;
case STREAM_DELETE_DATA: {
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
pInfo->updateResIndex = 0;
if (isIntervalWindow(pInfo)) {
copyDataBlock(pInfo->pDeleteDataRes, pBlock);
generateIntervalTs(pInfo, pInfo->pDeleteDataRes, pInfo->pTableScanOp, pInfo->pUpdateRes);
prepareDataScan(pInfo, pInfo->pUpdateRes, START_TS_COLUMN_INDEX, &pInfo->updateResIndex);
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
} else {
generateScanRange(pInfo, pBlock, pInfo->pTableScanOp, pInfo->pUpdateRes);
prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
}
pInfo->pUpdateRes->info.type = STREAM_DELETE_DATA;
return pInfo->pUpdateRes;
} break;
default:
break;
}
return pBlock;
} else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) {
@ -1233,38 +1232,33 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
SDataBlockInfo* pBlockInfo = &pInfo->pRes->info;
blockDataCleanup(pInfo->pRes);
while (tqNextDataBlock(pInfo->streamBlockReader)) {
while (tqNextDataBlock(pInfo->tqReader)) {
SSDataBlock block = {0};
// todo refactor
int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader);
int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader);
uint64_t groupId = block.info.groupId;
uint64_t uid = block.info.uid;
int32_t numOfRows = block.info.rows;
if (code != TSDB_CODE_SUCCESS || numOfRows == 0) {
if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) {
pTaskInfo->code = code;
return NULL;
}
pInfo->pRes->info.groupId = groupId;
pInfo->pRes->info.rows = numOfRows;
pInfo->pRes->info.uid = uid;
pInfo->pRes->info.rows = block.info.rows;
pInfo->pRes->info.uid = block.info.uid;
pInfo->pRes->info.type = STREAM_NORMAL;
pInfo->pRes->info.capacity = numOfRows;
pInfo->pRes->info.capacity = block.info.rows;
// for generating rollup SMA result, each time is an independent time serie.
// TODO temporarily used, when the statement of "partition by tbname" is ready, remove this
if (pInfo->assignBlockUid) {
pInfo->pRes->info.groupId = uid;
} else {
pInfo->pRes->info.groupId = groupId;
pInfo->pRes->info.groupId = block.info.uid;
}
uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t));
uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &block.info.uid, sizeof(int64_t));
if (groupIdPre) {
pInfo->pRes->info.groupId = *groupIdPre;
} else {
pInfo->pRes->info.groupId = 0;
}
// todo extract method
@ -1333,11 +1327,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
}
}
}
return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes;
} else if (pInfo->blockType == STREAM_INPUT__DATA_SCAN) {
// check reader last status
// if not match, reset status
SSDataBlock* pResult = doTableScan(pInfo->pSnapshotReadOp);
SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
return pResult && pResult->info.rows > 0 ? pResult : NULL;
} else {
@ -1361,8 +1357,8 @@ static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) {
SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode,
SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId,
uint64_t taskId) {
SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
@ -1400,13 +1396,25 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
}
if (pHandle) {
SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo);
STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanDummy->info;
SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo);
STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanOp->info;
SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, 0);
if (pHandle->tqReader) {
if (pHandle->initTableReader) {
pSTInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
tsdbReaderOpen(pHandle->vnode, &pSTInfo->cond, tableList, &pSTInfo->dataReader, 0);
pSTInfo->dataReader = NULL;
if (tsdbReaderOpen(pHandle->vnode, &pSTInfo->cond, tableList, &pSTInfo->dataReader, NULL) < 0) {
ASSERT(0);
}
}
if (pHandle->initTqReader) {
ASSERT(pHandle->tqReader == NULL);
pInfo->tqReader = tqOpenReader(pHandle->vnode);
ASSERT(pInfo->tqReader);
} else {
ASSERT(pHandle->tqReader);
pInfo->tqReader = pHandle->tqReader;
}
if (pSTInfo->interval.interval > 0) {
@ -1414,18 +1422,17 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
} else {
pInfo->pUpdateInfo = NULL;
}
pInfo->pSnapshotReadOp = pTableScanDummy;
pInfo->pTableScanOp = pTableScanOp;
pInfo->interval = pSTInfo->interval;
pInfo->readHandle = *pHandle;
ASSERT(pHandle->reader);
pInfo->streamBlockReader = pHandle->reader;
pInfo->tableUid = pScanPhyNode->uid;
// set the extract column id to streamHandle
tqReadHandleSetColIdList((SStreamReader*)pHandle->reader, pColIds);
tqReaderSetColIdList(pInfo->tqReader, pColIds);
SArray* tableIdList = extractTableIdList(&pTaskInfo->tableqinfoList);
int32_t code = tqReadHandleSetTbUidList(pHandle->reader, tableIdList);
int32_t code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList);
if (code != 0) {
taosArrayDestroy(tableIdList);
goto _error;
@ -1449,7 +1456,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
pInfo->deleteDataIndex = 0;
pInfo->pDeleteDataRes = createPullDataBlock();
pOperator->name = "StreamBlockScanOperator";
pOperator->name = "StreamScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
@ -1458,7 +1465,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet =
createOperatorFpSet(operatorDummyOpenFn, doStreamBlockScan, NULL, NULL, operatorDummyCloseFn, NULL, NULL, NULL);
createOperatorFpSet(operatorDummyOpenFn, doStreamScan, NULL, NULL, operatorDummyCloseFn, NULL, NULL, NULL);
return pOperator;

123
source/libs/executor/src/timewindowoperator.c
View File

@ -1146,13 +1146,22 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
SOptrBasicInfo* pBInfo = &pInfo->binfo;
if (pOperator->status == OP_RES_TO_RETURN) {
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
doSetOperatorCompleted(pOperator);
return NULL;
}
while(1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pInfo->pCondition, pBInfo->pRes);
return pBInfo->pRes;
bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
if (!hasRemain) {
doSetOperatorCompleted(pOperator);
break;
}
if (pBInfo->pRes->info.rows > 0) {
break;
}
}
pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes;
}
int32_t order = TSDB_ORDER_ASC;
@ -1178,15 +1187,22 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
doSetOperatorCompleted(pOperator);
while(1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pInfo->pCondition, pBInfo->pRes);
bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
if (!hasRemain) {
doSetOperatorCompleted(pOperator);
break;
}
if (pBInfo->pRes->info.rows > 0) {
break;
}
}
size_t rows = pBInfo->pRes->info.rows;
pOperator->resultInfo.totalRows += rows;
return (rows == 0) ? NULL : pBInfo->pRes;
pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes;
}
static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
@ -1880,12 +1896,22 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
SExprSupp* pSup = &pOperator->exprSupp;
if (pOperator->status == OP_RES_TO_RETURN) {
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
doSetOperatorCompleted(pOperator);
}
while(1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pInfo->pCondition, pBInfo->pRes);
return pBInfo->pRes->info.rows > 0 ? pBInfo->pRes : NULL;
bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
if (!hasRemain) {
doSetOperatorCompleted(pOperator);
break;
}
if (pBInfo->pRes->info.rows > 0) {
break;
}
}
pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes;
}
int64_t st = taosGetTimestampUs();
@ -1914,15 +1940,22 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
doSetOperatorCompleted(pOperator);
while(1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pInfo->pCondition, pBInfo->pRes);
bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
if (!hasRemain) {
doSetOperatorCompleted(pOperator);
break;
}
if (pBInfo->pRes->info.rows > 0) {
break;
}
}
size_t rows = pBInfo->pRes->info.rows;
pOperator->resultInfo.totalRows += rows;
return (rows == 0) ? NULL : pBInfo->pRes;
pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
return (pBInfo->pRes->info.rows == 0)? NULL:pBInfo->pRes;
}
static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
@ -2235,7 +2268,7 @@ _error:
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols,
SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, int32_t tsSlotId,
SColumn* pStateKeyCol, SExecTaskInfo* pTaskInfo) {
SColumn* pStateKeyCol, SNode* pCondition, SExecTaskInfo* pTaskInfo) {
SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -2246,6 +2279,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf
pInfo->stateKey.type = pInfo->stateCol.type;
pInfo->stateKey.bytes = pInfo->stateCol.bytes;
pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes);
pInfo->pCondition = pCondition;
if (pInfo->stateKey.pData == NULL) {
goto _error;
}
@ -2289,7 +2323,7 @@ void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) {
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId,
STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) {
STimeWindowAggSupp* pTwAggSupp, SNode* pCondition, SExecTaskInfo* pTaskInfo) {
SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -2315,6 +2349,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo
pInfo->binfo.pRes = pResBlock;
pInfo->winSup.prevTs = INT64_MIN;
pInfo->reptScan = false;
pInfo->pCondition = pCondition;
pOperator->name = "SessionWindowAggOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION;
pOperator->blocking = true;
@ -2988,7 +3023,7 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark,
uint8_t type) {
ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
SStreamBlockScanInfo* pScanInfo = downstream->info;
SStreamScanInfo* pScanInfo = downstream->info;
pScanInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};
pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark);
}
@ -3154,7 +3189,7 @@ SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY start
*pIndex = index + 1;
return pWin;
} else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
*pIndex = index;
*pIndex = index + 1;
return pWin;
}
}
@ -3407,7 +3442,7 @@ void deleteWindow(SArray* pWinInfos, int32_t index) {
taosArrayRemove(pWinInfos, index);
}
static void doDeleteSessionWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result) {
static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result) {
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* startDatas = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
@ -3545,8 +3580,8 @@ typedef SResultWindowInfo* (*__get_win_info_)(void*);
SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; }
SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; }
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup,
SArray* pClosed, __get_win_info_ fn, bool delete) {
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn,
bool delete) {
// Todo(liuyao) save window to tdb
void** pIte = NULL;
size_t keyLen = 0;
@ -3665,13 +3700,13 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
} else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
// gap must be 0
doDeleteSessionWindows(&pInfo->streamAggSup, pBlock, 0, pWins);
doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins);
if (IS_FINAL_OP(pInfo)) {
int32_t childIndex = getChildIndex(pBlock);
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
// gap must be 0
doDeleteSessionWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL);
doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL);
rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator);
}
copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
@ -3708,8 +3743,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
// restore the value
pOperator->status = OP_RES_TO_RETURN;
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
getResWinForSession, pInfo->ignoreExpiredData);
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForSession,
pInfo->ignoreExpiredData);
closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData);
copyUpdateResult(pStUpdated, pUpdated);
taosHashCleanup(pStUpdated);
@ -3805,7 +3840,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
break;
} else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
// gap must be 0
doDeleteSessionWindows(&pInfo->streamAggSup, pBlock, 0, NULL);
doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, NULL);
copyDataBlock(pInfo->pDelRes, pBlock);
pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
break;
@ -4197,6 +4232,12 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId,
pSeUpdated, pInfo->pSeDeleted);
continue;
} else if (pBlock->info.type == STREAM_DELETE_DATA) {
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins);
copyDeleteWindowInfo(pWins, pInfo->pSeDeleted);
taosArrayDestroy(pWins);
continue;
} else if (pBlock->info.type == STREAM_GET_ALL) {
getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForState);
continue;
@ -4210,8 +4251,8 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
// restore the value
pOperator->status = OP_RES_TO_RETURN;
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
getResWinForState, pInfo->ignoreExpiredData);
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForState,
pInfo->ignoreExpiredData);
closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData);
copyUpdateResult(pSeUpdated, pUpdated);
taosHashCleanup(pSeUpdated);

28
source/libs/executor/test/CMakeLists.txt
View File

@ -1,18 +1,20 @@
MESSAGE(STATUS "build parser unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
IF(NOT TD_DARWIN)
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(executorTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
executorTest
PRIVATE os util common transport gtest taos_static qcom executor function planner scalar nodes vnode
)
ADD_EXECUTABLE(executorTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
executorTest
PRIVATE os util common transport gtest taos_static qcom executor function planner scalar nodes vnode
)
TARGET_INCLUDE_DIRECTORIES(
executorTest
PUBLIC "${TD_SOURCE_DIR}/include/libs/executor/"
PRIVATE "${TD_SOURCE_DIR}/source/libs/executor/inc"
)
TARGET_INCLUDE_DIRECTORIES(
executorTest
PUBLIC "${TD_SOURCE_DIR}/include/libs/executor/"
PRIVATE "${TD_SOURCE_DIR}/source/libs/executor/inc"
)
ENDIF ()

4
source/libs/function/src/tudf.c
View File

@ -75,6 +75,10 @@ static int32_t udfSpawnUdfd(SUdfdData* pData) {
#ifdef WINDOWS
GetModuleFileName(NULL, path, PATH_MAX);
taosDirName(path);
#elif defined(_TD_DARWIN_64)
uint32_t pathSize = sizeof(path);
_NSGetExecutablePath(path, &pathSize);
taosDirName(path);
#endif
} else {
strncpy(path, tsProcPath, strlen(tsProcPath));

6
source/libs/index/CMakeLists.txt
View File

@ -35,7 +35,7 @@ if (${BUILD_WITH_INVERTEDINDEX})
endif(${BUILD_WITH_INVERTEDINDEX})
# if (${BUILD_TEST})
# add_subdirectory(test)
# endif(${BUILD_TEST})
if (${BUILD_TEST})
add_subdirectory(test)
endif(${BUILD_TEST})

13
source/libs/index/inc/indexFst.h
View File

@ -53,7 +53,7 @@ typedef struct FstRange {
} FstRange;
typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State;
typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType;
typedef enum { Ordered, OutOfOrdered, DuplicateKey } FstOrderType;
FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data);
bool fstBoundWithDataExceededBy(FstBoundWithData* bound, FstSlice* slice);
@ -106,7 +106,7 @@ bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in);
void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate);
void* fstBuilerIntoInner(FstBuilder* b);
void fstBuilderFinish(FstBuilder* b);
OrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup);
FstOrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup);
CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn);
typedef struct FstTransitions {
@ -213,14 +213,18 @@ typedef struct FstNode {
// If this node is final and has a terminal output value, then it is, returned.
// Otherwise, a zero output is returned
#define FST_NODE_FINAL_OUTPUT(node) node->finalOutput
// Returns true if and only if this node corresponds to a final or "match",
// state in the finite state transducer.
#define FST_NODE_IS_FINAL(node) node->isFinal
// Returns the number of transitions in this node, The maximum number of
// transitions is 256.
#define FST_NODE_LEN(node) node->nTrans
// Returns true if and only if this node has zero transitions.
#define FST_NODE_IS_EMPTYE(node) (node->nTrans == 0)
// Return the address of this node.
#define FST_NODE_ADDR(node) node->start
@ -277,6 +281,8 @@ FStmBuilder* fstSearch(Fst* fst, FAutoCtx* ctx);
FStmStBuilder* fstSearchWithState(Fst* fst, FAutoCtx* ctx);
// into stream to expand later
//
FStmSt* stmBuilderIntoStm(FStmBuilder* sb);
bool fstVerify(Fst* fst);
@ -325,7 +331,8 @@ FStmBuilder* stmBuilderCreate(Fst* fst, FAutoCtx* aut);
void stmBuilderDestroy(FStmBuilder* b);
// set up bound range
// refator later: to simple code by marco
// refator later
// simple code by marco
void stmBuilderSetRange(FStmBuilder* b, FstSlice* val, RangeType type);
#ifdef __cplusplus

2
source/libs/index/inc/indexInt.h
View File

@ -45,6 +45,7 @@ extern "C" {
typedef enum { LT, LE, GT, GE, CONTAINS, EQ } RangeType;
typedef enum { kTypeValue, kTypeDeletion } STermValueType;
typedef enum { kRebuild, kFinished } SIdxStatus;
typedef struct SIndexStat {
int32_t totalAdded; //
@ -65,6 +66,7 @@ struct SIndex {
char* path;
int8_t status;
SIndexStat stat;
TdThreadMutex mtx;
tsem_t sem;

90
source/libs/index/src/index.c
View File

@ -63,7 +63,7 @@ static void indexDestroy(void* sIdx);
void indexInit() {
// refactor later
indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index");
indexRefMgt = taosOpenRef(10, indexDestroy);
indexRefMgt = taosOpenRef(1000, indexDestroy);
}
void indexCleanup() {
// refacto later
@ -101,15 +101,16 @@ static void indexWait(void* idx) {
}
int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
int ret = TSDB_CODE_SUCCESS;
taosThreadOnce(&isInit, indexInit);
SIndex* sIdx = taosMemoryCalloc(1, sizeof(SIndex));
if (sIdx == NULL) {
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
// sIdx->cache = (void*)idxCacheCreate(sIdx);
sIdx->tindex = idxTFileCreate(path);
if (sIdx->tindex == NULL) {
ret = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
@ -123,14 +124,14 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
idxAcquireRef(sIdx->refId);
*index = sIdx;
return 0;
return ret;
END:
if (sIdx != NULL) {
indexClose(sIdx);
}
*index = NULL;
return -1;
return ret;
}
void indexDestroy(void* handle) {
@ -231,7 +232,7 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result
}
int indexDelete(SIndex* index, SIndexMultiTermQuery* query) { return 1; }
int indexRebuild(SIndex* index, SIndexOpts* opts) { return 0; }
// int indexRebuild(SIndex* index, SIndexOpts* opts) { return 0; }
SIndexOpts* indexOptsCreate() { return NULL; }
void indexOptsDestroy(SIndexOpts* opts) { return; }
@ -273,33 +274,28 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy
tm->operType = oper;
tm->colType = colType;
#if 0
tm->colName = (char*)taosMemoryCalloc(1, nColName + 1);
memcpy(tm->colName, colName, nColName);
tm->nColName = nColName;
tm->colVal = (char*)taosMemoryCalloc(1, nColVal + 1);
memcpy(tm->colVal, colVal, nColVal);
tm->nColVal = nColVal;
#endif
#if 1
tm->colName = (char*)taosMemoryCalloc(1, nColName + 1);
memcpy(tm->colName, colName, nColName);
tm->nColName = nColName;
char* buf = NULL;
int32_t len = idxConvertDataToStr((void*)colVal, IDX_TYPE_GET_TYPE(colType), (void**)&buf);
assert(len != -1);
int32_t len = 0;
if (colVal != NULL && nColVal != 0) {
len = idxConvertDataToStr((void*)colVal, IDX_TYPE_GET_TYPE(colType), (void**)&buf);
} else if (colVal == NULL) {
buf = strndup(INDEX_DATA_NULL_STR, (int32_t)strlen(INDEX_DATA_NULL_STR));
len = (int32_t)strlen(INDEX_DATA_NULL_STR);
} else {
const char* emptyStr = " ";
buf = strndup(emptyStr, (int32_t)strlen(emptyStr));
len = (int32_t)strlen(emptyStr);
}
tm->colVal = buf;
tm->nColVal = len;
#endif
return tm;
}
void indexTermDestroy(SIndexTerm* p) {
taosMemoryFree(p->colName);
taosMemoryFree(p->colVal);
@ -320,6 +316,54 @@ void indexMultiTermDestroy(SIndexMultiTerm* terms) {
taosArrayDestroy(terms);
}
/*
* rebuild index
*/
static void idxSchedRebuildIdx(SSchedMsg* msg) {
// TODO, no need rebuild index
SIndex* idx = msg->ahandle;
int8_t st = kFinished;
atomic_store_8(&idx->status, st);
idxReleaseRef(idx->refId);
}
void indexRebuild(SIndexJson* idx, void* iter) {
// set up rebuild status
int8_t st = kRebuild;
atomic_store_8(&idx->status, st);
// task put into BG thread
SSchedMsg schedMsg = {0};
schedMsg.fp = idxSchedRebuildIdx;
schedMsg.ahandle = idx;
idxAcquireRef(idx->refId);
taosScheduleTask(indexQhandle, &schedMsg);
}
/*
* check index json status
**/
bool indexIsRebuild(SIndex* idx) {
// idx rebuild or not
return ((SIdxStatus)atomic_load_8(&idx->status)) == kRebuild ? true : false;
}
/*
* rebuild index
*/
void indexJsonRebuild(SIndexJson* idx, void* iter) {
// idx rebuild or not
indexRebuild(idx, iter);
}
/*
* check index json status
**/
bool indexJsonIsRebuild(SIndexJson* idx) {
// load idx rebuild or not
return ((SIdxStatus)atomic_load_8(&idx->status)) == kRebuild ? true : false;
}
static int idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) {
SIndexTerm* term = query->term;
const char* colName = term->colName;

6
source/libs/index/src/indexComm.c
View File

@ -29,7 +29,7 @@
#define INDEX_DATA_BIGINT_NULL 0x8000000000000000LL
#define INDEX_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL
#define INDEX_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN
#define INDEX_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN
#define INDEX_DATA_DOUBLE_NULL 0x7FFFFF0000000000LL // an NAN
#define INDEX_DATA_NCHAR_NULL 0xFFFFFFFF
#define INDEX_DATA_BINARY_NULL 0xFF
@ -374,6 +374,10 @@ int32_t idxConvertData(void* src, int8_t type, void** dst) {
return tlen;
}
int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) {
if (src == NULL) {
*dst = strndup(INDEX_DATA_NULL_STR, (int)strlen(INDEX_DATA_NULL_STR));
return (int32_t)strlen(INDEX_DATA_NULL_STR);
}
int tlen = tDataTypes[type].bytes;
int32_t bufSize = 64;
switch (type) {

32
source/libs/index/src/indexFilter.c
View File

@ -181,11 +181,9 @@ static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
param->colValType = l->node.resType.type;
memcpy(param->dbName, l->dbName, sizeof(l->dbName));
memcpy(param->colName, r->literal, strlen(r->literal));
// sprintf(param->colName, "%s_%s", l->colName, r->literal);
param->colValType = r->typeData;
param->status = SFLT_COARSE_INDEX;
return 0;
// memcpy(param->colName, l->colName, sizeof(l->colName));
}
static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
param->status = SFLT_COARSE_INDEX;
@ -274,6 +272,10 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx
SIF_ERR_JRET(sifInitParam(node->pLeft, &paramList[0], ctx));
if (nParam > 1) {
SIF_ERR_JRET(sifInitParam(node->pRight, &paramList[1], ctx));
// if (paramList[0].colValType == TSDB_DATA_TYPE_JSON &&
// ((SOperatorNode *)(node))->opType == OP_TYPE_JSON_CONTAINS) {
// return TSDB_CODE_QRY_OUT_OF_MEMORY;
//}
}
*params = paramList;
return TSDB_CODE_SUCCESS;
@ -511,11 +513,12 @@ static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *sta
}
return 0;
}
// typedef struct filterFuncDict {
static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
int32_t code = 0;
if (sifValidOp(node->opType) < 0) {
code = TSDB_CODE_QRY_INVALID_INPUT;
ctx->code = code;
output->status = SFLT_NOT_INDEX;
return code;
}
@ -532,7 +535,7 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
SIFParam *params = NULL;
SIF_ERR_RET(sifInitOperParams(&params, node, ctx));
if (params[0].status == SFLT_NOT_INDEX || (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) {
if (params[0].status == SFLT_NOT_INDEX && (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) {
output->status = SFLT_NOT_INDEX;
return code;
}
@ -737,23 +740,23 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) {
SIF_RET(code);
}
int32_t doFilterTag(const SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result) {
if (pFilterNode == NULL) {
return TSDB_CODE_SUCCESS;
int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SIdxFltStatus *status) {
SIdxFltStatus st = idxGetFltStatus(pFilterNode);
if (st == SFLT_NOT_INDEX) {
*status = st;
return 0;
}
SFilterInfo *filter = NULL;
// todo move to the initialization function
// SIF_ERR_RET(filterInitFromNode((SNode *)pFilterNode, &filter, 0));
SArray * output = taosArrayInit(8, sizeof(uint64_t));
SIFParam param = {.arg = *metaArg, .result = output};
SIF_ERR_RET(sifCalculate((SNode *)pFilterNode, &param));
taosArrayAddAll(result, param.result);
// taosArrayAddAll(result, param.result);
sifFreeParam(&param);
SIF_RET(TSDB_CODE_SUCCESS);
*status = st;
return TSDB_CODE_SUCCESS;
}
SIdxFltStatus idxGetFltStatus(SNode *pFilterNode) {
@ -761,10 +764,9 @@ SIdxFltStatus idxGetFltStatus(SNode *pFilterNode) {
if (pFilterNode == NULL) {
return SFLT_NOT_INDEX;
}
// SFilterInfo *filter = NULL;
// todo move to the initialization function
// SIF_ERR_RET(filterInitFromNode((SNode *)pFilterNode, &filter, 0));
SIF_ERR_RET(sifGetFltHint((SNode *)pFilterNode, &st));
if (sifGetFltHint((SNode *)pFilterNode, &st) != TSDB_CODE_SUCCESS) {
st = SFLT_NOT_INDEX;
}
return st;
}

54
source/libs/index/src/indexFst.c
View File

@ -289,22 +289,14 @@ void fstStateCompileForAnyTrans(IdxFstFile* w, CompiledAddr addr, FstBuilderNode
for (int32_t i = sz - 1; i >= 0; i--) {
FstTransition* t = taosArrayGet(node->trans, i);
idxFileWrite(w, (char*)&t->inp, 1);
// fstPackDeltaIn(w, addr, t->addr, tSize);
}
if (sz > TRANS_INDEX_THRESHOLD) {
// A value of 255 indicates that no transition exists for the byte
// at that index. (Except when there are 256 transitions.) Namely,
// any value greater than or equal to the number of transitions in
// this node indicates an absent transition.
// A value of 255 indicates that no transition exists for the byte at that idx
uint8_t* index = (uint8_t*)taosMemoryMalloc(sizeof(uint8_t) * 256);
memset(index, 255, sizeof(uint8_t) * 256);
/// for (uint8_t i = 0; i < 256; i++) {
// index[i] = 255;
///}
for (int32_t i = 0; i < sz; i++) {
FstTransition* t = taosArrayGet(node->trans, i);
index[t->inp] = i;
// fstPackDeltaIn(w, addr, t->addr, tSize);
}
idxFileWrite(w, (char*)index, 256);
taosMemoryFree(index);
@ -344,7 +336,7 @@ uint8_t fstStateCommInput(FstState* s, bool* null) {
*null = true;
return v;
}
// v = 0 indicate that common_input is None
// 0 indicate that common_input is None
return v == 0 ? 0 : COMMON_INPUT(v);
}
@ -522,7 +514,6 @@ uint64_t fstStateNtrans(FstState* s, FstSlice* slice) {
int32_t len;
uint8_t* data = fstSliceData(slice, &len);
n = data[len - 2];
// n = data[slice->end - 1]; // data[data.len() - 2]
return n == 1 ? 256 : n; // // "1" is never a normal legal value here, because if there, // is only 1 transition,
// then it is encoded in the state byte
}
@ -546,7 +537,6 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) {
int32_t dlen = 0;
uint8_t* data = fstSliceData(slice, &dlen);
uint64_t i = data[at + b];
// uint64_t i = slice->data[slice->start + at + b];
if (i >= node->nTrans) {
*null = true;
}
@ -558,16 +548,15 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) {
FstSlice t = fstSliceCopy(slice, start, end - 1);
int32_t len = 0;
uint8_t* data = fstSliceData(&t, &len);
int i = 0;
for (; i < len; i++) {
for (int i = 0; i < len; i++) {
uint8_t v = data[i];
if (v == b) {
fstSliceDestroy(&t);
return node->nTrans - i - 1; // bug
}
}
if (i == len) {
*null = true;
if (i + 1 == len) {
*null = true;
}
}
fstSliceDestroy(&t);
}
@ -737,16 +726,13 @@ bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr
return true;
} else if (sz != 1 || builderNode->isFinal) {
fstStateCompileForAnyTrans(w, addr, builderNode);
// AnyTrans->Compile(w, addr, node);
} else {
FstTransition* tran = taosArrayGet(builderNode->trans, 0);
if (tran->addr == lastAddr && tran->out == 0) {
fstStateCompileForOneTransNext(w, addr, tran->inp);
// OneTransNext::compile(w, lastAddr, tran->inp);
return true;
} else {
fstStateCompileForOneTrans(w, addr, tran);
// OneTrans::Compile(w, lastAddr, *tran);
return true;
}
}
@ -795,7 +781,7 @@ void fstBuilderDestroy(FstBuilder* b) {
}
bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in) {
OrderType t = fstBuilderCheckLastKey(b, bs, true);
FstOrderType t = fstBuilderCheckLastKey(b, bs, true);
if (t == Ordered) {
// add log info
fstBuilderInsertOutput(b, bs, in);
@ -812,12 +798,6 @@ void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in) {
fstUnFinishedNodesSetRootOutput(b->unfinished, in);
return;
}
// if (in != 0) { //if let Some(in) = in
// prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out);
//} else {
// prefixLen = fstUnFinishedNodesFindCommPrefix(b->unfinished, bs);
// out = 0;
//}
Output out;
uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out);
@ -835,7 +815,7 @@ void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in) {
return;
}
OrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup) {
FstOrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup) {
FstSlice* input = &bs;
if (fstSliceIsEmpty(&b->last)) {
fstSliceDestroy(&b->last);
@ -867,7 +847,6 @@ void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) {
fstBuilderNodeDestroy(bn);
assert(addr != NONE_ADDRESS);
// fstBuilderNodeDestroy(n);
}
fstUnFinishedNodesTopLastFreeze(b->unfinished, addr);
return;
@ -1044,8 +1023,6 @@ void fstDestroy(Fst* fst) {
}
bool fstGet(Fst* fst, FstSlice* b, Output* out) {
// dec lock range
// taosThreadMutexLock(&fst->mtx);
FstNode* root = fstGetRoot(fst);
Output tOut = 0;
int32_t len;
@ -1058,7 +1035,6 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
uint8_t inp = data[i];
Output res = 0;
if (false == fstNodeFindInput(root, inp, &res)) {
// taosThreadMutexUnlock(&fst->mtx);
return false;
}
@ -1069,7 +1045,6 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
taosArrayPush(nodes, &root);
}
if (!FST_NODE_IS_FINAL(root)) {
// taosThreadMutexUnlock(&fst->mtx);
return false;
} else {
tOut = tOut + FST_NODE_FINAL_OUTPUT(root);
@ -1080,8 +1055,6 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
fstNodeDestroy(*node);
}
taosArrayDestroy(nodes);
// fst->root = NULL;
// taosThreadMutexUnlock(&fst->mtx);
*out = tOut;
return true;
}
@ -1231,20 +1204,17 @@ bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min) {
FstNode* node = fstGetRoot(sws->fst);
Output out = 0;
// void* autState = sws->aut->start();
void* autState = automFuncs[aut->type].start(aut);
void* autState = automFuncs[aut->type].start(aut);
int32_t len;
uint8_t* data = fstSliceData(key, &len);
for (uint32_t i = 0; i < len; i++) {
uint8_t b = data[i];
uint64_t res = 0;
bool find = fstNodeFindInput(node, b, &res);
if (find == true) {
if (fstNodeFindInput(node, b, &res)) {
FstTransition trn;
fstNodeGetTransitionAt(node, res, &trn);
void* preState = autState;
// autState = sws->aut->accept(preState, b);
autState = automFuncs[aut->type].accept(aut, preState, b);
taosArrayPush(sws->inp, &b);
@ -1379,14 +1349,14 @@ FStmStRslt* stmStNextWith(FStmSt* sws, StreamCallback callback) {
return NULL;
}
FStmStRslt* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) {
FStmStRslt* swsResultCreate(FstSlice* data, FstOutput out, void* state) {
FStmStRslt* result = taosMemoryCalloc(1, sizeof(FStmStRslt));
if (result == NULL) {
return NULL;
}
result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1);
result->out = fOut;
result->out = out;
result->state = state;
return result;
}

8
source/libs/index/src/indexFstFile.c
View File

@ -71,9 +71,7 @@ static int idxFileCtxGetSize(IFileCtx* ctx) {
}
static int idxFileCtxDoFlush(IFileCtx* ctx) {
if (ctx->type == TFile) {
// taosFsyncFile(ctx->file.pFile);
taosFsyncFile(ctx->file.pFile);
// tfFlush(ctx->file.pFile);
} else {
// do nothing
}
@ -190,13 +188,11 @@ int idxFileRead(IdxFstFile* write, uint8_t* buf, uint32_t len) {
return 0;
}
IFileCtx* ctx = write->wrt;
int nRead = ctx->read(ctx, buf, len);
// assert(nRead == len);
return nRead;
return ctx->read(ctx, buf, len);
}
uint32_t idxFileMaskedCheckSum(IdxFstFile* write) {
// opt
//////
return write->summer;
}

5
source/libs/index/src/indexFstUtil.c
View File

@ -21,12 +21,12 @@ const CompiledAddr EMPTY_ADDRESS = 0;
const CompiledAddr NONE_ADDRESS = 1;
// This version number is written to every finite state transducer created by
// this crate. When a finite state transducer is read, its version number is
// this version. When a finite state transducer is read, its version number is
// checked against this value.
const uint64_t VERSION = 3;
// The threshold (in number of transitions) at which an index is created for
// a node's transitions. This speeds up lookup time at the expense of FST size
const uint64_t TRANS_INDEX_THRESHOLD = 32;
uint8_t packSize(uint64_t n) {
@ -52,7 +52,6 @@ uint8_t packSize(uint64_t n) {
uint64_t unpackUint64(uint8_t* ch, uint8_t sz) {
uint64_t n = 0;
for (uint8_t i = 0; i < sz; i++) {
//
n = n | (ch[i] << (8 * i));
}
return n;

50
source/libs/index/src/indexTfile.c
View File

@ -23,7 +23,7 @@
#include "tcoding.h"
#include "tcompare.h"
const static uint64_t tfileMagicNumber = 0xdb4775248b80fb57ull;
const static uint64_t FILE_MAGIC_NUMBER = 0xdb4775248b80fb57ull;
typedef struct TFileFstIter {
FStmBuilder* fb;
@ -457,7 +457,10 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt
} else if (0 != strncmp(ch, p, skip)) {
continue;
}
cond = cmpFn(ch + skip, tem->colVal, IDX_TYPE_GET_TYPE(tem->colType));
char* tBuf = taosMemoryCalloc(1, sz + 1);
memcpy(tBuf, ch, sz);
cond = cmpFn(tBuf + skip, tem->colVal, IDX_TYPE_GET_TYPE(tem->colType));
taosMemoryFree(tBuf);
}
if (MATCH == cond) {
tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total);
@ -545,9 +548,6 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn);
}
int32_t bufLimit = 64 * 4096, offset = 0;
// char* buf = taosMemoryCalloc(1, sizeof(char) * bufLimit);
// char* p = buf;
int32_t sz = taosArrayGetSize((SArray*)data);
int32_t fstOffset = tw->offset;
@ -561,6 +561,9 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
}
tfileWriteFstOffset(tw, fstOffset);
int32_t cap = 4 * 1024;
char* buf = taosMemoryCalloc(1, cap);
for (size_t i = 0; i < sz; i++) {
TFileValue* v = taosArrayGetP((SArray*)data, i);
@ -568,14 +571,18 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
// check buf has enough space or not
int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz);
char* buf = taosMemoryCalloc(1, ttsz * sizeof(char));
if (cap < ttsz) {
cap = ttsz;
buf = (char*)taosMemoryRealloc(buf, cap);
}
char* p = buf;
tfileSerialTableIdsToBuf(p, v->tableId);
tw->ctx->write(tw->ctx, buf, ttsz);
v->offset = tw->offset;
tw->offset += ttsz;
taosMemoryFree(buf);
memset(buf, 0, cap);
}
taosMemoryFree(buf);
tw->fb = fstBuilderCreate(tw->ctx, 0);
if (tw->fb == NULL) {
@ -866,13 +873,13 @@ static int tfileWriteData(TFileWriter* write, TFileValue* tval) {
//}
}
static int tfileWriteFooter(TFileWriter* write) {
char buf[sizeof(tfileMagicNumber) + 1] = {0};
char buf[sizeof(FILE_MAGIC_NUMBER) + 1] = {0};
void* pBuf = (void*)buf;
taosEncodeFixedU64((void**)(void*)&pBuf, tfileMagicNumber);
taosEncodeFixedU64((void**)(void*)&pBuf, FILE_MAGIC_NUMBER);
int nwrite = write->ctx->write(write->ctx, buf, (int32_t)strlen(buf));
indexInfo("tfile write footer size: %d", write->ctx->size(write->ctx));
assert(nwrite == sizeof(tfileMagicNumber));
assert(nwrite == sizeof(FILE_MAGIC_NUMBER));
return nwrite;
}
static int tfileReaderLoadHeader(TFileReader* reader) {
@ -896,7 +903,7 @@ static int tfileReaderLoadFst(TFileReader* reader) {
int size = ctx->size(ctx);
// current load fst into memory, refactor it later
int fstSize = size - reader->header.fstOffset - sizeof(tfileMagicNumber);
int fstSize = size - reader->header.fstOffset - sizeof(FILE_MAGIC_NUMBER);
char* buf = taosMemoryCalloc(1, fstSize);
if (buf == NULL) {
return -1;
@ -956,9 +963,8 @@ static int tfileReaderVerify(TFileReader* reader) {
IFileCtx* ctx = reader->ctx;
uint64_t tMagicNumber = 0;
char buf[sizeof(tMagicNumber) + 1] = {0};
int size = ctx->size(ctx);
char buf[sizeof(tMagicNumber) + 1] = {0};
int size = ctx->size(ctx);
if (size < sizeof(tMagicNumber) || size <= sizeof(reader->header)) {
return -1;
@ -967,25 +973,25 @@ static int tfileReaderVerify(TFileReader* reader) {
}
taosDecodeFixedU64(buf, &tMagicNumber);
return tMagicNumber == tfileMagicNumber ? 0 : -1;
return tMagicNumber == FILE_MAGIC_NUMBER ? 0 : -1;
}
void tfileReaderRef(TFileReader* reader) {
if (reader == NULL) {
void tfileReaderRef(TFileReader* rd) {
if (rd == NULL) {
return;
}
int ref = T_REF_INC(reader);
int ref = T_REF_INC(rd);
UNUSED(ref);
}
void tfileReaderUnRef(TFileReader* reader) {
if (reader == NULL) {
void tfileReaderUnRef(TFileReader* rd) {
if (rd == NULL) {
return;
}
int ref = T_REF_DEC(reader);
int ref = T_REF_DEC(rd);
if (ref == 0) {
// do nothing
tfileReaderDestroy(reader);
tfileReaderDestroy(rd);
}
}

204
source/libs/index/test/CMakeLists.txt
View File

@ -1,110 +1,112 @@
add_executable(idxTest "")
add_executable(idxFstTest "")
add_executable(idxFstUT "")
add_executable(idxUtilUT "")
add_executable(idxJsonUT "")
IF(NOT TD_DARWIN)
add_executable(idxTest "")
add_executable(idxFstTest "")
add_executable(idxFstUT "")
add_executable(idxUtilUT "")
add_executable(idxJsonUT "")
target_sources(idxTest
PRIVATE
"indexTests.cc"
)
target_sources(idxFstTest
PRIVATE
"fstTest.cc"
)
target_sources(idxTest
PRIVATE
"indexTests.cc"
)
target_sources(idxFstTest
PRIVATE
"fstTest.cc"
)
target_sources(idxFstUT
PRIVATE
"fstUT.cc"
)
target_sources(idxUtilUT
PRIVATE
"utilUT.cc"
)
target_sources(idxFstUT
PRIVATE
"fstUT.cc"
)
target_sources(idxUtilUT
PRIVATE
"utilUT.cc"
)
target_sources(idxJsonUT
PRIVATE
"jsonUT.cc"
)
target_include_directories (idxTest
PUBLIC
"${TD_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories (idxFstTest
PUBLIC
"${TD_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_sources(idxJsonUT
PRIVATE
"jsonUT.cc"
)
target_include_directories (idxTest
PUBLIC
"${TD_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories (idxFstTest
PUBLIC
"${TD_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories (idxFstUT
PUBLIC
"${TD_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories (idxFstUT
PUBLIC
"${TD_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories (idxUtilUT
PUBLIC
"${TD_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories (idxUtilUT
PUBLIC
"${TD_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories (idxJsonUT
PUBLIC
"${TD_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries (idxTest
os
util
common
gtest_main
index
)
target_link_libraries (idxFstTest
os
util
common
gtest_main
index
)
target_link_libraries (idxFstUT
os
util
common
gtest_main
index
)
target_include_directories (idxJsonUT
PUBLIC
"${TD_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries (idxTest
os
util
common
gtest_main
index
)
target_link_libraries (idxFstTest
os
util
common
gtest_main
index
)
target_link_libraries (idxFstUT
os
util
common
gtest_main
index
)
target_link_libraries (idxUtilUT
os
util
common
gtest_main
index
)
target_link_libraries (idxUtilUT
os
util
common
gtest_main
index
)
target_link_libraries (idxJsonUT
os
util
common
gtest_main
index
)
target_link_libraries (idxJsonUT
os
util
common
gtest_main
index
)
add_test(
NAME idxtest
COMMAND idxTest
)
add_test(
NAME idxJsonUT
COMMAND idxJsonUT
)
add_test(
NAME idxUtilUT
COMMAND idxUtilUT
)
add_test(
NAME idxFstUT
COMMAND idxFstUT
)
add_test(
NAME idxtest
COMMAND idxTest
)
add_test(
NAME idxJsonUT
COMMAND idxJsonUT
)
add_test(
NAME idxUtilUT
COMMAND idxUtilUT
)
add_test(
NAME idxFstUT
COMMAND idxFstUT
)
ENDIF ()

4
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -3605,7 +3605,7 @@ static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) {
int32_t code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsBuffer, pNode->buffer);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsCachelast, pNode->cachelast);
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsCachelast, pNode->cacheLast);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsCompressionLevel, pNode->compressionLevel);
@ -3667,7 +3667,7 @@ static int32_t jsonToDatabaseOptions(const SJson* pJson, void* pObj) {
int32_t code = tjsonGetIntValue(pJson, jkDatabaseOptionsBuffer, &pNode->buffer);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsCachelast, &pNode->cachelast);
code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsCachelast, &pNode->cacheLast);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsCompressionLevel, &pNode->compressionLevel);

1
source/libs/parser/inc/parAst.h
View File

@ -39,6 +39,7 @@ typedef struct SAstCreateContext {
typedef enum EDatabaseOptionType {
DB_OPTION_BUFFER = 1,
DB_OPTION_CACHELAST,
DB_OPTION_CACHELASTSIZE,
DB_OPTION_COMP,
DB_OPTION_DAYS,
DB_OPTION_FSYNC,

2
source/libs/parser/inc/sql.y
View File

@ -172,6 +172,7 @@ exists_opt(A) ::= .
db_options(A) ::= . { A = createDefaultDatabaseOptions(pCxt); }
db_options(A) ::= db_options(B) BUFFER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_BUFFER, &C); }
db_options(A) ::= db_options(B) CACHELAST NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELAST, &C); }
db_options(A) ::= db_options(B) CACHELASTSIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELASTSIZE, &C); }
db_options(A) ::= db_options(B) COMP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMP, &C); }
db_options(A) ::= db_options(B) DURATION NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); }
db_options(A) ::= db_options(B) DURATION NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); }
@ -198,6 +199,7 @@ alter_db_options(A) ::= alter_db_options(B) alter_db_option(C).
%destructor alter_db_option { }
alter_db_option(A) ::= BUFFER NK_INTEGER(B). { A.type = DB_OPTION_BUFFER; A.val = B; }
alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.val = B; }
alter_db_option(A) ::= CACHELASTSIZE NK_INTEGER(B). { A.type = DB_OPTION_CACHELASTSIZE; A.val = B; }
alter_db_option(A) ::= FSYNC NK_INTEGER(B). { A.type = DB_OPTION_FSYNC; A.val = B; }
alter_db_option(A) ::= KEEP integer_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; }
alter_db_option(A) ::= KEEP variable_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; }

11
source/libs/parser/src/parAstCreater.c
View File

@ -746,7 +746,8 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
SDatabaseOptions* pOptions = (SDatabaseOptions*)nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions);
pOptions->buffer = TSDB_DEFAULT_BUFFER_PER_VNODE;
pOptions->cachelast = TSDB_DEFAULT_CACHE_LAST_ROW;
pOptions->cacheLast = TSDB_DEFAULT_CACHE_LAST_ROW;
pOptions->cacheLastSize = TSDB_DEFAULT_LAST_ROW_MEM;
pOptions->compressionLevel = TSDB_DEFAULT_COMP_LEVEL;
pOptions->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
pOptions->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
@ -772,7 +773,8 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) {
SDatabaseOptions* pOptions = (SDatabaseOptions*)nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS);
CHECK_OUT_OF_MEM(pOptions);
pOptions->buffer = -1;
pOptions->cachelast = -1;
pOptions->cacheLast = -1;
pOptions->cacheLastSize = -1;
pOptions->compressionLevel = -1;
pOptions->daysPerFile = -1;
pOptions->fsyncPeriod = -1;
@ -800,7 +802,10 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti
((SDatabaseOptions*)pOptions)->buffer = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_CACHELAST:
((SDatabaseOptions*)pOptions)->cachelast = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
((SDatabaseOptions*)pOptions)->cacheLast = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_CACHELASTSIZE:
((SDatabaseOptions*)pOptions)->cacheLastSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_COMP:
((SDatabaseOptions*)pOptions)->compressionLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);

1
source/libs/parser/src/parTokenizer.c
View File

@ -53,6 +53,7 @@ static SKeyword keywordTable[] = {
{"BY", TK_BY},
{"CACHE", TK_CACHE},
{"CACHELAST", TK_CACHELAST},
{"CACHELASTSIZE", TK_CACHELASTSIZE},
{"CAST", TK_CAST},
{"CLIENT_VERSION", TK_CLIENT_VERSION},
{"CLUSTER", TK_CLUSTER},

14
source/libs/parser/src/parTranslater.c
View File

@ -2988,7 +2988,8 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS
pReq->compression = pStmt->pOptions->compressionLevel;
pReq->replications = pStmt->pOptions->replica;
pReq->strict = pStmt->pOptions->strict;
pReq->cacheLastRow = pStmt->pOptions->cachelast;
pReq->cacheLastRow = pStmt->pOptions->cacheLast;
pReq->lastRowMem = pStmt->pOptions->cacheLastSize;
pReq->schemaless = pStmt->pOptions->schemaless;
pReq->ignoreExist = pStmt->ignoreExists;
return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq);
@ -3149,9 +3150,13 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
int32_t code =
checkRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, TSDB_MAX_BUFFER_PER_VNODE);
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "cacheLast", pOptions->cachelast, TSDB_MIN_DB_CACHE_LAST_ROW,
code = checkRangeOption(pCxt, "cacheLast", pOptions->cacheLast, TSDB_MIN_DB_CACHE_LAST_ROW,
TSDB_MAX_DB_CACHE_LAST_ROW);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "cacheLastSize", pOptions->cacheLastSize, TSDB_MIN_DB_LAST_ROW_MEM,
TSDB_MAX_DB_LAST_ROW_MEM);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkRangeOption(pCxt, "compression", pOptions->compressionLevel, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL);
}
@ -3271,7 +3276,8 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt,
pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod;
pReq->walLevel = pStmt->pOptions->walLevel;
pReq->strict = pStmt->pOptions->strict;
pReq->cacheLastRow = pStmt->pOptions->cachelast;
pReq->cacheLastRow = pStmt->pOptions->cacheLast;
pReq->lastRowMem = pStmt->pOptions->cacheLastSize;
pReq->replications = pStmt->pOptions->replica;
return;
}
@ -6378,7 +6384,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
break;
case QUERY_NODE_INSERT_STMT:
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
pQuery->msgType = TDMT_VND_SUBMIT;
pQuery->msgType = TDMT_SCH_QUERY;
break;
case QUERY_NODE_VNODE_MODIF_STMT:
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;

5865
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

7
source/libs/parser/test/parInitialCTest.cpp
View File

@ -77,6 +77,7 @@ TEST_F(ParserInitialCTest, createDatabase) {
expect.ignoreExist = igExists;
expect.buffer = TSDB_DEFAULT_BUFFER_PER_VNODE;
expect.cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW;
expect.lastRowMem = TSDB_DEFAULT_LAST_ROW_MEM;
expect.compression = TSDB_DEFAULT_COMP_LEVEL;
expect.daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
expect.fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
@ -97,7 +98,8 @@ TEST_F(ParserInitialCTest, createDatabase) {
};
auto setDbBufferFunc = [&](int32_t buffer) { expect.buffer = buffer; };
auto setDbCachelastFunc = [&](int8_t CACHELAST) { expect.cacheLastRow = CACHELAST; };
auto setDbCachelastFunc = [&](int8_t cachelast) { expect.cacheLastRow = cachelast; };
auto setDbCachelastSize = [&](int8_t cachelastSize) { expect.lastRowMem = cachelastSize; };
auto setDbCompressionFunc = [&](int8_t compressionLevel) { expect.compression = compressionLevel; };
auto setDbDaysFunc = [&](int32_t daysPerFile) { expect.daysPerFile = daysPerFile; };
auto setDbFsyncFunc = [&](int32_t fsyncPeriod) { expect.fsyncPeriod = fsyncPeriod; };
@ -154,6 +156,7 @@ TEST_F(ParserInitialCTest, createDatabase) {
ASSERT_EQ(req.replications, expect.replications);
ASSERT_EQ(req.strict, expect.strict);
ASSERT_EQ(req.cacheLastRow, expect.cacheLastRow);
ASSERT_EQ(req.lastRowMem, expect.lastRowMem);
// ASSERT_EQ(req.schemaless, expect.schemaless);
ASSERT_EQ(req.ignoreExist, expect.ignoreExist);
ASSERT_EQ(req.numOfRetensions, expect.numOfRetensions);
@ -179,6 +182,7 @@ TEST_F(ParserInitialCTest, createDatabase) {
setCreateDbReqFunc("wxy_db", 1);
setDbBufferFunc(64);
setDbCachelastFunc(2);
setDbCachelastSize(20);
setDbCompressionFunc(1);
setDbDaysFunc(100 * 1440);
setDbFsyncFunc(100);
@ -200,6 +204,7 @@ TEST_F(ParserInitialCTest, createDatabase) {
run("CREATE DATABASE IF NOT EXISTS wxy_db "
"BUFFER 64 "
"CACHELAST 2 "
"CACHELASTSIZE 20 "
"COMP 1 "
"DURATION 100 "
"FSYNC 100 "

4
source/libs/planner/src/planOptimizer.c
View File

@ -890,7 +890,7 @@ static int32_t pushDownCondOptDealProject(SOptimizeContext* pCxt, SProjectLogicN
return code;
}
static int32_t pushDownCondOptDealLogicNode(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
static int32_t pushDownCondOptTrivialPushDown(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
if (NULL == pLogicNode->pConditions ||
OPTIMIZE_FLAG_TEST_MASK(pLogicNode->optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) {
return TSDB_CODE_SUCCESS;
@ -921,7 +921,7 @@ static int32_t pushDownCondOptimizeImpl(SOptimizeContext* pCxt, SLogicNode* pLog
break;
case QUERY_NODE_LOGIC_PLAN_SORT:
case QUERY_NODE_LOGIC_PLAN_PARTITION:
code = pushDownCondOptDealLogicNode(pCxt, pLogicNode);
code = pushDownCondOptTrivialPushDown(pCxt, pLogicNode);
break;
default:
break;

91
source/libs/planner/src/planSpliter.c
View File

@ -98,6 +98,43 @@ static int32_t splCreateExchangeNodeForSubplan(SSplitContext* pCxt, SLogicSubpla
return code;
}
static bool splIsChildSubplan(SLogicNode* pLogicNode, int32_t groupId) {
if (QUERY_NODE_LOGIC_PLAN_EXCHANGE == nodeType(pLogicNode)) {
return ((SExchangeLogicNode*)pLogicNode)->srcGroupId == groupId;
}
if (QUERY_NODE_LOGIC_PLAN_MERGE == nodeType(pLogicNode)) {
return ((SMergeLogicNode*)pLogicNode)->srcGroupId == groupId;
}
SNode* pChild;
FOREACH(pChild, pLogicNode->pChildren) {
bool isChild = splIsChildSubplan((SLogicNode*)pChild, groupId);
if (isChild) {
return isChild;
}
}
return false;
}
static int32_t splMountSubplan(SLogicSubplan* pParent, SNodeList* pChildren) {
SNode* pChild = NULL;
WHERE_EACH(pChild, pChildren) {
if (splIsChildSubplan(pParent->pNode, ((SLogicSubplan*)pChild)->id.groupId)) {
int32_t code = nodesListMakeAppend(&pParent->pChildren, pChild);
if (TSDB_CODE_SUCCESS == code) {
REPLACE_NODE(NULL);
ERASE_NODE(pChildren);
continue;
} else {
return code;
}
}
WHERE_NEXT;
}
return TSDB_CODE_SUCCESS;
}
static bool splMatchByNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pNode, FSplFindSplitNode func,
void* pInfo) {
if (func(pCxt, pSubplan, pNode, pInfo)) {
@ -982,43 +1019,6 @@ static int32_t singleTableJoinSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan
return code;
}
static bool unionIsChildSubplan(SLogicNode* pLogicNode, int32_t groupId) {
if (QUERY_NODE_LOGIC_PLAN_EXCHANGE == nodeType(pLogicNode)) {
return ((SExchangeLogicNode*)pLogicNode)->srcGroupId == groupId;
}
if (QUERY_NODE_LOGIC_PLAN_MERGE == nodeType(pLogicNode)) {
return ((SMergeLogicNode*)pLogicNode)->srcGroupId == groupId;
}
SNode* pChild;
FOREACH(pChild, pLogicNode->pChildren) {
bool isChild = unionIsChildSubplan((SLogicNode*)pChild, groupId);
if (isChild) {
return isChild;
}
}
return false;
}
static int32_t unionMountSubplan(SLogicSubplan* pParent, SNodeList* pChildren) {
SNode* pChild = NULL;
WHERE_EACH(pChild, pChildren) {
if (unionIsChildSubplan(pParent->pNode, ((SLogicSubplan*)pChild)->id.groupId)) {
int32_t code = nodesListMakeAppend(&pParent->pChildren, pChild);
if (TSDB_CODE_SUCCESS == code) {
REPLACE_NODE(NULL);
ERASE_NODE(pChildren);
continue;
} else {
return code;
}
}
WHERE_NEXT;
}
return TSDB_CODE_SUCCESS;
}
static SLogicSubplan* unionCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode, ESubplanType subplanType) {
SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN);
if (NULL == pSubplan) {
@ -1044,7 +1044,7 @@ static int32_t unionSplitSubplan(SSplitContext* pCxt, SLogicSubplan* pUnionSubpl
code = nodesListMakeStrictAppend(&pUnionSubplan->pChildren, (SNode*)pNewSubplan);
if (TSDB_CODE_SUCCESS == code) {
REPLACE_NODE(NULL);
code = unionMountSubplan(pNewSubplan, pSubplanChildren);
code = splMountSubplan(pNewSubplan, pSubplanChildren);
}
if (TSDB_CODE_SUCCESS != code) {
break;
@ -1219,14 +1219,27 @@ static int32_t insertSelectSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) {
return TSDB_CODE_SUCCESS;
}
SLogicSubplan* pNewSubplan = NULL;
SNodeList* pSubplanChildren = info.pSubplan->pChildren;
int32_t code = splCreateExchangeNodeForSubplan(pCxt, info.pSubplan, info.pQueryRoot, info.pSubplan->subplanType);
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, (SNode*)splCreateScanSubplan(pCxt, info.pQueryRoot, 0));
pNewSubplan = splCreateScanSubplan(pCxt, info.pQueryRoot, 0);
if (NULL == pNewSubplan) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, (SNode*)pNewSubplan);
}
if (TSDB_CODE_SUCCESS == code) {
code = splMountSubplan(pNewSubplan, pSubplanChildren);
}
if (TSDB_CODE_SUCCESS == code) {
info.pSubplan->subplanType = SUBPLAN_TYPE_MODIFY;
SPLIT_FLAG_SET_MASK(info.pSubplan->splitFlag, SPLIT_FLAG_INSERT_SPLIT);
}
++(pCxt->groupId);
pCxt->split = true;
return code;

60
source/libs/planner/test/CMakeLists.txt
View File

@ -1,38 +1,40 @@
MESSAGE(STATUS "build planner unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
IF(NOT TD_DARWIN)
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(plannerTest
ADD_EXECUTABLE(plannerTest
${SOURCE_LIST}
"${SOURCE_LIST}/../../../parser/test/mockCatalog.cpp"
"${SOURCE_LIST}/../../../parser/test/mockCatalogService.cpp"
)
TARGET_LINK_LIBRARIES(
plannerTest
PUBLIC os util common nodes planner parser catalog transport gtest function qcom
)
TARGET_INCLUDE_DIRECTORIES(
plannerTest
PUBLIC "${TD_SOURCE_DIR}/include/libs/planner/"
PRIVATE "${TD_SOURCE_DIR}/source/libs/planner/inc"
PRIVATE "${TD_SOURCE_DIR}/source/libs/parser/test"
)
if(${BUILD_WINGETOPT})
target_include_directories(
plannerTest
PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
)
target_link_libraries(plannerTest PUBLIC wingetopt)
endif()
add_test(
NAME plannerTest
COMMAND plannerTest
)
TARGET_LINK_LIBRARIES(
plannerTest
PUBLIC os util common nodes planner parser catalog transport gtest function qcom
)
TARGET_INCLUDE_DIRECTORIES(
plannerTest
PUBLIC "${TD_SOURCE_DIR}/include/libs/planner/"
PRIVATE "${TD_SOURCE_DIR}/source/libs/planner/inc"
PRIVATE "${TD_SOURCE_DIR}/source/libs/parser/test"
)
if(${BUILD_WINGETOPT})
target_include_directories(
plannerTest
PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
)
target_link_libraries(plannerTest PUBLIC wingetopt)
endif()
add_test(
NAME plannerTest
COMMAND plannerTest
)
ENDIF ()

6
source/libs/stream/inc/streamInc.h
View File

@ -17,6 +17,7 @@
#define _STREAM_INC_H_
#include "executor.h"
#include "tref.h"
#include "tstream.h"
#ifdef __cplusplus
@ -24,8 +25,9 @@ extern "C" {
#endif
typedef struct {
int8_t inited;
void* timer;
int8_t inited;
int32_t refPool;
void* timer;
} SStreamGlobalEnv;
static SStreamGlobalEnv streamEnv;

3
source/libs/stream/src/stream.c
View File

@ -76,9 +76,6 @@ void streamTriggerByTimer(void* param, void* tmrId) {
int32_t streamSetupTrigger(SStreamTask* pTask) {
if (pTask->triggerParam != 0) {
if (streamInit() < 0) {
return -1;
}
pTask->timer = taosTmrStart(streamTriggerByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer);
pTask->triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE;
}

1
source/libs/sync/inc/syncInt.h
View File

@ -67,7 +67,6 @@ typedef struct SSyncNode {
char path[TSDB_FILENAME_LEN];
char raftStorePath[TSDB_FILENAME_LEN * 2];
char configPath[TSDB_FILENAME_LEN * 2];
int32_t batchSize;
// sync io
SWal* pWal;

4
source/libs/sync/inc/syncRaftLog.h
View File

@ -32,8 +32,8 @@ typedef struct SSyncLogStoreData {
SSyncNode* pSyncNode;
SWal* pWal;
TdThreadMutex mutex;
SWalReadHandle* pWalHandle;
TdThreadMutex mutex;
SWalReader* pWalHandle;
// SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0
} SSyncLogStoreData;

17
source/libs/sync/src/syncAppendEntries.c
View File

@ -842,8 +842,8 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf),
"recv sync-append-entries-batch, fake match2, pre-index:%ld, pre-term:%lu, datalen:%d",
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen);
"recv sync-append-entries-batch, fake match2, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}",
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount);
syncNodeEventLog(ths, logBuf);
} while (0);
@ -876,7 +876,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
code = syncNodePreCommit(ths, pAppendEntry);
ASSERT(code == 0);
syncEntryDestory(pAppendEntry);
// syncEntryDestory(pAppendEntry);
}
// fsync once
@ -931,8 +931,8 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf),
"recv sync-append-entries-batch, not match, pre-index:%ld, pre-term:%lu, datalen:%d",
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen);
"recv sync-append-entries-batch, not match, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}",
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount);
syncNodeEventLog(ths, logBuf);
} while (0);
@ -976,8 +976,9 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, match, pre-index:%ld, pre-term:%lu, datalen:%d",
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen);
snprintf(logBuf, sizeof(logBuf),
"recv sync-append-entries, match, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}",
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount);
syncNodeEventLog(ths, logBuf);
} while (0);
@ -999,7 +1000,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
code = syncNodePreCommit(ths, pAppendEntry);
ASSERT(code == 0);
syncEntryDestory(pAppendEntry);
// syncEntryDestory(pAppendEntry);
}
// fsync once

48
source/libs/sync/src/syncAppendEntriesReply.c
View File

@ -174,8 +174,12 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie
SyncIndex newNextIndex = pMsg->matchIndex + 1;
SyncIndex newMatchIndex = pMsg->matchIndex;
if (ths->pLogStore->syncLogExist(ths->pLogStore, newNextIndex) &&
ths->pLogStore->syncLogExist(ths->pLogStore, newNextIndex - 1)) {
bool needStartSnapshot = false;
if (newMatchIndex >= SYNC_INDEX_BEGIN && !ths->pLogStore->syncLogExist(ths->pLogStore, newMatchIndex)) {
needStartSnapshot = true;
}
if (!needStartSnapshot) {
// update next-index, match-index
syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), newNextIndex);
syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), newMatchIndex);
@ -197,15 +201,36 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie
syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), newMatchIndex);
}
// event log, update next-index
do {
char host[64];
int16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
char logBuf[256];
snprintf(logBuf, sizeof(logBuf), "reset next-index:%ld, match-index:%ld for %s:%d", newNextIndex, newMatchIndex,
host, port);
syncNodeEventLog(ths, logBuf);
} while (0);
} else {
SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId));
if (nextIndex > SYNC_INDEX_BEGIN) {
--nextIndex;
if (ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex) &&
ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex - 1)) {
bool needStartSnapshot = false;
if (nextIndex >= SYNC_INDEX_BEGIN && !ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex)) {
needStartSnapshot = true;
}
if (nextIndex - 1 >= SYNC_INDEX_BEGIN && !ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex - 1)) {
needStartSnapshot = true;
}
if (!needStartSnapshot) {
// do nothing
} else {
SSyncRaftEntry* pEntry;
int32_t code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, nextIndex, &pEntry);
@ -227,6 +252,21 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie
nextIndex = SYNC_INDEX_BEGIN;
}
syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex);
// event log, update next-index
do {
char host[64];
int16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
SyncIndex newNextIndex = nextIndex;
SyncIndex newMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId));
char logBuf[256];
snprintf(logBuf, sizeof(logBuf), "reset2 next-index:%ld, match-index:%ld for %s:%d", newNextIndex, newMatchIndex,
host, port);
syncNodeEventLog(ths, logBuf);
} while (0);
}
return 0;

12
source/libs/sync/src/syncMain.c
View File

@ -1526,12 +1526,14 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) {
snprintf(logBuf, sizeof(logBuf),
"vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, "
"strategy:%d, batch:%d, "
"replica-num:%d, "
"lconfig:%ld, changing:%d, restore:%d, %s",
pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm,
pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex,
pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex,
pSyncNode->changing, pSyncNode->restoreFinish, printStr);
pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize,
pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing,
pSyncNode->restoreFinish, printStr);
} else {
snprintf(logBuf, sizeof(logBuf), "%s", str);
}
@ -1543,12 +1545,14 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) {
snprintf(s, len,
"vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, "
"strategy:%d, batch:%d, "
"replica-num:%d, "
"lconfig:%ld, changing:%d, restore:%d, %s",
pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm,
pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex,
pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex,
pSyncNode->changing, pSyncNode->restoreFinish, printStr);
pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize,
pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing,
pSyncNode->restoreFinish, printStr);
} else {
snprintf(s, len, "%s", str);
}

2
source/libs/sync/src/syncMessage.c
View File

@ -1605,7 +1605,7 @@ void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg) {
SyncAppendEntriesBatch* syncAppendEntriesBatchBuild(SSyncRaftEntry** entryPArr, int32_t arrSize, int32_t vgId) {
ASSERT(entryPArr != NULL);
ASSERT(arrSize > 0);
ASSERT(arrSize >= 0);
int32_t dataLen = 0;
int32_t metaArrayLen = sizeof(SOffsetAndContLen) * arrSize; // <offset, contLen>

8
source/libs/sync/src/syncRaftCfg.c
View File

@ -101,7 +101,7 @@ cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) {
char *syncCfg2Str(SSyncCfg *pSyncCfg) {
cJSON *pJson = syncCfg2Json(pSyncCfg);
char * serialized = cJSON_Print(pJson);
char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
@ -109,7 +109,7 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) {
char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) {
if (pSyncCfg != NULL) {
int32_t len = 512;
char * s = taosMemoryMalloc(len);
char *s = taosMemoryMalloc(len);
memset(s, 0, len);
snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex);
@ -206,7 +206,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) {
char *raftCfg2Str(SRaftCfg *pRaftCfg) {
cJSON *pJson = raftCfg2Json(pRaftCfg);
char * serialized = cJSON_Print(pJson);
char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
@ -285,7 +285,7 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) {
(pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring);
}
cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg");
cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg");
int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg));
ASSERT(code == 0);

12
source/libs/sync/src/syncRaftLog.c
View File

@ -62,7 +62,7 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
ASSERT(pData->pWal != NULL);
taosThreadMutexInit(&(pData->mutex), NULL);
pData->pWalHandle = walOpenReadHandle(pData->pWal);
pData->pWalHandle = walOpenReader(pData->pWal, NULL);
ASSERT(pData->pWalHandle != NULL);
pLogStore->appendEntry = logStoreAppendEntry;
@ -95,7 +95,7 @@ void logStoreDestory(SSyncLogStore* pLogStore) {
taosThreadMutexLock(&(pData->mutex));
if (pData->pWalHandle != NULL) {
walCloseReadHandle(pData->pWalHandle);
walCloseReader(pData->pWalHandle);
pData->pWalHandle = NULL;
}
taosThreadMutexUnlock(&(pData->mutex));
@ -255,7 +255,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
*ppEntry = NULL;
// SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
SWalReadHandle* pWalHandle = pData->pWalHandle;
SWalReader* pWalHandle = pData->pWalHandle;
if (pWalHandle == NULL) {
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
return -1;
@ -263,7 +263,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
taosThreadMutexLock(&(pData->mutex));
code = walReadWithHandle(pWalHandle, index);
code = walReadVer(pWalHandle, index);
if (code != 0) {
int32_t err = terrno;
const char* errStr = tstrerror(err);
@ -398,10 +398,10 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
taosThreadMutexLock(&(pData->mutex));
// SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
SWalReadHandle* pWalHandle = pData->pWalHandle;
SWalReader* pWalHandle = pData->pWalHandle;
ASSERT(pWalHandle != NULL);
int32_t code = walReadWithHandle(pWalHandle, index);
int32_t code = walReadVer(pWalHandle, index);
if (code != 0) {
int32_t err = terrno;
const char* errStr = tstrerror(err);

19
source/libs/sync/src/syncReplication.c
View File

@ -148,9 +148,17 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) {
// get entry batch
int32_t getCount = 0;
SyncIndex getEntryIndex = nextIndex;
for (int32_t i = 0; i < pSyncNode->batchSize; ++i) {
for (int32_t i = 0; i < pSyncNode->pRaftCfg->batchSize; ++i) {
SSyncRaftEntry* pEntry = NULL;
int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, getEntryIndex, &pEntry);
// event log
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "get index:%d, code:%d, %s", getEntryIndex, code, tstrerror(terrno));
syncNodeEventLog(pSyncNode, logBuf);
} while (0);
if (code == 0) {
ASSERT(pEntry != NULL);
entryPArr[i] = pEntry;
@ -162,12 +170,19 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) {
}
}
// event log
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "build batch:%d", getCount);
syncNodeEventLog(pSyncNode, logBuf);
} while (0);
// build msg
SyncAppendEntriesBatch* pMsg = syncAppendEntriesBatchBuild(entryPArr, getCount, pSyncNode->vgId);
ASSERT(pMsg != NULL);
// free entries
for (int32_t i = 0; i < pSyncNode->batchSize; ++i) {
for (int32_t i = 0; i < pSyncNode->pRaftCfg->batchSize; ++i) {
SSyncRaftEntry* pEntry = entryPArr[i];
if (pEntry != NULL) {
syncEntryDestory(pEntry);

16
source/libs/tfs/test/tfsTest.cpp
View File

@ -16,7 +16,11 @@
class TfsTest : public ::testing::Test {
protected:
#ifdef _TD_DARWIN_64
static void SetUpTestSuite() { root = "/private" TD_TMP_DIR_PATH "tfsTest"; }
#else
static void SetUpTestSuite() { root = TD_TMP_DIR_PATH "tfsTest"; }
#endif
static void TearDownTestSuite() {}
public:
@ -299,6 +303,17 @@ TEST_F(TfsTest, 04_File) {
TEST_F(TfsTest, 05_MultiDisk) {
int32_t code = 0;
#ifdef _TD_DARWIN_64
const char *root00 = "/private" TD_TMP_DIR_PATH "tfsTest00";
const char *root01 = "/private" TD_TMP_DIR_PATH "tfsTest01";
const char *root10 = "/private" TD_TMP_DIR_PATH "tfsTest10";
const char *root11 = "/private" TD_TMP_DIR_PATH "tfsTest11";
const char *root12 = "/private" TD_TMP_DIR_PATH "tfsTest12";
const char *root20 = "/private" TD_TMP_DIR_PATH "tfsTest20";
const char *root21 = "/private" TD_TMP_DIR_PATH "tfsTest21";
const char *root22 = "/private" TD_TMP_DIR_PATH "tfsTest22";
const char *root23 = "/private" TD_TMP_DIR_PATH "tfsTest23";
#else
const char *root00 = TD_TMP_DIR_PATH "tfsTest00";
const char *root01 = TD_TMP_DIR_PATH "tfsTest01";
const char *root10 = TD_TMP_DIR_PATH "tfsTest10";
@ -308,6 +323,7 @@ TEST_F(TfsTest, 05_MultiDisk) {
const char *root21 = TD_TMP_DIR_PATH "tfsTest21";
const char *root22 = TD_TMP_DIR_PATH "tfsTest22";
const char *root23 = TD_TMP_DIR_PATH "tfsTest23";
#endif
SDiskCfg dCfg[9] = {0};
tstrncpy(dCfg[0].dir, root01, TSDB_FILENAME_LEN);

4
source/libs/transport/src/transCli.c
View File

@ -573,8 +573,8 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
return;
}
if (nread < 0) {
tError("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread),
T_REF_VAL_GET(conn));
tWarn("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread),
T_REF_VAL_GET(conn));
conn->broken = true;
cliHandleExcept(conn);
}

5
source/libs/transport/src/transSvr.c
View File

@ -1,7 +1,6 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* * This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
@ -306,7 +305,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
return;
}
tError("%s conn %p read error: %s", transLabel(pTransInst), conn, uv_err_name(nread));
tWarn("%s conn %p read error: %s", transLabel(pTransInst), conn, uv_err_name(nread));
if (nread < 0) {
conn->broken = true;
if (conn->status == ConnAcquire) {

1
source/libs/wal/inc/walInt.h
View File

@ -19,6 +19,7 @@
#include "taoserror.h"
#include "tchecksum.h"
#include "tcoding.h"
#include "tcommon.h"
#include "tcompare.h"
#include "wal.h"

225
source/libs/wal/src/walRead.c
View File

@ -16,20 +16,29 @@
#include "taoserror.h"
#include "walInt.h"
SWalReadHandle *walOpenReadHandle(SWal *pWal) {
SWalReadHandle *pRead = taosMemoryMalloc(sizeof(SWalReadHandle));
static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer);
static int32_t walFetchBodyNew(SWalReader *pRead);
static int32_t walSkipFetchBodyNew(SWalReader *pRead);
SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) {
SWalReader *pRead = taosMemoryMalloc(sizeof(SWalReader));
if (pRead == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pRead->pWal = pWal;
pRead->pReadIdxTFile = NULL;
pRead->pReadLogTFile = NULL;
pRead->pIdxFile = NULL;
pRead->pLogFile = NULL;
pRead->curVersion = -1;
pRead->curFileFirstVer = -1;
pRead->capacity = 0;
pRead->status = 0;
if (cond)
pRead->cond = *cond;
else {
pRead->cond.scanMeta = 0;
pRead->cond.scanUncommited = 0;
}
taosThreadMutexInit(&pRead->mutex, NULL);
@ -39,26 +48,46 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) {
taosMemoryFree(pRead);
return NULL;
}
return pRead;
}
void walCloseReadHandle(SWalReadHandle *pRead) {
taosCloseFile(&pRead->pReadIdxTFile);
taosCloseFile(&pRead->pReadLogTFile);
void walCloseReader(SWalReader *pRead) {
taosCloseFile(&pRead->pIdxFile);
taosCloseFile(&pRead->pLogFile);
taosMemoryFreeClear(pRead->pHead);
taosMemoryFree(pRead);
}
int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) {
// TODO
return 0;
int32_t walNextValidMsg(SWalReader *pRead) {
int64_t fetchVer = pRead->curVersion;
int64_t endVer = pRead->cond.scanUncommited ? walGetLastVer(pRead->pWal) : walGetCommittedVer(pRead->pWal);
while (fetchVer <= endVer) {
if (walFetchHeadNew(pRead, fetchVer) < 0) {
return -1;
}
if (pRead->pHead->head.msgType == TDMT_VND_SUBMIT ||
(IS_META_MSG(pRead->pHead->head.msgType) && pRead->cond.scanMeta)) {
if (walFetchBodyNew(pRead) < 0) {
return -1;
}
return 0;
} else {
if (walSkipFetchBodyNew(pRead) < 0) {
return -1;
}
fetchVer++;
ASSERT(fetchVer == pRead->curVersion);
}
}
return -1;
}
static int64_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) {
static int64_t walReadSeekFilePos(SWalReader *pRead, int64_t fileFirstVer, int64_t ver) {
int64_t ret = 0;
TdFilePtr pIdxTFile = pRead->pReadIdxTFile;
TdFilePtr pLogTFile = pRead->pReadLogTFile;
TdFilePtr pIdxTFile = pRead->pIdxFile;
TdFilePtr pLogTFile = pRead->pLogFile;
// seek position
int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry);
@ -90,11 +119,11 @@ static int64_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i
return ret;
}
static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
static int32_t walReadChangeFile(SWalReader *pRead, int64_t fileFirstVer) {
char fnameStr[WAL_FILE_LEN];
taosCloseFile(&pRead->pReadIdxTFile);
taosCloseFile(&pRead->pReadLogTFile);
taosCloseFile(&pRead->pIdxFile);
taosCloseFile(&pRead->pLogFile);
walBuildLogName(pRead->pWal, fileFirstVer, fnameStr);
TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ);
@ -104,7 +133,7 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
return -1;
}
pRead->pReadLogTFile = pLogTFile;
pRead->pLogFile = pLogTFile;
walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr);
TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ);
@ -114,11 +143,11 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
return -1;
}
pRead->pReadIdxTFile = pIdxTFile;
pRead->pIdxFile = pIdxTFile;
return 0;
}
static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
static int32_t walReadSeekVer(SWalReader *pRead, int64_t ver) {
SWal *pWal = pRead->pWal;
if (ver == pRead->curVersion) {
return 0;
@ -153,9 +182,94 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
return 0;
}
void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity) { pRead->capacity = capacity; }
void walSetReaderCapacity(SWalReader *pRead, int32_t capacity) { pRead->capacity = capacity; }
int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalCkHead *pHead) {
static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) {
int64_t contLen;
if (pRead->curVersion != fetchVer) {
if (walReadSeekVer(pRead, fetchVer) < 0) return -1;
}
contLen = taosReadFile(pRead->pLogFile, pRead->pHead, sizeof(SWalCkHead));
if (contLen != sizeof(SWalCkHead)) {
if (contLen < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
} else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
}
pRead->curVersion = -1;
return -1;
}
return 0;
}
static int32_t walFetchBodyNew(SWalReader *pRead) {
SWalCont *pReadHead = &pRead->pHead->head;
int64_t ver = pReadHead->version;
if (pRead->capacity < pReadHead->bodyLen) {
void *ptr = taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen);
if (ptr == NULL) {
terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
return -1;
}
pRead->pHead = ptr;
pReadHead = &pRead->pHead->head;
pRead->capacity = pReadHead->bodyLen;
}
if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) {
if (pReadHead->bodyLen < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 ", since %s",
pRead->pHead->head.version, ver, tstrerror(terrno));
} else {
wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 ", since file corrupted",
pRead->pHead->head.version, ver);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
}
pRead->curVersion = -1;
ASSERT(0);
return -1;
}
if (pReadHead->version != ver) {
wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, ver);
pRead->curVersion = -1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
ASSERT(0);
return -1;
}
if (walValidBodyCksum(pRead->pHead) != 0) {
wError("wal fetch body error: % " PRId64 ", since body checksum not passed", ver);
pRead->curVersion = -1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
ASSERT(0);
return -1;
}
pRead->curVersion = ver + 1;
return 0;
}
static int32_t walSkipFetchBodyNew(SWalReader *pRead) {
int64_t code;
ASSERT(pRead->curVersion == pRead->pHead->head.version);
code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
pRead->curVersion = -1;
return -1;
}
pRead->curVersion++;
return 0;
}
int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) {
int64_t code;
// TODO: valid ver
@ -168,9 +282,9 @@ int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalCkHead *pHead) {
if (code < 0) return -1;
}
ASSERT(taosValidFile(pRead->pReadLogTFile) == true);
ASSERT(taosValidFile(pRead->pLogFile) == true);
code = taosReadFile(pRead->pReadLogTFile, pHead, sizeof(SWalCkHead));
code = taosReadFile(pRead->pLogFile, pHead, sizeof(SWalCkHead));
if (code != sizeof(SWalCkHead)) {
return -1;
}
@ -186,12 +300,12 @@ int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalCkHead *pHead) {
return 0;
}
int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalCkHead *pHead) {
int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead) {
int64_t code;
ASSERT(pRead->curVersion == pHead->head.version);
code = taosLSeekFile(pRead->pReadLogTFile, pHead->head.bodyLen, SEEK_CUR);
code = taosLSeekFile(pRead->pLogFile, pHead->head.bodyLen, SEEK_CUR);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
pRead->curVersion = -1;
@ -203,7 +317,7 @@ int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalCkHead *pHead) {
return 0;
}
int32_t walFetchBody(SWalReadHandle *pRead, SWalCkHead **ppHead) {
int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) {
SWalCont *pReadHead = &((*ppHead)->head);
int64_t ver = pReadHead->version;
@ -218,7 +332,7 @@ int32_t walFetchBody(SWalReadHandle *pRead, SWalCkHead **ppHead) {
pRead->capacity = pReadHead->bodyLen;
}
if (pReadHead->bodyLen != taosReadFile(pRead->pReadLogTFile, pReadHead->body, pReadHead->bodyLen)) {
if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) {
ASSERT(0);
return -1;
}
@ -241,9 +355,9 @@ int32_t walFetchBody(SWalReadHandle *pRead, SWalCkHead **ppHead) {
return 0;
}
int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalCont **ppHead) {
int32_t walReadWithHandle_s(SWalReader *pRead, int64_t ver, SWalCont **ppHead) {
taosThreadMutexLock(&pRead->mutex);
if (walReadWithHandle(pRead, ver) < 0) {
if (walReadVer(pRead, ver) < 0) {
taosThreadMutexUnlock(&pRead->mutex);
return -1;
}
@ -257,7 +371,7 @@ int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalCont **ppHea
return 0;
}
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
int32_t walReadVer(SWalReader *pRead, int64_t ver) {
int64_t code;
if (pRead->pWal->vers.firstVer == -1) {
@ -280,9 +394,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
return -1;
}
ASSERT(taosValidFile(pRead->pReadLogTFile) == true);
ASSERT(taosValidFile(pRead->pLogFile) == true);
code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalCkHead));
code = taosReadFile(pRead->pLogFile, pRead->pHead, sizeof(SWalCkHead));
if (code != sizeof(SWalCkHead)) {
if (code < 0)
terrno = TAOS_SYSTEM_ERROR(errno);
@ -310,7 +424,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
pRead->capacity = pRead->pHead->head.bodyLen;
}
if ((code = taosReadFile(pRead->pReadLogTFile, pRead->pHead->head.body, pRead->pHead->head.bodyLen)) !=
if ((code = taosReadFile(pRead->pLogFile, pRead->pHead->head.body, pRead->pHead->head.bodyLen)) !=
pRead->pHead->head.bodyLen) {
if (code < 0)
terrno = TAOS_SYSTEM_ERROR(errno);
@ -340,46 +454,3 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
return 0;
}
#if 0
int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
int code;
code = walSeekVer(pWal, ver);
if (code != 0) {
return code;
}
if (*ppHead == NULL) {
void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalHead));
if (ptr == NULL) {
return -1;
}
*ppHead = ptr;
}
if (tfRead(pWal->pWriteLogTFile, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
return -1;
}
// TODO: endian compatibility processing after read
if (walValidHeadCksum(*ppHead) != 0) {
return -1;
}
void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len);
if (ptr == NULL) {
taosMemoryFree(*ppHead);
*ppHead = NULL;
return -1;
}
if (tfRead(pWal->pWriteLogTFile, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) {
return -1;
}
// TODO: endian compatibility processing after read
if (walValidBodyCksum(*ppHead) != 0) {
return -1;
}
return 0;
}
int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) {
return 0;
}
#endif

9
source/libs/wal/src/walWrite.c
View File

@ -79,19 +79,21 @@ int32_t walCommit(SWal *pWal, int64_t ver) {
}
int32_t walRollback(SWal *pWal, int64_t ver) {
taosThreadMutexLock(&pWal->mutex);
int64_t code;
char fnameStr[WAL_FILE_LEN];
if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) {
terrno = TSDB_CODE_WAL_INVALID_VER;
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
taosThreadMutexLock(&pWal->mutex);
// find correct file
if (ver < walGetLastFileFirstVer(pWal)) {
// change current files
code = walChangeWrite(pWal, ver);
if (code < 0) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
@ -146,6 +148,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
ASSERT(taosValidFile(pLogTFile));
int64_t size = taosReadFile(pLogTFile, &head, sizeof(SWalCkHead));
if (size != sizeof(SWalCkHead)) {
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
code = walValidHeadCksum(&head);
@ -154,11 +157,13 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
if (code != 0) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
ASSERT(0);
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
if (head.head.version != ver) {
ASSERT(0);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
@ -167,12 +172,14 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
if (code < 0) {
ASSERT(0);
terrno = TAOS_SYSTEM_ERROR(errno);
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
code = taosFtruncateFile(pIdxTFile, idxOff);
if (code < 0) {
ASSERT(0);
terrno = TAOS_SYSTEM_ERROR(errno);
taosThreadMutexUnlock(&pWal->mutex);
return -1;
}
pWal->vers.lastVer = ver - 1;

14
source/libs/wal/test/walMetaTest.cpp
View File

@ -292,8 +292,8 @@ TEST_F(WalCleanDeleteEnv, roll) {
TEST_F(WalKeepEnv, readHandleRead) {
walResetEnv();
int code;
SWalReadHandle* pRead = walOpenReadHandle(pWal);
int code;
SWalReader* pRead = walOpenReader(pWal, NULL);
ASSERT(pRead != NULL);
int i;
@ -306,7 +306,7 @@ TEST_F(WalKeepEnv, readHandleRead) {
}
for (int i = 0; i < 1000; i++) {
int ver = taosRand() % 100;
code = walReadWithHandle(pRead, ver);
code = walReadVer(pRead, ver);
ASSERT_EQ(code, 0);
// printf("rrbody: \n");
@ -325,7 +325,7 @@ TEST_F(WalKeepEnv, readHandleRead) {
EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]);
}
}
walCloseReadHandle(pRead);
walCloseReader(pRead);
}
TEST_F(WalRetentionEnv, repairMeta1) {
@ -354,12 +354,12 @@ TEST_F(WalRetentionEnv, repairMeta1) {
ASSERT_EQ(pWal->vers.lastVer, 99);
SWalReadHandle* pRead = walOpenReadHandle(pWal);
SWalReader* pRead = walOpenReader(pWal, NULL);
ASSERT(pRead != NULL);
for (int i = 0; i < 1000; i++) {
int ver = taosRand() % 100;
code = walReadWithHandle(pRead, ver);
code = walReadVer(pRead, ver);
ASSERT_EQ(code, 0);
// printf("rrbody: \n");
@ -389,7 +389,7 @@ TEST_F(WalRetentionEnv, repairMeta1) {
for (int i = 0; i < 1000; i++) {
int ver = taosRand() % 200;
code = walReadWithHandle(pRead, ver);
code = walReadVer(pRead, ver);
ASSERT_EQ(code, 0);
// printf("rrbody: \n");

296
source/os/src/osAtomic.c
View File

@ -19,23 +19,19 @@
#ifdef WINDOWS
// add
int8_t interlocked_add_fetch_8(int8_t volatile* ptr, int8_t val) {
return _InterlockedExchangeAdd8(ptr, val) + val;
}
int8_t interlocked_add_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedExchangeAdd8(ptr, val) + val; }
int16_t interlocked_add_fetch_16(int16_t volatile* ptr, int16_t val) {
return _InterlockedExchangeAdd16(ptr, val) + val;
}
int32_t interlocked_add_fetch_32(int32_t volatile* ptr, int32_t val) {
return _InterlockedExchangeAdd(ptr, val) + val;
}
int32_t interlocked_add_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedExchangeAdd(ptr, val) + val; }
int64_t interlocked_add_fetch_64(int64_t volatile* ptr, int64_t val) {
return InterlockedExchangeAdd64(ptr, val) + val;
}
void* interlocked_add_fetch_ptr(void* volatile* ptr, void* val) {
void* interlocked_add_fetch_ptr(void* volatile* ptr, void* val) {
#ifdef WINDOWS
return (void*)(_InterlockedExchangeAdd((int32_t volatile*)(ptr), (int32_t)val) + (int32_t)val);
#else
@ -43,17 +39,11 @@ void* interlocked_add_fetch_ptr(void* volatile* ptr, void* val) {
#endif
}
int8_t interlocked_and_fetch_8(int8_t volatile* ptr, int8_t val) {
return _InterlockedAnd8(ptr, val) & val;
}
int8_t interlocked_and_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedAnd8(ptr, val) & val; }
int16_t interlocked_and_fetch_16(int16_t volatile* ptr, int16_t val) {
return _InterlockedAnd16(ptr, val) & val;
}
int16_t interlocked_and_fetch_16(int16_t volatile* ptr, int16_t val) { return _InterlockedAnd16(ptr, val) & val; }
int32_t interlocked_and_fetch_32(int32_t volatile* ptr, int32_t val) {
return _InterlockedAnd(ptr, val) & val;
}
int32_t interlocked_and_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedAnd(ptr, val) & val; }
int64_t interlocked_and_fetch_64(int64_t volatile* ptr, int64_t val) {
#ifdef WINDOWS
@ -96,17 +86,11 @@ void* interlocked_fetch_and_ptr(void* volatile* ptr, void* val) {
#endif
}
int8_t interlocked_or_fetch_8(int8_t volatile* ptr, int8_t val) {
return _InterlockedOr8(ptr, val) | val;
}
int8_t interlocked_or_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedOr8(ptr, val) | val; }
int16_t interlocked_or_fetch_16(int16_t volatile* ptr, int16_t val) {
return _InterlockedOr16(ptr, val) | val;
}
int16_t interlocked_or_fetch_16(int16_t volatile* ptr, int16_t val) { return _InterlockedOr16(ptr, val) | val; }
int32_t interlocked_or_fetch_32(int32_t volatile* ptr, int32_t val) {
return _InterlockedOr(ptr, val) | val;
}
int32_t interlocked_or_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedOr(ptr, val) | val; }
int64_t interlocked_or_fetch_64(int64_t volatile* ptr, int64_t val) {
#ifdef WINDOWS
@ -114,7 +98,7 @@ int64_t interlocked_or_fetch_64(int64_t volatile* ptr, int64_t val) {
do {
old = *ptr;
res = old | val;
} while(_InterlockedCompareExchange64(ptr, res, old) != old);
} while (_InterlockedCompareExchange64(ptr, res, old) != old);
return res;
#else
return _InterlockedOr64(ptr, val) & val;
@ -134,7 +118,7 @@ int64_t interlocked_fetch_or_64(int64_t volatile* ptr, int64_t val) {
int64_t old;
do {
old = *ptr;
} while(_InterlockedCompareExchange64(ptr, old | val, old) != old);
} while (_InterlockedCompareExchange64(ptr, old | val, old) != old);
return old;
#else
return _InterlockedOr64((int64_t volatile*)(ptr), (int64_t)(val));
@ -149,17 +133,11 @@ void* interlocked_fetch_or_ptr(void* volatile* ptr, void* val) {
#endif
}
int8_t interlocked_xor_fetch_8(int8_t volatile* ptr, int8_t val) {
return _InterlockedXor8(ptr, val) ^ val;
}
int8_t interlocked_xor_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedXor8(ptr, val) ^ val; }
int16_t interlocked_xor_fetch_16(int16_t volatile* ptr, int16_t val) {
return _InterlockedXor16(ptr, val) ^ val;
}
int16_t interlocked_xor_fetch_16(int16_t volatile* ptr, int16_t val) { return _InterlockedXor16(ptr, val) ^ val; }
int32_t interlocked_xor_fetch_32(int32_t volatile* ptr, int32_t val) {
return _InterlockedXor(ptr, val) ^ val;
}
int32_t interlocked_xor_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedXor(ptr, val) ^ val; }
int64_t interlocked_xor_fetch_64(int64_t volatile* ptr, int64_t val) {
#ifdef WINDOWS
@ -167,7 +145,7 @@ int64_t interlocked_xor_fetch_64(int64_t volatile* ptr, int64_t val) {
do {
old = *ptr;
res = old ^ val;
} while(_InterlockedCompareExchange64(ptr, res, old) != old);
} while (_InterlockedCompareExchange64(ptr, res, old) != old);
return res;
#else
return _InterlockedXor64(ptr, val) ^ val;
@ -202,13 +180,9 @@ void* interlocked_fetch_xor_ptr(void* volatile* ptr, void* val) {
#endif
}
int32_t interlocked_sub_fetch_32(int32_t volatile* ptr, int32_t val) {
return interlocked_add_fetch_32(ptr, -val);
}
int32_t interlocked_sub_fetch_32(int32_t volatile* ptr, int32_t val) { return interlocked_add_fetch_32(ptr, -val); }
int64_t interlocked_sub_fetch_64(int64_t volatile* ptr, int64_t val) {
return interlocked_add_fetch_64(ptr, -val);
}
int64_t interlocked_sub_fetch_64(int64_t volatile* ptr, int64_t val) { return interlocked_add_fetch_64(ptr, -val); }
void* interlocked_sub_fetch_ptr(void* volatile* ptr, void* val) {
#ifdef WINDOWS
@ -217,13 +191,9 @@ void* interlocked_sub_fetch_ptr(void* volatile* ptr, void* val) {
return (void*)interlocked_add_fetch_64((int64_t volatile*)ptr, (int64_t)val);
#endif
}
int32_t interlocked_fetch_sub_32(int32_t volatile* ptr, int32_t val) {
return _InterlockedExchangeAdd(ptr, -val);
}
int32_t interlocked_fetch_sub_32(int32_t volatile* ptr, int32_t val) { return _InterlockedExchangeAdd(ptr, -val); }
int64_t interlocked_fetch_sub_64(int64_t volatile* ptr, int64_t val) {
return _InterlockedExchangeAdd64(ptr, -val);
}
int64_t interlocked_fetch_sub_64(int64_t volatile* ptr, int64_t val) { return _InterlockedExchangeAdd64(ptr, -val); }
void* interlocked_fetch_sub_ptr(void* volatile* ptr, void* val) {
#ifdef WINDOWS
@ -236,45 +206,44 @@ void* interlocked_fetch_sub_ptr(void* volatile* ptr, void* val) {
#endif
#ifdef _TD_NINGSI_60
void* atomic_exchange_ptr_impl(void** ptr, void* val ) {
void *old;
void* atomic_exchange_ptr_impl(void** ptr, void* val) {
void* old;
do {
old = *ptr;
} while( !__sync_bool_compare_and_swap(ptr, old, val) );
} while (!__sync_bool_compare_and_swap(ptr, old, val));
return old;
}
int8_t atomic_exchange_8_impl(int8_t* ptr, int8_t val ) {
int8_t atomic_exchange_8_impl(int8_t* ptr, int8_t val) {
int8_t old;
do {
old = *ptr;
} while( !__sync_bool_compare_and_swap(ptr, old, val) );
} while (!__sync_bool_compare_and_swap(ptr, old, val));
return old;
}
int16_t atomic_exchange_16_impl(int16_t* ptr, int16_t val ) {
int16_t atomic_exchange_16_impl(int16_t* ptr, int16_t val) {
int16_t old;
do {
old = *ptr;
} while( !__sync_bool_compare_and_swap(ptr, old, val) );
} while (!__sync_bool_compare_and_swap(ptr, old, val));
return old;
}
int32_t atomic_exchange_32_impl(int32_t* ptr, int32_t val ) {
int32_t atomic_exchange_32_impl(int32_t* ptr, int32_t val) {
int32_t old;
do {
old = *ptr;
} while( !__sync_bool_compare_and_swap(ptr, old, val) );
} while (!__sync_bool_compare_and_swap(ptr, old, val));
return old;
}
int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val ) {
int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val) {
int64_t old;
do {
old = *ptr;
} while( !__sync_bool_compare_and_swap(ptr, old, val) );
} while (!__sync_bool_compare_and_swap(ptr, old, val));
return old;
}
#endif
int8_t atomic_load_8(int8_t volatile *ptr) {
int8_t atomic_load_8(int8_t volatile* ptr) {
#ifdef WINDOWS
return (*(int8_t volatile*)(ptr));
#elif defined(_TD_NINGSI_60)
@ -284,7 +253,7 @@ int8_t atomic_load_8(int8_t volatile *ptr) {
#endif
}
int16_t atomic_load_16(int16_t volatile *ptr) {
int16_t atomic_load_16(int16_t volatile* ptr) {
#ifdef WINDOWS
return (*(int16_t volatile*)(ptr));
#elif defined(_TD_NINGSI_60)
@ -294,7 +263,7 @@ int16_t atomic_load_16(int16_t volatile *ptr) {
#endif
}
int32_t atomic_load_32(int32_t volatile *ptr) {
int32_t atomic_load_32(int32_t volatile* ptr) {
#ifdef WINDOWS
return (*(int32_t volatile*)(ptr));
#elif defined(_TD_NINGSI_60)
@ -304,7 +273,7 @@ int32_t atomic_load_32(int32_t volatile *ptr) {
#endif
}
int64_t atomic_load_64(int64_t volatile *ptr) {
int64_t atomic_load_64(int64_t volatile* ptr) {
#ifdef WINDOWS
return (*(int64_t volatile*)(ptr));
#elif defined(_TD_NINGSI_60)
@ -314,7 +283,7 @@ int64_t atomic_load_64(int64_t volatile *ptr) {
#endif
}
void* atomic_load_ptr(void *ptr) {
void* atomic_load_ptr(void* ptr) {
#ifdef WINDOWS
return (*(void* volatile*)(ptr));
#elif defined(_TD_NINGSI_60)
@ -324,57 +293,57 @@ void* atomic_load_ptr(void *ptr) {
#endif
}
void atomic_store_8(int8_t volatile *ptr, int8_t val) {
void atomic_store_8(int8_t volatile* ptr, int8_t val) {
#ifdef WINDOWS
((*(int8_t volatile*)(ptr)) = (int8_t)(val));
#elif defined(_TD_NINGSI_60)
(*(ptr)=(val));
(*(ptr) = (val));
#else
__atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST);
#endif
}
void atomic_store_16(int16_t volatile *ptr, int16_t val) {
void atomic_store_16(int16_t volatile* ptr, int16_t val) {
#ifdef WINDOWS
((*(int16_t volatile*)(ptr)) = (int16_t)(val));
#elif defined(_TD_NINGSI_60)
(*(ptr)=(val));
(*(ptr) = (val));
#else
__atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST);
#endif
}
void atomic_store_32(int32_t volatile *ptr, int32_t val) {
void atomic_store_32(int32_t volatile* ptr, int32_t val) {
#ifdef WINDOWS
((*(int32_t volatile*)(ptr)) = (int32_t)(val));
#elif defined(_TD_NINGSI_60)
(*(ptr)=(val));
(*(ptr) = (val));
#else
__atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST);
#endif
}
void atomic_store_64(int64_t volatile *ptr, int64_t val) {
void atomic_store_64(int64_t volatile* ptr, int64_t val) {
#ifdef WINDOWS
((*(int64_t volatile*)(ptr)) = (int64_t)(val));
#elif defined(_TD_NINGSI_60)
(*(ptr)=(val));
(*(ptr) = (val));
#else
__atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST);
#endif
}
void atomic_store_ptr(void *ptr, void *val) {
void atomic_store_ptr(void* ptr, void* val) {
#ifdef WINDOWS
((*(void* volatile*)(ptr)) = (void*)(val));
#elif defined(_TD_NINGSI_60)
(*(ptr)=(val));
(*(ptr) = (val));
#else
__atomic_store_n((void **)(ptr), (val), __ATOMIC_SEQ_CST);
__atomic_store_n((void**)(ptr), (val), __ATOMIC_SEQ_CST);
#endif
}
int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val) {
int8_t atomic_exchange_8(int8_t volatile* ptr, int8_t val) {
#ifdef WINDOWS
return _InterlockedExchange8((int8_t volatile*)(ptr), (int8_t)(val));
#elif defined(_TD_NINGSI_60)
@ -384,7 +353,7 @@ int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val) {
#endif
}
int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val) {
int16_t atomic_exchange_16(int16_t volatile* ptr, int16_t val) {
#ifdef WINDOWS
return _InterlockedExchange16((int16_t volatile*)(ptr), (int16_t)(val));
#elif defined(_TD_NINGSI_60)
@ -394,7 +363,7 @@ int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val) {
#endif
}
int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val) {
int32_t atomic_exchange_32(int32_t volatile* ptr, int32_t val) {
#ifdef WINDOWS
return _InterlockedExchange((int32_t volatile*)(ptr), (int32_t)(val));
#elif defined(_TD_NINGSI_60)
@ -404,7 +373,7 @@ int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val) {
#endif
}
int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val) {
int64_t atomic_exchange_64(int64_t volatile* ptr, int64_t val) {
#ifdef WINDOWS
return _InterlockedExchange64((int64_t volatile*)(ptr), (int64_t)(val));
#elif defined(_TD_NINGSI_60)
@ -414,21 +383,21 @@ int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val) {
#endif
}
void* atomic_exchange_ptr(void *ptr, void *val) {
void* atomic_exchange_ptr(void* ptr, void* val) {
#ifdef WINDOWS
#ifdef _WIN64
#ifdef _WIN64
return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val));
#else
return _InlineInterlockedExchangePointer((void* volatile*)(ptr), (void*)(val));
#endif
#elif defined(_TD_NINGSI_60)
return atomic_exchange_ptr_impl((void *)ptr, (void*)val);
#else
return __atomic_exchange_n((void **)(ptr), (val), __ATOMIC_SEQ_CST);
return _InlineInterlockedExchangePointer((void* volatile*)(ptr), (void*)(val));
#endif
#elif defined(_TD_NINGSI_60)
return atomic_exchange_ptr_impl((void*)ptr, (void*)val);
#else
return __atomic_exchange_n((void**)(ptr), (val), __ATOMIC_SEQ_CST);
#endif
}
int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval) {
int8_t atomic_val_compare_exchange_8(int8_t volatile* ptr, int8_t oldval, int8_t newval) {
#ifdef WINDOWS
return _InterlockedCompareExchange8((int8_t volatile*)(ptr), (int8_t)(newval), (int8_t)(oldval));
#elif defined(_TD_NINGSI_60)
@ -438,7 +407,7 @@ int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t
#endif
}
int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval) {
int16_t atomic_val_compare_exchange_16(int16_t volatile* ptr, int16_t oldval, int16_t newval) {
#ifdef WINDOWS
return _InterlockedCompareExchange16((int16_t volatile*)(ptr), (int16_t)(newval), (int16_t)(oldval));
#elif defined(_TD_NINGSI_60)
@ -448,7 +417,7 @@ int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, in
#endif
}
int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, int32_t newval) {
int32_t atomic_val_compare_exchange_32(int32_t volatile* ptr, int32_t oldval, int32_t newval) {
#ifdef WINDOWS
return _InterlockedCompareExchange((int32_t volatile*)(ptr), (int32_t)(newval), (int32_t)(oldval));
#elif defined(_TD_NINGSI_60)
@ -458,7 +427,7 @@ int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, in
#endif
}
int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, int64_t newval) {
int64_t atomic_val_compare_exchange_64(int64_t volatile* ptr, int64_t oldval, int64_t newval) {
#ifdef WINDOWS
return _InterlockedCompareExchange64((int64_t volatile*)(ptr), (int64_t)(newval), (int64_t)(oldval));
#elif defined(_TD_NINGSI_60)
@ -468,17 +437,17 @@ int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, in
#endif
}
void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval) {
void* atomic_val_compare_exchange_ptr(void* ptr, void* oldval, void* newval) {
#ifdef WINDOWS
return _InterlockedCompareExchangePointer((void* volatile*)(ptr), (void*)(newval), (void*)(oldval));
#elif defined(_TD_NINGSI_60)
return __sync_val_compare_and_swap(ptr, oldval, newval);
#else
return __sync_val_compare_and_swap((void **)ptr, oldval, newval);
return __sync_val_compare_and_swap((void**)ptr, oldval, newval);
#endif
}
int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val) {
int8_t atomic_add_fetch_8(int8_t volatile* ptr, int8_t val) {
#ifdef WINDOWS
return interlocked_add_fetch_8((int8_t volatile*)(ptr), (int8_t)(val));
#elif defined(_TD_NINGSI_60)
@ -488,7 +457,7 @@ int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val) {
#endif
}
int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val) {
int16_t atomic_add_fetch_16(int16_t volatile* ptr, int16_t val) {
#ifdef WINDOWS
return interlocked_add_fetch_16((int16_t volatile*)(ptr), (int16_t)(val));
#elif defined(_TD_NINGSI_60)
@ -498,7 +467,7 @@ int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val) {
#endif
}
int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val) {
int32_t atomic_add_fetch_32(int32_t volatile* ptr, int32_t val) {
#ifdef WINDOWS
return interlocked_add_fetch_32((int32_t volatile*)(ptr), (int32_t)(val));
#elif defined(_TD_NINGSI_60)
@ -508,7 +477,7 @@ int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val) {
#endif
}
int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val) {
int64_t atomic_add_fetch_64(int64_t volatile* ptr, int64_t val) {
#ifdef WINDOWS
return interlocked_add_fetch_64((int64_t volatile*)(ptr), (int64_t)(val));
#elif defined(_TD_NINGSI_60)
@ -518,17 +487,17 @@ int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val) {
#endif
}
void* atomic_add_fetch_ptr(void *ptr, int64_t val) {
void* atomic_add_fetch_ptr(void* ptr, int64_t val) {
#ifdef WINDOWS
return interlocked_add_fetch_ptr((void* volatile*)(ptr), (void*)(val));
#elif defined(_TD_NINGSI_60)
return __sync_add_and_fetch((ptr), (val));
#else
return __atomic_add_fetch((void **)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
return __atomic_add_fetch((void**)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
#endif
}
int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val) {
int8_t atomic_fetch_add_8(int8_t volatile* ptr, int8_t val) {
#ifdef WINDOWS
return _InterlockedExchangeAdd8((int8_t volatile*)(ptr), (int8_t)(val));
#elif defined(_TD_NINGSI_60)
@ -538,7 +507,7 @@ int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val) {
#endif
}
int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val) {
int16_t atomic_fetch_add_16(int16_t volatile* ptr, int16_t val) {
#ifdef WINDOWS
return _InterlockedExchangeAdd16((int16_t volatile*)(ptr), (int16_t)(val));
#elif defined(_TD_NINGSI_60)
@ -548,7 +517,7 @@ int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val) {
#endif
}
int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val) {
int32_t atomic_fetch_add_32(int32_t volatile* ptr, int32_t val) {
#ifdef WINDOWS
return _InterlockedExchangeAdd((int32_t volatile*)(ptr), (int32_t)(val));
#elif defined(_TD_NINGSI_60)
@ -558,7 +527,7 @@ int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val) {
#endif
}
int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val) {
int64_t atomic_fetch_add_64(int64_t volatile* ptr, int64_t val) {
#ifdef WINDOWS
return _InterlockedExchangeAdd64((int64_t volatile*)(ptr), (int64_t)(val));
#elif defined(_TD_NINGSI_60)
@ -568,17 +537,17 @@ int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val) {
#endif
}
void* atomic_fetch_add_ptr(void *ptr, void *val) {
void* atomic_fetch_add_ptr(void* ptr, void* val) {
#ifdef WINDOWS
return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val));
#elif defined(_TD_NINGSI_60)
return __sync_fetch_and_add((ptr), (val));
#else
return __atomic_fetch_add((void **)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
return __atomic_fetch_add((void**)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
#endif
}
int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val) {
int8_t atomic_sub_fetch_8(int8_t volatile* ptr, int8_t val) {
#ifdef WINDOWS
return interlocked_add_fetch_8((int8_t volatile*)(ptr), -(int8_t)(val));
#elif defined(_TD_NINGSI_60)
@ -588,7 +557,7 @@ int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val) {
#endif
}
int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val) {
int16_t atomic_sub_fetch_16(int16_t volatile* ptr, int16_t val) {
#ifdef WINDOWS
return interlocked_add_fetch_16((int16_t volatile*)(ptr), -(int16_t)(val));
#elif defined(_TD_NINGSI_60)
@ -598,7 +567,7 @@ int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val) {
#endif
}
int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val) {
int32_t atomic_sub_fetch_32(int32_t volatile* ptr, int32_t val) {
#ifdef WINDOWS
return interlocked_sub_fetch_32(ptr, val);
#elif defined(_TD_NINGSI_60)
@ -608,7 +577,7 @@ int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val) {
#endif
}
int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val) {
int64_t atomic_sub_fetch_64(int64_t volatile* ptr, int64_t val) {
#ifdef WINDOWS
return interlocked_sub_fetch_64(ptr, val);
#elif defined(_TD_NINGSI_60)
@ -618,19 +587,19 @@ int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val) {
#endif
}
void* atomic_sub_fetch_ptr(void *ptr, int64_t val) {
void* atomic_sub_fetch_ptr(void* ptr, int64_t val) {
#ifdef WINDOWS
return interlocked_sub_fetch_ptr(ptr, val);
#elif defined(_TD_NINGSI_60)
return __sync_sub_and_fetch((ptr), (val));
#elif defined(_TD_DARWIN_64)
return __atomic_sub_fetch((void **)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
return __atomic_sub_fetch((void**)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
#else
return __atomic_sub_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST);
return __atomic_sub_fetch((void**)(ptr), (val), __ATOMIC_SEQ_CST);
#endif
}
int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val) {
int8_t atomic_fetch_sub_8(int8_t volatile* ptr, int8_t val) {
#ifdef WINDOWS
return _InterlockedExchangeAdd8((int8_t volatile*)(ptr), -(int8_t)(val));
#elif defined(_TD_NINGSI_60)
@ -640,7 +609,7 @@ int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val) {
#endif
}
int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val) {
int16_t atomic_fetch_sub_16(int16_t volatile* ptr, int16_t val) {
#ifdef WINDOWS
return _InterlockedExchangeAdd16((int16_t volatile*)(ptr), -(int16_t)(val));
#elif defined(_TD_NINGSI_60)
@ -650,7 +619,7 @@ int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val) {
#endif
}
int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val) {
int32_t atomic_fetch_sub_32(int32_t volatile* ptr, int32_t val) {
#ifdef WINDOWS
return interlocked_fetch_sub_32(ptr, val);
#elif defined(_TD_NINGSI_60)
@ -660,7 +629,7 @@ int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val) {
#endif
}
int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val) {
int64_t atomic_fetch_sub_64(int64_t volatile* ptr, int64_t val) {
#ifdef WINDOWS
return _InterlockedExchangeAdd64((int64_t volatile*)(ptr), -(int64_t)(val));
#elif defined(_TD_NINGSI_60)
@ -670,19 +639,19 @@ int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val) {
#endif
}
void* atomic_fetch_sub_ptr(void *ptr, void* val) {
void* atomic_fetch_sub_ptr(void* ptr, void* val) {
#ifdef WINDOWS
return interlocked_fetch_sub_ptr(ptr, val);
#elif defined(_TD_NINGSI_60)
return __sync_fetch_and_sub((ptr), (val));
#elif defined(_TD_DARWIN_64)
return __atomic_fetch_sub((void **)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
return __atomic_fetch_sub((void**)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
#else
return __atomic_fetch_sub((void **)(ptr), (val), __ATOMIC_SEQ_CST);
return __atomic_fetch_sub((void**)(ptr), (val), __ATOMIC_SEQ_CST);
#endif
}
int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val) {
int8_t atomic_and_fetch_8(int8_t volatile* ptr, int8_t val) {
#ifdef WINDOWS
return interlocked_and_fetch_8((int8_t volatile*)(ptr), (int8_t)(val));
#elif defined(_TD_NINGSI_60)
@ -692,7 +661,7 @@ int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val) {
#endif
}
int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val) {
int16_t atomic_and_fetch_16(int16_t volatile* ptr, int16_t val) {
#ifdef WINDOWS
return interlocked_and_fetch_16((int16_t volatile*)(ptr), (int16_t)(val));
#elif defined(_TD_NINGSI_60)
@ -702,7 +671,7 @@ int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val) {
#endif
}
int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val) {
int32_t atomic_and_fetch_32(int32_t volatile* ptr, int32_t val) {
#ifdef WINDOWS
return interlocked_and_fetch_32((int32_t volatile*)(ptr), (int32_t)(val));
#elif defined(_TD_NINGSI_60)
@ -712,7 +681,7 @@ int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val) {
#endif
}
int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val) {
int64_t atomic_and_fetch_64(int64_t volatile* ptr, int64_t val) {
#ifdef WINDOWS
return interlocked_and_fetch_64((int64_t volatile*)(ptr), (int64_t)(val));
#elif defined(_TD_NINGSI_60)
@ -722,19 +691,19 @@ int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val) {
#endif
}
void* atomic_and_fetch_ptr(void *ptr, void *val) {
void* atomic_and_fetch_ptr(void* ptr, void* val) {
#ifdef WINDOWS
return interlocked_and_fetch_ptr((void* volatile*)(ptr), (void*)(val));
#elif defined(_TD_NINGSI_60)
return __sync_and_and_fetch((ptr), (val));
#elif defined(_TD_DARWIN_64)
return (void*)__atomic_and_fetch((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
return (void*)__atomic_and_fetch((size_t*)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
#else
return __atomic_and_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST);
return __atomic_and_fetch((void**)(ptr), (val), __ATOMIC_SEQ_CST);
#endif
}
int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val) {
int8_t atomic_fetch_and_8(int8_t volatile* ptr, int8_t val) {
#ifdef WINDOWS
return _InterlockedAnd8((int8_t volatile*)(ptr), (int8_t)(val));
#elif defined(_TD_NINGSI_60)
@ -744,7 +713,7 @@ int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val) {
#endif
}
int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val) {
int16_t atomic_fetch_and_16(int16_t volatile* ptr, int16_t val) {
#ifdef WINDOWS
return _InterlockedAnd16((int16_t volatile*)(ptr), (int16_t)(val));
#elif defined(_TD_NINGSI_60)
@ -754,7 +723,7 @@ int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val) {
#endif
}
int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val) {
int32_t atomic_fetch_and_32(int32_t volatile* ptr, int32_t val) {
#ifdef WINDOWS
return _InterlockedAnd((int32_t volatile*)(ptr), (int32_t)(val));
#elif defined(_TD_NINGSI_60)
@ -764,7 +733,7 @@ int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val) {
#endif
}
int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val) {
int64_t atomic_fetch_and_64(int64_t volatile* ptr, int64_t val) {
#ifdef WINDOWS
return interlocked_fetch_and_64((int64_t volatile*)(ptr), (int64_t)(val));
#elif defined(_TD_NINGSI_60)
@ -774,19 +743,19 @@ int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val) {
#endif
}
void* atomic_fetch_and_ptr(void *ptr, void *val) {
void* atomic_fetch_and_ptr(void* ptr, void* val) {
#ifdef WINDOWS
return interlocked_fetch_and_ptr((void* volatile*)(ptr), (void*)(val));
#elif defined(_TD_NINGSI_60)
return __sync_fetch_and_and((ptr), (val));
#elif defined(_TD_DARWIN_64)
return (void*)__atomic_fetch_and((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
return (void*)__atomic_fetch_and((size_t*)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
#else
return __atomic_fetch_and((void **)(ptr), (val), __ATOMIC_SEQ_CST);
return __atomic_fetch_and((void**)(ptr), (val), __ATOMIC_SEQ_CST);
#endif
}
int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val) {
int8_t atomic_or_fetch_8(int8_t volatile* ptr, int8_t val) {
#ifdef WINDOWS
return interlocked_or_fetch_8((int8_t volatile*)(ptr), (int8_t)(val));
#elif defined(_TD_NINGSI_60)
@ -796,7 +765,7 @@ int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val) {
#endif
}
int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val) {
int16_t atomic_or_fetch_16(int16_t volatile* ptr, int16_t val) {
#ifdef WINDOWS
return interlocked_or_fetch_16((int16_t volatile*)(ptr), (int16_t)(val));
#elif defined(_TD_NINGSI_60)
@ -806,7 +775,7 @@ int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val) {
#endif
}
int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val) {
int32_t atomic_or_fetch_32(int32_t volatile* ptr, int32_t val) {
#ifdef WINDOWS
return interlocked_or_fetch_32((int32_t volatile*)(ptr), (int32_t)(val));
#elif defined(_TD_NINGSI_60)
@ -816,7 +785,7 @@ int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val) {
#endif
}
int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val) {
int64_t atomic_or_fetch_64(int64_t volatile* ptr, int64_t val) {
#ifdef WINDOWS
return interlocked_or_fetch_64((int64_t volatile*)(ptr), (int64_t)(val));
#elif defined(_TD_NINGSI_60)
@ -826,19 +795,19 @@ int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val) {
#endif
}
void* atomic_or_fetch_ptr(void *ptr, void *val) {
void* atomic_or_fetch_ptr(void* ptr, void* val) {
#ifdef WINDOWS
return interlocked_or_fetch_ptr((void* volatile*)(ptr), (void*)(val));
#elif defined(_TD_NINGSI_60)
return __sync_or_and_fetch((ptr), (val));
#elif defined(_TD_DARWIN_64)
return (void*)__atomic_or_fetch((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
return (void*)__atomic_or_fetch((size_t*)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
#else
return __atomic_or_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST);
return __atomic_or_fetch((void**)(ptr), (val), __ATOMIC_SEQ_CST);
#endif
}
int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val) {
int8_t atomic_fetch_or_8(int8_t volatile* ptr, int8_t val) {
#ifdef WINDOWS
return _InterlockedOr8((int8_t volatile*)(ptr), (int8_t)(val));
#elif defined(_TD_NINGSI_60)
@ -848,7 +817,7 @@ int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val) {
#endif
}
int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val) {
int16_t atomic_fetch_or_16(int16_t volatile* ptr, int16_t val) {
#ifdef WINDOWS
return _InterlockedOr16((int16_t volatile*)(ptr), (int16_t)(val));
#elif defined(_TD_NINGSI_60)
@ -858,7 +827,7 @@ int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val) {
#endif
}
int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val) {
int32_t atomic_fetch_or_32(int32_t volatile* ptr, int32_t val) {
#ifdef WINDOWS
return _InterlockedOr((int32_t volatile*)(ptr), (int32_t)(val));
#elif defined(_TD_NINGSI_60)
@ -868,7 +837,7 @@ int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val) {
#endif
}
int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val) {
int64_t atomic_fetch_or_64(int64_t volatile* ptr, int64_t val) {
#ifdef WINDOWS
return interlocked_fetch_or_64((int64_t volatile*)(ptr), (int64_t)(val));
#elif defined(_TD_NINGSI_60)
@ -878,19 +847,19 @@ int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val) {
#endif
}
void* atomic_fetch_or_ptr(void *ptr, void *val) {
void* atomic_fetch_or_ptr(void* ptr, void* val) {
#ifdef WINDOWS
return interlocked_fetch_or_ptr((void* volatile*)(ptr), (void*)(val));
#elif defined(_TD_NINGSI_60)
return __sync_fetch_and_or((ptr), (val));
#elif defined(_TD_DARWIN_64)
return (void*)__atomic_fetch_or((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
return (void*)__atomic_fetch_or((size_t*)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
#else
return __atomic_fetch_or((void **)(ptr), (val), __ATOMIC_SEQ_CST);
return __atomic_fetch_or((void**)(ptr), (val), __ATOMIC_SEQ_CST);
#endif
}
int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val) {
int8_t atomic_xor_fetch_8(int8_t volatile* ptr, int8_t val) {
#ifdef WINDOWS
return interlocked_xor_fetch_8((int8_t volatile*)(ptr), (int8_t)(val));
#elif defined(_TD_NINGSI_60)
@ -900,7 +869,7 @@ int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val) {
#endif
}
int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val) {
int16_t atomic_xor_fetch_16(int16_t volatile* ptr, int16_t val) {
#ifdef WINDOWS
return interlocked_xor_fetch_16((int16_t volatile*)(ptr), (int16_t)(val));
#elif defined(_TD_NINGSI_60)
@ -910,7 +879,7 @@ int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val) {
#endif
}
int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val) {
int32_t atomic_xor_fetch_32(int32_t volatile* ptr, int32_t val) {
#ifdef WINDOWS
return interlocked_xor_fetch_32((int32_t volatile*)(ptr), (int32_t)(val));
#elif defined(_TD_NINGSI_60)
@ -920,7 +889,7 @@ int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val) {
#endif
}
int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val) {
int64_t atomic_xor_fetch_64(int64_t volatile* ptr, int64_t val) {
#ifdef WINDOWS
return interlocked_xor_fetch_64((int64_t volatile*)(ptr), (int64_t)(val));
#elif defined(_TD_NINGSI_60)
@ -930,19 +899,19 @@ int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val) {
#endif
}
void* atomic_xor_fetch_ptr(void *ptr, void *val) {
void* atomic_xor_fetch_ptr(void* ptr, void* val) {
#ifdef WINDOWS
return interlocked_xor_fetch_ptr((void* volatile*)(ptr), (void*)(val));
#elif defined(_TD_NINGSI_60)
return __sync_xor_and_fetch((ptr), (val));
#elif defined(_TD_DARWIN_64)
return (void*)__atomic_xor_fetch((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
return (void*)__atomic_xor_fetch((size_t*)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
#else
return __atomic_xor_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST);
return __atomic_xor_fetch((void**)(ptr), (val), __ATOMIC_SEQ_CST);
#endif
}
int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val) {
int8_t atomic_fetch_xor_8(int8_t volatile* ptr, int8_t val) {
#ifdef WINDOWS
return _InterlockedXor8((int8_t volatile*)(ptr), (int8_t)(val));
#elif defined(_TD_NINGSI_60)
@ -952,7 +921,7 @@ int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val) {
#endif
}
int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val) {
int16_t atomic_fetch_xor_16(int16_t volatile* ptr, int16_t val) {
#ifdef WINDOWS
return _InterlockedXor16((int16_t volatile*)(ptr), (int16_t)(val));
#elif defined(_TD_NINGSI_60)
@ -962,7 +931,7 @@ int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val) {
#endif
}
int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val) {
int32_t atomic_fetch_xor_32(int32_t volatile* ptr, int32_t val) {
#ifdef WINDOWS
return _InterlockedXor((int32_t volatile*)(ptr), (int32_t)(val));
#elif defined(_TD_NINGSI_60)
@ -972,7 +941,7 @@ int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val) {
#endif
}
int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val) {
int64_t atomic_fetch_xor_64(int64_t volatile* ptr, int64_t val) {
#ifdef WINDOWS
return interlocked_fetch_xor_64((int64_t volatile*)(ptr), (int64_t)(val));
#elif defined(_TD_NINGSI_60)
@ -982,15 +951,14 @@ int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val) {
#endif
}
void* atomic_fetch_xor_ptr(void *ptr, void *val) {
void* atomic_fetch_xor_ptr(void* ptr, void* val) {
#ifdef WINDOWS
return interlocked_fetch_xor_ptr((void* volatile*)(ptr), (void*)(val));
#elif defined(_TD_NINGSI_60)
return __sync_fetch_and_xor((ptr), (val));
#elif defined(_TD_DARWIN_64)
return (void*)__atomic_fetch_xor((size_t *)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
return (void*)__atomic_fetch_xor((size_t*)(ptr), (size_t)(val), __ATOMIC_SEQ_CST);
#else
return __atomic_fetch_xor((void **)(ptr), (val), __ATOMIC_SEQ_CST);
return __atomic_fetch_xor((void**)(ptr), (val), __ATOMIC_SEQ_CST);
#endif
}

84
source/os/src/osString.c
View File

@ -29,9 +29,8 @@ char *strsep(char **stringp, const char *delim) {
char * s;
const char *spanp;
int32_t c, sc;
char *tok;
if ((s = *stringp) == NULL)
return (NULL);
char * tok;
if ((s = *stringp) == NULL) return (NULL);
for (tok = s;;) {
c = *s++;
spanp = delim;
@ -51,10 +50,10 @@ char *strsep(char **stringp, const char *delim) {
/* Duplicate a string, up to at most size characters */
char *strndup(const char *s, size_t size) {
size_t l;
char *s2;
char * s2;
l = strlen(s);
if (l > size) l=size;
s2 = malloc(l+1);
if (l > size) l = size;
s2 = malloc(l + 1);
if (s2) {
strncpy(s2, s, l);
s2[l] = '\0';
@ -63,13 +62,12 @@ char *strndup(const char *s, size_t size) {
}
/* Copy no more than N characters of SRC to DEST, returning the address of
the terminating '\0' in DEST, if any, or else DEST + N. */
char *stpncpy (char *dest, const char *src, size_t n) {
size_t size = strnlen (src, n);
memcpy (dest, src, size);
char *stpncpy(char *dest, const char *src, size_t n) {
size_t size = strnlen(src, n);
memcpy(dest, src, size);
dest += size;
if (size == n)
return dest;
return memset (dest, '\0', n - size);
if (size == n) return dest;
return memset(dest, '\0', n - size);
}
#endif
@ -113,10 +111,9 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) {
#endif
}
TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) {
assert(taosMemorySize(target_ucs4)>=len_ucs4*sizeof(TdUcs4));
return memcpy(target_ucs4, source_ucs4, len_ucs4*sizeof(TdUcs4));
TdUcs4 *tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) {
assert(taosMemorySize(target_ucs4) >= len_ucs4 * sizeof(TdUcs4));
return memcpy(target_ucs4, source_ucs4, len_ucs4 * sizeof(TdUcs4));
}
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
@ -137,7 +134,7 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
#endif
}
bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) {
bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n");
return -1;
@ -146,7 +143,7 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4
iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
size_t ucs4_input_len = mbsLength;
size_t outLeft = ucs4_max_len;
if (iconv(cd, (char**)&mbs, &ucs4_input_len, (char**)&ucs4, &outLeft) == -1) {
if (iconv(cd, (char **)&mbs, &ucs4_input_len, (char **)&ucs4, &outLeft) == -1) {
iconv_close(cd);
return false;
}
@ -195,7 +192,7 @@ int32_t taosUcs4len(TdUcs4 *ucs4) {
return n;
}
//dst buffer size should be at least 2*len + 1
// dst buffer size should be at least 2*len + 1
int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len) {
if (!dst) {
return -1;
@ -214,7 +211,7 @@ int32_t taosHexDecode(const char *src, char *dst, int32_t len) {
}
uint8_t hn, ln, out;
for (int i = 0, j = 0; i < len * 2; i += 2, ++j ) {
for (int i = 0, j = 0; i < len * 2; i += 2, ++j) {
hn = src[i] > '9' ? src[i] - 'a' + 10 : src[i] - '0';
ln = src[i + 1] > '9' ? src[i + 1] - 'a' + 10 : src[i + 1] - '0';
@ -238,25 +235,22 @@ int32_t taosWcharToMb(char *pStr, TdWchar wchar) { return wctomb(pStr, wchar); }
int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size) { return wcstombs(pStrs, pWchars, size); }
char *taosStrCaseStr(const char *str, const char *pattern) {
size_t i;
size_t i;
if (!*pattern)
return (char*)str;
if (!*pattern) return (char *)str;
for (; *str; str++) {
if (toupper(*str) == toupper(*pattern)) {
for (i = 1;; i++) {
if (!pattern[i])
return (char*)str;
if (toupper(str[i]) != toupper(pattern[i]))
break;
}
}
}
return NULL;
for (; *str; str++) {
if (toupper(*str) == toupper(*pattern)) {
for (i = 1;; i++) {
if (!pattern[i]) return (char *)str;
if (toupper(str[i]) != toupper(pattern[i])) break;
}
}
}
return NULL;
}
int64_t taosStr2Int64(const char *str, char** pEnd, int32_t radix) {
int64_t taosStr2Int64(const char *str, char **pEnd, int32_t radix) {
int64_t tmp = strtoll(str, pEnd, radix);
#ifdef TD_CHECK_STR_TO_INT_ERROR
assert(errno != ERANGE);
@ -265,7 +259,7 @@ int64_t taosStr2Int64(const char *str, char** pEnd, int32_t radix) {
return tmp;
}
uint64_t taosStr2UInt64(const char *str, char** pEnd, int32_t radix) {
uint64_t taosStr2UInt64(const char *str, char **pEnd, int32_t radix) {
uint64_t tmp = strtoull(str, pEnd, radix);
#ifdef TD_CHECK_STR_TO_INT_ERROR
assert(errno != ERANGE);
@ -274,7 +268,7 @@ uint64_t taosStr2UInt64(const char *str, char** pEnd, int32_t radix) {
return tmp;
}
int32_t taosStr2Int32(const char *str, char** pEnd, int32_t radix) {
int32_t taosStr2Int32(const char *str, char **pEnd, int32_t radix) {
int32_t tmp = strtol(str, pEnd, radix);
#ifdef TD_CHECK_STR_TO_INT_ERROR
assert(errno != ERANGE);
@ -283,7 +277,7 @@ int32_t taosStr2Int32(const char *str, char** pEnd, int32_t radix) {
return tmp;
}
uint32_t taosStr2UInt32(const char *str, char** pEnd, int32_t radix) {
uint32_t taosStr2UInt32(const char *str, char **pEnd, int32_t radix) {
uint32_t tmp = strtol(str, pEnd, radix);
#ifdef TD_CHECK_STR_TO_INT_ERROR
assert(errno != ERANGE);
@ -292,7 +286,7 @@ uint32_t taosStr2UInt32(const char *str, char** pEnd, int32_t radix) {
return tmp;
}
int16_t taosStr2Int16(const char *str, char** pEnd, int32_t radix) {
int16_t taosStr2Int16(const char *str, char **pEnd, int32_t radix) {
int32_t tmp = strtol(str, pEnd, radix);
#ifdef TD_CHECK_STR_TO_INT_ERROR
assert(errno != ERANGE);
@ -303,7 +297,7 @@ int16_t taosStr2Int16(const char *str, char** pEnd, int32_t radix) {
return (int16_t)tmp;
}
uint16_t taosStr2UInt16(const char *str, char** pEnd, int32_t radix) {
uint16_t taosStr2UInt16(const char *str, char **pEnd, int32_t radix) {
uint32_t tmp = strtoul(str, pEnd, radix);
#ifdef TD_CHECK_STR_TO_INT_ERROR
assert(errno != ERANGE);
@ -313,7 +307,7 @@ uint16_t taosStr2UInt16(const char *str, char** pEnd, int32_t radix) {
return (uint16_t)tmp;
}
int8_t taosStr2Int8(const char *str, char** pEnd, int32_t radix) {
int8_t taosStr2Int8(const char *str, char **pEnd, int32_t radix) {
int32_t tmp = strtol(str, pEnd, radix);
#ifdef TD_CHECK_STR_TO_INT_ERROR
assert(errno != ERANGE);
@ -324,7 +318,7 @@ int8_t taosStr2Int8(const char *str, char** pEnd, int32_t radix) {
return tmp;
}
uint8_t taosStr2UInt8(const char *str, char** pEnd, int32_t radix) {
uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix) {
uint32_t tmp = strtoul(str, pEnd, radix);
#ifdef TD_CHECK_STR_TO_INT_ERROR
assert(errno != ERANGE);
@ -334,7 +328,7 @@ uint8_t taosStr2UInt8(const char *str, char** pEnd, int32_t radix) {
return tmp;
}
double taosStr2Double(const char *str, char** pEnd) {
double taosStr2Double(const char *str, char **pEnd) {
double tmp = strtod(str, pEnd);
#ifdef TD_CHECK_STR_TO_INT_ERROR
assert(errno != ERANGE);
@ -344,7 +338,7 @@ double taosStr2Double(const char *str, char** pEnd) {
return tmp;
}
float taosStr2Float(const char *str, char** pEnd) {
float taosStr2Float(const char *str, char **pEnd) {
float tmp = strtof(str, pEnd);
#ifdef TD_CHECK_STR_TO_INT_ERROR
assert(errno != ERANGE);
@ -353,4 +347,4 @@ float taosStr2Float(const char *str, char** pEnd) {
assert(tmp != NAN);
#endif
return tmp;
}
}

58
source/os/src/osTimezone.c
View File

@ -857,19 +857,27 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) {
return;
}
buf[n] = '\0';
for (int i = n - 1; i >= 0; --i) {
if (buf[i] == '/') {
if (tz) {
tz = buf + i + 1;
break;
}
tz = buf + i + 1;
}
}
if (!tz || 0 == strchr(tz, '/')) {
char *zi = strstr(buf, "zoneinfo");
if (!zi) {
printf("parsing /etc/localtime failed");
return;
}
tz = zi + strlen("zoneinfo") + 1;
//for (int i = n - 1; i >= 0; --i) {
// if (buf[i] == '/') {
// if (tz) {
// tz = buf + i + 1;
// break;
// }
// tz = buf + i + 1;
// }
//}
//if (!tz || 0 == strchr(tz, '/')) {
// printf("parsing /etc/localtime failed");
// return;
//}
setenv("TZ", tz, 1);
tzset();
@ -900,7 +908,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) {
int n = readlink("/etc/localtime", buf, sizeof(buf));
if (n < 0) {
printf("read /etc/localtime error, reason:%s", strerror(errno));
if (taosCheckExistFile("/etc/timezone")) {
/*
* NOTE: do not remove it.
@ -962,19 +970,27 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) {
return;
}
buf[n] = '\0';
for (int i = n - 1; i >= 0; --i) {
if (buf[i] == '/') {
if (tz) {
tz = buf + i + 1;
break;
}
tz = buf + i + 1;
}
}
if (!tz || 0 == strchr(tz, '/')) {
char *zi = strstr(buf, "zoneinfo");
if (!zi) {
printf("parsing /etc/localtime failed");
return;
}
tz = zi + strlen("zoneinfo") + 1;
//for (int i = n - 1; i >= 0; --i) {
// if (buf[i] == '/') {
// if (tz) {
// tz = buf + i + 1;
// break;
// }
// tz = buf + i + 1;
// }
//}
//if (!tz || 0 == strchr(tz, '/')) {
// printf("parsing /etc/localtime failed");
// return;
//}
setenv("TZ", tz, 1);
tzset();

4
source/util/src/tpagedbuf.c
View File

@ -518,8 +518,8 @@ void releaseBufPageInfo(SDiskbasedBuf* pBuf, SPageInfo* pi) {
#ifdef BUF_PAGE_DEBUG
uDebug("page_releaseBufPageInfo pageId:%d, used:%d, offset:%"PRId64, pi->pageId, pi->used, pi->offset);
#endif
assert(pi->pData != NULL && pi->used == true);
// assert(pi->pData != NULL);
// assert(pi->pData != NULL && pi->used == true);
assert(pi->pData != NULL);
pi->used = false;
pBuf->statis.releasePages += 1;
}

14
source/util/src/tutil.c
View File

@ -64,6 +64,20 @@ int32_t strdequote(char *z) {
return j + 1; // only one quote, do nothing
}
char *strDupUnquo(const char *src) {
if (src == NULL) return NULL;
if (src[0] != '`') return strdup(src);
int32_t len = (int32_t)strlen(src);
if (src[len - 1] != '`') return NULL;
char *ret = taosMemoryMalloc(len);
if (ret == NULL) return NULL;
for (int32_t i = 0; i < len - 1; i++) {
ret[i] = src[i + 1];
}
ret[len - 1] = 0;
return ret;
}
size_t strtrim(char *z) {
int32_t i = 0;
int32_t j = 0;

4
tests/script/jenkins/basic.txt
View File

@ -163,8 +163,8 @@
# --- sma
./test.sh -f tsim/sma/drop_sma.sim
./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim
#./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
#./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
# --- valgrind
./test.sh -f tsim/valgrind/checkError1.sim

8
tests/script/loop.sh
View File

@ -13,7 +13,7 @@ CMD_NAME=
LOOP_TIMES=5
SLEEP_TIME=0
while getopts "f:t:s:" arg
while getopts "hf:t:s:" arg
do
case $arg in
f)
@ -25,6 +25,12 @@ do
s)
SLEEP_TIME=$OPTARG
;;
h)
echo "Usage: $(basename $0) -f [cmd name] "
echo " -t [loop times] "
echo " -s [sleep time] "
exit 0
;;
?)
echo "unknow argument"
;;

22
tests/script/tsim/stream/session0.sim
View File

@ -141,7 +141,7 @@ if $data01 != 7 then
goto loop1
endi
if $data02 != 9 then
if $data02 != 18 then
print =====data02=$data02
goto loop1
endi
@ -151,22 +151,22 @@ if $data03 != 4 then
goto loop1
endi
if $data04 != 1.100000000 then
if $data04 != 1.000000000 then
print ======$data04
return -1
endi
if $data05 != 0.816496581 then
if $data05 != 1.154700538 then
print ======$data05
return -1
endi
if $data06 != 3 then
if $data06 != 4 then
print ======$data06
return -1
endi
if $data07 != 1.100000000 then
if $data07 != 1.000000000 then
print ======$data07
return -1
endi
@ -235,7 +235,7 @@ sql create stream streams4 trigger at_once watermark 1d into streamt4 as select
# sql create stream streams6 trigger at_once watermark 1d into streamt6 as select _wstartts, bottom(b,3), a,c from t1 session(ts,10s);
# sql create stream streams7 trigger at_once watermark 1d into streamt7 as select _wstartts, spread(a), elapsed(ts), hyperloglog(a) from t1 session(ts,10s);
sql create stream streams7 trigger at_once watermark 1d into streamt7 as select _wstartts, spread(a), hyperloglog(a) from t1 session(ts,10s);
sql create stream streams8 trigger at_once watermark 1d into streamt8 as select _wstartts, histogram(a,"user_input", "[1,3,5,7]", 1), histogram(a,"user_input", "[1,3,5,7]", 0) from t1 session(ts,10s);
# sql create stream streams8 trigger at_once watermark 1d into streamt8 as select _wstartts, histogram(a,"user_input", "[1,3,5,7]", 1), histogram(a,"user_input", "[1,3,5,7]", 0) from t1 session(ts,10s);
sql insert into t1 values(1648791213001,1,1,1,1.0);
sql insert into t1 values(1648791213002,2,3,2,3.4);
sql insert into t1 values(1648791213003,4,9,3,4.8);
@ -288,10 +288,10 @@ if $rows == 0 then
goto loop3
endi
sql select * from streamt8;
if $rows == 0 then
print ======$rows
goto loop3
endi
#sql select * from streamt8;
#if $rows == 0 then
# print ======$rows
# goto loop3
#endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT

162
tests/script/tsim/sync/vnodesnapshot.sim Normal file
View File

@ -0,0 +1,162 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
system sh/exec.sh -n dnode4 -s start
$loop_cnt = 0
check_dnode_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6]
if $data[0][0] != 1 then
return -1
endi
if $data[0][4] != ready then
goto check_dnode_ready
endi
sql connect
sql create dnode $hostname port 7200
sql create dnode $hostname port 7300
sql create dnode $hostname port 7400
$loop_cnt = 0
check_dnode_ready_1:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnodes not ready!
return -1
endi
sql show dnodes
print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6]
if $data[0][4] != ready then
goto check_dnode_ready_1
endi
if $data[1][4] != ready then
goto check_dnode_ready_1
endi
if $data[2][4] != ready then
goto check_dnode_ready_1
endi
if $data[3][4] != ready then
goto check_dnode_ready_1
endi
$replica = 3
$vgroups = 1
print ============= create database
sql create database db replica $replica vgroups $vgroups
$loop_cnt = 0
check_db_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 100 then
print ====> db not ready!
return -1
endi
sql show databases
print ===> rows: $rows
print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] $data[2][7] $data[2][8] $data[2][9] $data[2][6] $data[2][11] $data[2][12] $data[2][13] $data[2][14] $data[2][15] $data[2][16] $data[2][17] $data[2][18] $data[2][19]
if $rows != 3 then
return -1
endi
if $data[2][19] != ready then
goto check_db_ready
endi
sql use db
$loop_cnt = 0
check_vg_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 300 then
print ====> vgroups not ready!
return -1
endi
sql show vgroups
print ===> rows: $rows
print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[0][10] $data[0][11]
if $rows != $vgroups then
return -1
endi
if $data[0][4] == leader then
if $data[0][6] == follower then
if $data[0][8] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][3]
endi
endi
elif $data[0][6] == leader then
if $data[0][4] == follower then
if $data[0][8] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][5]
endi
endi
elif $data[0][8] == leader then
if $data[0][4] == follower then
if $data[0][6] == follower then
print ---- vgroup $data[0][0] leader locate on dnode $data[0][7]
endi
endi
else
goto check_vg_ready
endi
vg_ready:
print ====> create stable/child table
sql create table stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int)
sql show stables
if $rows != 1 then
return -1
endi
sql create table ct1 using stb tags(1000)
system sh/exec.sh -n dnode4 -s stop -x SIGINT
sql insert into ct1 values(now+0s, 10, 2.0, 3.0)
sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3)
#sql flush database db;
#system sh/exec.sh -n dnode4 -s start
sql insert into ct1 values(now+1s, 81, 8.1, 8.1)(now+2s, -92, -9.2, -9.2)(now+3s, -73, -7.3, -7.3)
#system sh/exec.sh -n dnode1 -s stop -x SIGINT
#system sh/exec.sh -n dnode2 -s stop -x SIGINT
#system sh/exec.sh -n dnode3 -s stop -x SIGINT
#system sh/exec.sh -n dnode4 -s stop -x SIGINT

2
tests/script/tsim/user/basic.sim
View File

@ -22,7 +22,7 @@ sql_error ALTER USER root SYSINFO 1
sql_error ALTER USER root enable 0
sql_error ALTER USER root enable 1
sql_error create database db vgroups 1;
#sql_error create database db vgroups 1;
sql_error GRANT read ON db.* to root;
sql_error GRANT read ON *.* to root;
sql_error REVOKE read ON db.* from root;

42
tests/script/tsim/valgrind/basic1.sim
View File

@ -1,5 +1,6 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c debugflag -v 131
system sh/exec.sh -n dnode1 -s start -v
sql connect
@ -18,26 +19,29 @@ if $rows != 1 then
return -1
endi
print =============== step2: create alter drop show user
sql create user u1 pass 'taosdata'
sql show users
sql alter user u1 sysinfo 1
sql alter user u1 enable 1
sql alter user u1 pass 'taosdata'
sql drop user u1
sql_error alter user u2 sysinfo 0
print =============== step3: create drop dnode
sql create dnode $hostname port 7200
sql drop dnode 2
sql alter dnode 1 'debugflag 143'
print =============== step4: create alter drop show database
sql create database db vgroups 1
sql show databases
sql show db.vgroups
sql drop database db
print =============== step2: create show database
sql create database d1 vgroups 1 buffer 3
sql show databases
sql use d1
sql show vgroups
print =============== step3: create show stable
sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned)
sql show stables
if $rows != 1 then
return -1
endi
print =============== step4: create show table
sql create table ct1 using stb tags(1000)
sql show tables
if $rows != 1 then
return -1
endi
print =============== step5: insert data
print =============== step6: select data
_OVER:
system sh/exec.sh -n dnode1 -s stop -x SIGINT

11
tests/script/tsim/valgrind/basic2.sim
View File

@ -1,5 +1,6 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c debugflag -v 131
system sh/exec.sh -n dnode1 -s start -v
sql connect
@ -18,8 +19,14 @@ if $rows != 1 then
return -1
endi
print =============== step2: create db
sql create database db vgroups 1
print =============== step2
sql create database db vgroups 1 buffer 3
sql use db
sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned)
sql create table ct1 using stb tags(1000)
sql insert into ct1 values(now+0s, 10, 2.0, 3.0)
sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3)
_OVER:
system sh/exec.sh -n dnode1 -s stop -x SIGINT

14
tests/script/tsim/valgrind/checkError1.sim
View File

@ -1,23 +1,26 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c debugflag -v 131
system sh/exec.sh -n dnode1 -s start -v
sql connect
print =============== step1: show dnodes
$x = 0
step1:
$x = $x + 1
sleep 1000
if $x == 10 then
print ----> dnode not ready!
print ---> dnode not ready!
return -1
endi
sql show dnodes
print ----> $data00 $data01 $data02 $data03 $data04 $data05
print ---> $data00 $data01 $data02 $data03 $data04 $data05
if $rows != 1 then
return -1
endi
if $data(1)[4] != ready then
goto step1
endi
print =============== step2: create alter drop show user
sql create user u1 pass 'taosdata'
@ -28,7 +31,10 @@ sql alter user u1 pass 'taosdata'
sql drop user u1
sql_error alter user u2 sysinfo 0
print =============== step3:
print =============== step3: create drop dnode
sql create dnode $hostname port 7200
sql drop dnode 2
sql alter dnode 1 'debugflag 131'
print =============== stop
system sh/exec.sh -n dnode1 -s stop -x SIGINT

37
tests/script/tsim/valgrind/checkError2.sim
View File

@ -1,5 +1,6 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c debugflag -v 131
system sh/exec.sh -n dnode1 -s start -v
sql connect
@ -9,17 +10,45 @@ step1:
$x = $x + 1
sleep 1000
if $x == 10 then
print ----> dnode not ready!
print ---> dnode not ready!
return -1
endi
sql show dnodes
print ----> $data00 $data01 $data02 $data03 $data04 $data05
print ---> $data00 $data01 $data02 $data03 $data04 $data05
if $rows != 1 then
return -1
endi
if $data(1)[4] != ready then
goto step1
endi
print =============== step2: create db
sql create database db vgroups 1
sql create database d1 vgroups 1 buffer 3
sql show databases
sql use d1
sql show vgroups
print =============== step3: create show stable
sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned)
sql show stables
if $rows != 1 then
return -1
endi
print =============== step3: create show table
sql create table ct1 using stb tags(1000)
#sql show tables
#if $rows != 1 then
# return -1
#endi
print =============== step5: insert data
sql insert into ct1 values(now+0s, 10, 2.0, 3.0)
sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3)
print =============== step6: select data
#sql select * from ct1
#sql select * from stb
_OVER:
system sh/exec.sh -n dnode1 -s stop -x SIGINT
@ -29,7 +58,7 @@ print ----> start to check if there are ERRORS in vagrind log file for each dnod
system_content sh/checkValgrind.sh -n dnode1
print cmd return result ----> [ $system_content ]
if $system_content <= 10 then
if $system_content <= 0 then
return 0
endi

156
tests/script/tsim/valgrind/checkError3.sim Normal file
View File

@ -0,0 +1,156 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/exec.sh -n dnode1 -s start -v
system sh/exec.sh -n dnode2 -s start -v
sql connect
print =============== add dnode2 into cluster
sql create dnode $hostname port 7200
$x = 0
step1:
$x = $x + 1
sleep 1000
if $x == 10 then
print ---> dnode not ready!
return -1
endi
sql show dnodes
print ---> $data00 $data01 $data02 $data03 $data04 $data05
print ---> $data10 $data11 $data12 $data13 $data14 $data15
if $rows != 2 then
return -1
endi
if $data(1)[4] != ready then
goto step1
endi
if $data(2)[4] != ready then
goto step1
endi
print =============== create database, stable, table
sql create database db vgroups 3
sql use db
sql create table stb (ts timestamp, c int) tags (t int)
sql create table t0 using stb tags (0)
sql create table tba (ts timestamp, c1 binary(10), c2 nchar(10));
print =============== run show xxxx
sql show dnodes
if $rows != 2 then
return -1
endi
sql show mnodes
if $rows != 1 then
return -1
endi
sql show databases
if $rows != 3 then
return -1
endi
sql show stables
if $rows != 1 then
return -1
endi
sql show tables
if $rows != 2 then
return -1
endi
sql show users
if $rows != 1 then
return -1
endi
sql show vgroups
if $rows != 3 then
return -1
endi
print =============== run select * from information_schema.xxxx
sql select * from information_schema.`dnodes`
if $rows != 2 then
return -1
endi
sql select * from information_schema.`mnodes`
if $rows != 1 then
return -1
endi
sql select * from information_schema.user_databases
if $rows != 3 then
return -1
endi
sql select * from information_schema.user_stables
if $rows != 1 then
return -1
endi
sql select * from information_schema.user_tables
if $rows != 31 then
return -1
endi
sql select * from information_schema.user_users
if $rows != 1 then
return -1
endi
sql select * from information_schema.`vgroups`
if $rows != 3 then
return -1
endi
sql show variables;
if $rows != 4 then
return -1
endi
sql show dnode 1 variables;
if $rows <= 0 then
return -1
endi
sql show local variables;
if $rows <= 0 then
return -1
endi
print ==== stop dnode1 and dnode2, and restart dnodes
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
print =============== check dnode1
system_content sh/checkValgrind.sh -n dnode1
print cmd return result ----> [ $system_content ]
if $system_content <= 0 then
return 0
endi
$null=
if $system_content == $null then
return 0
endi
return -1
print =============== check dnode2
system_content sh/checkValgrind.sh -n dnode2
print cmd return result ----> [ $system_content ]
if $system_content <= 0 then
return 0
endi
$null=
if $system_content == $null then
return 0
endi
return -1

Some files were not shown because too many files have changed in this diff Show More