Merge branch '3.0' into fix/TD-17155
This commit is contained in:
Ganlin Zhao
commit
6eb40c4090
|
|
@ -97,7 +97,7 @@ ELSE ()
|
|||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
|
||||
ENDIF ()
|
||||
|
||||
IF (${SANITIZER} MATCHES "true")
|
||||
IF (${BUILD_SANITIZER})
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
|
||||
MESSAGE(STATUS "Will compile with Address Sanitizer!")
|
||||
|
|
|
|||
|
|
@ -77,6 +77,12 @@ ELSE ()
|
|||
ENDIF ()
|
||||
ENDIF ()
|
||||
|
||||
option(
|
||||
BUILD_SANITIZER
|
||||
"If build addr2line"
|
||||
OFF
|
||||
)
|
||||
|
||||
option(
|
||||
BUILD_ADDR2LINE
|
||||
"If build addr2line"
|
||||
|
|
|
|||
|
|
@ -379,11 +379,11 @@ We still use the hypothetical environment from Chapter 4. There are three measur
|
|||
|
||||
### Storage resource estimation
|
||||
|
||||
Assuming that the number of sensor devices that generate data and need to be stored is `n`, the frequency of data generation is `t` per second, and the length of each record is `L` bytes, the scale of data generated per day is `n * t * L` bytes. Assuming the compression ratio is `C`, the daily data size is `(n * t * L)/C` bytes. The storage resources are estimated to accommodate the data scale for 1.5 years. In the production environment, the compression ratio C of TDengine is generally between 5 and 7.
|
||||
Assuming that the number of sensor devices that generate data and need to be stored is `n`, the frequency of data generation is `t` per second, and the length of each record is `L` bytes, the scale of data generated per day is `86400 * n * t * L` bytes. Assuming the compression ratio is `C`, the daily data size is `(86400 * n * t * L)/C` bytes. The storage resources are estimated to accommodate the data scale for 1.5 years. In the production environment, the compression ratio C of TDengine is generally between 5 and 7.
|
||||
With additional 20% redundancy, you can calculate the required storage resources:
|
||||
|
||||
```matlab
|
||||
(n * t * L) * (365 * 1.5) * (1+20%)/C
|
||||
(86400 * n * t * L) * (365 * 1.5) * (1+20%)/C
|
||||
````
|
||||
Substituting in the above formula, the raw data generated every year is 11.8TB without considering the label information. Note that tag information is associated with each timeline in TDengine, not every record. The amount of data to be recorded is somewhat reduced relative to the generated data, and label data can be ignored as a whole. Assuming a compression ratio of 5, the size of the retained data ends up being 2.56 TB.
|
||||
|
||||
|
|
|
|||
|
|
@ -367,10 +367,10 @@ WHERE ts>=1510560000 AND ts<=1515000009
|
|||
|
||||
### 存储资源估算
|
||||
|
||||
假设产生数据并需要存储的传感器设备数量为 `n`，数据生成的频率为`t`条/秒，每条记录的长度为 `L` bytes，则每天产生的数据规模为 `n×t×L` bytes。假设压缩比为 C，则每日产生数据规模为 `(n×t×L)/C` bytes。存储资源预估为能够容纳 1.5 年的数据规模，生产环境下 TDengine 的压缩比 C 一般在 5 ~ 7 之间，同时为最后结果增加 20% 的冗余，可计算得到需要存储资源：
|
||||
假设产生数据并需要存储的传感器设备数量为 `n`，数据生成的频率为`t`条/秒，每条记录的长度为 `L` bytes，则每天产生的数据规模为 `86400×n×t×L` bytes。假设压缩比为 C，则每日产生数据规模为 `(86400×n×t×L)/C` bytes。存储资源预估为能够容纳 1.5 年的数据规模，生产环境下 TDengine 的压缩比 C 一般在 5 ~ 7 之间，同时为最后结果增加 20% 的冗余，可计算得到需要存储资源：
|
||||
|
||||
```matlab
|
||||
(n×t×L)×(365×1.5)×(1+20%)/C
|
||||
(86400×n×t×L)×(365×1.5)×(1+20%)/C
|
||||
```
|
||||
|
||||
结合以上的计算公式，将参数带入计算公式，在不考虑标签信息的情况下，每年产生的原始数据规模是 11.8TB。需要注意的是，由于标签信息在 TDengine 中关联到每个时间线，并不是每条记录。所以需要记录的数据量规模相对于产生的数据有一定的降低，而这部分标签数据整体上可以忽略不记。假设压缩比为 5，则保留的数据规模最终为 2.56 TB。
|
||||
|
|
|
|||
|
|
@ -241,7 +241,7 @@ int32_t create_topic() {
|
|||
taos_free_result(pRes);
|
||||
|
||||
pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");
|
||||
// pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");
|
||||
/*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/
|
||||
if (taos_errno(pRes) != 0) {
|
||||
printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
|
||||
return -1;
|
||||
|
|
@ -302,7 +302,7 @@ tmq_t* build_consumer() {
|
|||
tmq_conf_set(conf, "msg.with.table.name", "true");
|
||||
tmq_conf_set(conf, "enable.auto.commit", "true");
|
||||
|
||||
tmq_conf_set(conf, "experimental.snapshot.enable", "false");
|
||||
/*tmq_conf_set(conf, "experimental.snapshot.enable", "true");*/
|
||||
|
||||
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
|
||||
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
|
||||
|
|
|
|||
|
|
@ -32,6 +32,18 @@ enum {
|
|||
TMQ_CONF__RESET_OFFSET__LATEST = -1,
|
||||
};
|
||||
|
||||
// clang-format off
|
||||
#define IS_META_MSG(x) ( \
|
||||
x == TDMT_VND_CREATE_STB \
|
||||
|| x == TDMT_VND_ALTER_STB \
|
||||
|| x == TDMT_VND_DROP_STB \
|
||||
|| x == TDMT_VND_CREATE_TABLE \
|
||||
|| x == TDMT_VND_ALTER_TABLE \
|
||||
|| x == TDMT_VND_DROP_TABLE \
|
||||
|| x == TDMT_VND_DROP_TTL_TABLE \
|
||||
)
|
||||
// clang-format on
|
||||
|
||||
enum {
|
||||
TMQ_MSG_TYPE__DUMMY = 0,
|
||||
TMQ_MSG_TYPE__POLL_RSP,
|
||||
|
|
@ -105,6 +117,20 @@ typedef struct SSDataBlock {
|
|||
SDataBlockInfo info;
|
||||
} SSDataBlock;
|
||||
|
||||
enum {
|
||||
FETCH_TYPE__DATA = 1,
|
||||
FETCH_TYPE__META,
|
||||
FETCH_TYPE__NONE,
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
int8_t fetchType;
|
||||
union {
|
||||
SSDataBlock data;
|
||||
void* meta;
|
||||
};
|
||||
} SFetchRet;
|
||||
|
||||
typedef struct SVarColAttr {
|
||||
int32_t* offset; // start position for each entry in the list
|
||||
uint32_t length; // used buffer size that contain the valid data
|
||||
|
|
|
|||
|
|
@ -2826,8 +2826,8 @@ typedef struct {
|
|||
|
||||
static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp) {
|
||||
int32_t tlen = 0;
|
||||
tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
|
||||
tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
|
||||
// tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
|
||||
// tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
|
||||
tlen += taosEncodeFixedI16(buf, pRsp->resMsgType);
|
||||
tlen += taosEncodeFixedI32(buf, pRsp->metaRspLen);
|
||||
tlen += taosEncodeBinary(buf, pRsp->metaRsp, pRsp->metaRspLen);
|
||||
|
|
@ -2835,8 +2835,8 @@ static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp
|
|||
}
|
||||
|
||||
static FORCE_INLINE void* tDecodeSMqMetaRsp(const void* buf, SMqMetaRsp* pRsp) {
|
||||
buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
|
||||
buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
|
||||
// buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
|
||||
// buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
|
||||
buf = taosDecodeFixedI16(buf, &pRsp->resMsgType);
|
||||
buf = taosDecodeFixedI32(buf, &pRsp->metaRspLen);
|
||||
buf = taosDecodeBinary(buf, &pRsp->metaRsp, pRsp->metaRspLen);
|
||||
|
|
|
|||
|
|
@ -30,15 +30,15 @@ struct SRpcMsg;
|
|||
struct SSubplan;
|
||||
|
||||
typedef struct SReadHandle {
|
||||
void* reader;
|
||||
void* tqReader;
|
||||
void* meta;
|
||||
void* config;
|
||||
void* vnode;
|
||||
void* mnd;
|
||||
SMsgCb* pMsgCb;
|
||||
|
||||
// int8_t initTsdbReader;
|
||||
bool tqReader;
|
||||
bool initMetaReader;
|
||||
bool initTableReader;
|
||||
bool initTqReader;
|
||||
} SReadHandle;
|
||||
|
||||
typedef enum {
|
||||
|
|
@ -52,7 +52,7 @@ typedef enum {
|
|||
* @param streamReadHandle
|
||||
* @return
|
||||
*/
|
||||
qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle);
|
||||
qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers);
|
||||
|
||||
/**
|
||||
* Switch the stream scan to snapshot mode
|
||||
|
|
@ -176,6 +176,9 @@ int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts);
|
|||
|
||||
int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts);
|
||||
|
||||
void* qExtractReaderFromStreamScanner(void* scanner);
|
||||
int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -223,7 +223,7 @@ typedef struct {
|
|||
SEpSet epSet;
|
||||
} SStreamChildEpInfo;
|
||||
|
||||
struct SStreamTask {
|
||||
typedef struct SStreamTask {
|
||||
int64_t streamId;
|
||||
int32_t taskId;
|
||||
int8_t isDataScan;
|
||||
|
|
@ -277,7 +277,7 @@ struct SStreamTask {
|
|||
|
||||
// msg handle
|
||||
SMsgCb* pMsgCb;
|
||||
};
|
||||
} SStreamTask;
|
||||
|
||||
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
|
||||
int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo);
|
||||
|
|
@ -288,6 +288,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
|
|||
void tFreeSStreamTask(SStreamTask* pTask);
|
||||
|
||||
static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) {
|
||||
#if 0
|
||||
while (1) {
|
||||
int8_t inputStatus =
|
||||
atomic_val_compare_exchange_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL, TASK_INPUT_STATUS__PROCESSING);
|
||||
|
|
@ -296,6 +297,7 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem
|
|||
}
|
||||
ASSERT(0);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
|
||||
SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem);
|
||||
|
|
@ -316,8 +318,10 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem
|
|||
atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE, TASK_TRIGGER_STATUS__ACTIVE);
|
||||
}
|
||||
|
||||
#if 0
|
||||
// TODO: back pressure
|
||||
atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL);
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ typedef struct {
|
|||
EWalType level; // wal level
|
||||
} SWalCfg;
|
||||
|
||||
typedef struct SWalVer {
|
||||
typedef struct {
|
||||
int64_t firstVer;
|
||||
int64_t verInSnapshotting;
|
||||
int64_t snapshotVer;
|
||||
|
|
@ -149,17 +149,22 @@ typedef struct SWal {
|
|||
SWalCkHead writeHead;
|
||||
} SWal; // WAL HANDLE
|
||||
|
||||
typedef struct SWalReadHandle {
|
||||
typedef struct {
|
||||
int8_t scanUncommited;
|
||||
int8_t scanMeta;
|
||||
} SWalFilterCond;
|
||||
|
||||
typedef struct {
|
||||
SWal *pWal;
|
||||
TdFilePtr pReadLogTFile;
|
||||
TdFilePtr pReadIdxTFile;
|
||||
TdFilePtr pLogFile;
|
||||
TdFilePtr pIdxFile;
|
||||
int64_t curFileFirstVer;
|
||||
int64_t curVersion;
|
||||
int64_t capacity;
|
||||
int64_t status; // if cursor valid
|
||||
TdThreadMutex mutex;
|
||||
SWalFilterCond cond;
|
||||
SWalCkHead *pHead;
|
||||
} SWalReadHandle;
|
||||
} SWalReader;
|
||||
|
||||
// module initialization
|
||||
int32_t walInit();
|
||||
|
|
@ -178,7 +183,6 @@ void walFsync(SWal *, bool force);
|
|||
|
||||
// apis for lifecycle management
|
||||
int32_t walCommit(SWal *, int64_t ver);
|
||||
// truncate after
|
||||
int32_t walRollback(SWal *, int64_t ver);
|
||||
// notify that previous logs can be pruned safely
|
||||
int32_t walBeginSnapshot(SWal *, int64_t ver);
|
||||
|
|
@ -187,15 +191,16 @@ int32_t walRestoreFromSnapshot(SWal *, int64_t ver);
|
|||
// int32_t walDataCorrupted(SWal*);
|
||||
|
||||
// read
|
||||
SWalReadHandle *walOpenReadHandle(SWal *);
|
||||
void walCloseReadHandle(SWalReadHandle *);
|
||||
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
|
||||
SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond);
|
||||
void walCloseReader(SWalReader *pRead);
|
||||
int32_t walReadVer(SWalReader *pRead, int64_t ver);
|
||||
int32_t walNextValidMsg(SWalReader *pRead);
|
||||
|
||||
// only for tq usage
|
||||
void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity);
|
||||
int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalCkHead *pHead);
|
||||
int32_t walFetchBody(SWalReadHandle *pRead, SWalCkHead **ppHead);
|
||||
int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalCkHead *pHead);
|
||||
void walSetReaderCapacity(SWalReader *pRead, int32_t capacity);
|
||||
int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead);
|
||||
int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead);
|
||||
int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead);
|
||||
|
||||
typedef struct {
|
||||
int64_t refId;
|
||||
|
|
@ -207,10 +212,11 @@ void walCloseRef(SWalRef *);
|
|||
int32_t walRefVer(SWalRef *, int64_t ver);
|
||||
int32_t walUnrefVer(SWal *);
|
||||
|
||||
// help function for raft
|
||||
bool walLogExist(SWal *, int64_t ver);
|
||||
bool walIsEmpty(SWal *);
|
||||
|
||||
// lifecycle check
|
||||
bool walIsEmpty(SWal *);
|
||||
int64_t walGetFirstVer(SWal *);
|
||||
int64_t walGetSnapshotVer(SWal *);
|
||||
int64_t walGetLastVer(SWal *);
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ void taosIp2String(uint32_t ip, char *str);
|
|||
void taosIpPort2String(uint32_t ip, uint16_t port, char *str);
|
||||
|
||||
void *tmemmem(const char *haystack, int hlen, const char *needle, int nlen);
|
||||
char *strDupUnquo(const char *src);
|
||||
|
||||
static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) {
|
||||
T_MD5_CTX context;
|
||||
|
|
|
|||
|
|
@ -62,7 +62,6 @@ struct tmq_conf_t {
|
|||
char* ip;
|
||||
char* user;
|
||||
char* pass;
|
||||
/*char* db;*/
|
||||
tmq_commit_cb* commitCb;
|
||||
void* commitCbUserParam;
|
||||
};
|
||||
|
|
@ -338,7 +337,7 @@ tmq_list_t* tmq_list_new() {
|
|||
|
||||
int32_t tmq_list_append(tmq_list_t* list, const char* src) {
|
||||
SArray* container = &list->container;
|
||||
char* topic = strdup(src);
|
||||
char* topic = strDupUnquo(src);
|
||||
if (taosArrayPush(container, &topic) == NULL) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -238,10 +238,12 @@ SArray *mmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
|
|
|
|||
|
|
@ -377,10 +377,12 @@ SArray *vmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_SET_VNODE_STANDBY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
|
|
|
|||
|
|
@ -146,19 +146,37 @@ int32_t tsdbLastrowReaderClose(void *pReader);
|
|||
|
||||
// tq
|
||||
|
||||
typedef struct STqReadHandle SStreamReader;
|
||||
typedef struct STqReader {
|
||||
int64_t ver;
|
||||
const SSubmitReq *pMsg;
|
||||
SSubmitBlk *pBlock;
|
||||
SSubmitMsgIter msgIter;
|
||||
SSubmitBlkIter blkIter;
|
||||
|
||||
SStreamReader *tqInitSubmitMsgScanner(SMeta *pMeta);
|
||||
SWalReader *pWalReader;
|
||||
|
||||
void tqReadHandleSetColIdList(SStreamReader *pReadHandle, SArray *pColIdList);
|
||||
int32_t tqReadHandleSetTbUidList(SStreamReader *pHandle, const SArray *tbUidList);
|
||||
int32_t tqReadHandleAddTbUidList(SStreamReader *pHandle, const SArray *tbUidList);
|
||||
int32_t tqReadHandleRemoveTbUidList(SStreamReader *pHandle, const SArray *tbUidList);
|
||||
SMeta *pVnodeMeta;
|
||||
SHashObj *tbIdHash;
|
||||
SArray *pColIdList; // SArray<int16_t>
|
||||
|
||||
int32_t tqReadHandleSetMsg(SStreamReader *pHandle, SSubmitReq *pMsg, int64_t ver);
|
||||
bool tqNextDataBlock(SStreamReader *pHandle);
|
||||
bool tqNextDataBlockFilterOut(SStreamReader *pHandle, SHashObj *filterOutUids);
|
||||
int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, SStreamReader *pHandle);
|
||||
int32_t cachedSchemaVer;
|
||||
int64_t cachedSchemaSuid;
|
||||
SSchemaWrapper *pSchemaWrapper;
|
||||
STSchema *pSchema;
|
||||
} STqReader;
|
||||
|
||||
STqReader *tqOpenReader(SVnode *pVnode);
|
||||
void tqCloseReader(STqReader *);
|
||||
|
||||
void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList);
|
||||
int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList);
|
||||
int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList);
|
||||
int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
|
||||
|
||||
int32_t tqReaderSetDataMsg(STqReader *pReader, SSubmitReq *pMsg, int64_t ver);
|
||||
bool tqNextDataBlock(STqReader *pReader);
|
||||
bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
|
||||
int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader);
|
||||
|
||||
// sma
|
||||
int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
|
||||
|
|
|
|||
|
|
@ -40,38 +40,10 @@ extern "C" {
|
|||
#define tqDebug(...) do { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0)
|
||||
#define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
|
||||
|
||||
#define IS_META_MSG(x) ( \
|
||||
x == TDMT_VND_CREATE_STB \
|
||||
|| x == TDMT_VND_ALTER_STB \
|
||||
|| x == TDMT_VND_DROP_STB \
|
||||
|| x == TDMT_VND_CREATE_TABLE \
|
||||
|| x == TDMT_VND_ALTER_TABLE \
|
||||
|| x == TDMT_VND_DROP_TABLE \
|
||||
|| x == TDMT_VND_DROP_TTL_TABLE \
|
||||
)
|
||||
// clang-format on
|
||||
|
||||
typedef struct STqOffsetStore STqOffsetStore;
|
||||
|
||||
// tqRead
|
||||
|
||||
struct STqReadHandle {
|
||||
int64_t ver;
|
||||
const SSubmitReq* pMsg;
|
||||
SSubmitBlk* pBlock;
|
||||
SSubmitMsgIter msgIter;
|
||||
SSubmitBlkIter blkIter;
|
||||
|
||||
SMeta* pVnodeMeta;
|
||||
SHashObj* tbIdHash;
|
||||
SArray* pColIdList; // SArray<int16_t>
|
||||
|
||||
int32_t cachedSchemaVer;
|
||||
int64_t cachedSchemaSuid;
|
||||
SSchemaWrapper* pSchemaWrapper;
|
||||
STSchema* pSchema;
|
||||
};
|
||||
|
||||
// tqPush
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -111,7 +83,7 @@ typedef struct {
|
|||
typedef struct {
|
||||
int8_t subType;
|
||||
|
||||
SStreamReader* pExecReader[5];
|
||||
STqReader* pExecReader[5];
|
||||
union {
|
||||
STqExecCol execCol;
|
||||
STqExecTb execTb;
|
||||
|
|
@ -127,8 +99,8 @@ typedef struct {
|
|||
int32_t epoch;
|
||||
int8_t fetchMeta;
|
||||
|
||||
// reader
|
||||
SWalReadHandle* pWalReader;
|
||||
// TODO remove
|
||||
SWalReader* pWalReader;
|
||||
|
||||
// push
|
||||
STqPushHandle pushHandle;
|
||||
|
|
|
|||
|
|
@ -439,6 +439,7 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv
|
|||
tDecoderInit(&dc, pData, nData);
|
||||
tDecodeSSchemaWrapper(&dc, pSchemaWrapper);
|
||||
tDecoderClear(&dc);
|
||||
tdbFree(pData);
|
||||
|
||||
// convert
|
||||
STSchemaBuilder sb = {0};
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
#include "meta.h"
|
||||
|
||||
static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
|
||||
static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
|
||||
static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME);
|
||||
static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
|
||||
static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME);
|
||||
|
|
@ -92,7 +93,65 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
|
|||
} else if (type == TSDB_DATA_TYPE_BOOL) {
|
||||
int val = *(int *)(&pTagVal->i64);
|
||||
int len = sizeof(val);
|
||||
term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_INT, key, nKey, (const char *)&val, len);
|
||||
term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BOOL, key, nKey, (const char *)&val, len);
|
||||
}
|
||||
if (term != NULL) {
|
||||
indexMultiTermAdd(terms, term);
|
||||
}
|
||||
}
|
||||
indexJsonPut(pMeta->pTagIvtIdx, terms, tuid);
|
||||
indexMultiTermDestroy(terms);
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) {
|
||||
#ifdef USE_INVERTED_INDEX
|
||||
if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
|
||||
return -1;
|
||||
}
|
||||
void * data = pCtbEntry->ctbEntry.pTags;
|
||||
const char *tagName = pSchema->name;
|
||||
|
||||
tb_uid_t suid = pCtbEntry->ctbEntry.suid;
|
||||
tb_uid_t tuid = pCtbEntry->uid;
|
||||
const void *pTagData = pCtbEntry->ctbEntry.pTags;
|
||||
int32_t nTagData = 0;
|
||||
|
||||
SArray *pTagVals = NULL;
|
||||
if (tTagToValArray((const STag *)data, &pTagVals) != 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
SIndexMultiTerm *terms = indexMultiTermCreate();
|
||||
int16_t nCols = taosArrayGetSize(pTagVals);
|
||||
for (int i = 0; i < nCols; i++) {
|
||||
STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
|
||||
char type = pTagVal->type;
|
||||
|
||||
char * key = pTagVal->pKey;
|
||||
int32_t nKey = strlen(key);
|
||||
|
||||
SIndexTerm *term = NULL;
|
||||
if (type == TSDB_DATA_TYPE_NULL) {
|
||||
term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0);
|
||||
} else if (type == TSDB_DATA_TYPE_NCHAR) {
|
||||
if (pTagVal->nData > 0) {
|
||||
char * val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
|
||||
int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
|
||||
memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
|
||||
type = TSDB_DATA_TYPE_VARCHAR;
|
||||
term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, val, len);
|
||||
} else if (pTagVal->nData == 0) {
|
||||
term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, pTagVal->pData, 0);
|
||||
}
|
||||
} else if (type == TSDB_DATA_TYPE_DOUBLE) {
|
||||
double val = *(double *)(&pTagVal->i64);
|
||||
int len = sizeof(val);
|
||||
term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, (const char *)&val, len);
|
||||
} else if (type == TSDB_DATA_TYPE_BOOL) {
|
||||
int val = *(int *)(&pTagVal->i64);
|
||||
int len = sizeof(val);
|
||||
term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_BOOL, key, nKey, (const char *)&val, len);
|
||||
}
|
||||
if (term != NULL) {
|
||||
indexMultiTermAdd(terms, term);
|
||||
|
|
@ -434,9 +493,33 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
|
|||
|
||||
if (type) *type = e.type;
|
||||
|
||||
if (e.type == TSDB_CHILD_TABLE) {
|
||||
void *tData = NULL;
|
||||
int tLen = 0;
|
||||
|
||||
if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) {
|
||||
version = *(int64_t *)tData;
|
||||
STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = version};
|
||||
if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) {
|
||||
SDecoder tdc = {0};
|
||||
SMetaEntry stbEntry = {0};
|
||||
|
||||
tDecoderInit(&tdc, tData, tLen);
|
||||
metaDecodeEntry(&tdc, &stbEntry);
|
||||
const SSchema *pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0];
|
||||
if (pTagColumn->type == TSDB_DATA_TYPE_JSON) {
|
||||
metaDelJsonVarFromIdx(pMeta, &e, pTagColumn);
|
||||
}
|
||||
tDecoderClear(&tdc);
|
||||
}
|
||||
tdbFree(tData);
|
||||
}
|
||||
}
|
||||
|
||||
tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pMeta->txn);
|
||||
tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, &pMeta->txn);
|
||||
tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), &pMeta->txn);
|
||||
|
||||
if (e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e);
|
||||
|
||||
if (e.type == TSDB_CHILD_TABLE) {
|
||||
|
|
|
|||
|
|
@ -277,6 +277,8 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo
|
|||
pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY;
|
||||
}
|
||||
pItem->level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2);
|
||||
smaInfo("vgId:%d table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 ", finally maxdelay:%"PRIi32, SMA_VID(pSma),
|
||||
pRSmaInfo->suid, idx + 1, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay);
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
_err:
|
||||
|
|
@ -325,14 +327,14 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
|
|||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
SStreamReader *pReadHandle = tqInitSubmitMsgScanner(pMeta);
|
||||
if (!pReadHandle) {
|
||||
STqReader *pReader = tqOpenReader(pVnode);
|
||||
if (!pReader) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
goto _err;
|
||||
}
|
||||
|
||||
SReadHandle handle = {
|
||||
.reader = pReadHandle,
|
||||
.tqReader = pReader,
|
||||
.meta = pMeta,
|
||||
.pMsgCb = pMsgCb,
|
||||
.vnode = pVnode,
|
||||
|
|
@ -364,7 +366,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
|
|||
return TSDB_CODE_SUCCESS;
|
||||
_err:
|
||||
tdFreeRSmaInfo(pRSmaInfo);
|
||||
taosMemoryFree(pReadHandle);
|
||||
taosMemoryFree(pReader);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -28,8 +28,12 @@ int32_t tqInit() {
|
|||
atomic_store_8(&tqMgmt.inited, 0);
|
||||
return -1;
|
||||
}
|
||||
if (streamInit() < 0) {
|
||||
return -1;
|
||||
}
|
||||
atomic_store_8(&tqMgmt.inited, 1);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -42,6 +46,7 @@ void tqCleanUp() {
|
|||
|
||||
if (old == 1) {
|
||||
taosTmrCleanUp(tqMgmt.timer);
|
||||
streamCleanUp();
|
||||
atomic_store_8(&tqMgmt.inited, 0);
|
||||
}
|
||||
}
|
||||
|
|
@ -144,7 +149,6 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
|
|||
SEncoder encoder;
|
||||
tEncoderInit(&encoder, abuf, len);
|
||||
tEncodeSMqDataRsp(&encoder, pRsp);
|
||||
/*tEncodeSMqDataBlkRsp(&abuf, pRsp);*/
|
||||
|
||||
SRpcMsg rsp = {
|
||||
.info = pMsg->info,
|
||||
|
|
@ -361,8 +365,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
|||
ASSERT(IS_META_MSG(pHead->msgType));
|
||||
tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType);
|
||||
SMqMetaRsp metaRsp = {0};
|
||||
metaRsp.reqOffset = pReq->reqOffset.version;
|
||||
metaRsp.rspOffset = fetchVer;
|
||||
/*metaRsp.reqOffset = pReq->reqOffset.version;*/
|
||||
/*metaRsp.rspOffset = fetchVer;*/
|
||||
/*metaRsp.rspOffsetNew.version = fetchVer;*/
|
||||
tqOffsetResetToLog(&metaRsp.reqOffsetNew, pReq->reqOffset.version);
|
||||
tqOffsetResetToLog(&metaRsp.rspOffsetNew, fetchVer);
|
||||
metaRsp.resMsgType = pHead->msgType;
|
||||
metaRsp.metaRspLen = pHead->bodyLen;
|
||||
metaRsp.metaRsp = pHead->body;
|
||||
|
|
@ -439,27 +446,39 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
|
|||
pHandle->execHandle.subType = req.subType;
|
||||
pHandle->fetchMeta = req.withMeta;
|
||||
|
||||
pHandle->pWalReader = walOpenReadHandle(pTq->pVnode->pWal);
|
||||
for (int32_t i = 0; i < 5; i++) {
|
||||
pHandle->execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
|
||||
}
|
||||
pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
|
||||
/*for (int32_t i = 0; i < 5; i++) {*/
|
||||
/*pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);*/
|
||||
/*}*/
|
||||
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
|
||||
pHandle->execHandle.execCol.qmsg = req.qmsg;
|
||||
req.qmsg = NULL;
|
||||
for (int32_t i = 0; i < 5; i++) {
|
||||
SReadHandle handle = {
|
||||
.reader = pHandle->execHandle.pExecReader[i],
|
||||
.tqReader = pHandle->execHandle.pExecReader[i],
|
||||
.meta = pTq->pVnode->pMeta,
|
||||
.vnode = pTq->pVnode,
|
||||
.tqReader = true,
|
||||
.initTableReader = true,
|
||||
.initTqReader = true,
|
||||
};
|
||||
pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle);
|
||||
ASSERT(pHandle->execHandle.execCol.task[i]);
|
||||
void* scanner = NULL;
|
||||
qExtractStreamScanner(pHandle->execHandle.execCol.task[i], &scanner);
|
||||
ASSERT(scanner);
|
||||
pHandle->execHandle.pExecReader[i] = qExtractReaderFromStreamScanner(scanner);
|
||||
ASSERT(pHandle->execHandle.pExecReader[i]);
|
||||
}
|
||||
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) {
|
||||
for (int32_t i = 0; i < 5; i++) {
|
||||
pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);
|
||||
}
|
||||
pHandle->execHandle.execDb.pFilterOutTbUid =
|
||||
taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
|
||||
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
|
||||
for (int32_t i = 0; i < 5; i++) {
|
||||
pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);
|
||||
}
|
||||
pHandle->execHandle.execTb.suid = req.suid;
|
||||
SArray* tbUidList = taosArrayInit(0, sizeof(int64_t));
|
||||
vnodeGetCtbIdList(pTq->pVnode, req.suid, tbUidList);
|
||||
|
|
@ -469,7 +488,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
|
|||
tqDebug("vg %d, idx %d, uid: %ld", TD_VID(pTq->pVnode), i, tbUid);
|
||||
}
|
||||
for (int32_t i = 0; i < 5; i++) {
|
||||
tqReadHandleSetTbUidList(pHandle->execHandle.pExecReader[i], tbUidList);
|
||||
tqReaderSetTbUidList(pHandle->execHandle.pExecReader[i], tbUidList);
|
||||
}
|
||||
taosArrayDestroy(tbUidList);
|
||||
}
|
||||
|
|
@ -522,19 +541,16 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
|
|||
if (pTask->execType != TASK_EXEC__NONE) {
|
||||
// expand runners
|
||||
if (pTask->isDataScan) {
|
||||
SStreamReader* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
|
||||
SReadHandle handle = {
|
||||
.reader = pStreamReader,
|
||||
.meta = pTq->pVnode->pMeta,
|
||||
.vnode = pTq->pVnode,
|
||||
.initTqReader = 1,
|
||||
};
|
||||
/*pTask->exec.inputHandle = pStreamReader;*/
|
||||
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
|
||||
ASSERT(pTask->exec.executor);
|
||||
} else {
|
||||
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL);
|
||||
ASSERT(pTask->exec.executor);
|
||||
}
|
||||
ASSERT(pTask->exec.executor);
|
||||
}
|
||||
|
||||
// sink
|
||||
|
|
|
|||
|
|
@ -135,8 +135,8 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR
|
|||
}
|
||||
} else if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
|
||||
pRsp->withSchema = 1;
|
||||
SStreamReader* pReader = pExec->pExecReader[workerId];
|
||||
tqReadHandleSetMsg(pReader, pReq, 0);
|
||||
STqReader* pReader = pExec->pExecReader[workerId];
|
||||
tqReaderSetDataMsg(pReader, pReq, 0);
|
||||
while (tqNextDataBlock(pReader)) {
|
||||
SSDataBlock block = {0};
|
||||
if (tqRetrieveDataBlock(&block, pReader) < 0) {
|
||||
|
|
@ -153,8 +153,8 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR
|
|||
}
|
||||
} else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
|
||||
pRsp->withSchema = 1;
|
||||
SStreamReader* pReader = pExec->pExecReader[workerId];
|
||||
tqReadHandleSetMsg(pReader, pReq, 0);
|
||||
STqReader* pReader = pExec->pExecReader[workerId];
|
||||
tqReaderSetDataMsg(pReader, pReq, 0);
|
||||
while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) {
|
||||
SSDataBlock block = {0};
|
||||
if (tqRetrieveDataBlock(&block, pReader) < 0) {
|
||||
|
|
|
|||
|
|
@ -77,14 +77,14 @@ int32_t tqMetaOpen(STQ* pTq) {
|
|||
STqHandle handle;
|
||||
tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
|
||||
tDecodeSTqHandle(&decoder, &handle);
|
||||
handle.pWalReader = walOpenReadHandle(pTq->pVnode->pWal);
|
||||
handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
|
||||
for (int32_t i = 0; i < 5; i++) {
|
||||
handle.execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
|
||||
handle.execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);
|
||||
}
|
||||
if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
|
||||
for (int32_t i = 0; i < 5; i++) {
|
||||
SReadHandle reader = {
|
||||
.reader = handle.execHandle.pExecReader[i],
|
||||
.tqReader = handle.execHandle.pExecReader[i],
|
||||
.meta = pTq->pVnode->pMeta,
|
||||
.pMsgCb = &pTq->pVnode->msgCb,
|
||||
.vnode = pTq->pVnode,
|
||||
|
|
|
|||
|
|
@ -15,6 +15,11 @@
|
|||
|
||||
#include "tq.h"
|
||||
|
||||
int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset) {
|
||||
/*if ()*/
|
||||
return 0;
|
||||
}
|
||||
|
||||
int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) {
|
||||
int32_t code = 0;
|
||||
taosThreadMutexLock(&pHandle->pWalReader->mutex);
|
||||
|
|
@ -73,53 +78,107 @@ END:
|
|||
return code;
|
||||
}
|
||||
|
||||
SStreamReader* tqInitSubmitMsgScanner(SMeta* pMeta) {
|
||||
SStreamReader* pReadHandle = taosMemoryMalloc(sizeof(SStreamReader));
|
||||
if (pReadHandle == NULL) {
|
||||
STqReader* tqOpenReader(SVnode* pVnode) {
|
||||
STqReader* pReader = taosMemoryMalloc(sizeof(STqReader));
|
||||
if (pReader == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
pReadHandle->pVnodeMeta = pMeta;
|
||||
pReadHandle->pMsg = NULL;
|
||||
pReadHandle->ver = -1;
|
||||
pReadHandle->pColIdList = NULL;
|
||||
pReadHandle->cachedSchemaVer = 0;
|
||||
pReadHandle->cachedSchemaSuid = 0;
|
||||
pReadHandle->pSchema = NULL;
|
||||
pReadHandle->pSchemaWrapper = NULL;
|
||||
pReadHandle->tbIdHash = NULL;
|
||||
return pReadHandle;
|
||||
|
||||
// TODO open
|
||||
/*pReader->pWalReader = walOpenReader(pVnode->pWal, NULL);*/
|
||||
|
||||
pReader->pVnodeMeta = pVnode->pMeta;
|
||||
pReader->pMsg = NULL;
|
||||
pReader->ver = -1;
|
||||
pReader->pColIdList = NULL;
|
||||
pReader->cachedSchemaVer = 0;
|
||||
pReader->cachedSchemaSuid = 0;
|
||||
pReader->pSchema = NULL;
|
||||
pReader->pSchemaWrapper = NULL;
|
||||
pReader->tbIdHash = NULL;
|
||||
return pReader;
|
||||
}
|
||||
|
||||
int32_t tqReadHandleSetMsg(SStreamReader* pReadHandle, SSubmitReq* pMsg, int64_t ver) {
|
||||
pReadHandle->pMsg = pMsg;
|
||||
|
||||
if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1;
|
||||
while (true) {
|
||||
if (tGetSubmitMsgNext(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1;
|
||||
if (pReadHandle->pBlock == NULL) break;
|
||||
void tqCloseReader(STqReader* pReader) {
|
||||
// close wal reader
|
||||
// free cached schema
|
||||
// free hash
|
||||
taosMemoryFree(pReader);
|
||||
}
|
||||
|
||||
if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1;
|
||||
pReadHandle->ver = ver;
|
||||
memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter));
|
||||
int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
|
||||
bool fromProcessedMsg = pReader->pMsg != NULL;
|
||||
|
||||
while (1) {
|
||||
if (!fromProcessedMsg) {
|
||||
if (walNextValidMsg(pReader->pWalReader) < 0) {
|
||||
ret->fetchType = FETCH_TYPE__NONE;
|
||||
return -1;
|
||||
}
|
||||
void* body = pReader->pWalReader->pHead->head.body;
|
||||
if (pReader->pWalReader->pHead->head.msgType != TDMT_VND_SUBMIT) {
|
||||
// TODO do filter
|
||||
ret->fetchType = FETCH_TYPE__META;
|
||||
ret->meta = pReader->pWalReader->pHead->head.body;
|
||||
return 0;
|
||||
} else {
|
||||
tqReaderSetDataMsg(pReader, body, pReader->pWalReader->pHead->head.version);
|
||||
}
|
||||
}
|
||||
|
||||
while (tqNextDataBlock(pReader)) {
|
||||
memset(&ret->data, 0, sizeof(SSDataBlock));
|
||||
int32_t code = tqRetrieveDataBlock(&ret->data, pReader);
|
||||
if (code != 0 || ret->data.info.rows == 0) {
|
||||
if (fromProcessedMsg) {
|
||||
ret->fetchType = FETCH_TYPE__NONE;
|
||||
return 0;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
ret->fetchType = FETCH_TYPE__DATA;
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool tqNextDataBlock(SStreamReader* pHandle) {
|
||||
if (pHandle->pMsg == NULL) return false;
|
||||
if (fromProcessedMsg) {
|
||||
ret->fetchType = FETCH_TYPE__NONE;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int32_t tqReaderSetDataMsg(STqReader* pReader, SSubmitReq* pMsg, int64_t ver) {
|
||||
pReader->pMsg = pMsg;
|
||||
|
||||
if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1;
|
||||
while (true) {
|
||||
if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) return -1;
|
||||
if (pReader->pBlock == NULL) break;
|
||||
}
|
||||
|
||||
if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1;
|
||||
pReader->ver = ver;
|
||||
memset(&pReader->blkIter, 0, sizeof(SSubmitBlkIter));
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool tqNextDataBlock(STqReader* pReader) {
|
||||
if (pReader->pMsg == NULL) return false;
|
||||
while (1) {
|
||||
if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
|
||||
if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) {
|
||||
return false;
|
||||
}
|
||||
if (pHandle->pBlock == NULL) {
|
||||
pHandle->pMsg = NULL;
|
||||
if (pReader->pBlock == NULL) {
|
||||
pReader->pMsg = NULL;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (pHandle->tbIdHash == NULL) {
|
||||
if (pReader->tbIdHash == NULL) {
|
||||
return true;
|
||||
}
|
||||
void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->msgIter.uid, sizeof(int64_t));
|
||||
void* ret = taosHashGet(pReader->tbIdHash, &pReader->msgIter.uid, sizeof(int64_t));
|
||||
/*tqDebug("search uid %ld", pHandle->msgIter.uid);*/
|
||||
if (ret != NULL) {
|
||||
/*tqDebug("find uid %ld", pHandle->msgIter.uid);*/
|
||||
|
|
@ -129,7 +188,7 @@ bool tqNextDataBlock(SStreamReader* pHandle) {
|
|||
return false;
|
||||
}
|
||||
|
||||
bool tqNextDataBlockFilterOut(SStreamReader* pHandle, SHashObj* filterOutUids) {
|
||||
bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) {
|
||||
while (1) {
|
||||
if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
|
||||
return false;
|
||||
|
|
@ -145,38 +204,38 @@ bool tqNextDataBlockFilterOut(SStreamReader* pHandle, SHashObj* filterOutUids) {
|
|||
return false;
|
||||
}
|
||||
|
||||
int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) {
|
||||
int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) {
|
||||
// TODO: cache multiple schema
|
||||
int32_t sversion = htonl(pHandle->pBlock->sversion);
|
||||
if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion ||
|
||||
pHandle->cachedSchemaSuid != pHandle->msgIter.suid) {
|
||||
if (pHandle->pSchema) taosMemoryFree(pHandle->pSchema);
|
||||
pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion);
|
||||
if (pHandle->pSchema == NULL) {
|
||||
int32_t sversion = htonl(pReader->pBlock->sversion);
|
||||
if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion ||
|
||||
pReader->cachedSchemaSuid != pReader->msgIter.suid) {
|
||||
if (pReader->pSchema) taosMemoryFree(pReader->pSchema);
|
||||
pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion);
|
||||
if (pReader->pSchema == NULL) {
|
||||
tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table",
|
||||
pHandle->msgIter.uid, pHandle->msgIter.suid, pHandle->cachedSchemaVer);
|
||||
pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer);
|
||||
/*ASSERT(0);*/
|
||||
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (pHandle->pSchemaWrapper) tDeleteSSchemaWrapper(pHandle->pSchemaWrapper);
|
||||
pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion, true);
|
||||
if (pHandle->pSchemaWrapper == NULL) {
|
||||
if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
|
||||
pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, true);
|
||||
if (pReader->pSchemaWrapper == NULL) {
|
||||
tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table",
|
||||
pHandle->msgIter.uid, pHandle->cachedSchemaVer);
|
||||
pReader->msgIter.uid, pReader->cachedSchemaVer);
|
||||
/*ASSERT(0);*/
|
||||
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
|
||||
return -1;
|
||||
}
|
||||
pHandle->cachedSchemaVer = sversion;
|
||||
pHandle->cachedSchemaSuid = pHandle->msgIter.suid;
|
||||
pReader->cachedSchemaVer = sversion;
|
||||
pReader->cachedSchemaSuid = pReader->msgIter.suid;
|
||||
}
|
||||
|
||||
STSchema* pTschema = pHandle->pSchema;
|
||||
SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper;
|
||||
STSchema* pTschema = pReader->pSchema;
|
||||
SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
|
||||
|
||||
int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
|
||||
int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList);
|
||||
|
||||
if (colNumNeed == 0) {
|
||||
int32_t colMeta = 0;
|
||||
|
|
@ -199,7 +258,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) {
|
|||
while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) {
|
||||
SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta];
|
||||
col_id_t colIdSchema = pColSchema->colId;
|
||||
col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pHandle->pColIdList, colNeed);
|
||||
col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed);
|
||||
if (colIdSchema < colIdNeed) {
|
||||
colMeta++;
|
||||
} else if (colIdSchema > colIdNeed) {
|
||||
|
|
@ -216,7 +275,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) {
|
|||
}
|
||||
}
|
||||
|
||||
if (blockDataEnsureCapacity(pBlock, pHandle->msgIter.numOfRows) < 0) {
|
||||
if (blockDataEnsureCapacity(pBlock, pReader->msgIter.numOfRows) < 0) {
|
||||
goto FAIL;
|
||||
}
|
||||
|
||||
|
|
@ -227,13 +286,12 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) {
|
|||
STSRow* row;
|
||||
int32_t curRow = 0;
|
||||
|
||||
tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter);
|
||||
tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter);
|
||||
|
||||
pBlock->info.groupId = 0;
|
||||
pBlock->info.uid = pHandle->msgIter.uid;
|
||||
pBlock->info.rows = pHandle->msgIter.numOfRows;
|
||||
pBlock->info.uid = pReader->msgIter.uid;
|
||||
pBlock->info.rows = pReader->msgIter.numOfRows;
|
||||
|
||||
while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
|
||||
while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) {
|
||||
tdSTSRowIterReset(&iter, row);
|
||||
// get all wanted col of that block
|
||||
for (int32_t i = 0; i < colActual; i++) {
|
||||
|
|
@ -255,9 +313,9 @@ FAIL:
|
|||
return -1;
|
||||
}
|
||||
|
||||
void tqReadHandleSetColIdList(SStreamReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; }
|
||||
void tqReaderSetColIdList(STqReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; }
|
||||
|
||||
int tqReadHandleSetTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
|
||||
int tqReaderSetTbUidList(STqReader* pHandle, const SArray* tbUidList) {
|
||||
if (pHandle->tbIdHash) {
|
||||
taosHashClear(pHandle->tbIdHash);
|
||||
}
|
||||
|
|
@ -276,7 +334,7 @@ int tqReadHandleSetTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int tqReadHandleAddTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
|
||||
int tqReaderAddTbUidList(STqReader* pHandle, const SArray* tbUidList) {
|
||||
if (pHandle->tbIdHash == NULL) {
|
||||
pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
|
||||
if (pHandle->tbIdHash == NULL) {
|
||||
|
|
@ -293,7 +351,7 @@ int tqReadHandleAddTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int tqReadHandleRemoveTbUidList(SStreamReader* pHandle, const SArray* tbUidList) {
|
||||
int tqReaderRemoveTbUidList(STqReader* pHandle, const SArray* tbUidList) {
|
||||
ASSERT(pHandle->tbIdHash != NULL);
|
||||
|
||||
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
|
||||
|
|
|
|||
|
|
@ -309,9 +309,11 @@ static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t
|
|||
if (code) goto _err;
|
||||
|
||||
// code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx);
|
||||
pDelIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ);
|
||||
SDelIdx *pIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ);
|
||||
if (code) goto _err;
|
||||
|
||||
*pDelIdx = *pIdx;
|
||||
|
||||
if (pDelIdxArray) {
|
||||
taosArrayDestroy(pDelIdxArray);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -168,8 +168,6 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel
|
|||
tb_uid_t suid;
|
||||
tb_uid_t uid;
|
||||
|
||||
taosArrayClear(pCommitter->aDelData);
|
||||
|
||||
if (pTbData) {
|
||||
suid = pTbData->suid;
|
||||
uid = pTbData->uid;
|
||||
|
|
@ -185,6 +183,8 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel
|
|||
|
||||
code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, NULL);
|
||||
if (code) goto _err;
|
||||
} else {
|
||||
taosArrayClear(pCommitter->aDelData);
|
||||
}
|
||||
|
||||
if (pTbData == NULL && pDelIdx == NULL) goto _exit;
|
||||
|
|
@ -205,7 +205,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel
|
|||
if (code) goto _err;
|
||||
|
||||
// put delIdx
|
||||
if (taosArrayPush(pCommitter->aDelIdx, &delIdx) == NULL) {
|
||||
if (taosArrayPush(pCommitter->aDelIdxN, &delIdx) == NULL) {
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
goto _err;
|
||||
}
|
||||
|
|
@ -602,7 +602,7 @@ static int32_t tsdbGetOvlpNRow(STbDataIter *pIter, SBlock *pBlock) {
|
|||
|
||||
iter.pRow = NULL;
|
||||
while (true) {
|
||||
pRow = tsdbTbDataIterGet(pIter);
|
||||
pRow = tsdbTbDataIterGet(&iter);
|
||||
|
||||
if (pRow == NULL) break;
|
||||
key = TSDBROW_KEY(pRow);
|
||||
|
|
@ -610,7 +610,7 @@ static int32_t tsdbGetOvlpNRow(STbDataIter *pIter, SBlock *pBlock) {
|
|||
c = tBlockCmprFn(&(SBlock){.maxKey = key, .minKey = key}, pBlock);
|
||||
if (c == 0) {
|
||||
nRow++;
|
||||
tsdbTbDataIterNext(pIter);
|
||||
tsdbTbDataIterNext(&iter);
|
||||
} else if (c > 0) {
|
||||
break;
|
||||
} else {
|
||||
|
|
@ -635,7 +635,7 @@ static int32_t tsdbMergeAsSubBlock(SCommitter *pCommitter, STbDataIter *pIter, S
|
|||
code = tsdbCommitterUpdateRowSchema(pCommitter, pBlockIdx->suid, pBlockIdx->uid, TSDBROW_SVERSION(pRow));
|
||||
if (code) goto _err;
|
||||
while (true) {
|
||||
if (pRow) break;
|
||||
if (pRow == NULL) break;
|
||||
code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema);
|
||||
if (code) goto _err;
|
||||
|
||||
|
|
|
|||
|
|
@ -520,7 +520,7 @@ static int32_t tsdbScanAndTryFixFS(STsdbFS *pFS, int8_t deepScan) {
|
|||
return code;
|
||||
|
||||
_err:
|
||||
tsdbError("vgId:%d tsdb can and try fix fs failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
|
||||
tsdbError("vgId:%d tsdb scan and try fix fs failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
|
|||
|
||||
// check if table exists (todo: refact)
|
||||
SMetaReader mr = {0};
|
||||
SMetaEntry me = {0};
|
||||
// SMetaEntry me = {0};
|
||||
metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0);
|
||||
if (metaGetTableEntryByUid(&mr, pMsgIter->uid) < 0) {
|
||||
metaReaderClear(&mr);
|
||||
|
|
@ -117,6 +117,8 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
|
|||
if (mr.me.type == TSDB_NORMAL_TABLE) {
|
||||
sverNew = mr.me.ntbEntry.schemaRow.version;
|
||||
} else {
|
||||
tDecoderClear(&mr.coder);
|
||||
|
||||
metaGetTableEntryByUid(&mr, mr.me.ctbEntry.suid);
|
||||
sverNew = mr.me.stbEntry.schemaRow.version;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1940,6 +1940,7 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo*
|
|||
}
|
||||
|
||||
ASSERT(0);
|
||||
return -1;
|
||||
}
|
||||
|
||||
static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo,
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb
|
|||
}
|
||||
|
||||
pDelFWriter->fDel.size = TSDB_FHDR_SIZE;
|
||||
pDelFWriter->fDel.size = 0;
|
||||
pDelFWriter->fDel.offset = 0;
|
||||
|
||||
*ppWriter = pDelFWriter;
|
||||
return code;
|
||||
|
|
@ -969,6 +969,8 @@ int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBl
|
|||
SBlockData *pBlockData1 = &(SBlockData){0};
|
||||
SBlockData *pBlockData2 = &(SBlockData){0};
|
||||
|
||||
tBlockDataInit(pBlockData1);
|
||||
tBlockDataInit(pBlockData2);
|
||||
for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) {
|
||||
code = tsdbReadSubColData(pReader, pBlockIdx, pBlock, iSubBlock, aColId, nCol, pBlockData1, ppBuf1, ppBuf2);
|
||||
if (code) goto _err;
|
||||
|
|
@ -1106,6 +1108,8 @@ int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *p
|
|||
SBlockData *pBlockData1 = &(SBlockData){0};
|
||||
SBlockData *pBlockData2 = &(SBlockData){0};
|
||||
|
||||
tBlockDataInit(pBlockData1);
|
||||
tBlockDataInit(pBlockData2);
|
||||
for (iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) {
|
||||
code = tsdbReadSubBlockData(pReader, pBlockIdx, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2);
|
||||
if (code) {
|
||||
|
|
|
|||
|
|
@ -821,16 +821,20 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
|
|||
int32_t code = 0;
|
||||
int32_t size;
|
||||
|
||||
ASSERT(pColDataSrc->nVal > 0);
|
||||
|
||||
pColDataDest->cid = pColDataSrc->cid;
|
||||
pColDataDest->type = pColDataSrc->type;
|
||||
pColDataDest->smaOn = pColDataSrc->smaOn;
|
||||
pColDataDest->nVal = pColDataSrc->nVal;
|
||||
pColDataDest->flag = pColDataSrc->flag;
|
||||
|
||||
if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) {
|
||||
size = BIT2_SIZE(pColDataSrc->nVal);
|
||||
code = tRealloc(&pColDataDest->pBitMap, size);
|
||||
if (code) goto _exit;
|
||||
memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size);
|
||||
}
|
||||
|
||||
if (IS_VAR_DATA_TYPE(pColDataDest->type)) {
|
||||
size = sizeof(int32_t) * pColDataSrc->nVal;
|
||||
|
|
|
|||
|
|
@ -348,7 +348,7 @@ typedef struct SessionWindowSupporter {
|
|||
uint8_t parentType;
|
||||
} SessionWindowSupporter;
|
||||
|
||||
typedef struct SStreamBlockScanInfo {
|
||||
typedef struct SStreamScanInfo {
|
||||
uint64_t tableUid; // queried super table uid
|
||||
SExprInfo* pPseudoExpr;
|
||||
int32_t numOfPseudoExpr;
|
||||
|
|
@ -365,7 +365,7 @@ typedef struct SStreamBlockScanInfo {
|
|||
int32_t blockType; // current block type
|
||||
int32_t validBlockIndex; // Is current data has returned?
|
||||
uint64_t numOfExec; // execution times
|
||||
void* streamBlockReader;// stream block reader handle
|
||||
STqReader* tqReader;
|
||||
|
||||
int32_t tsArrayIndex;
|
||||
SArray* tsArray;
|
||||
|
|
@ -374,7 +374,7 @@ typedef struct SStreamBlockScanInfo {
|
|||
|
||||
EStreamScanMode scanMode;
|
||||
SOperatorInfo* pStreamScanOp;
|
||||
SOperatorInfo* pSnapshotReadOp;
|
||||
SOperatorInfo* pTableScanOp;
|
||||
SArray* childIds;
|
||||
SessionWindowSupporter sessionSup;
|
||||
bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA.
|
||||
|
|
@ -383,7 +383,12 @@ typedef struct SStreamBlockScanInfo {
|
|||
SSDataBlock* pPullDataRes; // pull data SSDataBlock
|
||||
SSDataBlock* pDeleteDataRes; // delete data SSDataBlock
|
||||
int32_t deleteDataIndex;
|
||||
} SStreamBlockScanInfo;
|
||||
|
||||
// status for tmq
|
||||
//SSchemaWrapper schema;
|
||||
STqOffset offset;
|
||||
|
||||
} SStreamScanInfo;
|
||||
|
||||
typedef struct SSysTableScanInfo {
|
||||
SRetrieveMetaTableRsp* pRsp;
|
||||
|
|
@ -518,6 +523,7 @@ typedef struct SIndefOperatorInfo {
|
|||
SAggSupporter aggSup;
|
||||
SArray* pPseudoColInfo;
|
||||
SExprSupp scalarSup;
|
||||
SNode* pCondition;
|
||||
} SIndefOperatorInfo;
|
||||
|
||||
typedef struct SFillOperatorInfo {
|
||||
|
|
@ -528,6 +534,7 @@ typedef struct SFillOperatorInfo {
|
|||
SSDataBlock* existNewGroupBlock;
|
||||
bool multigroupResult;
|
||||
STimeWindow win;
|
||||
SNode* pCondition;
|
||||
} SFillOperatorInfo;
|
||||
|
||||
typedef struct SGroupbyOperatorInfo {
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
|
|||
} else {
|
||||
pOperator->status = OP_NOT_OPENED;
|
||||
|
||||
SStreamBlockScanInfo* pInfo = pOperator->info;
|
||||
SStreamScanInfo* pInfo = pOperator->info;
|
||||
pInfo->assignBlockUid = assignUid;
|
||||
|
||||
// TODO: if a block was set but not consumed,
|
||||
|
|
@ -45,7 +45,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
|
|||
pInfo->blockType = type;
|
||||
|
||||
if (type == STREAM_INPUT__DATA_SUBMIT) {
|
||||
if (tqReadHandleSetMsg(pInfo->streamBlockReader, input, 0) < 0) {
|
||||
if (tqReaderSetDataMsg(pInfo->tqReader, input, 0) < 0) {
|
||||
qError("submit msg messed up when initing stream block, %s" PRIx64, id);
|
||||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
}
|
||||
|
|
@ -105,7 +105,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO
|
|||
return code;
|
||||
}
|
||||
|
||||
qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) {
|
||||
qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) {
|
||||
if (msg == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -120,7 +120,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) {
|
|||
}
|
||||
|
||||
qTaskInfo_t pTaskInfo = NULL;
|
||||
code = qCreateExecTask(streamReadHandle, 0, 0, plan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM);
|
||||
code = qCreateExecTask(readers, 0, 0, plan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
// TODO: destroy SSubplan & pTaskInfo
|
||||
terrno = code;
|
||||
|
|
@ -130,7 +130,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) {
|
|||
return pTaskInfo;
|
||||
}
|
||||
|
||||
static SArray* filterQualifiedChildTables(const SStreamBlockScanInfo* pScanInfo, const SArray* tableIdList) {
|
||||
static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList) {
|
||||
SArray* qa = taosArrayInit(4, sizeof(tb_uid_t));
|
||||
|
||||
// let's discard the tables those are not created according to the queried super table.
|
||||
|
|
@ -169,16 +169,16 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
|
|||
}
|
||||
|
||||
int32_t code = 0;
|
||||
SStreamBlockScanInfo* pScanInfo = pInfo->info;
|
||||
SStreamScanInfo* pScanInfo = pInfo->info;
|
||||
if (isAdd) { // add new table id
|
||||
SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList);
|
||||
|
||||
qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa));
|
||||
code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa);
|
||||
code = tqReaderAddTbUidList(pScanInfo->tqReader, qa);
|
||||
taosArrayDestroy(qa);
|
||||
} else { // remove the table id in current list
|
||||
qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList));
|
||||
code = tqReadHandleRemoveTbUidList(pScanInfo->streamBlockReader, tableIdList);
|
||||
code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList);
|
||||
}
|
||||
|
||||
return code;
|
||||
|
|
|
|||
|
|
@ -236,6 +236,37 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le
|
|||
return decodeOperator(pTaskInfo->pRoot, pInput, len);
|
||||
}
|
||||
|
||||
int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) {
|
||||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
|
||||
SOperatorInfo* pOperator = pTaskInfo->pRoot;
|
||||
|
||||
while (1) {
|
||||
uint8_t type = pOperator->operatorType;
|
||||
if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
|
||||
*scanner = pOperator->info;
|
||||
return 0;
|
||||
} else {
|
||||
ASSERT(pOperator->numOfDownstream == 1);
|
||||
pOperator = pOperator->pDownstream[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void* qExtractReaderFromStreamScanner(void* scanner) {
|
||||
SStreamScanInfo* pInfo = scanner;
|
||||
return (void*)pInfo->tqReader;
|
||||
}
|
||||
|
||||
const SSchemaWrapper* qExtractSchemaFromStreamScanner(void* scanner) {
|
||||
SStreamScanInfo* pInfo = scanner;
|
||||
return pInfo->tqReader->pSchemaWrapper;
|
||||
}
|
||||
|
||||
const STqOffset* qExtractStatusFromStreamScanner(void* scanner) {
|
||||
SStreamScanInfo* pInfo = scanner;
|
||||
return &pInfo->offset;
|
||||
}
|
||||
|
||||
int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) {
|
||||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
|
||||
|
||||
|
|
|
|||
|
|
@ -2844,17 +2844,17 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
|
|||
}
|
||||
|
||||
int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) {
|
||||
int32_t type = pOperator->operatorType;
|
||||
uint8_t type = pOperator->operatorType;
|
||||
|
||||
pOperator->status = OP_OPENED;
|
||||
|
||||
if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
|
||||
SStreamBlockScanInfo* pScanInfo = pOperator->info;
|
||||
SStreamScanInfo* pScanInfo = pOperator->info;
|
||||
pScanInfo->blockType = STREAM_INPUT__DATA_SCAN;
|
||||
|
||||
pScanInfo->pSnapshotReadOp->status = OP_OPENED;
|
||||
pScanInfo->pTableScanOp->status = OP_OPENED;
|
||||
|
||||
STableScanInfo* pInfo = pScanInfo->pSnapshotReadOp->info;
|
||||
STableScanInfo* pInfo = pScanInfo->pTableScanOp->info;
|
||||
ASSERT(pInfo->scanMode == TABLE_SCAN__TABLE_ORDER);
|
||||
|
||||
if (uid == 0) {
|
||||
|
|
@ -2914,8 +2914,8 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) {
|
|||
int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts) {
|
||||
int32_t type = pOperator->operatorType;
|
||||
if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
|
||||
SStreamBlockScanInfo* pScanInfo = pOperator->info;
|
||||
STableScanInfo* pSnapShotScanInfo = pScanInfo->pSnapshotReadOp->info;
|
||||
SStreamScanInfo* pScanInfo = pOperator->info;
|
||||
STableScanInfo* pSnapShotScanInfo = pScanInfo->pTableScanOp->info;
|
||||
*uid = pSnapShotScanInfo->lastStatus.uid;
|
||||
*ts = pSnapShotScanInfo->lastStatus.ts;
|
||||
} else {
|
||||
|
|
@ -3364,7 +3364,7 @@ static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo* pInfo, SResultInf
|
|||
}
|
||||
}
|
||||
|
||||
static SSDataBlock* doFill(SOperatorInfo* pOperator) {
|
||||
static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
|
||||
SFillOperatorInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
||||
|
|
@ -3372,9 +3372,6 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
|
|||
SSDataBlock* pResBlock = pInfo->pRes;
|
||||
|
||||
blockDataCleanup(pResBlock);
|
||||
if (pOperator->status == OP_EXEC_DONE) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// todo handle different group data interpolation
|
||||
bool n = false;
|
||||
|
|
@ -3440,6 +3437,39 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
|
|||
}
|
||||
}
|
||||
|
||||
static SSDataBlock* doFill(SOperatorInfo* pOperator) {
|
||||
SFillOperatorInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
||||
if (pOperator->status == OP_EXEC_DONE) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
SSDataBlock* fillResult = NULL;
|
||||
while (true) {
|
||||
fillResult = doFillImpl(pOperator);
|
||||
if (fillResult != NULL) {
|
||||
doFilter(pInfo->pCondition, fillResult);
|
||||
}
|
||||
|
||||
if (fillResult == NULL) {
|
||||
doSetOperatorCompleted(pOperator);
|
||||
break;
|
||||
}
|
||||
|
||||
if (fillResult->info.rows > 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (fillResult != NULL) {
|
||||
size_t rows = fillResult->info.rows;
|
||||
pOperator->resultInfo.totalRows += rows;
|
||||
}
|
||||
|
||||
return fillResult;
|
||||
}
|
||||
|
||||
static void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) {
|
||||
for (int32_t i = 0; i < numOfExprs; ++i) {
|
||||
SExprInfo* pExprInfo = &pExpr[i];
|
||||
|
|
@ -3832,6 +3862,8 @@ static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) {
|
|||
}
|
||||
}
|
||||
|
||||
doFilter(pIndefInfo->pCondition, pInfo->pRes);
|
||||
|
||||
size_t rows = pInfo->pRes->info.rows;
|
||||
pOperator->resultInfo.totalRows += rows;
|
||||
|
||||
|
|
@ -3885,6 +3917,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
|
|||
|
||||
pInfo->binfo.pRes = pResBlock;
|
||||
pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr);
|
||||
pInfo->pCondition = pPhyNode->node.pConditions;
|
||||
|
||||
pOperator->name = "IndefinitOperator";
|
||||
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT;
|
||||
|
|
@ -3958,6 +3991,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
|
|||
|
||||
pInfo->pRes = pResBlock;
|
||||
pInfo->multigroupResult = multigroupResult;
|
||||
pInfo->pCondition = pPhyFillNode->node.pConditions;
|
||||
pOperator->name = "FillOperator";
|
||||
pOperator->blocking = false;
|
||||
pOperator->status = OP_NOT_OPENED;
|
||||
|
|
@ -4373,8 +4407,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
|
|||
pOptr = createGroupOperatorInfo(ops[0], pExprInfo, num, pResBlock, pColList, pAggNode->node.pConditions,
|
||||
pScalarExprInfo, numOfScalarExpr, pTaskInfo);
|
||||
} else {
|
||||
pOptr =
|
||||
createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions, pScalarExprInfo, numOfScalarExpr, pTaskInfo);
|
||||
pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions,
|
||||
pScalarExprInfo, numOfScalarExpr, pTaskInfo);
|
||||
}
|
||||
} else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) {
|
||||
SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
|
||||
|
|
@ -4541,7 +4575,8 @@ SArray* extractColumnInfo(SNodeList* pNodeList) {
|
|||
return pList;
|
||||
}
|
||||
|
||||
STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableListInfo, const char* idstr) {
|
||||
STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
|
||||
STableListInfo* pTableListInfo, const char* idstr) {
|
||||
int32_t code = getTableList(pHandle->meta, pHandle->vnode, &pTableScanNode->scan, pTableListInfo);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _error;
|
||||
|
|
@ -4587,9 +4622,9 @@ static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanI
|
|||
}
|
||||
return extractTbscanInStreamOpTree(pOperator->pDownstream[0], ppInfo);
|
||||
} else {
|
||||
SStreamBlockScanInfo* pInfo = pOperator->info;
|
||||
ASSERT(pInfo->pSnapshotReadOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
|
||||
*ppInfo = pInfo->pSnapshotReadOp->info;
|
||||
SStreamScanInfo* pInfo = pOperator->info;
|
||||
ASSERT(pInfo->pTableScanOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
|
||||
*ppInfo = pInfo->pTableScanOp->info;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
|
@ -4785,7 +4820,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
|
|||
(*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId,
|
||||
&(*pTaskInfo)->tableqinfoList, pPlan->user);
|
||||
|
||||
|
||||
if (NULL == (*pTaskInfo)->pRoot) {
|
||||
code = (*pTaskInfo)->code;
|
||||
goto _complete;
|
||||
|
|
|
|||
|
|
@ -780,7 +780,7 @@ _error:
|
|||
return NULL;
|
||||
}
|
||||
|
||||
static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) {
|
||||
static void doClearBufferedBlocks(SStreamScanInfo* pInfo) {
|
||||
size_t total = taosArrayGetSize(pInfo->pBlockLists);
|
||||
|
||||
pInfo->validBlockIndex = 0;
|
||||
|
|
@ -791,21 +791,20 @@ static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) {
|
|||
taosArrayClear(pInfo->pBlockLists);
|
||||
}
|
||||
|
||||
static bool isSessionWindow(SStreamBlockScanInfo* pInfo) {
|
||||
static bool isSessionWindow(SStreamScanInfo* pInfo) {
|
||||
return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
|
||||
pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
|
||||
}
|
||||
|
||||
static bool isStateWindow(SStreamBlockScanInfo* pInfo) {
|
||||
static bool isStateWindow(SStreamScanInfo* pInfo) {
|
||||
return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
|
||||
}
|
||||
|
||||
static bool isIntervalWindow(SStreamBlockScanInfo* pInfo) {
|
||||
static bool isIntervalWindow(SStreamScanInfo* pInfo) {
|
||||
return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL ||
|
||||
pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL;
|
||||
}
|
||||
|
||||
|
||||
static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) {
|
||||
uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t));
|
||||
if (groupId) {
|
||||
|
|
@ -827,17 +826,15 @@ static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) {
|
|||
*/
|
||||
}
|
||||
|
||||
static void setGroupId(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) {
|
||||
static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) {
|
||||
ASSERT(rowIndex < pBlock->info.rows);
|
||||
switch (pBlock->info.type)
|
||||
{
|
||||
switch (pBlock->info.type) {
|
||||
case STREAM_DELETE_DATA:
|
||||
case STREAM_RETRIEVE: {
|
||||
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex);
|
||||
uint64_t* groupCol = (uint64_t*)pColInfo->pData;
|
||||
pInfo->groupId = groupCol[rowIndex];
|
||||
}
|
||||
break;
|
||||
} break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
@ -854,7 +851,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) {
|
|||
pTableScanInfo->currentGroupId = -1;
|
||||
}
|
||||
|
||||
static bool prepareRangeScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) {
|
||||
static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) {
|
||||
if ((*pRowIndex) == pBlock->info.rows) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -882,11 +879,11 @@ static bool prepareRangeScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, i
|
|||
break;
|
||||
}
|
||||
|
||||
resetTableScanInfo(pInfo->pSnapshotReadOp->info, &win);
|
||||
resetTableScanInfo(pInfo->pTableScanOp->info, &win);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
|
||||
static bool prepareDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
|
||||
STimeWindow win = {
|
||||
.skey = INT64_MIN,
|
||||
.ekey = INT64_MAX,
|
||||
|
|
@ -907,11 +904,10 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3
|
|||
setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex);
|
||||
(*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, *pRowIndex, gap, NULL);
|
||||
} else {
|
||||
win =
|
||||
getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, pInfo->interval.precision, NULL);
|
||||
win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, pInfo->interval.precision, NULL);
|
||||
setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex);
|
||||
(*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, *pRowIndex, win.ekey, binarySearchForKey, NULL,
|
||||
TSDB_ORDER_ASC);
|
||||
(*pRowIndex) +=
|
||||
getNumOfRowsInTimeWindow(&pSDB->info, tsCols, *pRowIndex, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
|
||||
}
|
||||
needRead = true;
|
||||
} else if (isStateWindow(pInfo)) {
|
||||
|
|
@ -929,7 +925,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3
|
|||
if (!needRead) {
|
||||
return false;
|
||||
}
|
||||
resetTableScanInfo(pInfo->pSnapshotReadOp->info, &win);
|
||||
resetTableScanInfo(pInfo->pTableScanOp->info, &win);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -946,13 +942,13 @@ static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRow
|
|||
dest->info.rows++;
|
||||
}
|
||||
|
||||
static SSDataBlock* doRangeScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
|
||||
static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
|
||||
while (1) {
|
||||
SSDataBlock* pResult = NULL;
|
||||
pResult = doTableScan(pInfo->pSnapshotReadOp);
|
||||
pResult = doTableScan(pInfo->pTableScanOp);
|
||||
if (!pResult && prepareRangeScan(pInfo, pSDB, pRowIndex)) {
|
||||
// scan next window data
|
||||
pResult = doTableScan(pInfo->pSnapshotReadOp);
|
||||
pResult = doTableScan(pInfo->pTableScanOp);
|
||||
}
|
||||
if (!pResult) {
|
||||
blockDataCleanup(pSDB);
|
||||
|
|
@ -966,14 +962,14 @@ static SSDataBlock* doRangeScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB,
|
|||
}
|
||||
}
|
||||
|
||||
static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
|
||||
static SSDataBlock* doDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
|
||||
while (1) {
|
||||
SSDataBlock* pResult = NULL;
|
||||
pResult = doTableScan(pInfo->pSnapshotReadOp);
|
||||
pResult = doTableScan(pInfo->pTableScanOp);
|
||||
if (pResult == NULL) {
|
||||
if (prepareDataScan(pInfo, pSDB, tsColIndex, pRowIndex)) {
|
||||
// scan next window data
|
||||
pResult = doTableScan(pInfo->pSnapshotReadOp);
|
||||
pResult = doTableScan(pInfo->pTableScanOp);
|
||||
}
|
||||
}
|
||||
if (!pResult) {
|
||||
|
|
@ -997,8 +993,8 @@ static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, i
|
|||
return pResult;
|
||||
*/
|
||||
}
|
||||
|
||||
static void generateIntervalTs(SStreamBlockScanInfo* pInfo, SSDataBlock* pDelBlock, SOperatorInfo* pOperator, SSDataBlock* pUpdateRes) {
|
||||
static void generateIntervalTs(SStreamScanInfo* pInfo, SSDataBlock* pDelBlock, SOperatorInfo* pOperator,
|
||||
SSDataBlock* pUpdateRes) {
|
||||
if (pDelBlock->info.rows == 0) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -1014,8 +1010,10 @@ static void generateIntervalTs(SStreamBlockScanInfo* pInfo, SSDataBlock* pDelBlo
|
|||
|
||||
SColumnInfoData* pDestTsCol = taosArrayGet(pUpdateRes->pDataBlock, START_TS_COLUMN_INDEX);
|
||||
SColumnInfoData* pDestGpCol = taosArrayGet(pUpdateRes->pDataBlock, GROUPID_COLUMN_INDEX);
|
||||
for (int32_t i = pInfo->deleteDataIndex ; i < pDelBlock->info.rows &&
|
||||
i < pDelBlock->info.capacity - (endData[i] - startData[i])/pInfo->interval.interval - 1; i++) {
|
||||
for (int32_t i = pInfo->deleteDataIndex;
|
||||
i < pDelBlock->info.rows &&
|
||||
i < pDelBlock->info.capacity - (endData[i] - startData[i]) / pInfo->interval.interval - 1;
|
||||
i++) {
|
||||
uint64_t groupId = getGroupId(pOperator, uidCol[i]);
|
||||
for (TSKEY startTs = startData[i]; startTs <= endData[i];) {
|
||||
colDataAppend(pDestTsCol, pUpdateRes->info.rows, (const char*)&startTs, false);
|
||||
|
|
@ -1032,7 +1030,8 @@ static void generateIntervalTs(SStreamBlockScanInfo* pInfo, SSDataBlock* pDelBlo
|
|||
}
|
||||
}
|
||||
|
||||
static void generateScanRange(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SOperatorInfo* pOperator, SSDataBlock* pUpdateRes) {
|
||||
static void generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SOperatorInfo* pOperator,
|
||||
SSDataBlock* pUpdateRes) {
|
||||
if (pBlock->info.rows == 0) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -1053,12 +1052,14 @@ static void generateScanRange(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock,
|
|||
for (int32_t i = 0; i < pBlock->info.rows; i++) {
|
||||
uint64_t groupId = getGroupId(pOperator, uidCol[i]);
|
||||
// gap must be 0.
|
||||
SResultWindowInfo* pStartWin = getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, startData[i], endData[i], groupId, 0, &dummy);
|
||||
SResultWindowInfo* pStartWin =
|
||||
getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, startData[i], endData[i], groupId, 0, &dummy);
|
||||
if (!pStartWin) {
|
||||
// window has been closed.
|
||||
continue;
|
||||
}
|
||||
SResultWindowInfo* pEndWin = getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, endData[i], endData[i], groupId, 0, &dummy);
|
||||
SResultWindowInfo* pEndWin =
|
||||
getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, endData[i], endData[i], groupId, 0, &dummy);
|
||||
ASSERT(pEndWin);
|
||||
colDataAppend(pDestStartCol, i, (const char*)&pStartWin->win.skey, false);
|
||||
colDataAppend(pDestEndCol, i, (const char*)&pEndWin->win.ekey, false);
|
||||
|
|
@ -1066,7 +1067,7 @@ static void generateScanRange(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock,
|
|||
pUpdateRes->info.rows++;
|
||||
}
|
||||
}
|
||||
static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlock* pUpdateBlock) {
|
||||
static void setUpdateData(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlock* pUpdateBlock) {
|
||||
blockDataCleanup(pUpdateBlock);
|
||||
int32_t size = taosArrayGetSize(pInfo->tsArray);
|
||||
if (pInfo->tsArrayIndex < size) {
|
||||
|
|
@ -1075,11 +1076,11 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa
|
|||
blockDataEnsureCapacity(pUpdateBlock, size);
|
||||
|
||||
int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex);
|
||||
pInfo->groupId = getGroupId(pInfo->pSnapshotReadOp, pBlock->info.uid);
|
||||
pInfo->groupId = getGroupId(pInfo->pTableScanOp, pBlock->info.uid);
|
||||
int32_t i = 0;
|
||||
for (; i < size; i++) {
|
||||
rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, i + pInfo->tsArrayIndex);
|
||||
uint64_t id = getGroupId(pInfo->pSnapshotReadOp, pBlock->info.uid);
|
||||
uint64_t id = getGroupId(pInfo->pTableScanOp, pBlock->info.uid);
|
||||
if (pInfo->groupId != id) {
|
||||
break;
|
||||
}
|
||||
|
|
@ -1098,12 +1099,11 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa
|
|||
}
|
||||
|
||||
if (size == 0) {
|
||||
generateIntervalTs(pInfo, pInfo->pDeleteDataRes, pInfo->pSnapshotReadOp, pUpdateBlock);
|
||||
generateIntervalTs(pInfo, pInfo->pDeleteDataRes, pInfo->pTableScanOp, pUpdateBlock);
|
||||
}
|
||||
}
|
||||
|
||||
static void checkUpdateData(SStreamBlockScanInfo* pInfo, bool invertible, SSDataBlock* pBlock,
|
||||
bool out) {
|
||||
static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) {
|
||||
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
|
||||
ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP);
|
||||
TSKEY* ts = (TSKEY*)pColDataInfo->pData;
|
||||
|
|
@ -1124,10 +1124,10 @@ static void setBlockGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32
|
|||
}
|
||||
}
|
||||
|
||||
static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
||||
static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
|
||||
// NOTE: this operator does never check if current status is done or not
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
SStreamBlockScanInfo* pInfo = pOperator->info;
|
||||
SStreamScanInfo* pInfo = pOperator->info;
|
||||
|
||||
pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
|
||||
if (pTaskInfo->code != TSDB_CODE_SUCCESS || pOperator->status == OP_EXEC_DONE) {
|
||||
|
|
@ -1145,6 +1145,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
|||
|
||||
int32_t current = pInfo->validBlockIndex++;
|
||||
SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current);
|
||||
// TODO move into scan
|
||||
blockDataUpdateTsWindow(pBlock, 0);
|
||||
switch (pBlock->info.type) {
|
||||
case STREAM_RETRIEVE: {
|
||||
|
|
@ -1154,25 +1155,23 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
|||
pInfo->pullDataResIndex = 0;
|
||||
prepareDataScan(pInfo, pInfo->pPullDataRes, START_TS_COLUMN_INDEX, &pInfo->pullDataResIndex);
|
||||
updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo);
|
||||
}
|
||||
break;
|
||||
} break;
|
||||
case STREAM_DELETE_DATA: {
|
||||
pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
|
||||
pInfo->updateResIndex = 0;
|
||||
if (isIntervalWindow(pInfo)) {
|
||||
copyDataBlock(pInfo->pDeleteDataRes, pBlock);
|
||||
generateIntervalTs(pInfo, pInfo->pDeleteDataRes, pInfo->pSnapshotReadOp, pInfo->pUpdateRes);
|
||||
generateIntervalTs(pInfo, pInfo->pDeleteDataRes, pInfo->pTableScanOp, pInfo->pUpdateRes);
|
||||
prepareDataScan(pInfo, pInfo->pUpdateRes, START_TS_COLUMN_INDEX, &pInfo->updateResIndex);
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
|
||||
} else {
|
||||
generateScanRange(pInfo, pBlock, pInfo->pSnapshotReadOp, pInfo->pUpdateRes);
|
||||
generateScanRange(pInfo, pBlock, pInfo->pTableScanOp, pInfo->pUpdateRes);
|
||||
prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
|
||||
}
|
||||
pInfo->pUpdateRes->info.type = STREAM_DELETE_DATA;
|
||||
return pInfo->pUpdateRes;
|
||||
}
|
||||
break;
|
||||
} break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
@ -1233,38 +1232,33 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
|||
SDataBlockInfo* pBlockInfo = &pInfo->pRes->info;
|
||||
blockDataCleanup(pInfo->pRes);
|
||||
|
||||
while (tqNextDataBlock(pInfo->streamBlockReader)) {
|
||||
while (tqNextDataBlock(pInfo->tqReader)) {
|
||||
SSDataBlock block = {0};
|
||||
|
||||
// todo refactor
|
||||
int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader);
|
||||
int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader);
|
||||
|
||||
uint64_t groupId = block.info.groupId;
|
||||
uint64_t uid = block.info.uid;
|
||||
int32_t numOfRows = block.info.rows;
|
||||
|
||||
if (code != TSDB_CODE_SUCCESS || numOfRows == 0) {
|
||||
if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) {
|
||||
pTaskInfo->code = code;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
pInfo->pRes->info.groupId = groupId;
|
||||
pInfo->pRes->info.rows = numOfRows;
|
||||
pInfo->pRes->info.uid = uid;
|
||||
pInfo->pRes->info.rows = block.info.rows;
|
||||
pInfo->pRes->info.uid = block.info.uid;
|
||||
pInfo->pRes->info.type = STREAM_NORMAL;
|
||||
pInfo->pRes->info.capacity = numOfRows;
|
||||
pInfo->pRes->info.capacity = block.info.rows;
|
||||
|
||||
// for generating rollup SMA result, each time is an independent time serie.
|
||||
// TODO temporarily used, when the statement of "partition by tbname" is ready, remove this
|
||||
if (pInfo->assignBlockUid) {
|
||||
pInfo->pRes->info.groupId = uid;
|
||||
} else {
|
||||
pInfo->pRes->info.groupId = groupId;
|
||||
pInfo->pRes->info.groupId = block.info.uid;
|
||||
}
|
||||
|
||||
uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t));
|
||||
uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &block.info.uid, sizeof(int64_t));
|
||||
if (groupIdPre) {
|
||||
pInfo->pRes->info.groupId = *groupIdPre;
|
||||
} else {
|
||||
pInfo->pRes->info.groupId = 0;
|
||||
}
|
||||
|
||||
// todo extract method
|
||||
|
|
@ -1333,11 +1327,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes;
|
||||
|
||||
} else if (pInfo->blockType == STREAM_INPUT__DATA_SCAN) {
|
||||
// check reader last status
|
||||
// if not match, reset status
|
||||
SSDataBlock* pResult = doTableScan(pInfo->pSnapshotReadOp);
|
||||
SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
|
||||
return pResult && pResult->info.rows > 0 ? pResult : NULL;
|
||||
|
||||
} else {
|
||||
|
|
@ -1361,7 +1357,7 @@ static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) {
|
|||
SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode,
|
||||
SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId,
|
||||
uint64_t taskId) {
|
||||
SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo));
|
||||
SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo));
|
||||
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
|
||||
|
||||
if (pInfo == NULL || pOperator == NULL) {
|
||||
|
|
@ -1400,13 +1396,25 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
|
|||
}
|
||||
|
||||
if (pHandle) {
|
||||
SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo);
|
||||
STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanDummy->info;
|
||||
SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo);
|
||||
STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanOp->info;
|
||||
|
||||
SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, 0);
|
||||
if (pHandle->tqReader) {
|
||||
if (pHandle->initTableReader) {
|
||||
pSTInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
|
||||
tsdbReaderOpen(pHandle->vnode, &pSTInfo->cond, tableList, &pSTInfo->dataReader, 0);
|
||||
pSTInfo->dataReader = NULL;
|
||||
if (tsdbReaderOpen(pHandle->vnode, &pSTInfo->cond, tableList, &pSTInfo->dataReader, NULL) < 0) {
|
||||
ASSERT(0);
|
||||
}
|
||||
}
|
||||
|
||||
if (pHandle->initTqReader) {
|
||||
ASSERT(pHandle->tqReader == NULL);
|
||||
pInfo->tqReader = tqOpenReader(pHandle->vnode);
|
||||
ASSERT(pInfo->tqReader);
|
||||
} else {
|
||||
ASSERT(pHandle->tqReader);
|
||||
pInfo->tqReader = pHandle->tqReader;
|
||||
}
|
||||
|
||||
if (pSTInfo->interval.interval > 0) {
|
||||
|
|
@ -1414,18 +1422,17 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
|
|||
} else {
|
||||
pInfo->pUpdateInfo = NULL;
|
||||
}
|
||||
pInfo->pSnapshotReadOp = pTableScanDummy;
|
||||
|
||||
pInfo->pTableScanOp = pTableScanOp;
|
||||
pInfo->interval = pSTInfo->interval;
|
||||
|
||||
pInfo->readHandle = *pHandle;
|
||||
ASSERT(pHandle->reader);
|
||||
pInfo->streamBlockReader = pHandle->reader;
|
||||
pInfo->tableUid = pScanPhyNode->uid;
|
||||
|
||||
// set the extract column id to streamHandle
|
||||
tqReadHandleSetColIdList((SStreamReader*)pHandle->reader, pColIds);
|
||||
tqReaderSetColIdList(pInfo->tqReader, pColIds);
|
||||
SArray* tableIdList = extractTableIdList(&pTaskInfo->tableqinfoList);
|
||||
int32_t code = tqReadHandleSetTbUidList(pHandle->reader, tableIdList);
|
||||
int32_t code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList);
|
||||
if (code != 0) {
|
||||
taosArrayDestroy(tableIdList);
|
||||
goto _error;
|
||||
|
|
@ -1449,7 +1456,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
|
|||
pInfo->deleteDataIndex = 0;
|
||||
pInfo->pDeleteDataRes = createPullDataBlock();
|
||||
|
||||
pOperator->name = "StreamBlockScanOperator";
|
||||
pOperator->name = "StreamScanOperator";
|
||||
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
|
||||
pOperator->blocking = false;
|
||||
pOperator->status = OP_NOT_OPENED;
|
||||
|
|
@ -1458,7 +1465,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
|
|||
pOperator->pTaskInfo = pTaskInfo;
|
||||
|
||||
pOperator->fpSet =
|
||||
createOperatorFpSet(operatorDummyOpenFn, doStreamBlockScan, NULL, NULL, operatorDummyCloseFn, NULL, NULL, NULL);
|
||||
createOperatorFpSet(operatorDummyOpenFn, doStreamScan, NULL, NULL, operatorDummyCloseFn, NULL, NULL, NULL);
|
||||
|
||||
return pOperator;
|
||||
|
||||
|
|
|
|||
|
|
@ -1316,7 +1316,7 @@ bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId)
|
|||
// window has been closed
|
||||
return false;
|
||||
}
|
||||
SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId);
|
||||
// SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, p1->pageId);
|
||||
// dBufSetBufPageRecycled(pAggSup->pResultBuf, bufPage);
|
||||
taosHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
|
||||
return true;
|
||||
|
|
@ -1419,7 +1419,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
|
|||
ASSERT(pRecyPages != NULL);
|
||||
taosArrayPush(pRecyPages, &pPos->pageId);
|
||||
} else {
|
||||
SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId);
|
||||
// SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId);
|
||||
// dBufSetBufPageRecycled(pDiscBuf, bufPage);
|
||||
}
|
||||
char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
|
||||
|
|
@ -1446,7 +1446,7 @@ static void freeAllPages(SArray* pageIds, SDiskbasedBuf* pDiskBuf) {
|
|||
int32_t size = taosArrayGetSize(pageIds);
|
||||
for (int32_t i = 0; i < size; i++) {
|
||||
int32_t pageId = *(int32_t*)taosArrayGet(pageIds, i);
|
||||
SFilePage* bufPage = getBufPage(pDiskBuf, pageId);
|
||||
// SFilePage* bufPage = getBufPage(pDiskBuf, pageId);
|
||||
// dBufSetBufPageRecycled(pDiskBuf, bufPage);
|
||||
}
|
||||
taosArrayClear(pageIds);
|
||||
|
|
@ -3023,7 +3023,7 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num
|
|||
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark,
|
||||
uint8_t type) {
|
||||
ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
|
||||
SStreamBlockScanInfo* pScanInfo = downstream->info;
|
||||
SStreamScanInfo* pScanInfo = downstream->info;
|
||||
pScanInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};
|
||||
pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark);
|
||||
}
|
||||
|
|
@ -3189,7 +3189,7 @@ SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY start
|
|||
*pIndex = index + 1;
|
||||
return pWin;
|
||||
} else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) {
|
||||
*pIndex = index;
|
||||
*pIndex = index + 1;
|
||||
return pWin;
|
||||
}
|
||||
}
|
||||
|
|
@ -3442,7 +3442,7 @@ void deleteWindow(SArray* pWinInfos, int32_t index) {
|
|||
taosArrayRemove(pWinInfos, index);
|
||||
}
|
||||
|
||||
static void doDeleteSessionWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result) {
|
||||
static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result) {
|
||||
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
|
||||
TSKEY* startDatas = (TSKEY*)pStartTsCol->pData;
|
||||
SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
|
||||
|
|
@ -3580,8 +3580,8 @@ typedef SResultWindowInfo* (*__get_win_info_)(void*);
|
|||
SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; }
|
||||
SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; }
|
||||
|
||||
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup,
|
||||
SArray* pClosed, __get_win_info_ fn, bool delete) {
|
||||
int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn,
|
||||
bool delete) {
|
||||
// Todo(liuyao) save window to tdb
|
||||
void** pIte = NULL;
|
||||
size_t keyLen = 0;
|
||||
|
|
@ -3700,13 +3700,13 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
|
|||
} else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
|
||||
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
|
||||
// gap must be 0
|
||||
doDeleteSessionWindows(&pInfo->streamAggSup, pBlock, 0, pWins);
|
||||
doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins);
|
||||
if (IS_FINAL_OP(pInfo)) {
|
||||
int32_t childIndex = getChildIndex(pBlock);
|
||||
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
|
||||
SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
|
||||
// gap must be 0
|
||||
doDeleteSessionWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL);
|
||||
doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL);
|
||||
rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator);
|
||||
}
|
||||
copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
|
||||
|
|
@ -3743,8 +3743,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
|
|||
// restore the value
|
||||
pOperator->status = OP_RES_TO_RETURN;
|
||||
|
||||
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
|
||||
getResWinForSession, pInfo->ignoreExpiredData);
|
||||
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForSession,
|
||||
pInfo->ignoreExpiredData);
|
||||
closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData);
|
||||
copyUpdateResult(pStUpdated, pUpdated);
|
||||
taosHashCleanup(pStUpdated);
|
||||
|
|
@ -3840,7 +3840,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
|
|||
break;
|
||||
} else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
|
||||
// gap must be 0
|
||||
doDeleteSessionWindows(&pInfo->streamAggSup, pBlock, 0, NULL);
|
||||
doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, NULL);
|
||||
copyDataBlock(pInfo->pDelRes, pBlock);
|
||||
pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
|
||||
break;
|
||||
|
|
@ -4232,6 +4232,12 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
|
|||
doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId,
|
||||
pSeUpdated, pInfo->pSeDeleted);
|
||||
continue;
|
||||
} else if (pBlock->info.type == STREAM_DELETE_DATA) {
|
||||
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
|
||||
doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins);
|
||||
copyDeleteWindowInfo(pWins, pInfo->pSeDeleted);
|
||||
taosArrayDestroy(pWins);
|
||||
continue;
|
||||
} else if (pBlock->info.type == STREAM_GET_ALL) {
|
||||
getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForState);
|
||||
continue;
|
||||
|
|
@ -4245,8 +4251,8 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
|
|||
// restore the value
|
||||
pOperator->status = OP_RES_TO_RETURN;
|
||||
|
||||
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
|
||||
getResWinForState, pInfo->ignoreExpiredData);
|
||||
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForState,
|
||||
pInfo->ignoreExpiredData);
|
||||
closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData);
|
||||
copyUpdateResult(pSeUpdated, pUpdated);
|
||||
taosHashCleanup(pSeUpdated);
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
#define _STREAM_INC_H_
|
||||
|
||||
#include "executor.h"
|
||||
#include "tref.h"
|
||||
#include "tstream.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
|
@ -25,6 +26,7 @@ extern "C" {
|
|||
|
||||
typedef struct {
|
||||
int8_t inited;
|
||||
int32_t refPool;
|
||||
void* timer;
|
||||
} SStreamGlobalEnv;
|
||||
|
||||
|
|
|
|||
|
|
@ -76,9 +76,6 @@ void streamTriggerByTimer(void* param, void* tmrId) {
|
|||
|
||||
int32_t streamSetupTrigger(SStreamTask* pTask) {
|
||||
if (pTask->triggerParam != 0) {
|
||||
if (streamInit() < 0) {
|
||||
return -1;
|
||||
}
|
||||
pTask->timer = taosTmrStart(streamTriggerByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer);
|
||||
pTask->triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -67,7 +67,6 @@ typedef struct SSyncNode {
|
|||
char path[TSDB_FILENAME_LEN];
|
||||
char raftStorePath[TSDB_FILENAME_LEN * 2];
|
||||
char configPath[TSDB_FILENAME_LEN * 2];
|
||||
int32_t batchSize;
|
||||
|
||||
// sync io
|
||||
SWal* pWal;
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ typedef struct SSyncLogStoreData {
|
|||
SWal* pWal;
|
||||
|
||||
TdThreadMutex mutex;
|
||||
SWalReadHandle* pWalHandle;
|
||||
SWalReader* pWalHandle;
|
||||
|
||||
// SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0
|
||||
} SSyncLogStoreData;
|
||||
|
|
|
|||
|
|
@ -842,8 +842,8 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
|
|||
do {
|
||||
char logBuf[128];
|
||||
snprintf(logBuf, sizeof(logBuf),
|
||||
"recv sync-append-entries-batch, fake match2, pre-index:%ld, pre-term:%lu, datalen:%d",
|
||||
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen);
|
||||
"recv sync-append-entries-batch, fake match2, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}",
|
||||
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount);
|
||||
syncNodeEventLog(ths, logBuf);
|
||||
} while (0);
|
||||
|
||||
|
|
@ -876,7 +876,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
|
|||
code = syncNodePreCommit(ths, pAppendEntry);
|
||||
ASSERT(code == 0);
|
||||
|
||||
syncEntryDestory(pAppendEntry);
|
||||
// syncEntryDestory(pAppendEntry);
|
||||
}
|
||||
|
||||
// fsync once
|
||||
|
|
@ -931,8 +931,8 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
|
|||
do {
|
||||
char logBuf[128];
|
||||
snprintf(logBuf, sizeof(logBuf),
|
||||
"recv sync-append-entries-batch, not match, pre-index:%ld, pre-term:%lu, datalen:%d",
|
||||
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen);
|
||||
"recv sync-append-entries-batch, not match, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}",
|
||||
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount);
|
||||
syncNodeEventLog(ths, logBuf);
|
||||
} while (0);
|
||||
|
||||
|
|
@ -976,8 +976,9 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
|
|||
|
||||
do {
|
||||
char logBuf[128];
|
||||
snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, match, pre-index:%ld, pre-term:%lu, datalen:%d",
|
||||
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen);
|
||||
snprintf(logBuf, sizeof(logBuf),
|
||||
"recv sync-append-entries, match, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}",
|
||||
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount);
|
||||
syncNodeEventLog(ths, logBuf);
|
||||
} while (0);
|
||||
|
||||
|
|
@ -999,7 +1000,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
|
|||
code = syncNodePreCommit(ths, pAppendEntry);
|
||||
ASSERT(code == 0);
|
||||
|
||||
syncEntryDestory(pAppendEntry);
|
||||
// syncEntryDestory(pAppendEntry);
|
||||
}
|
||||
|
||||
// fsync once
|
||||
|
|
|
|||
|
|
@ -174,8 +174,12 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie
|
|||
SyncIndex newNextIndex = pMsg->matchIndex + 1;
|
||||
SyncIndex newMatchIndex = pMsg->matchIndex;
|
||||
|
||||
if (ths->pLogStore->syncLogExist(ths->pLogStore, newNextIndex) &&
|
||||
ths->pLogStore->syncLogExist(ths->pLogStore, newNextIndex - 1)) {
|
||||
bool needStartSnapshot = false;
|
||||
if (newMatchIndex >= SYNC_INDEX_BEGIN && !ths->pLogStore->syncLogExist(ths->pLogStore, newMatchIndex)) {
|
||||
needStartSnapshot = true;
|
||||
}
|
||||
|
||||
if (!needStartSnapshot) {
|
||||
// update next-index, match-index
|
||||
syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), newNextIndex);
|
||||
syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), newMatchIndex);
|
||||
|
|
@ -197,15 +201,36 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie
|
|||
syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), newMatchIndex);
|
||||
}
|
||||
|
||||
// event log, update next-index
|
||||
do {
|
||||
char host[64];
|
||||
int16_t port;
|
||||
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
|
||||
|
||||
char logBuf[256];
|
||||
snprintf(logBuf, sizeof(logBuf), "reset next-index:%ld, match-index:%ld for %s:%d", newNextIndex, newMatchIndex,
|
||||
host, port);
|
||||
syncNodeEventLog(ths, logBuf);
|
||||
|
||||
} while (0);
|
||||
|
||||
} else {
|
||||
SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId));
|
||||
|
||||
if (nextIndex > SYNC_INDEX_BEGIN) {
|
||||
--nextIndex;
|
||||
|
||||
if (ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex) &&
|
||||
ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex - 1)) {
|
||||
bool needStartSnapshot = false;
|
||||
if (nextIndex >= SYNC_INDEX_BEGIN && !ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex)) {
|
||||
needStartSnapshot = true;
|
||||
}
|
||||
if (nextIndex - 1 >= SYNC_INDEX_BEGIN && !ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex - 1)) {
|
||||
needStartSnapshot = true;
|
||||
}
|
||||
|
||||
if (!needStartSnapshot) {
|
||||
// do nothing
|
||||
|
||||
} else {
|
||||
SSyncRaftEntry* pEntry;
|
||||
int32_t code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, nextIndex, &pEntry);
|
||||
|
|
@ -227,6 +252,21 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie
|
|||
nextIndex = SYNC_INDEX_BEGIN;
|
||||
}
|
||||
syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex);
|
||||
|
||||
// event log, update next-index
|
||||
do {
|
||||
char host[64];
|
||||
int16_t port;
|
||||
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
|
||||
|
||||
SyncIndex newNextIndex = nextIndex;
|
||||
SyncIndex newMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId));
|
||||
char logBuf[256];
|
||||
snprintf(logBuf, sizeof(logBuf), "reset2 next-index:%ld, match-index:%ld for %s:%d", newNextIndex, newMatchIndex,
|
||||
host, port);
|
||||
syncNodeEventLog(ths, logBuf);
|
||||
|
||||
} while (0);
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -1526,12 +1526,14 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
|
|||
if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) {
|
||||
snprintf(logBuf, sizeof(logBuf),
|
||||
"vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, "
|
||||
"strategy:%d, batch:%d, "
|
||||
"replica-num:%d, "
|
||||
"lconfig:%ld, changing:%d, restore:%d, %s",
|
||||
pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm,
|
||||
pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex,
|
||||
pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex,
|
||||
pSyncNode->changing, pSyncNode->restoreFinish, printStr);
|
||||
pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize,
|
||||
pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing,
|
||||
pSyncNode->restoreFinish, printStr);
|
||||
} else {
|
||||
snprintf(logBuf, sizeof(logBuf), "%s", str);
|
||||
}
|
||||
|
|
@ -1543,12 +1545,14 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
|
|||
if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) {
|
||||
snprintf(s, len,
|
||||
"vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, "
|
||||
"strategy:%d, batch:%d, "
|
||||
"replica-num:%d, "
|
||||
"lconfig:%ld, changing:%d, restore:%d, %s",
|
||||
pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm,
|
||||
pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex,
|
||||
pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex,
|
||||
pSyncNode->changing, pSyncNode->restoreFinish, printStr);
|
||||
pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize,
|
||||
pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing,
|
||||
pSyncNode->restoreFinish, printStr);
|
||||
} else {
|
||||
snprintf(s, len, "%s", str);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1605,7 +1605,7 @@ void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg) {
|
|||
|
||||
SyncAppendEntriesBatch* syncAppendEntriesBatchBuild(SSyncRaftEntry** entryPArr, int32_t arrSize, int32_t vgId) {
|
||||
ASSERT(entryPArr != NULL);
|
||||
ASSERT(arrSize > 0);
|
||||
ASSERT(arrSize >= 0);
|
||||
|
||||
int32_t dataLen = 0;
|
||||
int32_t metaArrayLen = sizeof(SOffsetAndContLen) * arrSize; // <offset, contLen>
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
|
|||
ASSERT(pData->pWal != NULL);
|
||||
|
||||
taosThreadMutexInit(&(pData->mutex), NULL);
|
||||
pData->pWalHandle = walOpenReadHandle(pData->pWal);
|
||||
pData->pWalHandle = walOpenReader(pData->pWal, NULL);
|
||||
ASSERT(pData->pWalHandle != NULL);
|
||||
|
||||
pLogStore->appendEntry = logStoreAppendEntry;
|
||||
|
|
@ -95,7 +95,7 @@ void logStoreDestory(SSyncLogStore* pLogStore) {
|
|||
|
||||
taosThreadMutexLock(&(pData->mutex));
|
||||
if (pData->pWalHandle != NULL) {
|
||||
walCloseReadHandle(pData->pWalHandle);
|
||||
walCloseReader(pData->pWalHandle);
|
||||
pData->pWalHandle = NULL;
|
||||
}
|
||||
taosThreadMutexUnlock(&(pData->mutex));
|
||||
|
|
@ -255,7 +255,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
|
|||
*ppEntry = NULL;
|
||||
|
||||
// SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
|
||||
SWalReadHandle* pWalHandle = pData->pWalHandle;
|
||||
SWalReader* pWalHandle = pData->pWalHandle;
|
||||
if (pWalHandle == NULL) {
|
||||
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
|
||||
return -1;
|
||||
|
|
@ -263,7 +263,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
|
|||
|
||||
taosThreadMutexLock(&(pData->mutex));
|
||||
|
||||
code = walReadWithHandle(pWalHandle, index);
|
||||
code = walReadVer(pWalHandle, index);
|
||||
if (code != 0) {
|
||||
int32_t err = terrno;
|
||||
const char* errStr = tstrerror(err);
|
||||
|
|
@ -398,10 +398,10 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
|
|||
taosThreadMutexLock(&(pData->mutex));
|
||||
|
||||
// SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
|
||||
SWalReadHandle* pWalHandle = pData->pWalHandle;
|
||||
SWalReader* pWalHandle = pData->pWalHandle;
|
||||
ASSERT(pWalHandle != NULL);
|
||||
|
||||
int32_t code = walReadWithHandle(pWalHandle, index);
|
||||
int32_t code = walReadVer(pWalHandle, index);
|
||||
if (code != 0) {
|
||||
int32_t err = terrno;
|
||||
const char* errStr = tstrerror(err);
|
||||
|
|
|
|||
|
|
@ -148,9 +148,17 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) {
|
|||
// get entry batch
|
||||
int32_t getCount = 0;
|
||||
SyncIndex getEntryIndex = nextIndex;
|
||||
for (int32_t i = 0; i < pSyncNode->batchSize; ++i) {
|
||||
for (int32_t i = 0; i < pSyncNode->pRaftCfg->batchSize; ++i) {
|
||||
SSyncRaftEntry* pEntry = NULL;
|
||||
int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, getEntryIndex, &pEntry);
|
||||
|
||||
// event log
|
||||
do {
|
||||
char logBuf[128];
|
||||
snprintf(logBuf, sizeof(logBuf), "get index:%d, code:%d, %s", getEntryIndex, code, tstrerror(terrno));
|
||||
syncNodeEventLog(pSyncNode, logBuf);
|
||||
} while (0);
|
||||
|
||||
if (code == 0) {
|
||||
ASSERT(pEntry != NULL);
|
||||
entryPArr[i] = pEntry;
|
||||
|
|
@ -162,12 +170,19 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) {
|
|||
}
|
||||
}
|
||||
|
||||
// event log
|
||||
do {
|
||||
char logBuf[128];
|
||||
snprintf(logBuf, sizeof(logBuf), "build batch:%d", getCount);
|
||||
syncNodeEventLog(pSyncNode, logBuf);
|
||||
} while (0);
|
||||
|
||||
// build msg
|
||||
SyncAppendEntriesBatch* pMsg = syncAppendEntriesBatchBuild(entryPArr, getCount, pSyncNode->vgId);
|
||||
ASSERT(pMsg != NULL);
|
||||
|
||||
// free entries
|
||||
for (int32_t i = 0; i < pSyncNode->batchSize; ++i) {
|
||||
for (int32_t i = 0; i < pSyncNode->pRaftCfg->batchSize; ++i) {
|
||||
SSyncRaftEntry* pEntry = entryPArr[i];
|
||||
if (pEntry != NULL) {
|
||||
syncEntryDestory(pEntry);
|
||||
|
|
|
|||
|
|
@ -16,20 +16,33 @@
|
|||
#define _DEFAULT_SOURCE
|
||||
#include "tmsgcb.h"
|
||||
#include "taoserror.h"
|
||||
#include "trpc.h"
|
||||
|
||||
static SMsgCb defaultMsgCb;
|
||||
|
||||
void tmsgSetDefault(const SMsgCb* msgcb) { defaultMsgCb = *msgcb; }
|
||||
|
||||
int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg) {
|
||||
return (*msgcb->putToQueueFp)(msgcb->mgmt, qtype, pMsg);
|
||||
int32_t code = (*msgcb->putToQueueFp)(msgcb->mgmt, qtype, pMsg);
|
||||
if (code != 0) {
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
pMsg->pCont = NULL;
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype) {
|
||||
return (*msgcb->qsizeFp)(msgcb->mgmt, vgId, qtype);
|
||||
}
|
||||
|
||||
int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { return (*defaultMsgCb.sendReqFp)(epSet, pMsg); }
|
||||
int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) {
|
||||
int32_t code = (*defaultMsgCb.sendReqFp)(epSet, pMsg);
|
||||
if (code != 0) {
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
pMsg->pCont = NULL;
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
void tmsgSendRsp(SRpcMsg* pMsg) { return (*defaultMsgCb.sendRspFp)(pMsg); }
|
||||
|
||||
|
|
@ -573,7 +573,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
|
|||
return;
|
||||
}
|
||||
if (nread < 0) {
|
||||
tError("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread),
|
||||
tWarn("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread),
|
||||
T_REF_VAL_GET(conn));
|
||||
conn->broken = true;
|
||||
cliHandleExcept(conn);
|
||||
|
|
|
|||
|
|
@ -305,7 +305,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
|
|||
return;
|
||||
}
|
||||
|
||||
tError("%s conn %p read error: %s", transLabel(pTransInst), conn, uv_err_name(nread));
|
||||
tWarn("%s conn %p read error: %s", transLabel(pTransInst), conn, uv_err_name(nread));
|
||||
if (nread < 0) {
|
||||
conn->broken = true;
|
||||
if (conn->status == ConnAcquire) {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
#include "taoserror.h"
|
||||
#include "tchecksum.h"
|
||||
#include "tcoding.h"
|
||||
#include "tcommon.h"
|
||||
#include "tcompare.h"
|
||||
#include "wal.h"
|
||||
|
||||
|
|
|
|||
|
|
@ -16,20 +16,29 @@
|
|||
#include "taoserror.h"
|
||||
#include "walInt.h"
|
||||
|
||||
SWalReadHandle *walOpenReadHandle(SWal *pWal) {
|
||||
SWalReadHandle *pRead = taosMemoryMalloc(sizeof(SWalReadHandle));
|
||||
static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer);
|
||||
static int32_t walFetchBodyNew(SWalReader *pRead);
|
||||
static int32_t walSkipFetchBodyNew(SWalReader *pRead);
|
||||
|
||||
SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) {
|
||||
SWalReader *pRead = taosMemoryMalloc(sizeof(SWalReader));
|
||||
if (pRead == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
pRead->pWal = pWal;
|
||||
pRead->pReadIdxTFile = NULL;
|
||||
pRead->pReadLogTFile = NULL;
|
||||
pRead->pIdxFile = NULL;
|
||||
pRead->pLogFile = NULL;
|
||||
pRead->curVersion = -1;
|
||||
pRead->curFileFirstVer = -1;
|
||||
pRead->capacity = 0;
|
||||
pRead->status = 0;
|
||||
if (cond)
|
||||
pRead->cond = *cond;
|
||||
else {
|
||||
pRead->cond.scanMeta = 0;
|
||||
pRead->cond.scanUncommited = 0;
|
||||
}
|
||||
|
||||
taosThreadMutexInit(&pRead->mutex, NULL);
|
||||
|
||||
|
|
@ -39,26 +48,46 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) {
|
|||
taosMemoryFree(pRead);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return pRead;
|
||||
}
|
||||
|
||||
void walCloseReadHandle(SWalReadHandle *pRead) {
|
||||
taosCloseFile(&pRead->pReadIdxTFile);
|
||||
taosCloseFile(&pRead->pReadLogTFile);
|
||||
void walCloseReader(SWalReader *pRead) {
|
||||
taosCloseFile(&pRead->pIdxFile);
|
||||
taosCloseFile(&pRead->pLogFile);
|
||||
taosMemoryFreeClear(pRead->pHead);
|
||||
taosMemoryFree(pRead);
|
||||
}
|
||||
|
||||
int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) {
|
||||
// TODO
|
||||
int32_t walNextValidMsg(SWalReader *pRead) {
|
||||
int64_t fetchVer = pRead->curVersion;
|
||||
int64_t endVer = pRead->cond.scanUncommited ? walGetLastVer(pRead->pWal) : walGetCommittedVer(pRead->pWal);
|
||||
while (fetchVer <= endVer) {
|
||||
if (walFetchHeadNew(pRead, fetchVer) < 0) {
|
||||
return -1;
|
||||
}
|
||||
if (pRead->pHead->head.msgType == TDMT_VND_SUBMIT ||
|
||||
(IS_META_MSG(pRead->pHead->head.msgType) && pRead->cond.scanMeta)) {
|
||||
if (walFetchBodyNew(pRead) < 0) {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
} else {
|
||||
if (walSkipFetchBodyNew(pRead) < 0) {
|
||||
return -1;
|
||||
}
|
||||
fetchVer++;
|
||||
ASSERT(fetchVer == pRead->curVersion);
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
static int64_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) {
|
||||
static int64_t walReadSeekFilePos(SWalReader *pRead, int64_t fileFirstVer, int64_t ver) {
|
||||
int64_t ret = 0;
|
||||
|
||||
TdFilePtr pIdxTFile = pRead->pReadIdxTFile;
|
||||
TdFilePtr pLogTFile = pRead->pReadLogTFile;
|
||||
TdFilePtr pIdxTFile = pRead->pIdxFile;
|
||||
TdFilePtr pLogTFile = pRead->pLogFile;
|
||||
|
||||
// seek position
|
||||
int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry);
|
||||
|
|
@ -90,11 +119,11 @@ static int64_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i
|
|||
return ret;
|
||||
}
|
||||
|
||||
static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
|
||||
static int32_t walReadChangeFile(SWalReader *pRead, int64_t fileFirstVer) {
|
||||
char fnameStr[WAL_FILE_LEN];
|
||||
|
||||
taosCloseFile(&pRead->pReadIdxTFile);
|
||||
taosCloseFile(&pRead->pReadLogTFile);
|
||||
taosCloseFile(&pRead->pIdxFile);
|
||||
taosCloseFile(&pRead->pLogFile);
|
||||
|
||||
walBuildLogName(pRead->pWal, fileFirstVer, fnameStr);
|
||||
TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ);
|
||||
|
|
@ -104,7 +133,7 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
pRead->pReadLogTFile = pLogTFile;
|
||||
pRead->pLogFile = pLogTFile;
|
||||
|
||||
walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr);
|
||||
TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ);
|
||||
|
|
@ -114,11 +143,11 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
pRead->pReadIdxTFile = pIdxTFile;
|
||||
pRead->pIdxFile = pIdxTFile;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
|
||||
static int32_t walReadSeekVer(SWalReader *pRead, int64_t ver) {
|
||||
SWal *pWal = pRead->pWal;
|
||||
if (ver == pRead->curVersion) {
|
||||
return 0;
|
||||
|
|
@ -153,9 +182,94 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity) { pRead->capacity = capacity; }
|
||||
void walSetReaderCapacity(SWalReader *pRead, int32_t capacity) { pRead->capacity = capacity; }
|
||||
|
||||
int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalCkHead *pHead) {
|
||||
static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) {
|
||||
int64_t contLen;
|
||||
if (pRead->curVersion != fetchVer) {
|
||||
if (walReadSeekVer(pRead, fetchVer) < 0) return -1;
|
||||
}
|
||||
contLen = taosReadFile(pRead->pLogFile, pRead->pHead, sizeof(SWalCkHead));
|
||||
if (contLen != sizeof(SWalCkHead)) {
|
||||
if (contLen < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
} else {
|
||||
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
|
||||
}
|
||||
pRead->curVersion = -1;
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t walFetchBodyNew(SWalReader *pRead) {
|
||||
SWalCont *pReadHead = &pRead->pHead->head;
|
||||
int64_t ver = pReadHead->version;
|
||||
|
||||
if (pRead->capacity < pReadHead->bodyLen) {
|
||||
void *ptr = taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen);
|
||||
if (ptr == NULL) {
|
||||
terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
pRead->pHead = ptr;
|
||||
pReadHead = &pRead->pHead->head;
|
||||
pRead->capacity = pReadHead->bodyLen;
|
||||
}
|
||||
|
||||
if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) {
|
||||
if (pReadHead->bodyLen < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 ", since %s",
|
||||
pRead->pHead->head.version, ver, tstrerror(terrno));
|
||||
} else {
|
||||
wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 ", since file corrupted",
|
||||
pRead->pHead->head.version, ver);
|
||||
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
|
||||
}
|
||||
pRead->curVersion = -1;
|
||||
ASSERT(0);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (pReadHead->version != ver) {
|
||||
wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, ver);
|
||||
pRead->curVersion = -1;
|
||||
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
|
||||
ASSERT(0);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (walValidBodyCksum(pRead->pHead) != 0) {
|
||||
wError("wal fetch body error: % " PRId64 ", since body checksum not passed", ver);
|
||||
pRead->curVersion = -1;
|
||||
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
|
||||
ASSERT(0);
|
||||
return -1;
|
||||
}
|
||||
|
||||
pRead->curVersion = ver + 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t walSkipFetchBodyNew(SWalReader *pRead) {
|
||||
int64_t code;
|
||||
|
||||
ASSERT(pRead->curVersion == pRead->pHead->head.version);
|
||||
|
||||
code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR);
|
||||
if (code < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
pRead->curVersion = -1;
|
||||
return -1;
|
||||
}
|
||||
|
||||
pRead->curVersion++;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) {
|
||||
int64_t code;
|
||||
|
||||
// TODO: valid ver
|
||||
|
|
@ -168,9 +282,9 @@ int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalCkHead *pHead) {
|
|||
if (code < 0) return -1;
|
||||
}
|
||||
|
||||
ASSERT(taosValidFile(pRead->pReadLogTFile) == true);
|
||||
ASSERT(taosValidFile(pRead->pLogFile) == true);
|
||||
|
||||
code = taosReadFile(pRead->pReadLogTFile, pHead, sizeof(SWalCkHead));
|
||||
code = taosReadFile(pRead->pLogFile, pHead, sizeof(SWalCkHead));
|
||||
if (code != sizeof(SWalCkHead)) {
|
||||
return -1;
|
||||
}
|
||||
|
|
@ -186,12 +300,12 @@ int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalCkHead *pHead) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalCkHead *pHead) {
|
||||
int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead) {
|
||||
int64_t code;
|
||||
|
||||
ASSERT(pRead->curVersion == pHead->head.version);
|
||||
|
||||
code = taosLSeekFile(pRead->pReadLogTFile, pHead->head.bodyLen, SEEK_CUR);
|
||||
code = taosLSeekFile(pRead->pLogFile, pHead->head.bodyLen, SEEK_CUR);
|
||||
if (code < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
pRead->curVersion = -1;
|
||||
|
|
@ -203,7 +317,7 @@ int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalCkHead *pHead) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t walFetchBody(SWalReadHandle *pRead, SWalCkHead **ppHead) {
|
||||
int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) {
|
||||
SWalCont *pReadHead = &((*ppHead)->head);
|
||||
int64_t ver = pReadHead->version;
|
||||
|
||||
|
|
@ -218,7 +332,7 @@ int32_t walFetchBody(SWalReadHandle *pRead, SWalCkHead **ppHead) {
|
|||
pRead->capacity = pReadHead->bodyLen;
|
||||
}
|
||||
|
||||
if (pReadHead->bodyLen != taosReadFile(pRead->pReadLogTFile, pReadHead->body, pReadHead->bodyLen)) {
|
||||
if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) {
|
||||
ASSERT(0);
|
||||
return -1;
|
||||
}
|
||||
|
|
@ -241,9 +355,9 @@ int32_t walFetchBody(SWalReadHandle *pRead, SWalCkHead **ppHead) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalCont **ppHead) {
|
||||
int32_t walReadWithHandle_s(SWalReader *pRead, int64_t ver, SWalCont **ppHead) {
|
||||
taosThreadMutexLock(&pRead->mutex);
|
||||
if (walReadWithHandle(pRead, ver) < 0) {
|
||||
if (walReadVer(pRead, ver) < 0) {
|
||||
taosThreadMutexUnlock(&pRead->mutex);
|
||||
return -1;
|
||||
}
|
||||
|
|
@ -257,7 +371,7 @@ int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalCont **ppHea
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
|
||||
int32_t walReadVer(SWalReader *pRead, int64_t ver) {
|
||||
int64_t code;
|
||||
|
||||
if (pRead->pWal->vers.firstVer == -1) {
|
||||
|
|
@ -280,9 +394,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
ASSERT(taosValidFile(pRead->pReadLogTFile) == true);
|
||||
ASSERT(taosValidFile(pRead->pLogFile) == true);
|
||||
|
||||
code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalCkHead));
|
||||
code = taosReadFile(pRead->pLogFile, pRead->pHead, sizeof(SWalCkHead));
|
||||
if (code != sizeof(SWalCkHead)) {
|
||||
if (code < 0)
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
|
|
@ -310,7 +424,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
|
|||
pRead->capacity = pRead->pHead->head.bodyLen;
|
||||
}
|
||||
|
||||
if ((code = taosReadFile(pRead->pReadLogTFile, pRead->pHead->head.body, pRead->pHead->head.bodyLen)) !=
|
||||
if ((code = taosReadFile(pRead->pLogFile, pRead->pHead->head.body, pRead->pHead->head.bodyLen)) !=
|
||||
pRead->pHead->head.bodyLen) {
|
||||
if (code < 0)
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
|
|
@ -340,46 +454,3 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
|
|||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#if 0
|
||||
int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
|
||||
int code;
|
||||
code = walSeekVer(pWal, ver);
|
||||
if (code != 0) {
|
||||
return code;
|
||||
}
|
||||
if (*ppHead == NULL) {
|
||||
void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalHead));
|
||||
if (ptr == NULL) {
|
||||
return -1;
|
||||
}
|
||||
*ppHead = ptr;
|
||||
}
|
||||
if (tfRead(pWal->pWriteLogTFile, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
|
||||
return -1;
|
||||
}
|
||||
// TODO: endian compatibility processing after read
|
||||
if (walValidHeadCksum(*ppHead) != 0) {
|
||||
return -1;
|
||||
}
|
||||
void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len);
|
||||
if (ptr == NULL) {
|
||||
taosMemoryFree(*ppHead);
|
||||
*ppHead = NULL;
|
||||
return -1;
|
||||
}
|
||||
if (tfRead(pWal->pWriteLogTFile, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) {
|
||||
return -1;
|
||||
}
|
||||
// TODO: endian compatibility processing after read
|
||||
if (walValidBodyCksum(*ppHead) != 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) {
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -79,19 +79,21 @@ int32_t walCommit(SWal *pWal, int64_t ver) {
|
|||
}
|
||||
|
||||
int32_t walRollback(SWal *pWal, int64_t ver) {
|
||||
taosThreadMutexLock(&pWal->mutex);
|
||||
int64_t code;
|
||||
char fnameStr[WAL_FILE_LEN];
|
||||
if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) {
|
||||
terrno = TSDB_CODE_WAL_INVALID_VER;
|
||||
taosThreadMutexUnlock(&pWal->mutex);
|
||||
return -1;
|
||||
}
|
||||
taosThreadMutexLock(&pWal->mutex);
|
||||
|
||||
// find correct file
|
||||
if (ver < walGetLastFileFirstVer(pWal)) {
|
||||
// change current files
|
||||
code = walChangeWrite(pWal, ver);
|
||||
if (code < 0) {
|
||||
taosThreadMutexUnlock(&pWal->mutex);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -146,6 +148,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
|
|||
ASSERT(taosValidFile(pLogTFile));
|
||||
int64_t size = taosReadFile(pLogTFile, &head, sizeof(SWalCkHead));
|
||||
if (size != sizeof(SWalCkHead)) {
|
||||
taosThreadMutexUnlock(&pWal->mutex);
|
||||
return -1;
|
||||
}
|
||||
code = walValidHeadCksum(&head);
|
||||
|
|
@ -154,11 +157,13 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
|
|||
if (code != 0) {
|
||||
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
|
||||
ASSERT(0);
|
||||
taosThreadMutexUnlock(&pWal->mutex);
|
||||
return -1;
|
||||
}
|
||||
if (head.head.version != ver) {
|
||||
ASSERT(0);
|
||||
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
|
||||
taosThreadMutexUnlock(&pWal->mutex);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -167,12 +172,14 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
|
|||
if (code < 0) {
|
||||
ASSERT(0);
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
taosThreadMutexUnlock(&pWal->mutex);
|
||||
return -1;
|
||||
}
|
||||
code = taosFtruncateFile(pIdxTFile, idxOff);
|
||||
if (code < 0) {
|
||||
ASSERT(0);
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
taosThreadMutexUnlock(&pWal->mutex);
|
||||
return -1;
|
||||
}
|
||||
pWal->vers.lastVer = ver - 1;
|
||||
|
|
|
|||
|
|
@ -293,7 +293,7 @@ TEST_F(WalCleanDeleteEnv, roll) {
|
|||
TEST_F(WalKeepEnv, readHandleRead) {
|
||||
walResetEnv();
|
||||
int code;
|
||||
SWalReadHandle* pRead = walOpenReadHandle(pWal);
|
||||
SWalReader* pRead = walOpenReader(pWal, NULL);
|
||||
ASSERT(pRead != NULL);
|
||||
|
||||
int i;
|
||||
|
|
@ -306,7 +306,7 @@ TEST_F(WalKeepEnv, readHandleRead) {
|
|||
}
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
int ver = taosRand() % 100;
|
||||
code = walReadWithHandle(pRead, ver);
|
||||
code = walReadVer(pRead, ver);
|
||||
ASSERT_EQ(code, 0);
|
||||
|
||||
// printf("rrbody: \n");
|
||||
|
|
@ -325,7 +325,7 @@ TEST_F(WalKeepEnv, readHandleRead) {
|
|||
EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]);
|
||||
}
|
||||
}
|
||||
walCloseReadHandle(pRead);
|
||||
walCloseReader(pRead);
|
||||
}
|
||||
|
||||
TEST_F(WalRetentionEnv, repairMeta1) {
|
||||
|
|
@ -354,12 +354,12 @@ TEST_F(WalRetentionEnv, repairMeta1) {
|
|||
|
||||
ASSERT_EQ(pWal->vers.lastVer, 99);
|
||||
|
||||
SWalReadHandle* pRead = walOpenReadHandle(pWal);
|
||||
SWalReader* pRead = walOpenReader(pWal, NULL);
|
||||
ASSERT(pRead != NULL);
|
||||
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
int ver = taosRand() % 100;
|
||||
code = walReadWithHandle(pRead, ver);
|
||||
code = walReadVer(pRead, ver);
|
||||
ASSERT_EQ(code, 0);
|
||||
|
||||
// printf("rrbody: \n");
|
||||
|
|
@ -389,7 +389,7 @@ TEST_F(WalRetentionEnv, repairMeta1) {
|
|||
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
int ver = taosRand() % 200;
|
||||
code = walReadWithHandle(pRead, ver);
|
||||
code = walReadVer(pRead, ver);
|
||||
ASSERT_EQ(code, 0);
|
||||
|
||||
// printf("rrbody: \n");
|
||||
|
|
|
|||
|
|
@ -857,19 +857,27 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) {
|
|||
return;
|
||||
}
|
||||
buf[n] = '\0';
|
||||
for (int i = n - 1; i >= 0; --i) {
|
||||
if (buf[i] == '/') {
|
||||
if (tz) {
|
||||
tz = buf + i + 1;
|
||||
break;
|
||||
}
|
||||
tz = buf + i + 1;
|
||||
}
|
||||
}
|
||||
if (!tz || 0 == strchr(tz, '/')) {
|
||||
|
||||
char *zi = strstr(buf, "zoneinfo");
|
||||
if (!zi) {
|
||||
printf("parsing /etc/localtime failed");
|
||||
return;
|
||||
}
|
||||
tz = zi + strlen("zoneinfo") + 1;
|
||||
|
||||
//for (int i = n - 1; i >= 0; --i) {
|
||||
// if (buf[i] == '/') {
|
||||
// if (tz) {
|
||||
// tz = buf + i + 1;
|
||||
// break;
|
||||
// }
|
||||
// tz = buf + i + 1;
|
||||
// }
|
||||
//}
|
||||
//if (!tz || 0 == strchr(tz, '/')) {
|
||||
// printf("parsing /etc/localtime failed");
|
||||
// return;
|
||||
//}
|
||||
|
||||
setenv("TZ", tz, 1);
|
||||
tzset();
|
||||
|
|
@ -962,19 +970,27 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) {
|
|||
return;
|
||||
}
|
||||
buf[n] = '\0';
|
||||
for (int i = n - 1; i >= 0; --i) {
|
||||
if (buf[i] == '/') {
|
||||
if (tz) {
|
||||
tz = buf + i + 1;
|
||||
break;
|
||||
}
|
||||
tz = buf + i + 1;
|
||||
}
|
||||
}
|
||||
if (!tz || 0 == strchr(tz, '/')) {
|
||||
|
||||
char *zi = strstr(buf, "zoneinfo");
|
||||
if (!zi) {
|
||||
printf("parsing /etc/localtime failed");
|
||||
return;
|
||||
}
|
||||
tz = zi + strlen("zoneinfo") + 1;
|
||||
|
||||
//for (int i = n - 1; i >= 0; --i) {
|
||||
// if (buf[i] == '/') {
|
||||
// if (tz) {
|
||||
// tz = buf + i + 1;
|
||||
// break;
|
||||
// }
|
||||
// tz = buf + i + 1;
|
||||
// }
|
||||
//}
|
||||
//if (!tz || 0 == strchr(tz, '/')) {
|
||||
// printf("parsing /etc/localtime failed");
|
||||
// return;
|
||||
//}
|
||||
|
||||
setenv("TZ", tz, 1);
|
||||
tzset();
|
||||
|
|
|
|||
|
|
@ -518,8 +518,8 @@ void releaseBufPageInfo(SDiskbasedBuf* pBuf, SPageInfo* pi) {
|
|||
#ifdef BUF_PAGE_DEBUG
|
||||
uDebug("page_releaseBufPageInfo pageId:%d, used:%d, offset:%"PRId64, pi->pageId, pi->used, pi->offset);
|
||||
#endif
|
||||
assert(pi->pData != NULL && pi->used == true);
|
||||
// assert(pi->pData != NULL);
|
||||
// assert(pi->pData != NULL && pi->used == true);
|
||||
assert(pi->pData != NULL);
|
||||
pi->used = false;
|
||||
pBuf->statis.releasePages += 1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -64,6 +64,20 @@ int32_t strdequote(char *z) {
|
|||
return j + 1; // only one quote, do nothing
|
||||
}
|
||||
|
||||
char *strDupUnquo(const char *src) {
|
||||
if (src == NULL) return NULL;
|
||||
if (src[0] != '`') return strdup(src);
|
||||
int32_t len = (int32_t)strlen(src);
|
||||
if (src[len - 1] != '`') return NULL;
|
||||
char *ret = taosMemoryMalloc(len);
|
||||
if (ret == NULL) return NULL;
|
||||
for (int32_t i = 0; i < len - 1; i++) {
|
||||
ret[i] = src[i + 1];
|
||||
}
|
||||
ret[len - 1] = 0;
|
||||
return ret;
|
||||
}
|
||||
|
||||
size_t strtrim(char *z) {
|
||||
int32_t i = 0;
|
||||
int32_t j = 0;
|
||||
|
|
|
|||
|
|
@ -48,7 +48,12 @@ fi
|
|||
PYTHON_EXEC=python3.8
|
||||
|
||||
# First we need to set up a path for Python to find our own TAOS modules, so that "import" can work.
|
||||
export PYTHONPATH=$(pwd)/../../src/connector/python:$(pwd)
|
||||
# export PYTHONPATH=$(pwd)/../../src/connector/python:$(pwd)
|
||||
# NOTE: we are now pointing outside the current github, per Wade on 7/7/2022, we'll be keeping connectors outside
|
||||
# and there does not seem to be a module to reference that.
|
||||
PROJECT_PARENT=$(pwd)/../../..
|
||||
TAOS_PYTHON_PROJECT_DIR=$PROJECT_PARENT/taos-connector-python
|
||||
export PYTHONPATH=$TAOS_PYTHON_PROJECT_DIR:$(pwd)
|
||||
|
||||
# Then let us set up the library path so that our compiled SO file can be loaded by Python
|
||||
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR
|
||||
|
|
|
|||
|
|
@ -466,6 +466,7 @@ class ThreadCoordinator:
|
|||
self._te = None # No more executor, time to end
|
||||
Logging.debug("Main thread tapping all threads one last time...")
|
||||
self.tapAllThreads() # Let the threads run one last time
|
||||
#TODO: looks like we are not capturing the failures for the last step yet (i.e. calling registerFailure if neccessary)
|
||||
|
||||
Logging.debug("\r\n\n--> Main thread ready to finish up...")
|
||||
Logging.debug("Main thread joining all threads")
|
||||
|
|
@ -1290,6 +1291,7 @@ class Task():
|
|||
|
||||
def _isErrAcceptable(self, errno, msg):
|
||||
if errno in [
|
||||
# TDengine 2.x Error Codes:
|
||||
0x05, # TSDB_CODE_RPC_NOT_READY
|
||||
0x0B, # Unable to establish connection, more details in TD-1648
|
||||
# 0x200, # invalid SQL， TODO: re-examine with TD-934
|
||||
|
|
@ -1310,6 +1312,18 @@ class Task():
|
|||
0x14, # db not ready, errno changed
|
||||
0x600, # Invalid table ID, why?
|
||||
0x218, # Table does not exist
|
||||
|
||||
# TDengine 3.0 Error Codes:
|
||||
0x0333, # Object is creating # TODO: this really is NOT an acceptable error
|
||||
0x03A0, # STable already exists
|
||||
0x03A1, # STable [does] not exist
|
||||
0x03AA, # Tag already exists
|
||||
0x0603, # Table already exists
|
||||
0x2602, # Table does not exist
|
||||
0x260d, # Tags number not matched
|
||||
|
||||
|
||||
|
||||
1000 # REST catch-all error
|
||||
]:
|
||||
return True # These are the ALWAYS-ACCEPTABLE ones
|
||||
|
|
@ -1749,6 +1763,8 @@ class TdSuperTable:
|
|||
tagType = tags[tagName]
|
||||
if tagType == 'BINARY':
|
||||
tagStrs.append("'Beijing-Shanghai-LosAngeles'")
|
||||
elif tagType== 'VARCHAR':
|
||||
tagStrs.append("'London-Paris-Berlin'")
|
||||
elif tagType == 'FLOAT':
|
||||
tagStrs.append('9.9')
|
||||
elif tagType == 'INT':
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ CMD_NAME=
|
|||
LOOP_TIMES=5
|
||||
SLEEP_TIME=0
|
||||
|
||||
while getopts "f:t:s:" arg
|
||||
while getopts "hf:t:s:" arg
|
||||
do
|
||||
case $arg in
|
||||
f)
|
||||
|
|
@ -25,6 +25,12 @@ do
|
|||
s)
|
||||
SLEEP_TIME=$OPTARG
|
||||
;;
|
||||
h)
|
||||
echo "Usage: $(basename $0) -f [cmd name] "
|
||||
echo " -t [loop times] "
|
||||
echo " -s [sleep time] "
|
||||
exit 0
|
||||
;;
|
||||
?)
|
||||
echo "unknow argument"
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ if $data01 != 7 then
|
|||
goto loop1
|
||||
endi
|
||||
|
||||
if $data02 != 9 then
|
||||
if $data02 != 18 then
|
||||
print =====data02=$data02
|
||||
goto loop1
|
||||
endi
|
||||
|
|
@ -151,22 +151,22 @@ if $data03 != 4 then
|
|||
goto loop1
|
||||
endi
|
||||
|
||||
if $data04 != 1.100000000 then
|
||||
if $data04 != 1.000000000 then
|
||||
print ======$data04
|
||||
return -1
|
||||
endi
|
||||
|
||||
if $data05 != 0.816496581 then
|
||||
if $data05 != 1.154700538 then
|
||||
print ======$data05
|
||||
return -1
|
||||
endi
|
||||
|
||||
if $data06 != 3 then
|
||||
if $data06 != 4 then
|
||||
print ======$data06
|
||||
return -1
|
||||
endi
|
||||
|
||||
if $data07 != 1.100000000 then
|
||||
if $data07 != 1.000000000 then
|
||||
print ======$data07
|
||||
return -1
|
||||
endi
|
||||
|
|
@ -235,7 +235,7 @@ sql create stream streams4 trigger at_once watermark 1d into streamt4 as select
|
|||
# sql create stream streams6 trigger at_once watermark 1d into streamt6 as select _wstartts, bottom(b,3), a,c from t1 session(ts,10s);
|
||||
# sql create stream streams7 trigger at_once watermark 1d into streamt7 as select _wstartts, spread(a), elapsed(ts), hyperloglog(a) from t1 session(ts,10s);
|
||||
sql create stream streams7 trigger at_once watermark 1d into streamt7 as select _wstartts, spread(a), hyperloglog(a) from t1 session(ts,10s);
|
||||
sql create stream streams8 trigger at_once watermark 1d into streamt8 as select _wstartts, histogram(a,"user_input", "[1,3,5,7]", 1), histogram(a,"user_input", "[1,3,5,7]", 0) from t1 session(ts,10s);
|
||||
# sql create stream streams8 trigger at_once watermark 1d into streamt8 as select _wstartts, histogram(a,"user_input", "[1,3,5,7]", 1), histogram(a,"user_input", "[1,3,5,7]", 0) from t1 session(ts,10s);
|
||||
sql insert into t1 values(1648791213001,1,1,1,1.0);
|
||||
sql insert into t1 values(1648791213002,2,3,2,3.4);
|
||||
sql insert into t1 values(1648791213003,4,9,3,4.8);
|
||||
|
|
@ -288,10 +288,10 @@ if $rows == 0 then
|
|||
goto loop3
|
||||
endi
|
||||
|
||||
sql select * from streamt8;
|
||||
if $rows == 0 then
|
||||
print ======$rows
|
||||
goto loop3
|
||||
endi
|
||||
#sql select * from streamt8;
|
||||
#if $rows == 0 then
|
||||
# print ======$rows
|
||||
# goto loop3
|
||||
#endi
|
||||
|
||||
system sh/exec.sh -n dnode1 -s stop -x SIGINT
|
||||
|
|
|
|||
|
|
@ -145,9 +145,9 @@ system sh/exec.sh -n dnode4 -s stop -x SIGINT
|
|||
sql insert into ct1 values(now+0s, 10, 2.0, 3.0)
|
||||
sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3)
|
||||
|
||||
sql flush database db;
|
||||
#sql flush database db;
|
||||
|
||||
system sh/exec.sh -n dnode4 -s start
|
||||
#system sh/exec.sh -n dnode4 -s start
|
||||
|
||||
sql insert into ct1 values(now+1s, 81, 8.1, 8.1)(now+2s, -92, -9.2, -9.2)(now+3s, -73, -7.3, -7.3)
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ sql_error ALTER USER root SYSINFO 1
|
|||
sql_error ALTER USER root enable 0
|
||||
sql_error ALTER USER root enable 1
|
||||
|
||||
sql_error create database db vgroups 1;
|
||||
#sql_error create database db vgroups 1;
|
||||
sql_error GRANT read ON db.* to root;
|
||||
sql_error GRANT read ON *.* to root;
|
||||
sql_error REVOKE read ON db.* from root;
|
||||
|
|
|
|||
|
|
@ -19,43 +19,29 @@ if $rows != 1 then
|
|||
return -1
|
||||
endi
|
||||
|
||||
print =============== step2: create alter drop show user
|
||||
sql create user u1 pass 'taosdata'
|
||||
sql show users
|
||||
sql alter user u1 sysinfo 1
|
||||
sql alter user u1 enable 1
|
||||
sql alter user u1 pass 'taosdata'
|
||||
sql drop user u1
|
||||
sql_error alter user u2 sysinfo 0
|
||||
|
||||
print =============== step3: create drop dnode
|
||||
sql create dnode $hostname port 7200
|
||||
sql drop dnode 2
|
||||
sql alter dnode 1 'debugflag 143'
|
||||
|
||||
print =============== step4: create show database
|
||||
print =============== step2: create show database
|
||||
sql create database d1 vgroups 1 buffer 3
|
||||
sql show databases
|
||||
sql use d1
|
||||
sql show vgroups
|
||||
|
||||
print =============== step5: create show stable
|
||||
print =============== step3: create show stable
|
||||
sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned)
|
||||
sql show stables
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
print =============== step6: create show table
|
||||
print =============== step4: create show table
|
||||
sql create table ct1 using stb tags(1000)
|
||||
sql show tables
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
print =============== step7: insert data
|
||||
print =============== step5: insert data
|
||||
|
||||
print =============== step7: select data
|
||||
print =============== step6: select data
|
||||
|
||||
_OVER:
|
||||
system sh/exec.sh -n dnode1 -s stop -x SIGINT
|
||||
|
|
|
|||
|
|
@ -4,25 +4,107 @@ system sh/cfg.sh -n dnode1 -c debugflag -v 131
|
|||
system sh/exec.sh -n dnode1 -s start -v
|
||||
sql connect
|
||||
|
||||
print =============== step1: create drop show dnodes
|
||||
print =============== step1: show dnodes
|
||||
$x = 0
|
||||
step1:
|
||||
$x = $x + 1
|
||||
sleep 1000
|
||||
if $x == 10 then
|
||||
print ----> dnode not ready!
|
||||
print ---> dnode not ready!
|
||||
return -1
|
||||
endi
|
||||
sql show dnodes
|
||||
print ----> $data00 $data01 $data02 $data03 $data04 $data05
|
||||
print ---> $data00 $data01 $data02 $data03 $data04 $data05
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
if $data(1)[4] != ready then
|
||||
goto step1
|
||||
endi
|
||||
|
||||
print =============== step2: create alter drop show user
|
||||
sql create user u1 pass 'taosdata'
|
||||
sql show users
|
||||
sql alter user u1 sysinfo 1
|
||||
sql alter user u1 enable 1
|
||||
sql alter user u1 pass 'taosdata'
|
||||
sql drop user u1
|
||||
sql_error alter user u2 sysinfo 0
|
||||
|
||||
print =============== step3: create drop dnode
|
||||
sql create dnode $hostname port 7200
|
||||
sql drop dnode 2
|
||||
sql alter dnode 1 'debugflag 131'
|
||||
|
||||
print =============== step4:
|
||||
|
||||
print =============== run show xxxx
|
||||
sql show dnodes
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
print =============== step2: create db
|
||||
sql create database db vgroups 1 buffer 3
|
||||
sql use db
|
||||
sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned)
|
||||
sql show mnodes
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
_OVER:
|
||||
sql show databases
|
||||
if $rows != 2 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql show users
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
print =============== run select * from information_schema.xxxx
|
||||
sql select * from information_schema.`dnodes`
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select * from information_schema.`mnodes`
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select * from information_schema.user_users
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql show variables;
|
||||
if $rows != 4 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql show dnode 1 variables;
|
||||
if $rows <= 0 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql show local variables;
|
||||
if $rows <= 0 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
print =============== stop
|
||||
system sh/exec.sh -n dnode1 -s stop -x SIGINT
|
||||
|
||||
print =============== check
|
||||
print ----> start to check if there are ERRORS in vagrind log file for each dnode
|
||||
system_content sh/checkValgrind.sh -n dnode1
|
||||
|
||||
print cmd return result ----> [ $system_content ]
|
||||
if $system_content <= 0 then
|
||||
return 0
|
||||
endi
|
||||
|
||||
$null=
|
||||
if $system_content == $null then
|
||||
return 0
|
||||
endi
|
||||
|
||||
return -1
|
||||
|
|
|
|||
|
|
@ -5,20 +5,22 @@ system sh/exec.sh -n dnode1 -s start -v
|
|||
sql connect
|
||||
|
||||
print =============== step1: show dnodes
|
||||
|
||||
$x = 0
|
||||
step1:
|
||||
$x = $x + 1
|
||||
sleep 1000
|
||||
if $x == 10 then
|
||||
print ----> dnode not ready!
|
||||
print ---> dnode not ready!
|
||||
return -1
|
||||
endi
|
||||
sql show dnodes
|
||||
print ----> $data00 $data01 $data02 $data03 $data04 $data05
|
||||
print ---> $data00 $data01 $data02 $data03 $data04 $data05
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
if $data(1)[4] != ready then
|
||||
goto step1
|
||||
endi
|
||||
|
||||
print =============== step2: create alter drop show user
|
||||
sql create user u1 pass 'taosdata'
|
||||
|
|
@ -32,7 +34,7 @@ sql_error alter user u2 sysinfo 0
|
|||
print =============== step3: create drop dnode
|
||||
sql create dnode $hostname port 7200
|
||||
sql drop dnode 2
|
||||
sql alter dnode 1 'debugflag 143'
|
||||
sql alter dnode 1 'debugflag 131'
|
||||
|
||||
print =============== stop
|
||||
system sh/exec.sh -n dnode1 -s stop -x SIGINT
|
||||
|
|
|
|||
|
|
@ -10,14 +10,17 @@ step1:
|
|||
$x = $x + 1
|
||||
sleep 1000
|
||||
if $x == 10 then
|
||||
print ----> dnode not ready!
|
||||
print ---> dnode not ready!
|
||||
return -1
|
||||
endi
|
||||
sql show dnodes
|
||||
print ----> $data00 $data01 $data02 $data03 $data04 $data05
|
||||
print ---> $data00 $data01 $data02 $data03 $data04 $data05
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
if $data(1)[4] != ready then
|
||||
goto step1
|
||||
endi
|
||||
|
||||
print =============== step2: create db
|
||||
sql create database d1 vgroups 1 buffer 3
|
||||
|
|
@ -32,6 +35,21 @@ if $rows != 1 then
|
|||
return -1
|
||||
endi
|
||||
|
||||
print =============== step3: create show table
|
||||
sql create table ct1 using stb tags(1000)
|
||||
#sql show tables
|
||||
#if $rows != 1 then
|
||||
# return -1
|
||||
#endi
|
||||
|
||||
print =============== step5: insert data
|
||||
sql insert into ct1 values(now+0s, 10, 2.0, 3.0)
|
||||
sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3)
|
||||
|
||||
print =============== step6: select data
|
||||
#sql select * from ct1
|
||||
#sql select * from stb
|
||||
|
||||
_OVER:
|
||||
system sh/exec.sh -n dnode1 -s stop -x SIGINT
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,156 @@
|
|||
system sh/stop_dnodes.sh
|
||||
system sh/deploy.sh -n dnode1 -i 1
|
||||
system sh/deploy.sh -n dnode2 -i 2
|
||||
system sh/exec.sh -n dnode1 -s start -v
|
||||
system sh/exec.sh -n dnode2 -s start -v
|
||||
sql connect
|
||||
|
||||
print =============== add dnode2 into cluster
|
||||
sql create dnode $hostname port 7200
|
||||
|
||||
$x = 0
|
||||
step1:
|
||||
$x = $x + 1
|
||||
sleep 1000
|
||||
if $x == 10 then
|
||||
print ---> dnode not ready!
|
||||
return -1
|
||||
endi
|
||||
sql show dnodes
|
||||
print ---> $data00 $data01 $data02 $data03 $data04 $data05
|
||||
print ---> $data10 $data11 $data12 $data13 $data14 $data15
|
||||
if $rows != 2 then
|
||||
return -1
|
||||
endi
|
||||
if $data(1)[4] != ready then
|
||||
goto step1
|
||||
endi
|
||||
if $data(2)[4] != ready then
|
||||
goto step1
|
||||
endi
|
||||
|
||||
print =============== create database, stable, table
|
||||
sql create database db vgroups 3
|
||||
sql use db
|
||||
sql create table stb (ts timestamp, c int) tags (t int)
|
||||
sql create table t0 using stb tags (0)
|
||||
sql create table tba (ts timestamp, c1 binary(10), c2 nchar(10));
|
||||
|
||||
print =============== run show xxxx
|
||||
sql show dnodes
|
||||
if $rows != 2 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql show mnodes
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql show databases
|
||||
if $rows != 3 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql show stables
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql show tables
|
||||
if $rows != 2 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql show users
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql show vgroups
|
||||
if $rows != 3 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
print =============== run select * from information_schema.xxxx
|
||||
sql select * from information_schema.`dnodes`
|
||||
if $rows != 2 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select * from information_schema.`mnodes`
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select * from information_schema.user_databases
|
||||
if $rows != 3 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select * from information_schema.user_stables
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select * from information_schema.user_tables
|
||||
if $rows != 31 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select * from information_schema.user_users
|
||||
if $rows != 1 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql select * from information_schema.`vgroups`
|
||||
if $rows != 3 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql show variables;
|
||||
if $rows != 4 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql show dnode 1 variables;
|
||||
if $rows <= 0 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
sql show local variables;
|
||||
if $rows <= 0 then
|
||||
return -1
|
||||
endi
|
||||
|
||||
print ==== stop dnode1 and dnode2, and restart dnodes
|
||||
system sh/exec.sh -n dnode1 -s stop -x SIGINT
|
||||
system sh/exec.sh -n dnode2 -s stop -x SIGINT
|
||||
|
||||
print =============== check dnode1
|
||||
system_content sh/checkValgrind.sh -n dnode1
|
||||
print cmd return result ----> [ $system_content ]
|
||||
if $system_content <= 0 then
|
||||
return 0
|
||||
endi
|
||||
|
||||
$null=
|
||||
if $system_content == $null then
|
||||
return 0
|
||||
endi
|
||||
|
||||
return -1
|
||||
|
||||
print =============== check dnode2
|
||||
system_content sh/checkValgrind.sh -n dnode2
|
||||
print cmd return result ----> [ $system_content ]
|
||||
if $system_content <= 0 then
|
||||
return 0
|
||||
endi
|
||||
|
||||
$null=
|
||||
if $system_content == $null then
|
||||
return 0
|
||||
endi
|
||||
|
||||
return -1
|
||||
|
|
@ -230,7 +230,7 @@ class TDTestCase:
|
|||
work_sql += f"cast({arg} as bigint){opera}"
|
||||
|
||||
if not agg:
|
||||
work_sql+= f" from {tbname} order by ts"
|
||||
work_sql+= f" from {tbname} order by tbname ,ts"
|
||||
else:
|
||||
work_sql+= f" from {tbname} "
|
||||
tdSql.query(work_sql)
|
||||
|
|
@ -243,7 +243,7 @@ class TDTestCase:
|
|||
else:
|
||||
origin_sql += f"cast({arg} as bigint),"
|
||||
if not agg:
|
||||
origin_sql+= f" from {tbname} order by ts"
|
||||
origin_sql+= f" from {tbname} order by tbname ,ts"
|
||||
else:
|
||||
origin_sql+= f" from {tbname} "
|
||||
tdSql.query(origin_sql)
|
||||
|
|
|
|||
|
|
@ -323,12 +323,12 @@ class TDTestCase:
|
|||
# where json value is bool
|
||||
tdSql.query("select * from jsons1 where jtag->'tag1'=true")
|
||||
tdSql.checkRows(0)
|
||||
#tdSql.query("select * from jsons1 where jtag->'tag1'=false")
|
||||
#tdSql.checkRows(1)
|
||||
tdSql.query("select * from jsons1 where jtag->'tag1'=false")
|
||||
tdSql.checkRows(1)
|
||||
tdSql.query("select * from jsons1 where jtag->'tag1'!=false")
|
||||
tdSql.checkRows(0)
|
||||
#tdSql.query("select * from jsons1 where jtag->'tag1'>false")
|
||||
#tdSql.checkRows(0)
|
||||
tdSql.query("select * from jsons1 where jtag->'tag1'>false")
|
||||
tdSql.checkRows(0)
|
||||
|
||||
# where json value is null
|
||||
tdSql.query("select * from jsons1 where jtag->'tag1'=null")
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ class TDTestCase:
|
|||
paraDict = {'dbName': 'db2',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 4,
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
|
|
@ -44,7 +44,7 @@ class TDTestCase:
|
|||
topicNameList = ['topic1']
|
||||
expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
|
||||
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=1,replica=1)
|
||||
tdLog.info("create stb")
|
||||
tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
|
||||
tdLog.info("create ctb")
|
||||
|
|
|
|||
|
|
@ -64,8 +64,9 @@ class TDTestCase:
|
|||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
# tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
return
|
||||
|
||||
def tmqCase1(self):
|
||||
|
|
|
|||
|
|
@ -64,8 +64,9 @@ class TDTestCase:
|
|||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
# tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
return
|
||||
|
||||
def tmqCase1(self):
|
||||
|
|
|
|||
|
|
@ -64,9 +64,9 @@ class TDTestCase:
|
|||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
tdDnodes.starttaosd(1)
|
||||
# tdDnodes.stop(1)
|
||||
# tdDnodes.starttaosd(1)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
return
|
||||
|
||||
def tmqCase1(self):
|
||||
|
|
|
|||
|
|
@ -64,9 +64,10 @@ class TDTestCase:
|
|||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
tdDnodes.starttaosd(1)
|
||||
# tdDnodes.stop(1)
|
||||
# # tdDnodes.start(1)
|
||||
# tdDnodes.starttaosd(1)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
return
|
||||
|
||||
def tmqCase1(self):
|
||||
|
|
|
|||
|
|
@ -64,9 +64,10 @@ class TDTestCase:
|
|||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
tdDnodes.starttaosd(1)
|
||||
# tdDnodes.stop(1)
|
||||
# # tdDnodes.start(1)
|
||||
# tdDnodes.starttaosd(1)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
return
|
||||
|
||||
def tmqCase1(self):
|
||||
|
|
|
|||
|
|
@ -64,8 +64,9 @@ class TDTestCase:
|
|||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
# tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
return
|
||||
|
||||
def tmqCase1(self):
|
||||
|
|
|
|||
|
|
@ -64,8 +64,9 @@ class TDTestCase:
|
|||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
# tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
return
|
||||
|
||||
def tmqCase3(self):
|
||||
|
|
|
|||
|
|
@ -64,8 +64,9 @@ class TDTestCase:
|
|||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
# tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
return
|
||||
|
||||
def tmqCase3(self):
|
||||
|
|
|
|||
|
|
@ -64,8 +64,9 @@ class TDTestCase:
|
|||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
# tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
return
|
||||
|
||||
def tmqCase3(self):
|
||||
|
|
|
|||
|
|
@ -64,8 +64,9 @@ class TDTestCase:
|
|||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
# tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
return
|
||||
|
||||
def tmqCase3(self):
|
||||
|
|
|
|||
|
|
@ -64,8 +64,9 @@ class TDTestCase:
|
|||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
# tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
return
|
||||
|
||||
def tmqCase3(self):
|
||||
|
|
|
|||
|
|
@ -64,8 +64,9 @@ class TDTestCase:
|
|||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
# tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
tdSql.query("flush database %s"%(paraDict['dbName']))
|
||||
return
|
||||
|
||||
def tmqCase3(self):
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
#python3 ./test.py -f 2-query/last.py -Q 3
|
||||
|
|
@ -117,7 +117,7 @@ python3 ./test.py -f 2-query/distribute_agg_avg.py
|
|||
python3 ./test.py -f 2-query/distribute_agg_stddev.py
|
||||
python3 ./test.py -f 2-query/twa.py
|
||||
python3 ./test.py -f 2-query/irate.py
|
||||
#python3 ./test.py -f 2-query/and_or_for_byte.py
|
||||
python3 ./test.py -f 2-query/and_or_for_byte.py
|
||||
|
||||
python3 ./test.py -f 2-query/function_null.py
|
||||
python3 ./test.py -f 2-query/queryQnode.py
|
||||
|
|
@ -295,7 +295,7 @@ python3 ./test.py -f 2-query/Today.py -Q 3
|
|||
python3 ./test.py -f 2-query/max.py -Q 3
|
||||
python3 ./test.py -f 2-query/min.py -Q 3
|
||||
python3 ./test.py -f 2-query/count.py -Q 3
|
||||
python3 ./test.py -f 2-query/last.py -Q 3
|
||||
#python3 ./test.py -f 2-query/last.py -Q 3
|
||||
python3 ./test.py -f 2-query/first.py -Q 3
|
||||
python3 ./test.py -f 2-query/To_iso8601.py -Q 3
|
||||
python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3
|
||||
|
|
|
|||
|
|
@ -0,0 +1,55 @@
|
|||
#!/bin/bash
|
||||
|
||||
##################################################
|
||||
#
|
||||
# Do simulation test
|
||||
#
|
||||
##################################################
|
||||
|
||||
set -e
|
||||
#set -x
|
||||
|
||||
CMD_NAME=
|
||||
LOOP_TIMES=5
|
||||
SLEEP_TIME=0
|
||||
|
||||
while getopts "hf:t:s:" arg
|
||||
do
|
||||
case $arg in
|
||||
f)
|
||||
CMD_NAME=$OPTARG
|
||||
;;
|
||||
t)
|
||||
LOOP_TIMES=$OPTARG
|
||||
;;
|
||||
s)
|
||||
SLEEP_TIME=$OPTARG
|
||||
;;
|
||||
h)
|
||||
echo "Usage: $(basename $0) -f [cmd name] "
|
||||
echo " -t [loop times] "
|
||||
echo " -s [sleep time] "
|
||||
exit 0
|
||||
;;
|
||||
?)
|
||||
echo "unknow argument"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo LOOP_TIMES ${LOOP_TIMES}
|
||||
echo CMD_NAME ${CMD_NAME}
|
||||
echo SLEEP_TIME ${SLEEP_TIME}
|
||||
|
||||
GREEN='\033[1;32m'
|
||||
GREEN_DARK='\033[0;32m'
|
||||
GREEN_UNDERLINE='\033[4;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
for ((i=0; i<$LOOP_TIMES; i++ ))
|
||||
do
|
||||
echo -e $GREEN loop $i $NC
|
||||
echo -e $GREEN cmd $CMD_NAME $NC
|
||||
$CMD_NAME
|
||||
sleep ${SLEEP_TIME}
|
||||
done
|
||||
Loading…
Reference in New Issue