xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #21555 from taosdata/fix/liaohj_main 

other: merge main into 3.0.
This commit is contained in:
Haojun Liao 2023-06-02 13:11:12 +08:00 committed by GitHub
parent 7df4f8a89e bbf3f1ccd9
commit d45aa7ef84
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
83 changed files with 2776 additions and 575 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README-CN.md
View File

@ -15,7 +15,7 @@
[![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop)
[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201)
简体中文 | [English](README.md) | 很多职位正在热招中，请看[这里](https://www.taosdata.com/cn/careers/)
简体中文 | [English](README.md) | [TDengine 云服务](https://cloud.taosdata.com/?utm_medium=cn&utm_source=github) | 很多职位正在热招中，请看[这里](https://www.taosdata.com/cn/careers/)
# TDengine 简介

28
docs/zh/08-connector/26-rust.mdx
View File

@ -65,6 +65,13 @@ taos = "*"
taos = { version = "*", default-features = false, features = ["ws"] }
```
当仅启用 `ws` 特性时，可同时指定 `r2d2` 使得在同步（blocking/sync）模式下使用 [r2d2] 作为连接池：
```toml
[dependencies]
taos = { version = "*", default-features = false, features = ["r2d2", "ws"] }
```
</TabItem>
<TabItem value="native" label="仅原生连接">
@ -257,26 +264,24 @@ let conn: Taos = cfg.build();
### 连接池
在复杂应用中，建议启用连接池。[taos] 的连接池使用 [r2d2] 实现。
在复杂应用中，建议启用连接池。[taos] 的连接池默认（异步模式）使用 [deadpool] 实现。
如下，可以生成一个默认参数的连接池。
```rust
let pool = TaosBuilder::from_dsn(dsn)?.pool()?;
let pool: Pool<TaosBuilder> = TaosBuilder::from_dsn("taos:///")
.unwrap()
.pool()
.unwrap();
```
同样可以使用连接池的构造器，对连接池参数进行设置：
```rust
let dsn = "taos://localhost:6030";
let opts = PoolBuilder::new()
.max_size(5000) // max connections
.max_lifetime(Some(Duration::from_secs(60 * 60))) // lifetime of each connection
.min_idle(Some(1000)) // minimal idle connections
.connection_timeout(Duration::from_secs(2));
let pool = TaosBuilder::from_dsn(dsn)?.with_pool_builder(opts)?;
let pool: Pool<TaosBuilder> = Pool::builder(Manager::from_dsn(self.dsn.clone()).unwrap().0)
.max_size(88) // 最大连接数
.build()
.unwrap();
```
在应用代码中，使用 `pool.get()?` 来获取一个连接对象 [Taos]。
@ -516,6 +521,7 @@ consumer.unsubscribe().await;
其他相关结构体 API 使用说明请移步 Rust 文档托管网页：<https://docs.rs/taos>。
[taos]: https://github.com/taosdata/rust-connector-taos
[deadpool]: https://crates.io/crates/deadpool
[r2d2]: https://crates.io/crates/r2d2
[TaosBuilder]: https://docs.rs/taos/latest/taos/struct.TaosBuilder.html
[TaosCfg]: https://docs.rs/taos/latest/taos/struct.TaosCfg.html

25
docs/zh/14-reference/12-config/index.md
View File

@ -91,11 +91,30 @@ taos --dump-config
### maxShellConns
| 属性 | 说明 |
| -------- | ----------------------- |
| --------| ----------------------- |
| 适用范围 | 仅服务端适用 |
| 含义 | 一个 dnode 容许的连接数 |
| 含义 | 一个 dnode 容许的连接数 |
| 取值范围 | 10-50000000 |
| 缺省值 | 5000 |
| 缺省值 | 5000 |
### numOfRpcSessions
| 属性 | 说明 |
| --------| ---------------------- |
| 适用范围 | 客户端和服务端都适用 |
| 含义 | 一个客户端能创建的最大连接数|
| 取值范围 | 100-100000 |
| 缺省值 | 10000 |
### timeToGetAvailableConn
| 属性 | 说明 |
| -------- | --------------------|
| 适用范围 | 客户端和服务端都适用 |
| 含义 |获得可用连接的最长等待时间|
| 取值范围 | 10-50000000(单位为毫秒)|
| 缺省值 | 500000 |
### numOfRpcSessions

2
include/common/tdataformat.h
View File

@ -145,7 +145,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull);
// for stmt bind
int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind);
int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen);
void tColDataSortMerge(SArray *colDataArr);
// for raw block

2
include/common/tglobal.h
View File

@ -130,7 +130,7 @@ extern int32_t tsSlowLogScope;
// client
extern int32_t tsMinSlidingTime;
extern int32_t tsMinIntervalTime;
extern int32_t tsMaxMemUsedByInsert;
extern int32_t tsMaxInsertBatchRows;
// build info
extern char version[];

4
include/common/ttime.h
View File

@ -23,7 +23,7 @@
extern "C" {
#endif
#define TIME_IS_VAR_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y')
#define IS_CALENDAR_TIME_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y')
#define TIME_UNIT_NANOSECOND 'b'
#define TIME_UNIT_MICROSECOND 'u'
@ -74,7 +74,7 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) {
int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision);
int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision);
int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval);
int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision);
int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision);

14
include/libs/executor/executor.h
View File

@ -92,6 +92,8 @@ int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList
*/
void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId);
//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code);
int32_t qSetStreamOpOpen(qTaskInfo_t tinfo);
// todo refactor
@ -188,7 +190,17 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len);
int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len);
STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key);
void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order);
void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery);
STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key);
/**
* return the scan info, in the form of tuple of two items, including table uid and current timestamp
* @param tinfo
* @param uid
* @param ts
* @return
*/
int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts);
SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo);

14
include/libs/nodes/plannodes.h
View File

@ -112,6 +112,7 @@ typedef struct SJoinLogicNode {
SNode* pOnConditions;
bool isSingleTableJoin;
EOrder inputTsOrder;
SNode* pColEqualOnConditions;
} SJoinLogicNode;
typedef struct SAggLogicNode {
@ -406,6 +407,7 @@ typedef struct SSortMergeJoinPhysiNode {
SNode* pOnConditions;
SNodeList* pTargets;
EOrder inputTsOrder;
SNode* pColEqualOnConditions;
} SSortMergeJoinPhysiNode;
typedef struct SAggPhysiNode {
@ -448,7 +450,7 @@ typedef struct SMergePhysiNode {
bool ignoreGroupId;
} SMergePhysiNode;
typedef struct SWinodwPhysiNode {
typedef struct SWindowPhysiNode {
SPhysiNode node;
SNodeList* pExprs; // these are expression list of parameter expression of function
SNodeList* pFuncs;
@ -461,10 +463,10 @@ typedef struct SWinodwPhysiNode {
EOrder inputTsOrder;
EOrder outputTsOrder;
bool mergeDataBlock;
} SWinodwPhysiNode;
} SWindowPhysiNode;
typedef struct SIntervalPhysiNode {
SWinodwPhysiNode window;
SWindowPhysiNode window;
int64_t interval;
int64_t offset;
int64_t sliding;
@ -497,7 +499,7 @@ typedef struct SMultiTableIntervalPhysiNode {
} SMultiTableIntervalPhysiNode;
typedef struct SSessionWinodwPhysiNode {
SWinodwPhysiNode window;
SWindowPhysiNode window;
int64_t gap;
} SSessionWinodwPhysiNode;
@ -506,14 +508,14 @@ typedef SSessionWinodwPhysiNode SStreamSemiSessionWinodwPhysiNode;
typedef SSessionWinodwPhysiNode SStreamFinalSessionWinodwPhysiNode;
typedef struct SStateWinodwPhysiNode {
SWinodwPhysiNode window;
SWindowPhysiNode window;
SNode* pStateKey;
} SStateWinodwPhysiNode;
typedef SStateWinodwPhysiNode SStreamStateWinodwPhysiNode;
typedef struct SEventWinodwPhysiNode {
SWinodwPhysiNode window;
SWindowPhysiNode window;
SNode* pStartCond;
SNode* pEndCond;
} SEventWinodwPhysiNode;

3
include/libs/wal/wal.h
View File

@ -212,7 +212,8 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead);
int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead);
int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead);
SWalRef *walRefFirstVer(SWal *, SWalRef *);
void walRefFirstVer(SWal *, SWalRef *);
void walRefLastVer(SWal *, SWalRef *);
SWalRef *walRefCommittedVer(SWal *);
SWalRef *walOpenRef(SWal *);

3
packaging/tools/makepkg.sh
View File

@ -70,8 +70,7 @@ if [ "$pagMode" == "lite" ]; then
taostools_bin_files=""
else
if [ "$verMode" == "cloud" ]; then
taostools_bin_files=" ${build_dir}/bin/taosdump \
${build_dir}/bin/taosBenchmark"
taostools_bin_files=" ${build_dir}/bin/taosBenchmark"
else
wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \
&& echo "TDinsight.sh downloaded!" \

136
source/client/src/clientTmq.c
View File

@ -89,6 +89,7 @@ struct tmq_t {
void* commitCbUserParam;
// status
SRWLatch lock;
int8_t status;
int32_t epoch;
#if 0
@ -148,10 +149,10 @@ typedef struct {
SVgOffsetInfo offsetInfo;
int32_t vgId;
int32_t vgStatus;
int32_t vgSkipCnt; // here used to mark the slow vgroups
bool receivedInfoFromVnode;// has already received info from vnode
int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data
bool seekUpdated; // offset is updated by seek operator, therefore, not update by vnode rsp.
int32_t vgSkipCnt; // here used to mark the slow vgroups
bool receivedInfoFromVnode; // has already received info from vnode
int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data
bool seekUpdated; // offset is updated by seek operator, therefore, not update by vnode rsp.
SEpSet epSet;
} SMqClientVg;
@ -166,6 +167,7 @@ typedef struct {
int8_t tmqRspType;
int32_t epoch; // epoch can be used to guard the vgHandle
int32_t vgId;
char topicName[TSDB_TOPIC_FNAME_LEN];
SMqClientVg* vgHandle;
SMqClientTopic* topicHandle;
uint64_t reqId;
@ -178,8 +180,8 @@ typedef struct {
} SMqPollRspWrapper;
typedef struct {
int64_t refId;
int32_t epoch;
// int64_t refId;
// int32_t epoch;
tsem_t rspSem;
int32_t rspErr;
} SMqSubscribeCbParam;
@ -194,8 +196,9 @@ typedef struct {
typedef struct {
int64_t refId;
int32_t epoch;
SMqClientVg* pVg;
SMqClientTopic* pTopic;
char topicName[TSDB_TOPIC_FNAME_LEN];
// SMqClientVg* pVg;
// SMqClientTopic* pTopic;
int32_t vgId;
uint64_t requestId; // request id for debug purpose
} SMqPollCbParam;
@ -743,8 +746,8 @@ static void generateTimedTask(int64_t refId, int32_t type) {
*pTaskType = type;
taosWriteQitem(tmq->delayedTask, pTaskType);
tsem_post(&tmq->rspSem);
taosReleaseRef(tmqMgmt.rsetId, refId);
}
taosReleaseRef(tmqMgmt.rsetId, refId);
}
void tmqAssignAskEpTask(void* param, void* tmrId) {
@ -1071,6 +1074,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
pTmq->commitCb = conf->commitCb;
pTmq->commitCbUserParam = conf->commitCbUserParam;
pTmq->resetOffsetCfg = conf->resetOffset;
taosInitRWLatch(&pTmq->lock);
pTmq->hbBgEnable = conf->hbBgEnable;
@ -1119,7 +1123,7 @@ _failed:
}
int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
const int32_t MAX_RETRY_COUNT = 120 * 4; // let's wait for 4 mins at most
const int32_t MAX_RETRY_COUNT = 120 * 60; // let's wait for 2 mins at most
const SArray* container = &topic_list->container;
int32_t sz = taosArrayGetSize(container);
void* buf = NULL;
@ -1172,7 +1176,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
goto FAIL;
}
SMqSubscribeCbParam param = { .rspErr = 0, .refId = tmq->refId, .epoch = tmq->epoch };
SMqSubscribeCbParam param = { .rspErr = 0};
if (tsem_init(&param.rspSem, 0, 0) != 0) {
code = TSDB_CODE_TSC_INTERNAL_ERROR;
goto FAIL;
@ -1206,8 +1210,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
int32_t retryCnt = 0;
while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) {
if (retryCnt++ > MAX_RETRY_COUNT) {
tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, max retry reached:%d", tmq->consumerId, retryCnt);
code = TSDB_CODE_TSC_INTERNAL_ERROR;
tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt);
code = TSDB_CODE_MND_CONSUMER_NOT_READY;
goto FAIL;
}
@ -1242,12 +1246,40 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para
conf->commitCbUserParam = param;
}
static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
static SMqClientVg* getVgInfo(tmq_t* tmq, char* topicName, int32_t vgId){
int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics);
for(int i = 0; i < topicNumCur; i++){
SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i);
if(strcmp(pTopicCur->topicName, topicName) == 0){
int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs);
for (int32_t j = 0; j < vgNumCur; j++) {
SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j);
if(pVgCur->vgId == vgId){
return pVgCur;
}
}
}
}
return NULL;
}
static SMqClientTopic* getTopicInfo(tmq_t* tmq, char* topicName){
int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics);
for(int i = 0; i < topicNumCur; i++){
SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i);
if(strcmp(pTopicCur->topicName, topicName) == 0){
return pTopicCur;
}
}
return NULL;
}
int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
SMqPollCbParam* pParam = (SMqPollCbParam*)param;
int64_t refId = pParam->refId;
SMqClientVg* pVg = pParam->pVg;
SMqClientTopic* pTopic = pParam->pTopic;
// SMqClientVg* pVg = pParam->pVg;
// SMqClientTopic* pTopic = pParam->pTopic;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq == NULL) {
@ -1262,15 +1294,13 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
int32_t vgId = pParam->vgId;
uint64_t requestId = pParam->requestId;
taosMemoryFree(pParam);
if (code != 0) {
if (pMsg->pData) taosMemoryFree(pMsg->pData);
if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet);
// in case of consumer mismatch, wait for 500ms and retry
if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) {
taosMsleep(500);
// taosMsleep(500);
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER);
tscDebug("consumer:0x%" PRIx64 " wait for the re-balance, wait for 500ms and set status to be RECOVER",
tmq->consumerId);
@ -1284,8 +1314,8 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
pRspWrapper->tmqRspType = TMQ_MSG_TYPE__END_RSP;
taosWriteQitem(tmq->mqueue, pRspWrapper);
} else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert
taosMsleep(500);
// } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert
// taosMsleep(5);
} else{
tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId,
vgId, epoch, tstrerror(code), requestId);
@ -1307,6 +1337,8 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
taosMemoryFree(pParam);
return 0;
}
@ -1328,11 +1360,12 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
}
pRspWrapper->tmqRspType = rspType;
pRspWrapper->vgHandle = pVg;
pRspWrapper->topicHandle = pTopic;
// pRspWrapper->vgHandle = pVg;
// pRspWrapper->topicHandle = pTopic;
pRspWrapper->reqId = requestId;
pRspWrapper->pEpset = pMsg->pEpSet;
pRspWrapper->vgId = pVg->vgId;
pRspWrapper->vgId = vgId;
strcpy(pRspWrapper->topicName, pParam->topicName);
pMsg->pEpSet = NULL;
if (rspType == TMQ_MSG_TYPE__POLL_RSP) {
@ -1371,16 +1404,23 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
tsem_post(&tmq->rspSem);
taosReleaseRef(tmqMgmt.rsetId, refId);
taosMemoryFree(pParam);
return 0;
CREATE_MSG_FAIL:
if (epoch == tmq->epoch) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
taosWLockLatch(&tmq->lock);
SMqClientVg* pVg = getVgInfo(tmq, pParam->topicName, vgId);
if(pVg){
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
}
taosWUnLockLatch(&tmq->lock);
}
tsem_post(&tmq->rspSem);
taosReleaseRef(tmqMgmt.rsetId, refId);
taosMemoryFree(pParam);
return -1;
}
@ -1503,11 +1543,13 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
taosHashCleanup(pVgOffsetHashMap);
taosWLockLatch(&tmq->lock);
// destroy current buffered existed topics info
if (tmq->clientTopics) {
taosArrayDestroyEx(tmq->clientTopics, freeClientVgInfo);
}
tmq->clientTopics = newTopics;
taosWUnLockLatch(&tmq->lock);
int8_t flag = (topicNumGet == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY;
atomic_store_8(&tmq->status, flag);
@ -1523,7 +1565,7 @@ int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) {
if (tmq == NULL) {
terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED;
pParam->pUserFn(tmq, terrno, NULL, pParam->pParam);
// pParam->pUserFn(tmq, terrno, NULL, pParam->pParam);
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
@ -1682,8 +1724,9 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
pParam->refId = pTmq->refId;
pParam->epoch = pTmq->epoch;
pParam->pVg = pVg;
pParam->pTopic = pTopic;
// pParam->pVg = pVg; // pVg may be released,fix it
// pParam->pTopic = pTopic;
strcpy(pParam->topicName, pTopic->topicName);
pParam->vgId = pVg->vgId;
pParam->requestId = req.reqId;
@ -1718,6 +1761,9 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
// broadcast the poll request to all related vnodes
static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
if(atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER){
return 0;
}
int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
tscDebug("consumer:0x%" PRIx64 " start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics);
@ -1802,8 +1848,14 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp;
if (pDataRsp->head.epoch == consumerEpoch) {
SMqClientVg* pVg = pollRspWrapper->vgHandle;
SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
pollRspWrapper->vgHandle = pVg;
pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName);
if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){
tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId,
pollRspWrapper->topicName, pollRspWrapper->vgId);
return NULL;
}
// update the epset
if (pollRspWrapper->pEpset != NULL) {
SEp* pEp = GET_ACTIVE_EP(pollRspWrapper->pEpset);
@ -1862,8 +1914,16 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId);
if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) {
SMqClientVg* pVg = pollRspWrapper->vgHandle;
if (!pVg->seekUpdated) {
SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
pollRspWrapper->vgHandle = pVg;
pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName);
if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){
tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId,
pollRspWrapper->topicName, pollRspWrapper->vgId);
return NULL;
}
if(pollRspWrapper->metaRsp.rspOffset.type != 0){ // if offset is validate
pVg->offsetInfo.currentOffset = pollRspWrapper->metaRsp.rspOffset;
}
@ -1883,8 +1943,16 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) {
SMqClientVg* pVg = pollRspWrapper->vgHandle;
if (!pVg->seekUpdated) { // if offset is validate
SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
pollRspWrapper->vgHandle = pVg;
pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName);
if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){
tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId,
pollRspWrapper->topicName, pollRspWrapper->vgId);
return NULL;
}
if(pollRspWrapper->taosxRsp.rspOffset.type != 0){ // if offset is validate
pVg->offsetInfo.currentOffset = pollRspWrapper->taosxRsp.rspOffset;
}

50
source/common/src/tdataformat.c
View File

@ -2504,18 +2504,21 @@ _exit:
return code;
}
int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) {
int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen) {
int32_t code = 0;
if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) {
ASSERT(pColData->type == pBind->buffer_type);
}
if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type
for (int32_t i = 0; i < pBind->num; ++i) {
if (pBind->is_null && pBind->is_null[i]) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);
if (code) goto _exit;
} else if (pBind->length[i] > buffMaxLen) {
uError("var data length too big, len:%d, max:%d", pBind->length[i], buffMaxLen);
return TSDB_CODE_INVALID_PARA;
} else {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](
pColData, (uint8_t *)pBind->buffer + pBind->buffer_length * i, pBind->length[i]);
@ -3523,6 +3526,43 @@ static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum
}
}
static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
*(uint64_t *)sum = 0;
*(uint64_t *)max = 0;
*(uint64_t *)min = 0;
*numOfNull = 0;
switch (pColData->flag) {
case HAS_NONE:
case HAS_NULL:
case (HAS_NONE | HAS_NULL):
*numOfNull = pColData->nVal;
break;
case HAS_VALUE:
*numOfNull = 0;
break;
case (HAS_VALUE | HAS_NULL):
case (HAS_VALUE | HAS_NONE):
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
if (GET_BIT1(pColData->pBitMap, iVal) == 0) {
(*numOfNull)++;
}
}
break;
case (HAS_VALUE | HAS_NONE | HAS_NULL):
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
if (GET_BIT2(pColData->pBitMap, iVal) != 2) {
(*numOfNull)++;
}
}
break;
default:
ASSERT(0);
break;
}
}
void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull) = {
NULL,
tColDataCalcSMABool, // TSDB_DATA_TYPE_BOOL
@ -3532,14 +3572,14 @@ void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_
tColDataCalcSMABigInt, // TSDB_DATA_TYPE_BIGINT
tColDataCalcSMAFloat, // TSDB_DATA_TYPE_FLOAT
tColDataCalcSMADouble, // TSDB_DATA_TYPE_DOUBLE
NULL, // TSDB_DATA_TYPE_VARCHAR
tColDataCalcSMAVarType, // TSDB_DATA_TYPE_VARCHAR
tColDataCalcSMABigInt, // TSDB_DATA_TYPE_TIMESTAMP
NULL, // TSDB_DATA_TYPE_NCHAR
tColDataCalcSMAVarType, // TSDB_DATA_TYPE_NCHAR
tColDataCalcSMAUTinyInt, // TSDB_DATA_TYPE_UTINYINT
tColDataCalcSMATinyUSmallInt, // TSDB_DATA_TYPE_USMALLINT
tColDataCalcSMAUInt, // TSDB_DATA_TYPE_UINT
tColDataCalcSMAUBigInt, // TSDB_DATA_TYPE_UBIGINT
NULL, // TSDB_DATA_TYPE_JSON
tColDataCalcSMAVarType, // TSDB_DATA_TYPE_JSON
NULL, // TSDB_DATA_TYPE_VARBINARY
NULL, // TSDB_DATA_TYPE_DECIMAL
NULL, // TSDB_DATA_TYPE_BLOB

16
source/common/src/tglobal.c
View File

@ -142,8 +142,8 @@ int32_t tsCompressColData = -1;
// count/hyperloglog function always return values in case of all NULL data or Empty data set.
int32_t tsCountAlwaysReturnValue = 1;
// 10 ms for sliding time, the value will changed in case of time precision changed
int32_t tsMinSlidingTime = 10;
// 1 ms for sliding time, the value will changed in case of time precision changed
int32_t tsMinSlidingTime = 1;
// the maxinum number of distict query result
int32_t tsMaxNumOfDistinctResults = 1000 * 10000;
@ -151,8 +151,8 @@ int32_t tsMaxNumOfDistinctResults = 1000 * 10000;
// 1 database precision unit for interval time range, changed accordingly
int32_t tsMinIntervalTime = 1;
// maximum memory allowed to be allocated for a single csv load (in MB)
int32_t tsMaxMemUsedByInsert = 1024;
// maximum batch rows numbers imported from a single csv load
int32_t tsMaxInsertBatchRows = 1000000;
float tsSelectivityRatio = 1.0;
int32_t tsTagFilterResCacheSize = 1024 * 10;
@ -346,7 +346,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1;
// if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1;
// if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1;
if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1;
if (cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, true) != 0) return -1;
if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1;
if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1;
if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1;
@ -422,7 +422,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 10, 1000000, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1;
@ -776,7 +776,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
// tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval;
// tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32;
tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32;
tsMaxInsertBatchRows = cfgGetItem(pCfg, "maxInsertBatchRows")->i32;
tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32;
@ -1048,7 +1048,7 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) {
} else if (strcasecmp("maxNumOfDistinctRes", name) == 0) {
tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32;
} else if (strcasecmp("maxMemUsedByInsert", name) == 0) {
tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32;
tsMaxInsertBatchRows = cfgGetItem(pCfg, "maxInsertBatchRows")->i32;
} else if (strcasecmp("maxRetryWaitTime", name) == 0) {
tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32;
}

123
source/common/src/ttime.c
View File

@ -82,6 +82,7 @@ static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_
static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim);
static char* forwardToTimeStringEnd(char* str);
static bool checkTzPresent(const char* str, int32_t len);
static int32_t parseTimezone(char* str, int64_t* tzOffset);
static int32_t (*parseLocaltimeFp[])(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) = {
parseLocaltime, parseLocaltimeDst};
@ -92,13 +93,13 @@ int32_t taosParseTime(const char* timestr, int64_t* utime, int32_t len, int32_t
if (checkTzPresent(timestr, len)) {
return parseTimeWithTz(timestr, utime, timePrec, 'T');
} else {
return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 'T');
return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 'T');
}
} else {
if (checkTzPresent(timestr, len)) {
return parseTimeWithTz(timestr, utime, timePrec, 0);
} else {
return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 0);
return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 0);
}
}
}
@ -713,16 +714,12 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) {
return t;
}
if (unit != 'n' && unit != 'y') {
if (!IS_CALENDAR_TIME_DURATION(unit)) {
return t + duration;
}
// The following code handles the y/n time duration
int64_t numOfMonth = duration;
if (unit == 'y') {
numOfMonth *= 12;
}
int64_t numOfMonth = (unit == 'y')? duration*12:duration;
int64_t fraction = t % TSDB_TICK_PER_SECOND(precision);
struct tm tm;
@ -741,6 +738,7 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char
ekey = skey;
skey = tmp;
}
if (unit != 'n' && unit != 'y') {
return (int32_t)((ekey - skey) / interval);
}
@ -764,13 +762,16 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char
return (emon - smon) / (int32_t)interval;
}
int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision) {
int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) {
if (pInterval->sliding == 0 && pInterval->interval == 0) {
return t;
return ts;
}
int64_t start = t;
if (pInterval->slidingUnit == 'n' || pInterval->slidingUnit == 'y') {
int64_t start = ts;
int32_t precision = pInterval->precision;
if (IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) {
start /= (int64_t)(TSDB_TICK_PER_SECOND(precision));
struct tm tm;
time_t tt = (time_t)start;
@ -792,44 +793,72 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio
start = (int64_t)(taosMktime(&tm) * TSDB_TICK_PER_SECOND(precision));
} else {
int64_t delta = t - pInterval->interval;
int32_t factor = (delta >= 0) ? 1 : -1;
if (IS_CALENDAR_TIME_DURATION(pInterval->intervalUnit)) {
int64_t news = (ts / pInterval->sliding) * pInterval->sliding;
ASSERT(news <= ts);
start = (delta / pInterval->sliding + factor) * pInterval->sliding;
if (news <= ts) {
int64_t prev = news;
int64_t newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1;
if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') {
/*
* here we revised the start time of day according to the local time zone,
* but in case of DST, the start time of one day need to be dynamically decided.
*/
// todo refactor to extract function that is available for Linux/Windows/Mac platform
#if defined(WINDOWS) && _MSC_VER >= 1900
// see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019
int64_t timezone = _timezone;
int32_t daylight = _daylight;
char** tzname = _tzname;
#endif
if (newe < ts) { // move towards the greater endpoint
while(newe < ts && news < ts) {
news += pInterval->sliding;
newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1;
}
start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision));
}
int64_t end = 0;
// not enough time range
if (start < 0 || INT64_MAX - start > pInterval->interval - 1) {
end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1;
while (end < t) { // move forward to the correct time window
start += pInterval->sliding;
if (start < 0 || INT64_MAX - start > pInterval->interval - 1) {
end = start + pInterval->interval - 1;
prev = news;
} else {
end = INT64_MAX;
break;
while (newe >= ts) {
prev = news;
news -= pInterval->sliding;
newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1;
}
}
return prev;
}
} else {
end = INT64_MAX;
int64_t delta = ts - pInterval->interval;
int32_t factor = (delta >= 0) ? 1 : -1;
start = (delta / pInterval->sliding + factor) * pInterval->sliding;
if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') {
/*
* here we revised the start time of day according to the local time zone,
* but in case of DST, the start time of one day need to be dynamically decided.
*/
// todo refactor to extract function that is available for Linux/Windows/Mac platform
#if defined(WINDOWS) && _MSC_VER >= 1900
// see
// https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019
int64_t timezone = _timezone;
int32_t daylight = _daylight;
char** tzname = _tzname;
#endif
start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision));
}
int64_t end = 0;
// not enough time range
if (start < 0 || INT64_MAX - start > pInterval->interval - 1) {
end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1;
while (end < ts) { // move forward to the correct time window
start += pInterval->sliding;
if (start < 0 || INT64_MAX - start > pInterval->interval - 1) {
end = start + pInterval->interval - 1;
} else {
end = INT64_MAX;
break;
}
}
} else {
end = INT64_MAX;
}
}
}
@ -841,10 +870,10 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio
// try to move current window to the left-hande-side, due to the offset effect.
int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1;
int64_t newEnd = end;
while (newEnd >= t) {
end = newEnd;
newEnd = taosTimeAdd(newEnd, -pInterval->sliding, pInterval->slidingUnit, precision);
int64_t newe = end;
while (newe >= ts) {
end = newe;
newe = taosTimeAdd(newe, -pInterval->sliding, pInterval->slidingUnit, precision);
}
start = taosTimeAdd(end, -pInterval->interval, pInterval->intervalUnit, precision) + 1;

2
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -78,7 +78,7 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, pMsg);
if (code != 0) {
if (terrno != 0) code = terrno;
dGError("vgId:%d, msg:%p failed to query since %s", pVnode->vgId, pMsg, terrstr(code));
dGError("vgId:%d, msg:%p failed to query since %s", pVnode->vgId, pMsg, tstrerror(code));
vmSendRsp(pMsg, code);
}

4
source/dnode/mnode/impl/src/mndDb.c
View File

@ -929,7 +929,7 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) {
cfgRsp->walRetentionSize = pDb->cfg.walRetentionSize;
cfgRsp->walSegmentSize = pDb->cfg.walSegmentSize;
cfgRsp->numOfRetensions = pDb->cfg.numOfRetensions;
cfgRsp->pRetensions = pDb->cfg.pRetensions;
cfgRsp->pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL);
cfgRsp->schemaless = pDb->cfg.schemaless;
cfgRsp->sstTrigger = pDb->cfg.sstTrigger;
}
@ -972,6 +972,8 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) {
_OVER:
tFreeSDbCfgRsp(&cfgRsp);
if (code != 0) {
mError("db:%s, failed to get cfg since %s", cfgReq.db, terrstr());
}

2
source/dnode/vnode/inc/vnode.h
View File

@ -149,7 +149,7 @@ int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int3
void tsdbReaderSetId(STsdbReader *pReader, const char *idstr);
void tsdbReaderClose(STsdbReader *pReader);
int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext);
int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave);
int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA);
void tsdbReleaseDataBlock(STsdbReader *pReader);
SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);

23
source/dnode/vnode/src/inc/tq.h
View File

@ -98,18 +98,17 @@ typedef enum tq_handle_status {
} tq_handle_status;
typedef struct {
char subKey[TSDB_SUBSCRIBE_KEY_LEN];
int64_t consumerId;
int32_t epoch;
int8_t fetchMeta;
int64_t snapshotVer;
SWalReader* pWalReader;
SWalRef* pRef;
// STqPushHandle pushHandle; // push
STqExecHandle execHandle; // exec
SRpcMsg* msg;
int32_t noDataPollCnt;
tq_handle_status status;
char subKey[TSDB_SUBSCRIBE_KEY_LEN];
int64_t consumerId;
int32_t epoch;
int8_t fetchMeta;
int64_t snapshotVer;
SWalReader* pWalReader;
SWalRef* pRef;
// STqPushHandle pushHandle; // push
STqExecHandle execHandle; // exec
SRpcMsg* msg;
tq_handle_status status;
} STqHandle;
struct STQ {

38
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -317,42 +317,6 @@ _query:
tDecoderClear(&dc);
goto _exit;
}
{ // Traverse to find the previous qualified data
TBC *pCur;
tdbTbcOpen(pMeta->pTbDb, &pCur, NULL);
STbDbKey key = {.version = sver, .uid = INT64_MAX};
int c = 0;
tdbTbcMoveTo(pCur, &key, sizeof(key), &c);
if (c < 0) {
tdbTbcMoveToPrev(pCur);
}
void *pKey = NULL;
void *pVal = NULL;
int vLen = 0, kLen = 0;
while (1) {
int32_t ret = tdbTbcPrev(pCur, &pKey, &kLen, &pVal, &vLen);
if (ret < 0) break;
STbDbKey *tmp = (STbDbKey *)pKey;
if (tmp->uid != uid) {
continue;
}
SDecoder dcNew = {0};
SMetaEntry meNew = {0};
tDecoderInit(&dcNew, pVal, vLen);
metaDecodeEntry(&dcNew, &meNew);
pSchema = tCloneSSchemaWrapper(&meNew.stbEntry.schemaRow);
tDecoderClear(&dcNew);
tdbTbcClose(pCur);
tdbFree(pKey);
tdbFree(pVal);
goto _exit;
}
tdbFree(pKey);
tdbFree(pVal);
tdbTbcClose(pCur);
}
} else if (me.type == TSDB_CHILD_TABLE) {
uid = me.ctbEntry.suid;
tDecoderClear(&dc);
@ -377,7 +341,6 @@ _query:
tDecoderClear(&dc);
_exit:
tDecoderClear(&dc);
if (lock) {
metaULock(pMeta);
}
@ -385,7 +348,6 @@ _exit:
return pSchema;
_err:
tDecoderClear(&dc);
if (lock) {
metaULock(pMeta);
}

101
source/dnode/vnode/src/tq/tq.c
View File

@ -423,6 +423,7 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) {
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
SMqPollReq req = {0};
int code = 0;
if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen);
terrno = TSDB_CODE_INVALID_MSG;
@ -433,38 +434,45 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
int32_t reqEpoch = req.epoch;
STqOffsetVal reqOffset = req.reqOffset;
int32_t vgId = TD_VID(pTq->pVnode);
STqHandle* pHandle = NULL;
taosWLockLatch(&pTq->lock);
// 1. find handle
STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
if (pHandle == NULL) {
tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey);
terrno = TSDB_CODE_INVALID_MSG;
while (1) {
taosWLockLatch(&pTq->lock);
// 1. find handle
pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
if (pHandle == NULL) {
tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey);
terrno = TSDB_CODE_INVALID_MSG;
taosWUnLockLatch(&pTq->lock);
return -1;
}
// 2. check re-balance status
if (pHandle->consumerId != consumerId) {
tqError("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64,
consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId);
terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH;
taosWUnLockLatch(&pTq->lock);
return -1;
}
bool exec = tqIsHandleExec(pHandle);
if(!exec) {
tqSetHandleExec(pHandle);
// qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS);
tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle);
taosWUnLockLatch(&pTq->lock);
break;
}
taosWUnLockLatch(&pTq->lock);
return -1;
}
while (tqIsHandleExec(pHandle)) {
tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", consumerId, vgId, req.subKey);
taosMsleep(5);
tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 10ms and retry, pHandle:%p", consumerId, vgId, req.subKey, pHandle);
taosMsleep(10);
}
// 2. check re-balance status
if (pHandle->consumerId != consumerId) {
tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64,
consumerId, vgId, req.subKey, pHandle->consumerId);
terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH;
taosWUnLockLatch(&pTq->lock);
return -1;
}
tqSetHandleExec(pHandle);
taosWUnLockLatch(&pTq->lock);
// 3. update the epoch value
int32_t savedEpoch = pHandle->epoch;
if (savedEpoch < reqEpoch) {
tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch,
reqEpoch);
if (pHandle->epoch < reqEpoch) {
tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, pHandle->epoch, reqEpoch);
pHandle->epoch = reqEpoch;
}
@ -473,8 +481,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64,
consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId);
int code = tqExtractDataForMq(pTq, pHandle, &req, pMsg);
code = tqExtractDataForMq(pTq, pHandle, &req, pMsg);
tqSetHandleIdle(pHandle);
tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle);
return code;
}
@ -571,8 +581,8 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey));
if (pHandle) {
while (tqIsHandleExec(pHandle)) {
tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", vgId, pHandle->subKey);
taosMsleep(5);
tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 10ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle);
taosMsleep(10);
}
if (pHandle->pRef) {
@ -641,7 +651,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
tqDebug("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pVnode->config.vgId, req.subKey,
req.oldConsumerId, req.newConsumerId);
taosWLockLatch(&pTq->lock);
STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
if (pHandle == NULL) {
if (req.oldConsumerId != -1) {
@ -735,37 +744,35 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
ret = tqMetaSaveHandle(pTq, req.subKey, pHandle);
goto end;
} else {
while (tqIsHandleExec(pHandle)) {
tqDebug("sub req vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", vgId, pHandle->subKey);
taosMsleep(5);
}
taosWLockLatch(&pTq->lock);
if (pHandle->consumerId == req.newConsumerId) { // do nothing
tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs", req.vgId, req.newConsumerId);
atomic_add_fetch_32(&pHandle->epoch, 1);
tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs, should not reach here", req.vgId, req.newConsumerId);
} else {
tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId,
req.newConsumerId);
atomic_store_64(&pHandle->consumerId, req.newConsumerId);
atomic_store_32(&pHandle->epoch, 0);
}
// atomic_add_fetch_32(&pHandle->epoch, 1);
// kill executing task
qTaskInfo_t pTaskInfo = pHandle->execHandle.task;
if (pTaskInfo != NULL) {
qKillTask(pTaskInfo, TSDB_CODE_SUCCESS);
}
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
qStreamCloseTsdbReader(pTaskInfo);
}
// if(tqIsHandleExec(pHandle)) {
// qTaskInfo_t pTaskInfo = pHandle->execHandle.task;
// if (pTaskInfo != NULL) {
// qKillTask(pTaskInfo, TSDB_CODE_SUCCESS);
// }
// if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
// qStreamCloseTsdbReader(pTaskInfo);
// }
// }
// remove if it has been register in the push manager, and return one empty block to consumer
tqUnregisterPushHandle(pTq, pHandle);
taosWUnLockLatch(&pTq->lock);
ret = tqMetaSaveHandle(pTq, req.subKey, pHandle);
goto end;
}
end:
taosWUnLockLatch(&pTq->lock);
taosMemoryFree(req.qmsg);
return ret;
}

2
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -366,9 +366,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
taosArrayDestroy(tbUidList);
}
tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, vgId);
taosWLockLatch(&pTq->lock);
taosHashPut(pTq->pHandle, pKey, kLen, &handle, sizeof(STqHandle));
taosWUnLockLatch(&pTq->lock);
}
end:

2
source/dnode/vnode/src/tq/tqOffset.c
View File

@ -78,7 +78,6 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) {
// todo remove this
if (offset.val.type == TMQ_OFFSET__LOG) {
taosWLockLatch(&pStore->pTq->lock);
STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey));
if (pHandle) {
if (walSetRefVer(pHandle->pRef, offset.val.version) < 0) {
@ -86,7 +85,6 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) {
// offset.val.version);
}
}
taosWUnLockLatch(&pStore->pTq->lock);
}
taosMemoryFree(pMemBuf);

5
source/dnode/vnode/src/tq/tqPush.c
View File

@ -101,8 +101,11 @@ int32_t tqUnregisterPushHandle(STQ* pTq, void *handle) {
STqHandle *pHandle = (STqHandle*)handle;
int32_t vgId = TD_VID(pTq->pVnode);
if(taosHashGetSize(pTq->pPushMgr) <= 0) {
return 0;
}
int32_t ret = taosHashRemove(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey));
tqError("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId);
tqDebug("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId);
if(pHandle->msg != NULL) {
tqPushDataRsp(pHandle, vgId);

2
source/dnode/vnode/src/tq/tqRead.c
View File

@ -1059,7 +1059,6 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
int32_t vgId = TD_VID(pTq->pVnode);
// update the table list for each consumer handle
taosWLockLatch(&pTq->lock);
while (1) {
pIter = taosHashIterate(pTq->pHandle, pIter);
if (pIter == NULL) {
@ -1097,7 +1096,6 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
}
}
}
taosWUnLockLatch(&pTq->lock);
// update the table list handle for each stream scanner/wal reader
taosWLockLatch(&pTq->pStreamMeta->lock);

12
source/dnode/vnode/src/tq/tqScan.c
View File

@ -84,8 +84,10 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
qStreamSetOpen(task);
tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId);
if (qExecTask(task, &pDataBlock, &ts) != TSDB_CODE_SUCCESS) {
tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, terrstr());
code = qExecTask(task, &pDataBlock, &ts);
if (code != TSDB_CODE_SUCCESS) {
tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, tstrerror(code));
terrno = code;
return -1;
}
@ -128,8 +130,10 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
SSDataBlock* pDataBlock = NULL;
uint64_t ts = 0;
tqDebug("tmqsnap task start to execute");
if (qExecTask(task, &pDataBlock, &ts) < 0) {
tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr());
int code = qExecTask(task, &pDataBlock, &ts);
if (code != 0) {
tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, tstrerror(code));
terrno = code;
return -1;
}

42
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -15,8 +15,7 @@
#include "tq.h"
#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0)
#define NO_POLL_CNT 5
#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0)
static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq,
const SMqMetaRsp* pRsp, int32_t vgId);
@ -89,15 +88,13 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) {
return 0;
}
static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
SRpcMsg* pMsg, bool* pBlockReturned) {
static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, bool* pBlockReturned) {
uint64_t consumerId = pRequest->consumerId;
STqOffsetVal reqOffset = pRequest->reqOffset;
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey);
int32_t vgId = TD_VID(pTq->pVnode);
*pBlockReturned = false;
// In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value.
if (pOffset != NULL) {
*pOffsetVal = pOffset->val;
@ -121,21 +118,16 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
tqOffsetResetToData(pOffsetVal, 0, 0);
}
} else {
pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef);
if (pHandle->pRef == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
// offset set to previous version when init
walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef);
tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1);
}
} else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
walRefLastVer(pTq->pVnode->pWal, pHandle->pRef);
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
SMqDataRsp dataRsp = {0};
tqInitDataRsp(&dataRsp, pRequest);
tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
tqOffsetResetToLog(&dataRsp.rspOffset, pHandle->pRef->refVer);
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId,
pHandle->subKey, vgId, dataRsp.rspOffset.version);
int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId);
@ -146,7 +138,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
} else {
STaosxRsp taosxRsp = {0};
tqInitTaosxRsp(&taosxRsp, pRequest);
tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
tqOffsetResetToLog(&taosxRsp.rspOffset, pHandle->pRef->refVer);
int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId);
tDeleteSTaosxRsp(&taosxRsp);
@ -176,24 +168,18 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId);
code = tqScanData(pTq, pHandle, &dataRsp, pOffset);
if (code != 0) {
if(code != 0 && terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) {
goto end;
}
// till now, all data has been transferred to consumer, new data needs to push client once arrived.
if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG &&
dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) {
if (pHandle->noDataPollCnt >= NO_POLL_CNT) { // send poll result to client if no data 5 times to avoid lost data
pHandle->noDataPollCnt = 0;
// lock
taosWLockLatch(&pTq->lock);
code = tqRegisterPushHandle(pTq, pHandle, pMsg);
taosWUnLockLatch(&pTq->lock);
tDeleteMqDataRsp(&dataRsp);
return code;
} else {
pHandle->noDataPollCnt++;
}
if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.blockNum == 0) {
// lock
taosWLockLatch(&pTq->lock);
code = tqRegisterPushHandle(pTq, pHandle, pMsg);
taosWUnLockLatch(&pTq->lock);
tDeleteMqDataRsp(&dataRsp);
return code;
}
// NOTE: this pHandle->consumerId may have been changed already.

3
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -201,6 +201,9 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList,
void* tsdbCacherowsReaderClose(void* pReader) {
SCacheRowsReader* p = pReader;
if (p == NULL) {
return NULL;
}
if (p->pSchema != NULL) {
for (int32_t i = 0; i < p->pSchema->numOfCols; ++i) {

17
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -5030,7 +5030,8 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) {
return code;
}
static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) {
static bool doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) {
bool hasNullSMA = false;
// do fill all null column value SMA info
int32_t i = 0, j = 0;
int32_t size = (int32_t)taosArrayGetSize(pSup->pColAgg);
@ -5050,6 +5051,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_
taosArrayInsert(pSup->pColAgg, i, &nullColAgg);
i += 1;
size++;
hasNullSMA = true;
}
j += 1;
}
@ -5060,12 +5062,15 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_
SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows};
taosArrayInsert(pSup->pColAgg, i, &nullColAgg);
i += 1;
hasNullSMA = true;
}
j++;
}
return hasNullSMA;
}
int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave) {
int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool *hasNullSMA) {
SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg;
int32_t code = 0;
@ -5129,7 +5134,10 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock,
}
// do fill all null column value SMA info
doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg);
if (doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg)) {
*hasNullSMA = true;
return TSDB_CODE_SUCCESS;
}
size_t size = taosArrayGetSize(pSup->pColAgg);
int32_t i = 0, j = 0;
@ -5296,6 +5304,9 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
}
static int32_t getBucketIndex(int32_t startRow, int32_t bucketRange, int32_t numOfRows, int32_t numOfBucket) {
if (numOfRows < startRow) {
return 0;
}
int32_t bucketIndex = ((numOfRows - startRow) / bucketRange);
if (bucketIndex == numOfBucket) {
bucketIndex -= 1;

2
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -523,7 +523,7 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData,
for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) {
SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type) || ((pColData->flag & HAS_VALUE) == 0)) continue;
if ((!pColData->smaOn) || ((pColData->flag & HAS_VALUE) == 0)) continue;
SColumnDataAgg sma = {.colId = pColData->cid};
tColDataCalcSMA[pColData->type](pColData, &sma.sum, &sma.max, &sma.min, &sma.numOfNull);

0
source/libs/cache/test/cacheTests.cpp vendored Normal file
View File

4
source/libs/catalog/src/ctgUtil.c
View File

@ -1562,7 +1562,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
switch (pReq->type) {
case AUTH_TYPE_READ: {
if (pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) {
if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) {
req->singleType = AUTH_TYPE_READ;
CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res));
if (pRes->pass || res->metaNotExists) {
@ -1578,7 +1578,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
break;
}
case AUTH_TYPE_WRITE: {
if (pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) {
if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) {
req->singleType = AUTH_TYPE_WRITE;
CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res));
if (pRes->pass || res->metaNotExists) {

7
source/libs/executor/inc/executil.h
View File

@ -135,6 +135,10 @@ struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t i
static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) {
SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId);
if (!bufPage) {
uFatal("failed to get the buffer page:%d since %s", pos->pageId, terrstr());
return NULL;
}
if (forUpdate) {
setBufPageDirty(bufPage, true);
}
@ -182,4 +186,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle,
void printDataBlock(SSDataBlock* pBlock, const char* flag);
void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order);
void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery);
#endif // TDENGINE_EXECUTIL_H

2
source/libs/executor/inc/tfill.h
View File

@ -120,6 +120,8 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t
void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp);
void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput);
void taosFillUpdateStartTimestampInfo(SFillInfo* pFillInfo, int64_t ts);
bool taosFillNotStarted(const SFillInfo* pFillInfo);
SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr,
int32_t numOfNotFillCols, const struct SNodeListNode* val);
bool taosFillHasMoreResults(struct SFillInfo* pFillInfo);

39
source/libs/executor/src/executil.c
View File

@ -136,7 +136,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in
size_t keyLen = 0;
int32_t iter = 0;
int32_t bufLen = 0, offset = 0;
int64_t bufLen = 0, offset = 0;
// todo move away and record this during create window
while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) {
@ -1761,12 +1761,12 @@ int32_t convertFillType(int32_t mode) {
return type;
}
static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) {
void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) {
if (ascQuery) {
*w = getAlignQueryTimeWindow(pInterval, pInterval->precision, ts);
*w = getAlignQueryTimeWindow(pInterval, ts);
} else {
// the start position of the first time window in the endpoint that spreads beyond the queried last timestamp
*w = getAlignQueryTimeWindow(pInterval, pInterval->precision, ts);
*w = getAlignQueryTimeWindow(pInterval, ts);
int64_t key = w->skey;
while (key < ts) { // moving towards end
@ -1783,7 +1783,7 @@ static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindo
static STimeWindow doCalculateTimeWindow(int64_t ts, SInterval* pInterval) {
STimeWindow w = {0};
w.skey = taosTimeTruncate(ts, pInterval, pInterval->precision);
w.skey = taosTimeTruncate(ts, pInterval);
w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
return w;
}
@ -1817,6 +1817,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI
if (pRow) {
w = pRow->win;
}
// in case of typical time window, we can calculate time window directly.
if (w.skey > ts || w.ekey < ts) {
w = doCalculateTimeWindow(ts, pInterval);
@ -1831,6 +1832,34 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI
return w;
}
void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order) {
int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
if (!IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) {
tw->skey += pInterval->sliding * factor;
tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
return;
}
// convert key to second
int64_t key = convertTimePrecision(tw->skey, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000;
int64_t duration = pInterval->sliding;
if (pInterval->slidingUnit == 'y') {
duration *= 12;
}
struct tm tm;
time_t t = (time_t) key;
taosLocalTime(&t, &tm, NULL);
int mon = (int)(tm.tm_year * 12 + tm.tm_mon + duration * factor);
tm.tm_year = mon / 12;
tm.tm_mon = mon % 12;
tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision);
tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
}
bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo) {
return (pLimitInfo->limit.limit != -1 || pLimitInfo->limit.offset != -1 || pLimitInfo->slimit.limit != -1 ||
pLimitInfo->slimit.offset != -1);

5
source/libs/executor/src/executor.c
View File

@ -186,6 +186,11 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) {
doSetTaskId(pTaskInfo->pRoot, &pTaskInfo->storageAPI);
}
//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) {
// SExecTaskInfo* pTaskInfo = tinfo;
// pTaskInfo->code = code;
//}
int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) {
if (tinfo == NULL) {
return TSDB_CODE_APP_ERROR;

9
source/libs/executor/src/executorInt.c
View File

@ -442,15 +442,15 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB
}
/////////////////////////////////////////////////////////////////////////////////////////////
STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key) {
STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key) {
STimeWindow win = {0};
win.skey = taosTimeTruncate(key, pInterval, precision);
win.skey = taosTimeTruncate(key, pInterval);
/*
* if the realSkey > INT64_MAX - pInterval->interval, the query duration between
* realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges.
*/
win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
if (win.ekey < win.skey) {
win.ekey = INT64_MAX;
}
@ -726,7 +726,8 @@ void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultR
pCtx[j].resultInfo->numOfRes = pRow->numOfRows;
}
}
blockDataEnsureCapacity(pBlock, pBlock->info.rows + pCtx[j].resultInfo->numOfRes);
int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);
if (TAOS_FAILED(code)) {
qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code));

105
source/libs/executor/src/filloperator.c
View File

@ -61,25 +61,28 @@ typedef struct SFillOperatorInfo {
SExprSupp noFillExprSupp;
} SFillOperatorInfo;
static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order);
static void destroyFillOperatorInfo(void* param);
static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag);
static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo,
SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) {
SResultInfo* pResultInfo, int32_t order) {
pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows;
SSDataBlock* pResBlock = pInfo->pFinalRes;
int32_t order = TSDB_ORDER_ASC;
// int32_t order = TSDB_ORDER_ASC;
int32_t scanFlag = MAIN_SCAN;
getTableScanInfo(pOperator, &order, &scanFlag, false);
int64_t ekey = pInfo->existNewGroupBlock->info.window.ekey;
// getTableScanInfo(pOperator, &order, &scanFlag, false);
taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo));
blockDataCleanup(pInfo->pRes);
doApplyScalarCalculation(pOperator, pInfo->existNewGroupBlock, order, scanFlag);
taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ekey);
revisedFillStartKey(pInfo, pInfo->existNewGroupBlock, order);
int64_t ts = (order == TSDB_ORDER_ASC)? pInfo->existNewGroupBlock->info.window.ekey:pInfo->existNewGroupBlock->info.window.skey;
taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts);
taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes);
int32_t numOfResultRows = pResultInfo->capacity - pResBlock->info.rows;
@ -90,7 +93,7 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp
}
static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo,
SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) {
SResultInfo* pResultInfo, int32_t order) {
if (taosFillHasMoreResults(pInfo->pFillInfo)) {
int32_t numOfResultRows = pResultInfo->capacity - pInfo->pFinalRes->info.rows;
taosFillResultDataBlock(pInfo->pFillInfo, pInfo->pFinalRes, numOfResultRows);
@ -100,7 +103,7 @@ static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOpera
// handle the cached new group data block
if (pInfo->existNewGroupBlock) {
doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo);
doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, order);
}
}
@ -119,6 +122,53 @@ void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int
pInfo->pRes->info.id.groupId = pBlock->info.id.groupId;
}
// todo refactor: decide the start key according to the query time range.
static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order) {
if (order == TSDB_ORDER_ASC) {
int64_t skey = pBlock->info.window.skey;
if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the
ASSERT( taosFillNotStarted(pInfo->pFillInfo));
taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey);
} else if (pInfo->pFillInfo->start < skey) {
int64_t t = skey;
SInterval* pInterval = &pInfo->pFillInfo->interval;
while(1) {
int64_t prev = taosTimeAdd(t, -pInterval->sliding, pInterval->slidingUnit, pInterval->precision);
if (prev <= pInfo->pFillInfo->start) {
t = prev;
break;
}
t = prev;
}
// todo time window chosen problem: t or prev value?
taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t);
}
} else {
int64_t ekey = pBlock->info.window.ekey;
if (ekey > pInfo->pFillInfo->start) {
ASSERT( taosFillNotStarted(pInfo->pFillInfo));
taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, ekey);
} else if (ekey < pInfo->pFillInfo->start) {
int64_t t = ekey;
SInterval* pInterval = &pInfo->pFillInfo->interval;
while(1) {
int64_t prev = taosTimeAdd(t, pInterval->sliding, pInterval->slidingUnit, pInterval->precision);
if (prev >= pInfo->pFillInfo->start) {
t = prev;
break;
}
t = prev;
}
// todo time window chosen problem: t or prev value?
taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t);
}
}
}
static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
SFillOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -132,13 +182,19 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
int32_t scanFlag = MAIN_SCAN;
getTableScanInfo(pOperator, &order, &scanFlag, false);
doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo);
SOperatorInfo* pDownstream = pOperator->pDownstream[0];
// the scan order may be different from the output result order for agg interval operator.
if (pDownstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL) {
order = ((SIntervalAggOperatorInfo*) pDownstream->info)->resultTsOrder;
}
doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, order);
if (pResBlock->info.rows > 0) {
pResBlock->info.id.groupId = pInfo->curGroupId;
return pResBlock;
}
SOperatorInfo* pDownstream = pOperator->pDownstream[0];
while (1) {
SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream);
if (pBlock == NULL) {
@ -158,15 +214,16 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows);
doApplyScalarCalculation(pOperator, pBlock, order, scanFlag);
if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.id.groupId) {
if (pInfo->curGroupId == 0 || (pInfo->curGroupId == pInfo->pRes->info.id.groupId)) {
if (pInfo->curGroupId == 0 && taosFillNotStarted(pInfo->pFillInfo)) {
revisedFillStartKey(pInfo, pBlock, order);
}
pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block
pInfo->totalInputRows += pInfo->pRes->info.rows;
if (order == pInfo->pFillInfo->order) {
taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.ekey);
} else {
taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.skey);
}
int64_t ts = (order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey;
taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts);
taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes);
} else if (pInfo->curGroupId != pBlock->info.id.groupId) { // the new group data block
pInfo->existNewGroupBlock = pBlock;
@ -190,7 +247,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
return pResBlock;
}
doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo);
doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, order);
if (pResBlock->info.rows >= pOperator->resultInfo.threshold || pBlock == NULL) {
pResBlock->info.id.groupId = pInfo->curGroupId;
return pResBlock;
@ -198,7 +255,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
} else if (pInfo->existNewGroupBlock) { // try next group
blockDataCleanup(pResBlock);
doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo);
doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, order);
if (pResBlock->info.rows > pResultInfo->threshold) {
pResBlock->info.id.groupId = pInfo->curGroupId;
return pResBlock;
@ -256,11 +313,11 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t
const char* id, SInterval* pInterval, int32_t fillType, int32_t order) {
SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode);
int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey;
STimeWindow w = getAlignQueryTimeWindow(pInterval, pInterval->precision, startKey);
w = getFirstQualifiedTimeWindow(startKey, &w, pInterval, order);
int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey;
pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo,
// STimeWindow w = {0};
// getInitialStartTimeWindow(pInterval, startKey, &w, order == TSDB_ORDER_ASC);
pInfo->pFillInfo = taosCreateFillInfo(startKey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo,
pInfo->primaryTsCol, order, id);
if (order == TSDB_ORDER_ASC) {
@ -402,13 +459,13 @@ _error:
TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) {
STimeWindow win = {.skey = ts, .ekey = ts};
getNextIntervalWindow(pInterval, &win, TSDB_ORDER_ASC);
getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);
return win.skey;
}
TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) {
STimeWindow win = {.skey = ts, .ekey = ts};
getNextIntervalWindow(pInterval, &win, TSDB_ORDER_DESC);
getNextTimeWindow(pInterval, &win, TSDB_ORDER_DESC);
return win.skey;
}

335
source/libs/executor/src/joinoperator.c
View File

@ -30,11 +30,13 @@ typedef struct SJoinRowCtx {
bool rowRemains;
int64_t ts;
SArray* leftRowLocations;
SArray* rightRowLocations;
SArray* leftCreatedBlocks;
SArray* rightCreatedBlocks;
int32_t leftRowIdx;
int32_t rightRowIdx;
bool rightUseBuildTable;
SArray* rightRowLocations;
} SJoinRowCtx;
typedef struct SJoinOperatorInfo {
@ -50,7 +52,17 @@ typedef struct SJoinOperatorInfo {
int32_t rightPos;
SColumnInfo rightCol;
SNode* pCondAfterMerge;
SNode* pColEqualOnConditions;
SArray* leftEqOnCondCols;
char* leftEqOnCondKeyBuf;
int32_t leftEqOnCondKeyLen;
SArray* rightEqOnCondCols;
char* rightEqOnCondKeyBuf;
int32_t rightEqOnCondKeyLen;
SSHashObj* rightBuildTable;
SJoinRowCtx rowCtx;
} SJoinOperatorInfo;
@ -92,6 +104,100 @@ static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDown
setJoinColumnInfo(&pInfo->rightCol, rightTsCol);
}
static void extractEqualOnCondColsFromOper(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstreams, SOperatorNode* pOperNode,
SColumn* pLeft, SColumn* pRight) {
SColumnNode* pLeftNode = (SColumnNode*)pOperNode->pLeft;
SColumnNode* pRightNode = (SColumnNode*)pOperNode->pRight;
if (pLeftNode->dataBlockId == pRightNode->dataBlockId || pLeftNode->dataBlockId == pDownstreams[0]->resultDataBlockId) {
*pLeft = extractColumnFromColumnNode((SColumnNode*)pOperNode->pLeft);
*pRight = extractColumnFromColumnNode((SColumnNode*)pOperNode->pRight);
} else {
*pLeft = extractColumnFromColumnNode((SColumnNode*)pOperNode->pRight);
*pRight = extractColumnFromColumnNode((SColumnNode*)pOperNode->pLeft);
}
}
static void extractEqualOnCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownStream, SNode* pEqualOnCondNode,
SArray* leftTagEqCols, SArray* rightTagEqCols) {
SColumn left = {0};
SColumn right = {0};
if (nodeType(pEqualOnCondNode) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pEqualOnCondNode)->condType == LOGIC_COND_TYPE_AND) {
SNode* pNode = NULL;
FOREACH(pNode, ((SLogicConditionNode*)pEqualOnCondNode)->pParameterList) {
SOperatorNode* pOperNode = (SOperatorNode*)pNode;
extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right);
taosArrayPush(leftTagEqCols, &left);
taosArrayPush(rightTagEqCols, &right);
}
return;
}
if (nodeType(pEqualOnCondNode) == QUERY_NODE_OPERATOR) {
SOperatorNode* pOperNode = (SOperatorNode*)pEqualOnCondNode;
extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right);
taosArrayPush(leftTagEqCols, &left);
taosArrayPush(rightTagEqCols, &right);
}
}
static int32_t initTagColskeyBuf(int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) {
int32_t numOfGroupCols = taosArrayGetSize(pGroupColList);
for (int32_t i = 0; i < numOfGroupCols; ++i) {
SColumn* pCol = (SColumn*)taosArrayGet(pGroupColList, i);
(*keyLen) += pCol->bytes; // actual data + null_flag
}
int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols;
(*keyLen) += nullFlagSize;
(*keyBuf) = taosMemoryCalloc(1, (*keyLen));
if ((*keyBuf) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return TSDB_CODE_SUCCESS;
}
static int32_t fillKeyBufFromTagCols(SArray* pCols, SSDataBlock* pBlock, int32_t rowIndex, void* pKey) {
SColumnDataAgg* pColAgg = NULL;
size_t numOfGroupCols = taosArrayGetSize(pCols);
char* isNull = (char*)pKey;
char* pStart = (char*)pKey + sizeof(int8_t) * numOfGroupCols;
for (int32_t i = 0; i < numOfGroupCols; ++i) {
SColumn* pCol = (SColumn*) taosArrayGet(pCols, i);
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId);
// valid range check. todo: return error code.
if (pCol->slotId > taosArrayGetSize(pBlock->pDataBlock)) {
continue;
}
if (pBlock->pBlockAgg != NULL) {
pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched?
}
if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) {
isNull[i] = 1;
} else {
isNull[i] = 0;
char* val = colDataGetData(pColInfoData, rowIndex);
if (pCol->type == TSDB_DATA_TYPE_JSON) {
int32_t dataLen = getJsonValueLen(val);
memcpy(pStart, val, dataLen);
pStart += dataLen;
} else if (IS_VAR_DATA_TYPE(pCol->type)) {
varDataCopy(pStart, val);
pStart += varDataTLen(val);
} else {
memcpy(pStart, val, pCol->bytes);
pStart += pCol->bytes;
}
}
}
return (int32_t)(pStart - (char*)pKey);
}
SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream,
SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) {
SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo));
@ -153,6 +259,16 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
pInfo->inputOrder = TSDB_ORDER_DESC;
}
pInfo->pColEqualOnConditions = pJoinNode->pColEqualOnConditions;
if (pInfo->pColEqualOnConditions != NULL) {
pInfo->leftEqOnCondCols = taosArrayInit(4, sizeof(SColumn));
pInfo->rightEqOnCondCols = taosArrayInit(4, sizeof(SColumn));
extractEqualOnCondCols(pInfo, pDownstream, pInfo->pColEqualOnConditions, pInfo->leftEqOnCondCols, pInfo->rightEqOnCondCols);
initTagColskeyBuf(&pInfo->leftEqOnCondKeyLen, &pInfo->leftEqOnCondKeyBuf, pInfo->leftEqOnCondCols);
initTagColskeyBuf(&pInfo->rightEqOnCondKeyLen, &pInfo->rightEqOnCondKeyBuf, pInfo->rightEqOnCondCols);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->rightBuildTable = tSimpleHashInit(256, hashFn);
}
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, pDownstream, numOfDownstream);
if (code != TSDB_CODE_SUCCESS) {
@ -179,8 +295,28 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) {
pColumn->scale = pColumnNode->node.resType.scale;
}
static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) {
void* p = NULL;
int32_t iter = 0;
while ((p = tSimpleHashIterate(pBuildTable, p, &iter)) != NULL) {
SArray* rows = (*(SArray**)p);
taosArrayDestroy(rows);
}
tSimpleHashCleanup(pBuildTable);
}
void destroyMergeJoinOperator(void* param) {
SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param;
if (pJoinOperator->pColEqualOnConditions != NULL) {
mergeJoinDestoryBuildTable(pJoinOperator->rightBuildTable);
taosMemoryFreeClear(pJoinOperator->rightEqOnCondKeyBuf);
taosArrayDestroy(pJoinOperator->rightEqOnCondCols);
taosMemoryFreeClear(pJoinOperator->leftEqOnCondKeyBuf);
taosArrayDestroy(pJoinOperator->leftEqOnCondCols);
}
nodesDestroyNode(pJoinOperator->pCondAfterMerge);
pJoinOperator->pRes = blockDataDestroy(pJoinOperator->pRes);
@ -300,21 +436,122 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator
return 0;
}
static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations) {
for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) {
SRowLocation* rightRow = taosArrayGet(rightRowLocations, i);
int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightEqOnCondCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightEqOnCondKeyBuf);
SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightEqOnCondKeyBuf, keyLen);
if (!ppRows) {
SArray* rows = taosArrayInit(4, sizeof(SRowLocation));
taosArrayPush(rows, rightRow);
tSimpleHashPut(pInfo->rightBuildTable, pInfo->rightEqOnCondKeyBuf, keyLen, &rows, POINTER_BYTES);
} else {
taosArrayPush(*ppRows, rightRow);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* pRes, int32_t* nRows,
const SArray* leftRowLocations, int32_t leftRowIdx,
int32_t rightRowIdx, bool useBuildTableTSRange, SArray* rightRowLocations, bool* pReachThreshold) {
*pReachThreshold = false;
uint32_t limitRowNum = pOperator->resultInfo.threshold;
SJoinOperatorInfo* pJoinInfo = pOperator->info;
size_t leftNumJoin = taosArrayGetSize(leftRowLocations);
int32_t i,j;
for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) {
SRowLocation* leftRow = taosArrayGet(leftRowLocations, i);
SArray* pRightRows = NULL;
if (useBuildTableTSRange) {
int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftEqOnCondCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftEqOnCondKeyBuf);
SArray** ppRightRows = tSimpleHashGet(pJoinInfo->rightBuildTable, pJoinInfo->leftEqOnCondKeyBuf, keyLen);
if (!ppRightRows) {
continue;
}
pRightRows = *ppRightRows;
} else {
pRightRows = rightRowLocations;
}
size_t rightRowsSize = taosArrayGetSize(pRightRows);
for (j = rightRowIdx; j < rightRowsSize; ++j) {
if (*nRows >= limitRowNum) {
*pReachThreshold = true;
break;
}
SRowLocation* rightRow = taosArrayGet(pRightRows, j);
mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock,
rightRow->pos);
++*nRows;
}
if (*pReachThreshold) {
break;
}
}
if (*pReachThreshold) {
pJoinInfo->rowCtx.rowRemains = true;
pJoinInfo->rowCtx.leftRowIdx = i;
pJoinInfo->rowCtx.rightRowIdx = j;
}
return TSDB_CODE_SUCCESS;
}
static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* leftRowLocations, SArray* leftCreatedBlocks,
SArray* rightCreatedBlocks, bool rightUseBuildTable, SArray* rightRowLocations) {
for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) {
SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i);
blockDataDestroy(pBlock);
}
taosArrayDestroy(rightCreatedBlocks);
for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) {
SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i);
blockDataDestroy(pBlock);
}
if (rightRowLocations != NULL) {
taosArrayDestroy(rightRowLocations);
}
if (rightUseBuildTable) {
void* p = NULL;
int32_t iter = 0;
while ((p = tSimpleHashIterate(pJoinInfo->rightBuildTable, p, &iter)) != NULL) {
SArray* rows = (*(SArray**)p);
taosArrayDestroy(rows);
}
tSimpleHashClear(pJoinInfo->rightBuildTable);
}
taosArrayDestroy(leftCreatedBlocks);
taosArrayDestroy(leftRowLocations);
pJoinInfo->rowCtx.rowRemains = false;
pJoinInfo->rowCtx.leftRowLocations = NULL;
pJoinInfo->rowCtx.leftCreatedBlocks = NULL;
pJoinInfo->rowCtx.rightCreatedBlocks = NULL;
pJoinInfo->rowCtx.rightUseBuildTable = false;
pJoinInfo->rowCtx.rightRowLocations = NULL;
}
static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t timestamp, SSDataBlock* pRes,
int32_t* nRows) {
int32_t code = TSDB_CODE_SUCCESS;
SJoinOperatorInfo* pJoinInfo = pOperator->info;
SArray* leftRowLocations = NULL;
SArray* leftCreatedBlocks = NULL;
SArray* rightRowLocations = NULL;
SArray* leftCreatedBlocks = NULL;
SArray* rightCreatedBlocks = NULL;
int32_t leftRowIdx = 0;
int32_t rightRowIdx = 0;
int32_t i, j;
SSHashObj* rightTableHash = NULL;
bool rightUseBuildTable = false;
if (pJoinInfo->rowCtx.rowRemains) {
leftRowLocations = pJoinInfo->rowCtx.leftRowLocations;
leftCreatedBlocks = pJoinInfo->rowCtx.leftCreatedBlocks;
rightUseBuildTable = pJoinInfo->rowCtx.rightUseBuildTable;
rightRowLocations = pJoinInfo->rowCtx.rightRowLocations;
rightCreatedBlocks = pJoinInfo->rowCtx.rightCreatedBlocks;
leftRowIdx = pJoinInfo->rowCtx.leftRowIdx;
@ -330,78 +567,40 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t
pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks);
mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight,
pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks);
if (pJoinInfo->pColEqualOnConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) {
mergeJoinFillBuildTable(pJoinInfo, rightRowLocations);
rightUseBuildTable = true;
taosArrayDestroy(rightRowLocations);
rightRowLocations = NULL;
}
}
size_t leftNumJoin = taosArrayGetSize(leftRowLocations);
size_t rightNumJoin = taosArrayGetSize(rightRowLocations);
uint32_t maxRowNum = *nRows + (leftNumJoin - leftRowIdx - 1) * rightNumJoin + rightNumJoin - rightRowIdx;
uint32_t limitRowNum = maxRowNum;
if (maxRowNum > pOperator->resultInfo.threshold) {
limitRowNum = pOperator->resultInfo.threshold;
if (!pJoinInfo->rowCtx.rowRemains) {
code = blockDataEnsureCapacity(pRes, pOperator->resultInfo.threshold);
if (code != TSDB_CODE_SUCCESS) {
qError("%s can not ensure block capacity for join. left: %zu", GET_TASKID(pOperator->pTaskInfo),
leftNumJoin);
}
bool reachThreshold = false;
if (code == TSDB_CODE_SUCCESS) {
mergeJoinLeftRowsRightRows(pOperator, pRes, nRows, leftRowLocations, leftRowIdx,
rightRowIdx, rightUseBuildTable, rightRowLocations, &reachThreshold);
}
if (!reachThreshold) {
mergeJoinDestroyTSRangeCtx(pJoinInfo, leftRowLocations, leftCreatedBlocks, rightCreatedBlocks,
rightUseBuildTable, rightRowLocations);
} else {
pJoinInfo->rowCtx.rowRemains = true;
pJoinInfo->rowCtx.ts = timestamp;
pJoinInfo->rowCtx.leftRowLocations = leftRowLocations;
pJoinInfo->rowCtx.rightRowLocations = rightRowLocations;
pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks;
pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks;
}
}
code = blockDataEnsureCapacity(pRes, limitRowNum);
if (code != TSDB_CODE_SUCCESS) {
qError("%s can not ensure block capacity for join. left: %zu, right: %zu", GET_TASKID(pOperator->pTaskInfo),
leftNumJoin, rightNumJoin);
}
if (code == TSDB_CODE_SUCCESS) {
bool done = false;
for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) {
for (j = rightRowIdx; j < rightNumJoin; ++j) {
if (*nRows >= limitRowNum) {
done = true;
break;
}
SRowLocation* leftRow = taosArrayGet(leftRowLocations, i);
SRowLocation* rightRow = taosArrayGet(rightRowLocations, j);
mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock,
rightRow->pos);
++*nRows;
}
if (done) {
break;
}
}
if (maxRowNum > pOperator->resultInfo.threshold) {
pJoinInfo->rowCtx.leftRowIdx = i;
pJoinInfo->rowCtx.rightRowIdx = j;
}
}
if (maxRowNum <= pOperator->resultInfo.threshold) {
for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) {
SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i);
blockDataDestroy(pBlock);
}
taosArrayDestroy(rightCreatedBlocks);
taosArrayDestroy(rightRowLocations);
for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) {
SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i);
blockDataDestroy(pBlock);
}
taosArrayDestroy(leftCreatedBlocks);
taosArrayDestroy(leftRowLocations);
if (pJoinInfo->rowCtx.rowRemains) {
pJoinInfo->rowCtx.rowRemains = false;
pJoinInfo->rowCtx.leftRowLocations = NULL;
pJoinInfo->rowCtx.rightRowLocations = NULL;
pJoinInfo->rowCtx.leftCreatedBlocks = NULL;
pJoinInfo->rowCtx.rightCreatedBlocks = NULL;
}
pJoinInfo->rowCtx.rightUseBuildTable = rightUseBuildTable;
pJoinInfo->rowCtx.rightRowLocations = rightRowLocations;
}
return TSDB_CODE_SUCCESS;
}

54
source/libs/executor/src/scanoperator.c
View File

@ -88,39 +88,6 @@ static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
}
}
static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) {
int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') {
tw->skey += pInterval->sliding * factor;
tw->ekey = tw->skey + pInterval->interval - 1;
return;
}
int64_t key = tw->skey, interval = pInterval->interval;
// convert key to second
key = convertTimePrecision(key, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000;
if (pInterval->intervalUnit == 'y') {
interval *= 12;
}
struct tm tm;
time_t t = (time_t)key;
taosLocalTime(&t, &tm, NULL);
int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
tm.tm_year = mon / 12;
tm.tm_mon = mon % 12;
tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision);
mon = (int)(mon + interval);
tm.tm_year = mon / 12;
tm.tm_mon = mon % 12;
tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision);
tw->ekey -= 1;
}
static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo, int32_t order) {
STimeWindow w = {0};
@ -130,7 +97,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
}
if (order == TSDB_ORDER_ASC) {
w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey);
w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.skey);
ASSERT(w.ekey >= pBlockInfo->window.skey);
if (w.ekey < pBlockInfo->window.ekey) {
@ -149,7 +116,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
}
}
} else {
w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.ekey);
w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.ekey);
ASSERT(w.skey <= pBlockInfo->window.ekey);
if (w.skey > pBlockInfo->window.skey) {
@ -206,7 +173,7 @@ static int32_t insertTableToScanIgnoreList(STableScanInfo* pTableScanInfo, uint6
return TSDB_CODE_OUT_OF_MEMORY;
}
}
taosHashPut(pTableScanInfo->pIgnoreTables, &uid, sizeof(uid), &pTableScanInfo->scanTimes, sizeof(pTableScanInfo->scanTimes));
return TSDB_CODE_SUCCESS;
@ -215,7 +182,7 @@ static int32_t insertTableToScanIgnoreList(STableScanInfo* pTableScanInfo, uint6
static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* pBlockInfo, uint32_t* status) {
STableScanInfo* pTableScanInfo = pOperator->info;
int32_t code = TSDB_CODE_SUCCESS;
if (pTableScanInfo->base.pdInfo.pExprSup == NULL) {
return TSDB_CODE_SUCCESS;
}
@ -267,12 +234,13 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock,
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
bool allColumnsHaveAgg = true;
int32_t code = pAPI->tsdReader.tsdReaderRetrieveBlockSMAInfo(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg);
bool hasNullSMA = false;
int32_t code = pAPI->tsdReader.tsdReaderRetrieveBlockSMAInfo(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg, &hasNullSMA);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
if (!allColumnsHaveAgg) {
if (!allColumnsHaveAgg || hasNullSMA) {
return false;
}
return true;
@ -751,7 +719,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) {
STableScanInfo* pTableScanInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
// The read handle is not initialized yet, since no qualified tables exists
if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) {
return NULL;
@ -1651,6 +1619,10 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
qDebug("start to exec queue scan, %s", id);
if (isTaskKilled(pTaskInfo)) {
return NULL;
}
if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
if (pResult && pResult->info.rows > 0) {
@ -1663,7 +1635,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader);
pTSInfo->base.dataReader = NULL;
qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1);
if (pAPI->tqReaderFn.tqReaderSeek(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) {

21
source/libs/executor/src/tfill.c
View File

@ -517,11 +517,16 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey)
return;
}
// the endKey is now the aligned time window value. truncate time window isn't correct.
pFillInfo->end = endKey;
if (!FILL_IS_ASC_FILL(pFillInfo)) {
pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->interval.precision);
pFillInfo->end = taosTimeAdd(pFillInfo->end, pFillInfo->interval.interval, pFillInfo->interval.intervalUnit,pFillInfo->interval.precision);
#if 0
if (pFillInfo->order == TSDB_ORDER_ASC) {
ASSERT(pFillInfo->start <= pFillInfo->end);
} else {
ASSERT(pFillInfo->start >= pFillInfo->end);
}
#endif
pFillInfo->index = 0;
pFillInfo->numOfRows = numOfRows;
@ -531,6 +536,13 @@ void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput)
pFillInfo->pSrcBlock = (SSDataBlock*)pInput;
}
void taosFillUpdateStartTimestampInfo(SFillInfo* pFillInfo, int64_t ts) {
pFillInfo->start = ts;
pFillInfo->currentKey = ts;
}
bool taosFillNotStarted(const SFillInfo* pFillInfo) {return pFillInfo->start == pFillInfo->currentKey;}
bool taosFillHasMoreResults(SFillInfo* pFillInfo) {
int32_t remain = taosNumOfRemainRows(pFillInfo);
if (remain > 0) {
@ -562,9 +574,10 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma
ASSERT(numOfRes >= numOfRows);
} else { // reach the end of data
if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) ||
(ekey1 >= pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) {
(ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) {
return 0;
}
numOfRes = taosTimeCountInterval(ekey1, pFillInfo->currentKey, pFillInfo->interval.sliding,
pFillInfo->interval.slidingUnit, pFillInfo->interval.precision);
numOfRes += 1;

51
source/libs/executor/src/timewindowoperator.c
View File

@ -271,43 +271,6 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary
return num;
}
static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) {
int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') {
tw->skey += pInterval->sliding * factor;
tw->ekey = tw->skey + pInterval->interval - 1;
return;
}
int64_t key = tw->skey, interval = pInterval->interval;
// convert key to second
key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000;
if (pInterval->intervalUnit == 'y') {
interval *= 12;
}
struct tm tm;
time_t t = (time_t)key;
taosLocalTime(&t, &tm, NULL);
int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
tm.tm_year = mon / 12;
tm.tm_mon = mon % 12;
tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
mon = (int)(mon + interval);
tm.tm_year = mon / 12;
tm.tm_mon = mon % 12;
tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision);
tw->ekey -= 1;
}
void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) {
getNextTimeWindow(pInterval, pInterval->precision, order, tw);
}
void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs,
int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) {
SqlFunctionCtx* pCtx = pSup->pCtx;
@ -449,7 +412,7 @@ static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SEx
}
bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd, EStreamType blockType) {
if (pInterval->interval != pInterval->sliding &&
if (pInterval->interval != pInterval->sliding &&
((pWin->ekey < calStart || pWin->skey > calEnd) || (blockType == STREAM_PULL_DATA && pWin->skey < calStart) )) {
return false;
}
@ -466,7 +429,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext,
bool ascQuery = (order == TSDB_ORDER_ASC);
int32_t precision = pInterval->precision;
getNextTimeWindow(pInterval, precision, order, pNext);
getNextTimeWindow(pInterval, pNext, order);
// next time window is not in current block
if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) ||
@ -511,7 +474,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext,
if (ascQuery && primaryKeys[startPos] > pNext->ekey) {
TSKEY next = primaryKeys[startPos];
if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
pNext->skey = taosTimeTruncate(next, pInterval, precision);
pNext->skey = taosTimeTruncate(next, pInterval);
pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
} else {
pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
@ -520,7 +483,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext,
} else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) {
TSKEY next = primaryKeys[startPos];
if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') {
pNext->skey = taosTimeTruncate(next, pInterval, precision);
pNext->skey = taosTimeTruncate(next, pInterval);
pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
} else {
pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding;
@ -1370,14 +1333,14 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa
do {
if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i], pBlock->info.type)) {
getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);
continue;
}
uint64_t winGpId = pGpDatas[i];
SWinKey winRes = {.ts = win.skey, .groupId = winGpId};
void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));
if (chIds) {
getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);
continue;
}
bool res = doDeleteWindow(pOperator, win.skey, winGpId);
@ -1387,7 +1350,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa
if (pUpdatedMap) {
tSimpleHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));
}
getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);
} while (win.ekey <= endTsCols[i]);
}
}

161
source/libs/executor/test/timewindowTest.cpp Normal file
View File

@ -0,0 +1,161 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <iostream>
#include "taos.h"
#include "thash.h"
#include "tsimplehash.h"
#include "executor.h"
#include "ttime.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
namespace {
SInterval createInterval(int64_t interval, int64_t sliding, int64_t offset, char intervalUnit, char slidingUnit,
char offsetUnit, int8_t precision) {
SInterval v = {0};
v.interval = interval;
v.intervalUnit = intervalUnit;
v.sliding = sliding;
v.slidingUnit = slidingUnit;
v.offset = offset;
v.offsetUnit = offsetUnit;
v.precision = precision;
return v;
}
void printTimeWindow(STimeWindow* pWindow, int8_t precision, int64_t ts) {
char buf[64] = {0};
char bufs[64] = {0};
char bufe[64] = {0};
taosFormatUtcTime(buf, tListLen(buf), ts, precision);
taosFormatUtcTime(bufs, tListLen(bufs), pWindow->skey, precision);
taosFormatUtcTime(bufe, tListLen(bufe), pWindow->ekey, precision);
printf("%s [%s - %s]\n", buf, bufs, bufe);
}
} // namespace
TEST(testCase, timewindow_gen) {
// set correct time zone
osSetTimezone("UTC");
int32_t precision = TSDB_TIME_PRECISION_MILLI;
SInterval interval =
createInterval(10 * 86400 * 1000, 10 * 86400 * 1000, 0, 'd', 'd', 'd', precision);
int64_t key = 1659312000L * 1000; // 2022-8-1 00:00:00 // UTC+8 (ms)
STimeWindow w = {0};
getInitialStartTimeWindow(&interval, key, &w, true);
printTimeWindow(&w, precision, key);
getNextTimeWindow(&interval, &w, TSDB_ORDER_ASC);
printf("next\n");
printTimeWindow(&w, precision, key);
printf("---------------------------------------------------\n");
SInterval monthInterval =
createInterval(1, 1, 0, 'n', 'n', 'd', TSDB_TIME_PRECISION_MILLI);
getInitialStartTimeWindow(&monthInterval, key, &w, true);
printTimeWindow(&w, precision, key);
getNextTimeWindow(&monthInterval, &w, TSDB_ORDER_ASC);
printf("next\n");
printTimeWindow(&w, precision, key);
printf("----------------------------------------------------------\n");
SInterval slidingInterval = createInterval(1, 10*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI);
getInitialStartTimeWindow(&slidingInterval, key, &w, true);
printTimeWindow(&w, precision, key);
getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC);
printf("next\n");
printTimeWindow(&w, precision, key);
getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC);
printTimeWindow(&w, precision, key);
getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC);
printTimeWindow(&w, precision, key);
getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC);
printTimeWindow(&w, precision, key);
getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC);
printTimeWindow(&w, precision, key);
getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC);
printTimeWindow(&w, precision, key);
getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC);
printTimeWindow(&w, precision, key);
getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC);
printTimeWindow(&w, precision, key);
printf("-----------------calendar_interval_1n_sliding_1d-------\n");
SInterval calendar_interval_1n = createInterval(1, 1*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI);
int64_t k1 = 1664409600 * 1000L;
getInitialStartTimeWindow(&calendar_interval_1n, k1, &w, true);
printTimeWindow(&w, precision, k1);
printf("next\n");
getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC);
printTimeWindow(&w, precision, key);
getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC);
printTimeWindow(&w, precision, key);
getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC);
printTimeWindow(&w, precision, key);
getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC);
printTimeWindow(&w, precision, key);
printf("----------------interval_1d_clendar_sliding_1n---------\n");
SInterval interval_1d_calendar_sliding_1n = createInterval(1*86400*1000L, 1, 0, 'd', 'n', 'd', TSDB_TIME_PRECISION_MILLI);
k1 = 1664409600 * 1000L;
getInitialStartTimeWindow(&interval_1d_calendar_sliding_1n, k1, &w, true);
printTimeWindow(&w, precision, k1);
printf("next time window:\n");
getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC);
printTimeWindow(&w, precision, k1);
getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC);
printTimeWindow(&w, precision, k1);
getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC);
printTimeWindow(&w, precision, k1);
printf("----------------interval_1d_sliding_1d_calendar_offset_1n---------\n");
SInterval offset_1n = createInterval(10*86400*1000L, 10*86400*1000L, 1, 'd', 'd', 'n', TSDB_TIME_PRECISION_MILLI);
getInitialStartTimeWindow(&offset_1n, k1, &w, true);
printTimeWindow(&w, precision, k1);
}
#pragma GCC diagnostic pop

2
source/libs/function/src/builtinsimpl.c
View File

@ -499,7 +499,7 @@ static int64_t getNumOfElems(SqlFunctionCtx* pCtx) {
*/
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pInputCol = pInput->pData[0];
if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows && !IS_VAR_DATA_TYPE(pInputCol->info.type)) {
if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows) {
numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull;
} else {
if (pInputCol->hasNull) {

3
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -402,6 +402,7 @@ static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) {
COPY_SCALAR_FIELD(joinType);
CLONE_NODE_FIELD(pMergeCondition);
CLONE_NODE_FIELD(pOnConditions);
CLONE_NODE_FIELD(pColEqualOnConditions);
COPY_SCALAR_FIELD(isSingleTableJoin);
COPY_SCALAR_FIELD(inputTsOrder);
return TSDB_CODE_SUCCESS;
@ -589,7 +590,7 @@ static int32_t physiSysTableScanCopy(const SSystemTableScanPhysiNode* pSrc, SSys
return TSDB_CODE_SUCCESS;
}
static int32_t physiWindowCopy(const SWinodwPhysiNode* pSrc, SWinodwPhysiNode* pDst) {
static int32_t physiWindowCopy(const SWindowPhysiNode* pSrc, SWindowPhysiNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, physiNodeCopy);
CLONE_NODE_LIST_FIELD(pExprs);
CLONE_NODE_LIST_FIELD(pFuncs);

22
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -1428,6 +1428,7 @@ static int32_t jsonToLogicPlan(const SJson* pJson, void* pObj) {
static const char* jkJoinLogicPlanJoinType = "JoinType";
static const char* jkJoinLogicPlanOnConditions = "OnConditions";
static const char* jkJoinLogicPlanMergeCondition = "MergeConditions";
static const char* jkJoinLogicPlanColEqualOnConditions = "ColumnEqualOnConditions";
static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) {
const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj;
@ -1442,7 +1443,9 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinLogicPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions);
}
return code;
}
@ -1459,7 +1462,9 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinLogicPlanColEqualOnConditions, &pNode->pColEqualOnConditions);
}
return code;
}
@ -1890,6 +1895,7 @@ static const char* jkJoinPhysiPlanInputTsOrder = "InputTsOrder";
static const char* jkJoinPhysiPlanMergeCondition = "MergeCondition";
static const char* jkJoinPhysiPlanOnConditions = "OnConditions";
static const char* jkJoinPhysiPlanTargets = "Targets";
static const char* jkJoinPhysiPlanColEqualOnConditions = "ColumnEqualOnConditions";
static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) {
const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj;
@ -1910,7 +1916,9 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions);
}
return code;
}
@ -1933,7 +1941,9 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanColEqualOnConditions, &pNode->pColEqualOnConditions);
}
return code;
}
@ -2144,7 +2154,7 @@ static const char* jkWindowPhysiPlanOutputTsOrder = "outputTsOrder";
static const char* jkWindowPhysiPlanMergeDataBlock = "MergeDataBlock";
static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) {
const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj;
const SWindowPhysiNode* pNode = (const SWindowPhysiNode*)pObj;
int32_t code = physicPlanNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
@ -2185,7 +2195,7 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) {
}
static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) {
SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj;
SWindowPhysiNode* pNode = (SWindowPhysiNode*)pObj;
int32_t code = jsonToPhysicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {

4
source/libs/nodes/src/nodesEqualFuncs.c
View File

@ -82,9 +82,7 @@ static bool columnNodeEqual(const SColumnNode* a, const SColumnNode* b) {
COMPARE_STRING_FIELD(dbName);
COMPARE_STRING_FIELD(tableName);
COMPARE_STRING_FIELD(colName);
if (0 == a->tableId) {
COMPARE_STRING_FIELD(tableAlias);
}
COMPARE_STRING_FIELD(tableAlias);
return true;
}

14
source/libs/nodes/src/nodesMsgFuncs.c
View File

@ -2319,7 +2319,8 @@ enum {
PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION,
PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS,
PHY_SORT_MERGE_JOIN_CODE_TARGETS,
PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER
PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER,
PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS
};
static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -2341,7 +2342,9 @@ static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, pNode->inputTsOrder);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pColEqualOnConditions);
}
return code;
}
@ -2370,6 +2373,9 @@ static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER:
code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder));
break;
case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pColEqualOnConditions);
break;
default:
break;
}
@ -2642,7 +2648,7 @@ enum {
};
static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj;
const SWindowPhysiNode* pNode = (const SWindowPhysiNode*)pObj;
int32_t code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_BASE_NODE, physiNodeToMsg, &pNode->node);
if (TSDB_CODE_SUCCESS == code) {
@ -2683,7 +2689,7 @@ static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
}
static int32_t msgToPhysiWindowNode(STlvDecoder* pDecoder, void* pObj) {
SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj;
SWindowPhysiNode* pNode = (SWindowPhysiNode*)pObj;
int32_t code = TSDB_CODE_SUCCESS;
STlv* pTlv = NULL;

4
source/libs/nodes/src/nodesToSQLFuncs.c
View File

@ -120,9 +120,9 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) {
}
if (colNode->tableAlias[0]) {
*len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->colName);
*len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->node.userAlias);
} else {
*len += snprintf(buf + *len, bufSize - *len, "%s", colNode->colName);
*len += snprintf(buf + *len, bufSize - *len, "%s", colNode->node.userAlias);
}
return TSDB_CODE_SUCCESS;

14
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -608,7 +608,7 @@ static void destroyPhysiNode(SPhysiNode* pNode) {
nodesDestroyNode(pNode->pSlimit);
}
static void destroyWinodwPhysiNode(SWinodwPhysiNode* pNode) {
static void destroyWinodwPhysiNode(SWindowPhysiNode* pNode) {
destroyPhysiNode((SPhysiNode*)pNode);
nodesDestroyList(pNode->pExprs);
nodesDestroyList(pNode->pFuncs);
@ -1089,6 +1089,7 @@ void nodesDestroyNode(SNode* pNode) {
destroyLogicNode((SLogicNode*)pLogicNode);
nodesDestroyNode(pLogicNode->pMergeCondition);
nodesDestroyNode(pLogicNode->pOnConditions);
nodesDestroyNode(pLogicNode->pColEqualOnConditions);
break;
}
case QUERY_NODE_LOGIC_PLAN_AGG: {
@ -1221,6 +1222,7 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pPhyNode->pMergeCondition);
nodesDestroyNode(pPhyNode->pOnConditions);
nodesDestroyList(pPhyNode->pTargets);
nodesDestroyNode(pPhyNode->pColEqualOnConditions);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: {
@ -1258,7 +1260,7 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode);
destroyWinodwPhysiNode((SWindowPhysiNode*)pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_FILL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: {
@ -1274,19 +1276,19 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION:
destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode);
destroyWinodwPhysiNode((SWindowPhysiNode*)pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: {
SStateWinodwPhysiNode* pPhyNode = (SStateWinodwPhysiNode*)pNode;
destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode);
destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode);
nodesDestroyNode(pPhyNode->pStateKey);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: {
SEventWinodwPhysiNode* pPhyNode = (SEventWinodwPhysiNode*)pNode;
destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode);
destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode);
nodesDestroyNode(pPhyNode->pStartCond);
nodesDestroyNode(pPhyNode->pEndCond);
break;
@ -2074,7 +2076,7 @@ char* nodesGetNameFromColumnNode(SNode* pNode) {
return "NULL";
}
return ((SColumnNode*)pNode)->colName;
return ((SColumnNode*)pNode)->node.userAlias;
}
int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots) {

15
source/libs/parser/src/parAstCreater.c
View File

@ -259,8 +259,19 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) {
strcpy(pExpr->userAlias, ((SColumnNode*)pExpr)->colName);
} else {
int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n);
strncpy(pExpr->aliasName, pRawExpr->p, len);
pExpr->aliasName[len] = '\0';
// See TS-3398.
// Len of pRawExpr->p could be larger than len of aliasName[TSDB_COL_NAME_LEN].
// If aliasName is truncated, hash value of aliasName could be the same.
T_MD5_CTX ctx;
tMD5Init(&ctx);
tMD5Update(&ctx, (uint8_t*)pRawExpr->p, pRawExpr->n);
tMD5Final(&ctx);
char* p = pExpr->aliasName;
for (uint8_t i = 0; i < tListLen(ctx.digest); ++i) {
sprintf(p, "%02x", ctx.digest[i]);
p += 2;
}
strncpy(pExpr->userAlias, pRawExpr->p, len);
pExpr->userAlias[len] = '\0';
}

2
source/libs/parser/src/parAstParser.c
View File

@ -586,7 +586,7 @@ static int32_t collectMetaKeyFromShowCreateTable(SCollectMetaKeyCxt* pCxt, SShow
code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_READ,
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ,
pCxt->pMetaCache);
}
return code;

2
source/libs/parser/src/parAuthenticator.c
View File

@ -175,7 +175,7 @@ static int32_t authShowTables(SAuthCxt* pCxt, SShowStmt* pStmt) {
static int32_t authShowCreateTable(SAuthCxt* pCxt, SShowCreateTableStmt* pStmt) {
SNode* pTagCond = NULL;
// todo check tag condition for subtable
return checkAuth(pCxt, pStmt->dbName, NULL, AUTH_TYPE_READ, &pTagCond);
return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ, &pTagCond);
}
static int32_t authCreateTable(SAuthCxt* pCxt, SCreateTableStmt* pStmt) {

4
source/libs/parser/src/parInsertSql.c
View File

@ -1605,7 +1605,7 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt
(*pNumOfRows)++;
}
if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxMemUsedByInsert * 1024 * 1024) {
if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxInsertBatchRows) {
pStmt->fileProcessing = true;
break;
}
@ -1614,6 +1614,8 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt
}
taosMemoryFree(pLine);
parserDebug("0x%" PRIx64 " %d rows have been parsed", pCxt->pComCxt->requestId, *pNumOfRows);
if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) &&
(!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) && !pStmt->fileProcessing) {
code = buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL);

7
source/libs/parser/src/parInsertStmt.c
View File

@ -267,7 +267,10 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in
pBind = bind + c;
}
tColDataAddValueByBind(pCol, pBind);
code = tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1);
if (code) {
goto _return;
}
}
qDebug("stmt all %d columns bind %d rows data", boundInfo->numOfBound, rowNum);
@ -310,7 +313,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu
pBind = bind;
}
tColDataAddValueByBind(pCol, pBind);
tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1);
qDebug("stmt col %d bind %d rows data", colIdx, rowNum);

37
source/libs/parser/src/parInsertUtil.c
View File

@ -272,6 +272,41 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat
return code;
}
static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) {
int32_t code = TSDB_CODE_SUCCESS;
SSubmitTbData* pTmp = taosMemoryCalloc(1, sizeof(SSubmitTbData));
if (NULL == pTmp) {
code = TSDB_CODE_OUT_OF_MEMORY;
} else {
pTmp->flags = pSrc->flags;
pTmp->suid = pSrc->suid;
pTmp->uid = pSrc->uid;
pTmp->sver = pSrc->sver;
pTmp->pCreateTbReq = NULL;
if (pTmp->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
pTmp->aCol = taosArrayInit(128, sizeof(SColData));
if (NULL == pTmp->aCol) {
code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pTmp);
}
} else {
pTmp->aRowP = taosArrayInit(128, POINTER_BYTES);
if (NULL == pTmp->aRowP) {
code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pTmp);
}
}
}
taosMemoryFree(pSrc);
if (TSDB_CODE_SUCCESS == code) {
*pDst = pTmp;
}
return code;
}
static void resetColValues(SArray* pValues) {
int32_t num = taosArrayGetSize(pValues);
for (int32_t i = 0; i < num; ++i) {
@ -381,7 +416,7 @@ static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCx
}
}
taosArrayPush(pVgCxt->pData->aSubmitTbData, pTableCxt->pData);
taosMemoryFreeClear(pTableCxt->pData);
rebuildTableData(pTableCxt->pData, &pTableCxt->pData);
qDebug("add tableDataCxt uid:%" PRId64 " to vgId:%d", pTableCxt->pMeta->uid, pVgCxt->vgId);

12
source/libs/parser/src/parTranslater.c
View File

@ -834,7 +834,7 @@ static void setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SColum
strcpy(pCol->node.aliasName, pCol->colName);
}
if ('\0' == pCol->node.userAlias[0]) {
strcpy(pCol->node.userAlias, pCol->colName);
strcpy(pCol->node.userAlias, pExpr->userAlias);
}
pCol->node.resType = pExpr->resType;
}
@ -1770,6 +1770,7 @@ static int32_t rewriteFuncToValue(STranslateContext* pCxt, char* pLiteral, SNode
return TSDB_CODE_OUT_OF_MEMORY;
}
strcpy(pVal->node.aliasName, ((SExprNode*)*pNode)->aliasName);
strcpy(pVal->node.userAlias, ((SExprNode*)*pNode)->userAlias);
pVal->node.resType = ((SExprNode*)*pNode)->resType;
if (NULL == pLiteral) {
pVal->isNull = true;
@ -2768,6 +2769,7 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) {
} else {
len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->aliasName);
strncpy(pFunc->node.aliasName, buf, TMIN(len, sizeof(pFunc->node.aliasName) - 1));
len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->userAlias);
strncpy(pFunc->node.userAlias, buf, TMIN(len, sizeof(pFunc->node.userAlias) - 1));
}
@ -3214,7 +3216,7 @@ static int32_t checkFill(STranslateContext* pCxt, SFillNode* pFill, SValueNode*
int64_t timeRange = TABS(pFill->timeRange.skey - pFill->timeRange.ekey);
int64_t intervalRange = 0;
if (TIME_IS_VAR_DURATION(pInterval->unit)) {
if (IS_CALENDAR_TIME_DURATION(pInterval->unit)) {
int64_t f = 1;
if (pInterval->unit == 'n') {
f = 30LL * MILLISECOND_PER_DAY;
@ -3295,7 +3297,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode*
uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision;
SValueNode* pInter = (SValueNode*)pInterval->pInterval;
bool valInter = TIME_IS_VAR_DURATION(pInter->unit);
bool valInter = IS_CALENDAR_TIME_DURATION(pInter->unit);
if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime,
getPrecisionStr(precision));
@ -3309,7 +3311,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode*
if (pInter->unit == 'n' && pOffset->unit == 'y') {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT);
}
bool fixed = !TIME_IS_VAR_DURATION(pOffset->unit) && !valInter;
bool fixed = !IS_CALENDAR_TIME_DURATION(pOffset->unit) && !valInter;
if ((fixed && pOffset->datum.i >= pInter->datum.i) ||
(!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >=
getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) {
@ -3325,7 +3327,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode*
const static int32_t INTERVAL_SLIDING_FACTOR = 100;
SValueNode* pSliding = (SValueNode*)pInterval->pSliding;
if (TIME_IS_VAR_DURATION(pSliding->unit)) {
if (IS_CALENDAR_TIME_DURATION(pSliding->unit)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT);
}
if ((pSliding->datum.i < convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, precision)) ||

11
source/libs/parser/src/parUtil.c
View File

@ -499,7 +499,7 @@ int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName)
static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type,
char* pStr) {
return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL != pTable && '\0' == pTable[0]) ? NULL : pTable,
return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL == pTable || '\0' == pTable[0]) ? "``" : pTable,
type);
}
@ -525,6 +525,9 @@ static void getStringFromAuthStr(const char* pStart, char* pStr, char** pNext) {
strncpy(pStr, pStart, p - pStart);
*pNext = ++p;
}
if (*pStart == '`' && *(pStart + 1) == '`') {
*pStr = 0;
}
}
static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUserAuth) {
@ -533,7 +536,11 @@ static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUser
pUserAuth->tbName.acctId = getIntegerFromAuthStr(p, &p);
getStringFromAuthStr(p, pUserAuth->tbName.dbname, &p);
getStringFromAuthStr(p, pUserAuth->tbName.tname, &p);
pUserAuth->tbName.type = TSDB_TABLE_NAME_T;
if (pUserAuth->tbName.tname[0]) {
pUserAuth->tbName.type = TSDB_TABLE_NAME_T;
} else {
pUserAuth->tbName.type = TSDB_DB_NAME_T;
}
pUserAuth->type = getIntegerFromAuthStr(p, &p);
}

2
source/libs/parser/test/parSelectTest.cpp
View File

@ -472,6 +472,8 @@ TEST_F(ParserSelectTest, joinSemanticCheck) {
run("SELECT * FROM (SELECT tag1, SUM(c1) s FROM st1 GROUP BY tag1) t1, st1 t2 where t1.tag1 = t2.tag1",
TSDB_CODE_PAR_NOT_SUPPORT_JOIN);
run("SELECT count(*) FROM t1 a join t1 b on a.ts=b.ts where a.ts=b.ts");
}
} // namespace ParserTest

33
source/libs/planner/src/planLogicCreater.c
View File

@ -37,19 +37,24 @@ typedef struct SRewriteExprCxt {
int32_t errCode;
SNodeList* pExprs;
bool* pOutputs;
bool isPartitionBy;
} SRewriteExprCxt;
static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol) {
static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol, bool isPartitionBy) {
switch (pFunc->funcType) {
case FUNCTION_TYPE_TBNAME:
pCol->colType = COLUMN_TYPE_TBNAME;
break;
case FUNCTION_TYPE_WSTART:
pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
if (!isPartitionBy) {
pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
}
pCol->colType = COLUMN_TYPE_WINDOW_START;
break;
case FUNCTION_TYPE_WEND:
pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
if (!isPartitionBy) {
pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
}
pCol->colType = COLUMN_TYPE_WINDOW_END;
break;
case FUNCTION_TYPE_WDURATION:
@ -100,9 +105,10 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) {
SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode);
pCol->node.resType = pToBeRewrittenExpr->resType;
strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName);
strcpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias);
strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName);
if (QUERY_NODE_FUNCTION == nodeType(pExpr)) {
setColumnInfo((SFunctionNode*)pExpr, pCol);
setColumnInfo((SFunctionNode*)pExpr, pCol, pCxt->isPartitionBy);
}
nodesDestroyNode(*pNode);
*pNode = (SNode*)pCol;
@ -141,7 +147,8 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) {
static int32_t rewriteExprForSelect(SNode* pExpr, SSelectStmt* pSelect, ESqlClause clause) {
nodesWalkExpr(pExpr, doNameExpr, NULL);
SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL};
bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false;
SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL, .isPartitionBy = isPartitionBy};
cxt.errCode = nodesListMakeAppend(&cxt.pExprs, pExpr);
if (TSDB_CODE_SUCCESS == cxt.errCode) {
nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt);
@ -169,7 +176,8 @@ static int32_t cloneRewriteExprs(SNodeList* pExprs, bool* pOutputs, SNodeList**
static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause,
SNodeList** pRewriteExprs) {
nodesWalkExprs(pExprs, doNameExpr, NULL);
SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL};
bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false;
SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = isPartitionBy};
if (NULL != pRewriteExprs) {
cxt.pOutputs = taosMemoryCalloc(LIST_LENGTH(pExprs), sizeof(bool));
if (NULL == cxt.pOutputs) {
@ -186,14 +194,14 @@ static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ES
static int32_t rewriteExpr(SNodeList* pExprs, SNode** pTarget) {
nodesWalkExprs(pExprs, doNameExpr, NULL);
SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL};
SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = false};
nodesRewriteExpr(pTarget, doRewriteExpr, &cxt);
return cxt.errCode;
}
static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) {
nodesWalkExprs(pExprs, doNameExpr, NULL);
SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL};
SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = false};
nodesRewriteExprs(pTarget, doRewriteExpr, &cxt);
return cxt.errCode;
}
@ -543,11 +551,16 @@ static SNode* createGroupingSetNode(SNode* pExpr) {
return (SNode*)pGroupingSet;
}
static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) {
static EGroupAction getDistinctGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) {
return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) ? GROUP_ACTION_KEEP
: GROUP_ACTION_NONE;
}
static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) {
return ((pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) && !pSelect->isDistinct) ? GROUP_ACTION_KEEP
: GROUP_ACTION_NONE;
}
static EDataOrderLevel getRequireDataOrder(bool needTimeline, SSelectStmt* pSelect) {
return needTimeline ? (NULL != pSelect->pPartitionByList ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL)
: DATA_ORDER_LEVEL_NONE;
@ -1166,7 +1179,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe
return TSDB_CODE_OUT_OF_MEMORY;
}
pAgg->node.groupAction = GROUP_ACTION_CLEAR;
pAgg->node.groupAction = GROUP_ACTION_CLEAR;//getDistinctGroupAction(pCxt, pSelect);
pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE;
pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE;

89
source/libs/planner/src/planOptimizer.c
View File

@ -740,6 +740,85 @@ static int32_t pushDownCondOptJoinExtractMergeCond(SOptimizeContext* pCxt, SJoin
return code;
}
static bool pushDownCondOptIsTableColumn(SNode* pNode, SNodeList* pTableCols) {
if (QUERY_NODE_COLUMN != nodeType(pNode)) {
return false;
}
SColumnNode* pCol = (SColumnNode*)pNode;
return pushDownCondOptBelongThisTable(pNode, pTableCols);
}
static bool pushDownCondOptIsColEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) {
if (QUERY_NODE_OPERATOR != nodeType(pCond)) {
return false;
}
SOperatorNode* pOper = (SOperatorNode*)pCond;
if (OP_TYPE_EQUAL != pOper->opType) {
return false;
}
if (QUERY_NODE_COLUMN != nodeType(pOper->pLeft) || QUERY_NODE_COLUMN != nodeType(pOper->pRight)) {
return false;
}
SColumnNode* pLeft = (SColumnNode*)(pOper->pLeft);
SColumnNode* pRight = (SColumnNode*)(pOper->pRight);
//TODO: add cast to operator and remove this restriction of optimization
if (pLeft->node.resType.type != pRight->node.resType.type || pLeft->node.resType.bytes != pRight->node.resType.bytes) {
return false;
}
SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets;
SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets;
if (pushDownCondOptIsTableColumn(pOper->pLeft, pLeftCols)) {
return pushDownCondOptIsTableColumn(pOper->pRight, pRightCols);
} else if (pushDownCondOptIsTableColumn(pOper->pLeft, pRightCols)) {
return pushDownCondOptIsTableColumn(pOper->pRight, pLeftCols);
}
return false;
}
static int32_t pushDownCondOptJoinExtractColEqualOnLogicCond(SJoinLogicNode* pJoin) {
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions);
int32_t code = TSDB_CODE_SUCCESS;
SNodeList* pEqualOnConds = NULL;
SNode* pCond = NULL;
FOREACH(pCond, pLogicCond->pParameterList) {
if (pushDownCondOptIsColEqualOnCond(pJoin, pCond)) {
code = nodesListMakeAppend(&pEqualOnConds, nodesCloneNode(pCond));
}
}
SNode* pTempTagEqCond = NULL;
if (TSDB_CODE_SUCCESS == code) {
code = nodesMergeConds(&pTempTagEqCond, &pEqualOnConds);
}
if (TSDB_CODE_SUCCESS == code) {
pJoin->pColEqualOnConditions = pTempTagEqCond;
return TSDB_CODE_SUCCESS;
} else {
nodesDestroyList(pEqualOnConds);
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
return TSDB_CODE_SUCCESS;
}
static int32_t pushDownCondOptJoinExtractColEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
if (NULL == pJoin->pOnConditions) {
pJoin->pColEqualOnConditions = NULL;
return TSDB_CODE_SUCCESS;
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) &&
LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) {
return pushDownCondOptJoinExtractColEqualOnLogicCond(pJoin);
}
if (pushDownCondOptIsColEqualOnCond(pJoin, pJoin->pOnConditions)) {
pJoin->pColEqualOnConditions = nodesCloneNode(pJoin->pOnConditions);
}
return TSDB_CODE_SUCCESS;
}
static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) {
return TSDB_CODE_SUCCESS;
@ -774,6 +853,10 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p
code = pushDownCondOptJoinExtractMergeCond(pCxt, pJoin);
}
if (TSDB_CODE_SUCCESS == code) {
code = pushDownCondOptJoinExtractColEqualOnCond(pCxt, pJoin);
}
if (TSDB_CODE_SUCCESS == code) {
OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE);
pCxt->optimized = true;
@ -1259,8 +1342,8 @@ static bool smaIndexOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pW
.sliding = pIndex->sliding,
.slidingUnit = pIndex->slidingUnit,
.precision = pScan->node.precision};
return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval, pScan->node.precision)) &&
(pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval, pScan->node.precision));
return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval)) &&
(pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval));
}
return true;
}
@ -2234,7 +2317,7 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) {
if (QUERY_NODE_COLUMN == nodeType(pPar)) {
SColumnNode* pCol = (SColumnNode*)pPar;
if (pCol->colType != COLUMN_TYPE_COLUMN) {
return false;
return false;
}
}
if (hasSelectFunc || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pAggFunc->pParameterList, 0))) {

8
source/libs/planner/src/planPhysiCreater.c
View File

@ -705,6 +705,9 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
pJoinLogicNode->pOnConditions, &pJoin->pOnConditions);
}
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pColEqualOnConditions) {
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColEqualOnConditions, &pJoin->pColEqualOnConditions);
}
if (TSDB_CODE_SUCCESS == code) {
code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin);
}
@ -1147,7 +1150,7 @@ static int32_t createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNo
}
}
static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWinodwPhysiNode* pWindow,
static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowPhysiNode* pWindow,
SWindowLogicNode* pWindowLogicNode) {
pWindow->triggerType = pWindowLogicNode->triggerType;
pWindow->watermark = pWindowLogicNode->watermark;
@ -1644,6 +1647,9 @@ static int32_t createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode,
if (TSDB_CODE_SUCCESS == code) {
code = nodesListStrictAppend(pChildren, (SNode*)pChild);
}
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
if (TSDB_CODE_SUCCESS == code) {

17
source/libs/wal/src/walRef.c
View File

@ -63,21 +63,22 @@ int32_t walSetRefVer(SWalRef *pRef, int64_t ver) {
return 0;
}
SWalRef *walRefFirstVer(SWal *pWal, SWalRef *pRef) {
if (pRef == NULL) {
pRef = walOpenRef(pWal);
if (pRef == NULL) {
return NULL;
}
}
void walRefFirstVer(SWal *pWal, SWalRef *pRef) {
taosThreadMutexLock(&pWal->mutex);
int64_t ver = walGetFirstVer(pWal);
pRef->refVer = ver;
taosThreadMutexUnlock(&pWal->mutex);
wDebug("vgId:%d, wal ref version %" PRId64 " for first", pWal->cfg.vgId, ver);
}
return pRef;
void walRefLastVer(SWal *pWal, SWalRef *pRef) {
taosThreadMutexLock(&pWal->mutex);
int64_t ver = walGetLastVer(pWal);
pRef->refVer = ver;
taosThreadMutexUnlock(&pWal->mutex);
wDebug("vgId:%d, wal ref version %" PRId64 " for last", pWal->cfg.vgId, ver);
}
SWalRef *walRefCommittedVer(SWal *pWal) {

2
source/util/src/terror.c
View File

@ -382,7 +382,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, "Tag conditon too many
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, "Query not ready")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, "Query should response")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, "Multiple retrieval of this query")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many time window in query")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many groups/time window in query")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit has reached")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR, "System error")

14
source/util/src/tlog.c
View File

@ -253,15 +253,15 @@ static void taosKeepOldLog(char *oldName) {
(void)taosRenameFile(oldName, fileName);
if (tsLogKeepDays < 0) {
char compressFileName[LOG_FILE_NAME_LEN + 20];
snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec);
if (taosCompressFile(fileName, compressFileName) == 0) {
(void)taosRemoveFile(fileName);
}
char compressFileName[LOG_FILE_NAME_LEN + 20];
snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec);
if (taosCompressFile(fileName, compressFileName) == 0) {
(void)taosRemoveFile(fileName);
}
taosRemoveOldFiles(tsLogDir, TABS(tsLogKeepDays));
if (tsLogKeepDays > 0) {
taosRemoveOldFiles(tsLogDir, tsLogKeepDays);
}
}
static void *taosThreadToOpenNewFile(void *param) {

1
source/util/src/tpagedbuf.c
View File

@ -482,6 +482,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
SPageInfo** pInfo = (SPageInfo**)((*pi)->pn->data);
if (*pInfo != *pi) {
terrno = TSDB_CODE_APP_ERROR;
uError("inconsistently data in paged buffer, pInfo:%p, pi:%p, %s", *pInfo, *pi, pBuf->id);
return NULL;
}

3
tests/parallel_test/cases.task
View File

@ -743,6 +743,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaBasic.py -N 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py
@ -1352,6 +1353,7 @@
,,y,script,./test.sh -f tsim/query/multi_order_by.sim
,,y,script,./test.sh -f tsim/query/sys_tbname.sim
,,y,script,./test.sh -f tsim/query/groupby.sim
,,y,script,./test.sh -f tsim/query/groupby_distinct.sim
,,y,script,./test.sh -f tsim/query/event.sim
,,y,script,./test.sh -f tsim/query/forceFill.sim
,,y,script,./test.sh -f tsim/query/emptyTsRange.sim
@ -1360,6 +1362,7 @@
,,y,script,./test.sh -f tsim/query/tableCount.sim
,,y,script,./test.sh -f tsim/query/tag_scan.sim
,,y,script,./test.sh -f tsim/query/nullColSma.sim
,,y,script,./test.sh -f tsim/query/bug3398.sim
,,y,script,./test.sh -f tsim/qnode/basic1.sim
,,y,script,./test.sh -f tsim/snode/basic1.sim
,,y,script,./test.sh -f tsim/mnode/basic1.sim

27
tests/pytest/crash_gen/crash_gen_main.py
View File

@ -1722,12 +1722,14 @@ class TaskCreateDb(StateTransitionTask):
vg_nums = random.randint(1, 8)
cache_model = Dice.choice(['none', 'last_row', 'last_value', 'both'])
buffer = random.randint(3, 128)
walRetentionPeriod = random.randint(1, 10000)
dbName = self._db.getName()
self.execWtSql(wt, "create database {} {} {} vgroups {} cachemodel '{}' buffer {} ".format(dbName, repStr,
self.execWtSql(wt, "create database {} {} {} vgroups {} cachemodel '{}' buffer {} wal_retention_period {} ".format(dbName, repStr,
updatePostfix,
vg_nums,
cache_model,
buffer))
buffer,
walRetentionPeriod))
if dbName == "db_0" and Config.getConfig().use_shadow_db:
self.execWtSql(wt, "create database {} {} {} ".format("db_s", repStr, updatePostfix))
@ -2041,18 +2043,19 @@ class TdSuperTable:
for topic in current_topic_list:
topic_list.append(topic)
consumer.subscribe(topic_list)
# consumer with random work life
time_start = time.time()
while 1:
res = consumer.poll(1)
consumer.commit(res)
if time.time() - time_start > random.randint(5, 50):
break
try:
consumer.subscribe(topic_list)
# consumer with random work life
time_start = time.time()
while 1:
res = consumer.poll(1)
consumer.commit(res)
if time.time() - time_start > random.randint(5, 50):
break
consumer.unsubscribe()
except TmqError as e:
consumer.close()
except TmqError as err: # topic deleted by other threads
pass
return

93
tests/script/api/batchprepare.c
View File

@ -16,8 +16,8 @@
int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT};
int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR};
int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT};
int32_t optrIdxList[] = {0, 7};
int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_NCHAR};
int32_t optrIdxList[] = {5, 11};
typedef struct {
char* oper;
@ -123,6 +123,7 @@ int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos);
int queryColumnTest(TAOS_STMT *stmt, TAOS *taos);
int queryMiscTest(TAOS_STMT *stmt, TAOS *taos);
int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos);
int insertVarLenErr(TAOS_STMT *stmt, TAOS *taos);
enum {
TTYPE_INSERT = 1,
@ -190,6 +191,7 @@ CaseCfg gCase[] = {
{"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2},
{"query:NG-TBNEXISTS",tListLen(fullColList), fullColList, TTYPE_INSERT_NG,0, false, false, insertNonExistsTb, 10, 10, 1, 3, 0, 0, 1, -1},
{"query:NG-VARLENERR",tListLen(fullColList), fullColList, TTYPE_INSERT_NG,0, false, true, insertVarLenErr, 10, 10, 1, 3, 0, 0, 1, -1},
// {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2},
// {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2},
@ -319,7 +321,7 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper
#if 0
CaseCtrl gCaseCtrl = { // query case with specified col&oper
.bindNullNum = 1,
.bindNullNum = 0,
.printCreateTblSql = true,
.printQuerySql = true,
.printStmtSql = true,
@ -329,18 +331,19 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper
.bindTagNum = 0,
.bindRowNum = 0,
.bindColTypeNum = 0,
.bindColTypeList = NULL,
.bindColTypeList = bindColTypeList,
.optrIdxListNum = 0,
.optrIdxList = NULL,
.optrIdxList = optrIdxList,
.checkParamNum = false,
.printRes = true,
.runTimes = 0,
.caseRunIdx = -1,
//.optrIdxListNum = tListLen(optrIdxList),
//.optrIdxList = optrIdxList,
//.bindColTypeNum = tListLen(bindColTypeList),
//.bindColTypeList = bindColTypeList,
.caseIdx = 8,
.optrIdxListNum = tListLen(optrIdxList),
.optrIdxList = optrIdxList,
.bindColTypeNum = tListLen(bindColTypeList),
.bindColTypeList = bindColTypeList,
.caseRunIdx = -1,
.caseIdx = 24,
.caseNum = 1,
.caseRunNum = 1,
};
@ -1450,14 +1453,17 @@ void bpShowBindParam(TAOS_MULTI_BIND *bind, int32_t num) {
}
}
int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, bool expectFail) {
static int32_t n = 0;
bpCheckColFields(stmt, bind);
if (!expectFail) {
bpCheckColFields(stmt, bind);
}
if (gCurCase->bindRowNum > 1) {
if (0 == (n++%2)) {
if (taos_stmt_bind_param_batch(stmt, bind)) {
if (expectFail) return 0;
printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt));
bpShowBindParam(bind, gCurCase->bindColNum);
exit(1);
@ -1465,6 +1471,7 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
} else {
for (int32_t i = 0; i < gCurCase->bindColNum; ++i) {
if (taos_stmt_bind_single_param_batch(stmt, bind+i, i)) {
if (expectFail) continue;
printf("!!!taos_stmt_bind_single_param_batch %d error:%s\n", taos_stmt_errstr(stmt), i);
bpShowBindParam(bind, gCurCase->bindColNum);
exit(1);
@ -1474,12 +1481,14 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
} else {
if (0 == (n++%2)) {
if (taos_stmt_bind_param_batch(stmt, bind)) {
if (expectFail) return 0;
printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt));
bpShowBindParam(bind, gCurCase->bindColNum);
exit(1);
}
} else {
if (taos_stmt_bind_param(stmt, bind)) {
if (expectFail) return 0;
printf("!!!taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt));
bpShowBindParam(bind, gCurCase->bindColNum);
exit(1);
@ -1542,7 +1551,7 @@ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) {
}
for (int32_t b = 0; b <bindTimes; ++b) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) {
exit(1);
}
@ -1594,7 +1603,7 @@ int insertMBSETest2(TAOS_STMT *stmt, TAOS *taos) {
}
}
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) {
exit(1);
}
@ -1652,7 +1661,7 @@ int insertMBMETest1(TAOS_STMT *stmt, TAOS *taos) {
}
for (int32_t b = 0; b <bindTimes; ++b) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) {
exit(1);
}
@ -1702,7 +1711,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos) {
}
for (int32_t b = 0; b <bindTimes; ++b) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) {
exit(1);
}
@ -1770,7 +1779,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) {
}
}
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) {
exit(1);
}
@ -1822,7 +1831,7 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos) {
}
}
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) {
exit(1);
}
@ -1883,7 +1892,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) {
}
for (int32_t b = 0; b <bindTimes; ++b) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) {
exit(1);
}
@ -1949,7 +1958,7 @@ int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos) {
}
for (int32_t b = 0; b <bindTimes; ++b) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) {
exit(1);
}
@ -2016,7 +2025,7 @@ int insertAUTOTest2(TAOS_STMT *stmt, TAOS *taos) {
if (gCaseCtrl.checkParamNum) {
bpCheckParamNum(stmt);
}
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) {
exit(1);
}
@ -2076,7 +2085,7 @@ int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos) {
bpCheckParamNum(stmt);
}
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) {
exit(1);
}
@ -2130,7 +2139,7 @@ int queryColumnTest(TAOS_STMT *stmt, TAOS *taos) {
bpCheckParamNum(stmt);
}
if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) {
if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum, false)) {
exit(1);
}
@ -2178,7 +2187,7 @@ int queryMiscTest(TAOS_STMT *stmt, TAOS *taos) {
bpCheckParamNum(stmt);
}
if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) {
if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum, false)) {
exit(1);
}
@ -2245,6 +2254,42 @@ int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos) {
return 0;
}
void bpAddWrongVarBuffLen(TAOS_MULTI_BIND* pBind) {
for (int32_t i = 0; i < gCurCase->bindColNum; ++i) {
if (pBind[i].buffer_type == TSDB_DATA_TYPE_BINARY || pBind[i].buffer_type == TSDB_DATA_TYPE_NCHAR) {
*pBind[i].length += 100;
}
}
}
int insertVarLenErr(TAOS_STMT *stmt, TAOS *taos) {
BindData data = {0};
prepareInsertData(&data);
int code = taos_stmt_prepare(stmt, data.sql, 0);
if (code != 0){
printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt));
exit(1);
}
bpCheckIsInsert(stmt, 1);
code = bpSetTableNameTags(&data, 0, "t0", stmt);
if (code != 0){
printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt));
exit(1);
}
bpAddWrongVarBuffLen(data.pBind);
if (bpBindParam(stmt, data.pBind, true)) {
exit(1);
}
destroyData(&data);
return 0;
}
int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) {

50
tests/script/tsim/compute/last_row.sim
View File

@ -213,4 +213,54 @@ if $rows != 2 then
return -1
endi
print =======================> regresss bug in last_row query
sql drop database if exists db;
sql create database if not exists db vgroups 1 cachemodel 'both';
sql create table db.stb (ts timestamp, c0 bigint) tags(t1 int);
sql insert into db.stb_0 using db.stb tags(1) values ('2023-11-23 19:06:40.000', 491173569);
sql insert into db.stb_2 using db.stb tags(3) values ('2023-11-25 19:30:00.000', 2080726142);
sql insert into db.stb_3 using db.stb tags(4) values ('2023-11-26 06:48:20.000', 1907405128);
sql insert into db.stb_4 using db.stb tags(5) values ('2023-11-24 22:56:40.000', 220783803);
sql create table db.stb_1 using db.stb tags(2);
sql insert into db.stb_1 (ts) values('2023-11-26 13:11:40.000');
sql insert into db.stb_1 (ts, c0) values('2023-11-26 13:11:39.000', 11);
sql select tbname,ts,last_row(c0) from db.stb;
if $rows != 1 then
return -1
endi
if $data00 != @stb_1@ then
return -1
endi
if $data01 != @23-11-26 13:11:40.000@ then
return -1
endi
if $data02 != NULL then
return -1
endi
sql alter database db cachemodel 'none';
sql reset query cache;
sql select tbname,last_row(c0, ts) from db.stb;
if $rows != 1 then
return -1
endi
if $data00 != @stb_1@ then
return -1
endi
if $data02 != @23-11-26 13:11:40.000@ then
return -1
endi
if $data01 != NULL then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT

3
tests/script/tsim/parser/function.sim
View File

@ -954,11 +954,14 @@ endi
print =========================>TD-5190
sql select _wstart, stddev(f1) from st1 where ts>'2021-07-01 1:1:1' and ts<'2021-07-30 00:00:00' interval(1d) fill(NULL);
if $rows != 29 then
print expect 29, actual: $rows
return -1
endi
if $data00 != @21-07-01 00:00:00.000@ then
return -1
endi
if $data01 != NULL then
return -1
endi

8
tests/script/tsim/parser/join_manyblocks.sim
View File

@ -6,8 +6,8 @@ sql connect
$dbPrefix = join_m_db
$tbPrefix = join_tb
$mtPrefix = join_mt
$tbNum = 3
$rowNum = 2000
$tbNum = 20
$rowNum = 200
$totalNum = $tbNum * $rowNum
print =============== join_manyBlocks.sim
@ -78,8 +78,8 @@ print ==============> td-3313
sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1;
print $row
if $row != 6000 then
print expect 6000, actual: $row
if $row != 4000 then
print expect 4000, actual: $row
return -1
endi

3
tests/script/tsim/parser/sliding.sim
View File

@ -450,10 +450,11 @@ endi
print ====================>check boundary check crash at client side
sql select count(*) from sliding_mt0 where ts>now and ts < now-1h;
sql select sum(c1) from sliding_tb0 interval(1a) sliding(1a);
print ========================query on super table
print ========================error case
sql_error select sum(c1) from sliding_tb0 interval(1a) sliding(1a);
sql_error select sum(c1) from sliding_tb0 interval(10a) sliding(12a);
sql_error select sum(c1) from sliding_tb0 sliding(1n) interval(1y);
sql_error select sum(c1) from sliding_tb0 interval(-1y) sliding(1n);

30
tests/script/tsim/query/bug3398.sim Normal file
View File

@ -0,0 +1,30 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
print =============== create database
sql create database test
print =============== create super table and child table
sql use test
sql CREATE STABLE st (day timestamp, c2 int) TAGS (vin binary(32))
sql insert into test.g using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1)
sql insert into test.t using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1)
sql insert into test.tg using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1)
sql select sum(case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end), sum(case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end) from test.t t, test.g g, test.tg tg where t.day = g.day and t.day = tg.day and t.day between '2021-05-03' and '2023-05-04' and t.vin = 'TAG1' and t.vin = g.vin and t.vin = tg.vin group by t.day;
print $rows $data00 $data01
if $rows != 1 then
return -1
endi
if $data00 != 2.000000000 then
return -1
endi
if $data01 != 3.000000000 then
return -1
endi

30
tests/script/tsim/query/groupby_distinct.sim Normal file
View File

@ -0,0 +1,30 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
sql drop database if exists db1;
sql create database db1;
sql use db1;
sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create table tba1 using sta tags(1, 1, 1);
sql insert into tba1 values ('2022-04-26 15:15:08', 1, "a");
sql insert into tba1 values ('2022-04-26 15:15:07', 1, "b");
sql insert into tba1 values ('2022-04-26 15:15:06', 1, "a");
sql insert into tba1 values ('2022-04-26 15:15:05', 1, "b");
sql insert into tba1 values ('2022-04-26 15:15:04', 1, "c");
sql insert into tba1 values ('2022-04-26 15:15:03', 1, "c");
sql insert into tba1 values ('2022-04-26 15:15:02', 1, "d");
sql insert into tba1 values ('2022-04-26 15:15:01', 1, "d");
sql select distinct avg(f1) as avgv from sta group by f2;
if $rows != 1 then
return -1
endi
sql select distinct avg(f1) as avgv from sta group by f2 limit 1,10;
if $rows != 0 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT

36
tests/script/tsim/query/interval.sim
View File

@ -170,6 +170,42 @@ endi
# return -1
#endi
print ================ step10
print -------- create database and table
sql create database if not exists test
sql use test
sql create stable st (ts timestamp, c2 int) tags(tg int)
print -------- insert 300 rows data
$i = 0
while $i < 300
$t = 1577807983000
$cc = $i * 1000
$t = $t + $cc
sql select $i % 3
if $data00 != 0.000000000 then
goto continue_while
endi
sql select $i % 4
if $data00 == 0.000000000 then
goto continue_while
endi
sql insert into t1 using st tags(1) values ( $t , $i )
continue_while:
$i = $i + 1
endw
$ms1 = 1577808120000
$ms2 = 1577808000000
sql select * from (select _wstart, last(ts) as ts, avg(c2) as av from t1 where ts <= $ms1 and ts >= $ms2 interval(10s) sliding(1s) fill(NULL)) order by ts asc
print ----> select asc rows: $rows
$asc_rows = $rows
sql select * from (select _wstart, last(ts) as ts, avg(c2) as av from t1 where ts <= $ms1 and ts >= $ms2 interval(10s) sliding(1s) fill(NULL)) order by ts desc
print ----> select desc rows: $rows
$desc_rows = $rows
if $desc_rows != $asc_rows then
return -1
endi
print =============== clear
#sql drop database $db
#sql select * from information_schema.ins_databases

83
tests/script/tsim/query/partitionby.sim
View File

@ -24,18 +24,85 @@ sql use $db
sql create table $mt1 (ts timestamp, f1 int) TAGS(tag1 int, tag2 binary(500))
sql create table tb0 using $mt1 tags(0, 'a');
sql create table tb1 using $mt1 tags(1, 'b');
sql create table tb2 using $mt1 tags(1, 'a');
sql create table tb3 using $mt1 tags(1, 'a');
sql create table tb4 using $mt1 tags(3, 'b');
sql create table tb5 using $mt1 tags(3, 'a');
sql create table tb6 using $mt1 tags(3, 'b');
sql create table tb7 using $mt1 tags(3, 'b');
sql create table tb2 using $mt1 tags(2, 'a');
sql create table tb3 using $mt1 tags(3, 'a');
sql create table tb4 using $mt1 tags(4, 'b');
sql create table tb5 using $mt1 tags(5, 'a');
sql create table tb6 using $mt1 tags(6, 'b');
sql create table tb7 using $mt1 tags(7, 'b');
sql select * from $mt1 partition by tag1,tag2 limit 1;
if $rows != 0 then
return -1
endi
sql insert into tb0 values ('2022-04-26 15:15:08', 1);
sql insert into tb1 values ('2022-04-26 15:15:07', 2);
sql insert into tb2 values ('2022-04-26 15:15:06', 3);
sql insert into tb3 values ('2022-04-26 15:15:05', 4);
sql insert into tb4 values ('2022-04-26 15:15:04', 5);
sql insert into tb5 values ('2022-04-26 15:15:03', 6);
sql insert into tb6 values ('2022-04-26 15:15:02', 7);
sql insert into tb7 values ('2022-04-26 15:15:01', 8);
sql select _wstart as ts, count(*) from $mt1 partition by tag1 interval(1s) order by _wstart;
if $rows != 8 then
return -1
endi
if $data00 != @22-04-26 15:15:01.000@ then
return -1
endi
if $data01 != 1 then
return -1
endi
if $data10 != @22-04-26 15:15:02.000@ then
return -1
endi
if $data11 != 1 then
return -1
endi
if $data20 != @22-04-26 15:15:03.000@ then
return -1
endi
if $data21 != 1 then
return -1
endi
if $data30 != @22-04-26 15:15:04.000@ then
return -1
endi
if $data31 != 1 then
return -1
endi
if $data40 != @22-04-26 15:15:05.000@ then
return -1
endi
if $data41 != 1 then
return -1
endi
if $data50 != @22-04-26 15:15:06.000@ then
return -1
endi
if $data51 != 1 then
return -1
endi
if $data60 != @22-04-26 15:15:07.000@ then
return -1
endi
if $data61 != 1 then
return -1
endi
if $data70 != @22-04-26 15:15:08.000@ then
return -1
endi
if $data71 != 1 then
return -1
endi
sql select * from (select _wstart as ts, count(*) from $mt1 partition by tag1 interval(1s) order by _wstart) order by ts;
sql select _wstart as ts, count(*) from $mt1 interval(1s) order by _wstart;
sql select * from (select _wstart as ts, count(*) from $mt1 interval(1s) order by _wstart) order by ts;
sql select diff(a) from (select _wstart as ts, count(*) a from $mt1 interval(1s) order by _wstart);
sql select diff(a) from (select _wstart as ts, count(*) a from $mt1 partition by tag1 interval(1s) order by _wstart);
sql insert into tb0 values (now, 0);
sql insert into tb1 values (now, 1);
sql insert into tb2 values (now, 2);
@ -54,7 +121,7 @@ sql select count(*) from (select ts from $mt1 where ts is not null partition by
if $rows != 1 then
return -1
endi
if $data00 != 2 then
if $data00 != 4 then
return -1
endi
@ -62,7 +129,7 @@ sql select count(*) from (select ts from $mt1 where ts is not null partition by
if $rows != 1 then
return -1
endi
if $data00 != 8 then
if $data00 != 16 then
return -1
endi

144
tests/system-test/2-query/count_null.py Normal file
View File

@ -0,0 +1,144 @@
import taos
import sys
from util.log import *
from util.sql import *
from util.cases import *
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
#tdSql.init(conn.cursor())
tdSql.init(conn.cursor(), logSql) # output sql.txt file
def check_results(self):
tdSql.query(f"select count(*) from tb1")
tdSql.checkData(0, 0, 20000)
tdSql.query(f"select count(c1) from tb1")
tdSql.checkData(0, 0, 0)
tdSql.query(f"select count(c2) from tb1")
tdSql.checkData(0, 0, 0)
tdSql.query(f"select count(c3) from tb1")
tdSql.checkData(0, 0, 0)
tdSql.query(f"select count(c4) from tb1")
tdSql.checkData(0, 0, 0)
tdSql.query(f"select count(c5) from tb1")
tdSql.checkData(0, 0, 0)
tdSql.query(f"select count(c6) from tb1")
tdSql.checkData(0, 0, 0)
tdSql.query(f"select count(c7) from tb1")
tdSql.checkData(0, 0, 0)
tdSql.query(f"select count(c8) from tb1")
tdSql.checkData(0, 0, 0)
tdSql.query(f"select count(*) from tb2")
tdSql.checkData(0, 0, 20000)
tdSql.query(f"select count(c1) from tb2")
tdSql.checkData(0, 0, 20000)
tdSql.query(f"select count(c2) from tb2")
tdSql.checkData(0, 0, 20000)
tdSql.query(f"select count(c3) from tb2")
tdSql.checkData(0, 0, 20000)
tdSql.query(f"select count(c4) from tb2")
tdSql.checkData(0, 0, 20000)
tdSql.query(f"select count(c5) from tb2")
tdSql.checkData(0, 0, 20000)
tdSql.query(f"select count(c6) from tb2")
tdSql.checkData(0, 0, 20000)
tdSql.query(f"select count(c7) from tb2")
tdSql.checkData(0, 0, 0)
tdSql.query(f"select count(c8) from tb2")
tdSql.checkData(0, 0, 0)
for i in range (3, 6):
tdSql.query(f"select count(*) from tb{i}")
tdSql.checkData(0, 0, 20000)
tdSql.query(f"select count(c1) from tb{i}")
tdSql.checkData(0, 0, 10000)
tdSql.query(f"select count(c2) from tb{i}")
tdSql.checkData(0, 0, 10000)
tdSql.query(f"select count(c3) from tb{i}")
tdSql.checkData(0, 0, 10000)
tdSql.query(f"select count(c4) from tb{i}")
tdSql.checkData(0, 0, 10000)
tdSql.query(f"select count(c5) from tb{i}")
tdSql.checkData(0, 0, 10000)
tdSql.query(f"select count(c6) from tb{i}")
tdSql.checkData(0, 0, 10000)
tdSql.query(f"select count(c7) from tb{i}")
tdSql.checkData(0, 0, 10000)
tdSql.query(f"select count(c8) from tb{i}")
tdSql.checkData(0, 0, 10000)
def run(self):
dbname = 'db'
tbnames = ['tb1', 'tb2', 'tb3', 'tb4', 'tb5', 'tb6']
num_rows = 20000
num_tables = 6
ts_base = 1685548800000
tdSql.prepare()
tdLog.printNoPrefix("==========step1:create table")
for i in range (num_tables):
tdSql.execute(
f'''create table if not exists {dbname}.{tbnames[i]}
(ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10))
'''
)
tdLog.printNoPrefix("==========step2:insert data")
for i in range(num_rows):
tdSql.execute(f"insert into {dbname}.{tbnames[0]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)")
for i in range(num_rows):
tdSql.execute(f"insert into {dbname}.{tbnames[1]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)")
for i in range(num_rows):
if i % 2 == 0:
tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)")
else:
tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')")
for i in range(num_rows):
if i % 2 == 0:
tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, null, null, null, null, null, null, null, 'binary', 'nchar')")
else:
tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)")
for i in range(num_rows):
if i < num_rows / 2:
tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)")
else:
tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')")
for i in range(num_rows):
if i >= num_rows / 2:
tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)")
else:
tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')")
tdLog.printNoPrefix("==========step3:check result in memory")
self.check_results();
tdLog.printNoPrefix("==========step3:check result from disk")
tdSql.execute(f"flush database db")
self.check_results();
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

296
tests/system-test/2-query/smaBasic.py Normal file
View File

@ -0,0 +1,296 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import random
import time
import taos
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
# get col value and total max min ...
def getColsValue(self, i, j):
# c1 value
if random.randint(1, 10) == 5:
c1 = None
else:
c1 = 1
# c2 value
if j % 3200 == 0:
c2 = 8764231
elif random.randint(1, 10) == 5:
c2 = None
else:
c2 = random.randint(-87654297, 98765321)
value = f"({self.ts}, "
self.ts += 1
# c1
if c1 is None:
value += "null,"
else:
self.c1Cnt += 1
value += f"{c1},"
# c2
if c2 is None:
value += "null)"
else:
value += f"{c2})"
# total count
self.c2Cnt += 1
# max
if self.c2Max is None:
self.c2Max = c2
else:
if c2 > self.c2Max:
self.c2Max = c2
# min
if self.c2Min is None:
self.c2Min = c2
else:
if c2 < self.c2Min:
self.c2Min = c2
# sum
if self.c2Sum is None:
self.c2Sum = c2
else:
self.c2Sum += c2
return value
# insert data
def insertData(self):
tdLog.info("insert data ....")
sqls = ""
for i in range(self.childCnt):
# insert child table
values = ""
pre_insert = f"insert into t{i} values "
for j in range(self.childRow):
if values == "":
values = self.getColsValue(i, j)
else:
values += "," + self.getColsValue(i, j)
# batch insert
if j % self.batchSize == 0 and values != "":
sql = pre_insert + values
tdSql.execute(sql)
values = ""
# append last
if values != "":
sql = pre_insert + values
tdSql.execute(sql)
values = ""
sql = "flush database db;"
tdLog.info(sql)
tdSql.execute(sql)
# insert finished
tdLog.info(f"insert data successfully.\n"
f" inserted child table = {self.childCnt}\n"
f" inserted child rows = {self.childRow}\n"
f" total inserted rows = {self.childCnt*self.childRow}\n")
return
# prepareEnv
def prepareEnv(self):
# init
self.ts = 1600000000000
self.childCnt = 5
self.childRow = 1000000
self.batchSize = 5000
# total
self.c1Cnt = 0
self.c2Cnt = 0
self.c2Max = None
self.c2Min = None
self.c2Sum = None
# create database db
sql = f"create database db vgroups 5 replica 3"
tdLog.info(sql)
tdSql.execute(sql)
sql = f"use db"
tdSql.execute(sql)
# create super talbe st
sql = f"create table st(ts timestamp, c1 int, c2 bigint) tags(area int)"
tdLog.info(sql)
tdSql.execute(sql)
# create child table
for i in range(self.childCnt):
sql = f"create table t{i} using st tags({i}) "
tdSql.execute(sql)
# insert data
self.insertData()
# query sql value
def queryValue(self, sql):
tdSql.query(sql)
return tdSql.getData(0, 0)
# sum
def checkCorrentSum(self):
# query count
sql = "select sum(c1) from st"
val = self.queryValue(sql)
# c1Sum is equal c1Cnt
if val != self.c1Cnt:
tdLog.exit(f"Sum Not Expect. expect={self.c1Cnt} query={val} sql:{sql}")
return
# not
sql1 = "select sum(c1) from st where c2 = 8764231"
val1 = self.queryValue(sql1)
sql2 = "select sum(c1) from st where c2 != 8764231"
val2 = self.queryValue(sql2)
sql3 = "select sum(c1) from st where c2 is null"
val3 = self.queryValue(sql3)
if val != val1 + val2 + val3:
tdLog.exit(f"Sum Not Equal. val != val1 + val2 + val3. val={val} val1={val1} val2={val2} val2={val3} sql1={sql1} sql2={sql2} sql2={sql3}")
return
# over than
sql1 = "select sum(c1) from st where c2 > 8000"
val1 = self.queryValue(sql1)
sql2 = "select sum(c1) from st where c2 <= 8000"
val2 = self.queryValue(sql2)
sql3 = "select sum(c1) from st where c2 is null"
val3 = self.queryValue(sql3)
if val != val1 + val2 + val3:
tdLog.exit(f"Sum Not Equal. val != val1 + val2 + val3. val={val} val1={val1} val2={val2} val2={val3} sql1={sql1} sql2={sql2} sql2={sql3}")
return
tdLog.info(f"check correct sum on c1 successfully.")
# check result
def checkResult(self, fun, val, val1, val2, sql1, sql2):
if fun == "count":
if val != val1 + val2:
tdLog.exit(f"{fun} NOT SAME. val != val1 + val2. val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}")
return
elif fun == "max":
if val != max([val1, val2]):
tdLog.exit(f"{fun} NOT SAME . val != max(val1 ,val2) val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}")
return
elif fun == "min":
if val != min([val1, val2]):
tdLog.exit(f"{fun} NOT SAME . val != min(val1 ,val2) val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}")
return
# sum
def checkCorrentFun(self, fun, expectVal):
# query
sql = f"select {fun}(c2) from st"
val = self.queryValue(sql)
if val != expectVal:
tdLog.exit(f"{fun} Not Expect. expect={expectVal} query={val} sql:{sql}")
return
# not
sql1 = f"select {fun}(c2) from st where c2 = 8764231"
val1 = self.queryValue(sql1)
sql2 = f"select {fun}(c2) from st where c2 != 8764231"
val2 = self.queryValue(sql2)
self.checkResult(fun, val, val1, val2, sql1, sql2)
# over than
sql1 = f"select {fun}(c2) from st where c2 > 8000"
val1 = self.queryValue(sql1)
sql2 = f"select {fun}(c2) from st where c2 <= 8000"
val2 = self.queryValue(sql2)
self.checkResult(fun, val, val1, val2, sql1, sql2)
# successful
tdLog.info(f"check correct {fun} on c2 successfully.")
# check query corrent
def checkCorrect(self):
# count
self.checkCorrentFun("count", self.c2Cnt)
# max
self.checkCorrentFun("max", self.c2Max)
# min
self.checkCorrentFun("min", self.c2Min)
# sum
self.checkCorrentSum()
# c2 sum
sql = "select sum(c2) from st"
val = self.queryValue(sql)
# c1Sum is equal c1Cnt
if val != self.c2Sum:
tdLog.exit(f"c2 Sum Not Expect. expect={self.c2Sum} query={val} sql:{sql}")
return
def checkPerformance(self):
# have sma caculate
sql1 = "select count(*) from st"
stime = time.time()
tdSql.execute(sql1, 1)
spend1 = time.time() - stime
# no sma caculate
sql2 = "select count(*) from st where c2 != 8764231 or c2 is null"
stime = time.time()
tdSql.execute(sql2, 1)
spend2 = time.time() - stime
time1 = "%.2f"%(spend1*1000)
time2 = "%.2f"%(spend2*1000)
if spend2 < spend1 * 8:
tdLog.exit(f"performance not passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}")
return
tdLog.info(f"performance passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}")
# init
def init(self, conn, logSql, replicaVar=1):
seed = time.clock_gettime(time.CLOCK_REALTIME)
random.seed(seed)
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
# run
def run(self):
# prepare env
self.prepareEnv()
# query
self.checkCorrect()
# performance
self.checkPerformance()
# stop
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

37
tests/system-test/eco-system/main.py Normal file
View File

@ -0,0 +1,37 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import re
from util.log import *
from util.cases import *
from util.sql import *
from util.common import *
from util.sqlset import *
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.setsql = TDSetSql()
def run(self):
tdLog.info(" ------ eco-system main -------")
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

146
tests/system-test/eco-system/manager/drop_table.py Normal file
View File

@ -0,0 +1,146 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from util.log import *
from util.cases import *
from util.sql import *
from util.common import *
from util.sqlset import *
import random
import time
import traceback
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), True)
# describe table
def describe_table(self, tbname):
columns = []
tags = []
sql = f"describe {tbname}"
row_cnt = tdSql.query(sql)
for i in range(0, row_cnt):
col_name = tdSql.queryResult[i][0]
type_name = tdSql.queryResult[i][3]
if type_name == "TAG":
tags.append(col_name)
else:
columns.append(col_name)
return columns,tags
# show tables
def show_tables(self):
sql = "show tables;"
row_cnt = tdSql.query(sql)
tables = []
for i in range(0, row_cnt):
tb_name = tdSql.queryResult[i][0]
tables.append(tb_name)
# execute sql
def execute(self, sql):
try:
tdSql.execute(sql, 3)
tdLog.info(f" exec ok. {sql}")
except:
tdLog.info(f" exe failed. {sql}")
traceback.print_exc()
# query
def query_table(self, columns, tags):
if len(columns) < 5 :
return
if len(tags) < 5:
return
sel_cols = random.sample(columns, random.randint(1,int(len(columns)-1)))
sel_tags = random.sample(tags, random.randint(1, int(len(tags)-1)))
field_cols = ",".join(sel_cols)
field_tags = ",".join(sel_tags)
#sql = f"select {field_cols},{field_tags} from meters ;"
sql = f"select {field_cols},{field_tags} from meters"
try:
tdLog.info( " query sql:" + sql)
tdSql.query("select * from meters limit 1")
except:
tdLog.info( " query failed :" + sql)
traceback.print_exc()
# change table schema
def drop_table(self, change_cnt):
# init
tables = self.show_tables()
for i in range(change_cnt):
col_idx = random.randint(0, ncol - 1)
tag_idx = random.randint(0, ntag - 1)
cols = list(self.column_dict.keys())
tags = list(self.tag_dict.keys())
# column
key = cols[col_idx]
value = self.column_dict[key]
sql = f'alter table meters drop column {key}'
self.execute(sql)
sql = f'alter table meters add column {key} {value}'
self.execute(sql)
# column
key = tags[col_idx]
value = self.tag_dict[key]
sql = f'alter table meters drop tag {key}'
self.execute(sql)
sql = f'alter table meters add tag {key} {value}'
self.execute(sql)
# drop and rename
if i % 5 == 0:
# update columns
#columns,tags = self.describe_table("meters")
tdLog.info(f" ======= describe table column count = {len(cols)} tags= {len(tags)}======")
self.query_table(cols, tags)
# run
def run(self):
# seed
random.seed(int(time.time()))
self.dbname = "schema_change"
# switch db
tdSql.execute(f"use {self.dbname};")
# change meters
self.drop_table(1000000)
# stop
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

181
tests/system-test/eco-system/manager/same_column.py Normal file
View File

@ -0,0 +1,181 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from util.log import *
from util.cases import *
from util.sql import *
from util.common import *
from util.sqlset import *
import random
import time
import traceback
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), True)
self.setsql = TDSetSql()
self.column_dict = {
'bc': 'bool',
'fc': 'float',
'dc': 'double',
'ti': 'tinyint',
'si': 'smallint',
'ic': 'int',
'bi': 'bigint',
'uit': 'tinyint unsigned',
'usi': 'smallint unsigned',
'ui': 'int unsigned',
'ubi': 'bigint unsigned',
'bin': 'binary(32)',
'nch': 'nchar(64)'
}
self.tag_dict = {
'groupid': 'tinyint',
'location': 'binary(16)',
'tfc': 'float',
'tdc': 'double',
'tti': 'tinyint',
'tsi': 'smallint',
'tic': 'int',
'tbi': 'bigint',
'tuit': 'tinyint unsigned',
'tusi': 'smallint unsigned',
'tui': 'int unsigned',
'tubi': 'bigint unsigned',
'tbin': 'binary(32)',
'tnch': 'nchar(64)'
}
# describe table
def describe_table(self, tbname):
columns = []
tags = []
sql = f"describe {tbname}"
row_cnt = tdSql.query(sql)
for i in range(20, row_cnt):
col_name = tdSql.queryResult[i][0]
type_name = tdSql.queryResult[i][3]
if type_name == "TAG":
tags.append(col_name)
else:
columns.append(col_name)
return columns,tags
def drop_tag(self, tags, cnt):
for i in range(cnt):
tag_cnt = len(tags)
sel = random.randint(1, tag_cnt-1)
sql = f"alter table meters drop tag `{tags[sel]}` "
try:
tdSql.execute(sql)
tdLog.info(sql)
del tags[sel]
except:
tdLog.info(f" drop tags failed. {sql}")
traceback.print_exc()
# execute sql
def execute(self, sql):
try:
tdSql.execute(sql, 3)
tdLog.info(f" exec ok. {sql}")
except:
tdLog.info(f" exe failed. {sql}")
traceback.print_exc()
# query
def query_table(self, columns, tags):
if len(columns) < 5 :
return
if len(tags) < 5:
return
sel_cols = random.sample(columns, random.randint(1,int(len(columns)-1)))
sel_tags = random.sample(tags, random.randint(1, int(len(tags)-1)))
field_cols = ",".join(sel_cols)
field_tags = ",".join(sel_tags)
#sql = f"select {field_cols},{field_tags} from meters ;"
sql = f"select {field_cols},{field_tags} from meters"
try:
tdLog.info( " query sql:" + sql)
tdSql.query("select * from meters limit 1")
except:
tdLog.info( " query failed :" + sql)
traceback.print_exc()
# change table schema
def change_columns(self, change_cnt):
# init
ncol = len(self.column_dict)
ntag = len(self.tag_dict)
for i in range(change_cnt):
col_idx = random.randint(0, ncol - 1)
tag_idx = random.randint(0, ntag - 1)
cols = list(self.column_dict.keys())
tags = list(self.tag_dict.keys())
# column
key = cols[col_idx]
value = self.column_dict[key]
sql = f'alter table meters drop column {key}'
self.execute(sql)
sql = f'alter table meters add column {key} {value}'
self.execute(sql)
# column
key = tags[col_idx]
value = self.tag_dict[key]
sql = f'alter table meters drop tag {key}'
self.execute(sql)
sql = f'alter table meters add tag {key} {value}'
self.execute(sql)
# drop and rename
if i % 5 == 0:
# update columns
#columns,tags = self.describe_table("meters")
tdLog.info(f" ======= describe table column count = {len(cols)} tags= {len(tags)}======")
self.query_table(cols, tags)
# run
def run(self):
# seed
random.seed(int(time.time()))
self.dbname = "schema_change"
# switch db
tdSql.execute(f"use {self.dbname};")
# change meters
self.change_columns(1000000)
# stop
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

239
tests/system-test/eco-system/manager/schema_change.py Normal file
View File

@ -0,0 +1,239 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from util.log import *
from util.cases import *
from util.sql import *
from util.common import *
from util.sqlset import *
import random
import time
import traceback
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), True)
self.setsql = TDSetSql()
self.column_dict = {
'col0': 'int',
'col1': 'tinyint',
'col2': 'smallint',
'col3': 'int',
'col4': 'bigint',
'col5': 'tinyint unsigned',
'col6': 'smallint unsigned',
'col7': 'int unsigned',
'col8': 'bigint unsigned',
'col9': 'float',
'col10': 'double',
'col11': 'bool',
'col12': 'varchar(20)',
'col13': 'nchar(20)'
}
self.tag_dict = {
't1': 'tinyint',
't2': 'smallint',
't3': 'int',
't4': 'bigint',
't5': 'tinyint unsigned',
't6': 'smallint unsigned',
't7': 'int unsigned',
't8': 'bigint unsigned',
't9': 'float',
't10': 'double',
't11': 'bool',
't12': 'varchar(20)',
't13': 'nchar(20)',
't14': 'timestamp'
}
# delete
def delete_col(self, columns, cnt, max_col):
# delte for random
for i in range(cnt):
col_cnt = len(columns)
if col_cnt == 0:
return
sel = random.randint(0, col_cnt - 1)
sql = f"alter table meters drop column `{columns[sel]}`"
try:
tdSql.execute(sql)
tdLog.info(f" drop cur col={len(columns)} max_col={max_col} {sql}")
del columns[sel]
except:
tdLog.info(f" drop column failed. {sql}")
traceback.print_exc()
# describe table
def describe_table(self, tbname):
columns = []
tags = []
sql = f"describe {tbname}"
row_cnt = tdSql.query(sql)
for i in range(20, row_cnt):
col_name = tdSql.queryResult[i][0]
type_name = tdSql.queryResult[i][3]
if type_name == "TAG":
tags.append(col_name)
else:
columns.append(col_name)
return columns,tags
def renames(self, tags, cnt):
col_cnt = len(tags)
if col_cnt < 10:
return
for i in range(cnt):
sel = random.randint(1, col_cnt-3)
new_name = tags[sel] + "n"
sql = f"alter table meters rename tag `{tags[sel]}` `{new_name}` "
try:
tdSql.execute(sql)
tdLog.info(sql)
tags[sel] = new_name
except:
tdLog.info(f" rename tag failed. {sql}")
traceback.print_exc()
def drop_tag(self, tags, cnt):
for i in range(cnt):
tag_cnt = len(tags)
sel = random.randint(1, tag_cnt-1)
sql = f"alter table meters drop tag `{tags[sel]}` "
try:
tdSql.execute(sql)
tdLog.info(sql)
del tags[sel]
except:
tdLog.info(f" drop tags failed. {sql}")
traceback.print_exc()
# query
def query_table(self, columns, tags):
if len(columns) < 10 :
return
if len(tags) < 10:
return
sel_cols = random.sample(columns, random.randint(2,int(len(columns)/10)))
sel_tags = random.sample(tags, random.randint(1,int(len(tags)/10)))
field_cols = ",".join(sel_cols)
field_tags = ",".join(sel_tags)
#sql = f"select {field_cols},{field_tags} from meters ;"
sql = f"select {field_cols},{field_tags} from meters"
try:
tdLog.info( " query sql:" + sql)
tdSql.query("select * from meters limit 1")
except:
tdLog.info( " query failed :" + sql)
traceback.print_exc()
# change table schema
def change_schema(self, change_cnt):
# init
columns, tags = self.describe_table("meters")
max_col = random.randint(200, 2000)
tdLog.info(f" ----------- set max column = {max_col} -------------")
for i in range(change_cnt):
col_cnt = len(self.column_dict)
icol = random.randint(0, col_cnt-1)
key = f"col{icol}"
col_name = key + f"_{i}_{random.randint(1,100)}"
col_type = self.column_dict[key]
sql = f'alter table meters add column `{col_name}` {col_type}'
sql_tag = f'alter table meters add tag `t_{col_name}` {col_type}'
try:
tdSql.execute(sql)
tdLog.info(f" add cur col={len(columns)} max_col={max_col} {sql}")
columns.append(col_name)
if random.randint(1, 4) == 2:
tdSql.execute(sql_tag)
tdLog.info(f" add tag tag_cnt={len(tags)} {sql_tag}")
except:
tdLog.info(f" add column failed. {sql}")
traceback.print_exc()
col_cnt = len(columns)
# delete
if col_cnt > max_col + 100:
self.delete_col(columns, random.randint(1, 30), max_col)
elif col_cnt >= max_col + 30:
self.delete_col(columns, random.randint(1, 4), max_col)
max_col = random.randint(200, 2000)
tdLog.info(f" ----------- set max column = {max_col} -------------")
elif col_cnt > max_col:
self.delete_col(columns, random.randint(1, 3), max_col)
if i % 50 == 0:
sql = f"flush database {self.dbname};"
tdSql.execute(sql)
tdLog.info(f" ***** {sql} *****")
# query
if i % 70 == 0:
self.query_table(columns, tags)
# drop and rename
if i % 10 == 0:
# update columns
columns,tags = self.describe_table("meters")
tdLog.info(f" ======= describe table column count = {len(columns)} tags= {len(tags)}======")
if random.randint(1,3) == 2:
self.query_table(columns, tags)
if len(tags) > 50:
self.drop_tag(tags, random.randint(1, 30))
self.renames(tags, random.randint(1, 10))
# sleep
#time.sleep(0.3)
# run
def run(self):
# seed
random.seed(int(time.time()))
self.dbname = "schema_change"
# switch db
tdSql.execute(f"use {self.dbname};")
# change meters
self.change_schema(1000000)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

151
tests/system-test/eco-system/schemaless/insert.py Normal file
View File

@ -0,0 +1,151 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from util.log import *
from util.cases import *
from util.sql import *
from util.common import *
from util.sqlset import *
import random
import time
import traceback
import taos
import string
from taos import schemaless
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), True)
self.setsql = TDSetSql()
self.conn = conn
self.schema = {}
def random_string(self, count):
letters = string.ascii_letters
return ''.join(random.choice(letters) for i in range(count))
def genCol(self, col_name, isTag):
col_types = ["str","f64","f32","i8","u8","i16","u16","i32","u32","i64","u64"]
if self.schema.get(col_name) == None:
col_type = random.choice(col_types)
self.schema[col_name] = col_type
else:
col_type = self.schema[col_name]
is_num = True
val = ""
if col_type == "str":
val = self.random_string(random.randint(1, 10))
is_num = False
elif col_type == "f64":
val = random.randrange(-100000000000000, 1000000000000)/3*2.25678
elif col_type == "f32":
val = random.randrange(-100000000, 1000000000)/3*1.2345
elif col_type == "i8":
val = random.randint(-128, 127)
elif col_type == "u8":
val = random.randint(0, 256)
elif col_type == "i16":
val = random.randint(-32768, 32767)
elif col_type == "u16":
val = random.randint(0, 256*256)
elif col_type == "i32":
val = random.randint(-256*256*256*128, 256*256*256*128)
elif col_type == "u32":
val = random.randint(0, 256*256*256*256)
elif col_type == "i64":
val = random.randint(-256*256*256*256*256*256*256*128, 256*256*256*256*256*256*256*128)
elif col_type == "u64":
val = random.randint(0, 256*256*256*256*256*256*256*256)
else:
val = 100
if isTag:
col_val = val
elif is_num:
col_val = f'{val}{col_type}'
else:
col_val = '"' + val + '"'
return f'{col_name}={col_val}'
# cols
def genCols(self, pre, max, index, isTag):
col_cnt = random.randint(1, max)
cols = []
for i in range(col_cnt):
col_name = f'{pre}_{index}_{i}'
cols.append(self.genCol(col_name, isTag))
return ",".join(cols)
# execute sql
def insert(self,sql,i):
print("schema less insert")
try:
self.conn.schemaless_insert([sql], schemaless.SmlProtocol.LINE_PROTOCOL, schemaless.SmlPrecision.MILLI_SECONDS)
tdLog.info(f" exec ok i={i} {sql}")
except:
tdLog.info(f" exe failed. i={i} {sql}")
traceback.print_exc()
def genTags(self, i):
tags = f"t1={i},t2=abc,t3=work"
return tags
# change table schema
def schemaless_insert(self, change_cnt):
# init
ts = 1683194263000
for i in range(change_cnt):
index = int(i/10000) % 600
cols = self.genCols("c", 5, index, False)
tags = self.genTags(index)
sql = f'{self.stable},{tags} {cols} {ts + i}'
self.insert(sql, i)
# run
def run(self):
# seed
#random.seed(int(time.time()))
self.dbname = "eco_system"
self.stable = "sml_stb"
# switch db
tdSql.execute(f"use {self.dbname};")
tdSql.execute(f"drop table if exists {self.stable};")
# change meters
try:
self.schemaless_insert(1000000)
except:
traceback.print_exc()
print(self.schema)
# stop
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())