diff --git a/README-CN.md b/README-CN.md index 81d64941af..44542747eb 100644 --- a/README-CN.md +++ b/README-CN.md @@ -15,7 +15,7 @@ [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) -简体中文 | [English](README.md) | 很多职位正在热招中,请看[这里](https://www.taosdata.com/cn/careers/) +简体中文 | [English](README.md) | [TDengine 云服务](https://cloud.taosdata.com/?utm_medium=cn&utm_source=github) | 很多职位正在热招中,请看[这里](https://www.taosdata.com/cn/careers/) # TDengine 简介 diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index d4ca25be81..41a429b026 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -65,6 +65,13 @@ taos = "*" taos = { version = "*", default-features = false, features = ["ws"] } ``` +当仅启用 `ws` 特性时,可同时指定 `r2d2` 使得在同步(blocking/sync)模式下使用 [r2d2] 作为连接池: + +```toml +[dependencies] +taos = { version = "*", default-features = false, features = ["r2d2", "ws"] } +``` + @@ -257,26 +264,24 @@ let conn: Taos = cfg.build(); ### 连接池 -在复杂应用中,建议启用连接池。[taos] 的连接池使用 [r2d2] 实现。 +在复杂应用中,建议启用连接池。[taos] 的连接池默认(异步模式)使用 [deadpool] 实现。 如下,可以生成一个默认参数的连接池。 ```rust -let pool = TaosBuilder::from_dsn(dsn)?.pool()?; +let pool: Pool = TaosBuilder::from_dsn("taos:///") + .unwrap() + .pool() + .unwrap(); ``` 同样可以使用连接池的构造器,对连接池参数进行设置: ```rust -let dsn = "taos://localhost:6030"; - -let opts = PoolBuilder::new() - .max_size(5000) // max connections - .max_lifetime(Some(Duration::from_secs(60 * 60))) // lifetime of each connection - .min_idle(Some(1000)) // minimal idle connections - .connection_timeout(Duration::from_secs(2)); - -let pool = TaosBuilder::from_dsn(dsn)?.with_pool_builder(opts)?; +let pool: Pool = Pool::builder(Manager::from_dsn(self.dsn.clone()).unwrap().0) + .max_size(88) // 最大连接数 + .build() + .unwrap(); ``` 在应用代码中,使用 `pool.get()?` 来获取一个连接对象 [Taos]。 @@ -516,6 +521,7 @@ consumer.unsubscribe().await; 其他相关结构体 API 使用说明请移步 Rust 文档托管网页:。 [taos]: https://github.com/taosdata/rust-connector-taos +[deadpool]: https://crates.io/crates/deadpool [r2d2]: https://crates.io/crates/r2d2 [TaosBuilder]: https://docs.rs/taos/latest/taos/struct.TaosBuilder.html [TaosCfg]: https://docs.rs/taos/latest/taos/struct.TaosCfg.html diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 3ede85f86d..68f44d1e65 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -91,11 +91,30 @@ taos --dump-config ### maxShellConns | 属性 | 说明 | -| -------- | ----------------------- | +| --------| ----------------------- | | 适用范围 | 仅服务端适用 | -| 含义 | 一个 dnode 容许的连接数 | +| 含义 | 一个 dnode 容许的连接数 | | 取值范围 | 10-50000000 | -| 缺省值 | 5000 | +| 缺省值 | 5000 | + +### numOfRpcSessions + +| 属性 | 说明 | +| --------| ---------------------- | +| 适用范围 | 客户端和服务端都适用 | +| 含义 | 一个客户端能创建的最大连接数| +| 取值范围 | 100-100000 | +| 缺省值 | 10000 | + +### timeToGetAvailableConn + +| 属性 | 说明 | +| -------- | --------------------| +| 适用范围 | 客户端和服务端都适用 | +| 含义 |获得可用连接的最长等待时间| +| 取值范围 | 10-50000000(单位为毫秒)| +| 缺省值 | 500000 | + ### numOfRpcSessions diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 8be5cb4d41..e04bdd1b07 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -145,7 +145,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull); // for stmt bind -int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind); +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen); void tColDataSortMerge(SArray *colDataArr); // for raw block diff --git a/include/common/tglobal.h b/include/common/tglobal.h index ced94e0f11..a5611aeee7 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -130,7 +130,7 @@ extern int32_t tsSlowLogScope; // client extern int32_t tsMinSlidingTime; extern int32_t tsMinIntervalTime; -extern int32_t tsMaxMemUsedByInsert; +extern int32_t tsMaxInsertBatchRows; // build info extern char version[]; diff --git a/include/common/ttime.h b/include/common/ttime.h index f189959f22..de74e48100 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -23,7 +23,7 @@ extern "C" { #endif -#define TIME_IS_VAR_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') +#define IS_CALENDAR_TIME_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') #define TIME_UNIT_NANOSECOND 'b' #define TIME_UNIT_MICROSECOND 'u' @@ -74,7 +74,7 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); -int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision); +int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval); int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision); int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index e46757f6f8..3f53976c67 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -92,6 +92,8 @@ int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList */ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); +//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); + int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); // todo refactor @@ -188,7 +190,17 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len); int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len); -STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key); +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order); +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery); +STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key); +/** + * return the scan info, in the form of tuple of two items, including table uid and current timestamp + * @param tinfo + * @param uid + * @param ts + * @return + */ +int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts); SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 197a5ecaf9..02459ed951 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -112,6 +112,7 @@ typedef struct SJoinLogicNode { SNode* pOnConditions; bool isSingleTableJoin; EOrder inputTsOrder; + SNode* pColEqualOnConditions; } SJoinLogicNode; typedef struct SAggLogicNode { @@ -406,6 +407,7 @@ typedef struct SSortMergeJoinPhysiNode { SNode* pOnConditions; SNodeList* pTargets; EOrder inputTsOrder; + SNode* pColEqualOnConditions; } SSortMergeJoinPhysiNode; typedef struct SAggPhysiNode { @@ -448,7 +450,7 @@ typedef struct SMergePhysiNode { bool ignoreGroupId; } SMergePhysiNode; -typedef struct SWinodwPhysiNode { +typedef struct SWindowPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of parameter expression of function SNodeList* pFuncs; @@ -461,10 +463,10 @@ typedef struct SWinodwPhysiNode { EOrder inputTsOrder; EOrder outputTsOrder; bool mergeDataBlock; -} SWinodwPhysiNode; +} SWindowPhysiNode; typedef struct SIntervalPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; int64_t interval; int64_t offset; int64_t sliding; @@ -497,7 +499,7 @@ typedef struct SMultiTableIntervalPhysiNode { } SMultiTableIntervalPhysiNode; typedef struct SSessionWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; int64_t gap; } SSessionWinodwPhysiNode; @@ -506,14 +508,14 @@ typedef SSessionWinodwPhysiNode SStreamSemiSessionWinodwPhysiNode; typedef SSessionWinodwPhysiNode SStreamFinalSessionWinodwPhysiNode; typedef struct SStateWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; SNode* pStateKey; } SStateWinodwPhysiNode; typedef SStateWinodwPhysiNode SStreamStateWinodwPhysiNode; typedef struct SEventWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; SNode* pStartCond; SNode* pEndCond; } SEventWinodwPhysiNode; diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index a081527144..1aa08ff802 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -212,7 +212,8 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead); int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead); int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead); -SWalRef *walRefFirstVer(SWal *, SWalRef *); +void walRefFirstVer(SWal *, SWalRef *); +void walRefLastVer(SWal *, SWalRef *); SWalRef *walRefCommittedVer(SWal *); SWalRef *walOpenRef(SWal *); diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 2f1e803689..b0537e8bcf 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -70,8 +70,7 @@ if [ "$pagMode" == "lite" ]; then taostools_bin_files="" else if [ "$verMode" == "cloud" ]; then - taostools_bin_files=" ${build_dir}/bin/taosdump \ - ${build_dir}/bin/taosBenchmark" + taostools_bin_files=" ${build_dir}/bin/taosBenchmark" else wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ && echo "TDinsight.sh downloaded!" \ diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 59d411cde8..ccb09e3584 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -89,6 +89,7 @@ struct tmq_t { void* commitCbUserParam; // status + SRWLatch lock; int8_t status; int32_t epoch; #if 0 @@ -148,10 +149,10 @@ typedef struct { SVgOffsetInfo offsetInfo; int32_t vgId; int32_t vgStatus; - int32_t vgSkipCnt; // here used to mark the slow vgroups - bool receivedInfoFromVnode;// has already received info from vnode - int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data - bool seekUpdated; // offset is updated by seek operator, therefore, not update by vnode rsp. + int32_t vgSkipCnt; // here used to mark the slow vgroups + bool receivedInfoFromVnode; // has already received info from vnode + int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data + bool seekUpdated; // offset is updated by seek operator, therefore, not update by vnode rsp. SEpSet epSet; } SMqClientVg; @@ -166,6 +167,7 @@ typedef struct { int8_t tmqRspType; int32_t epoch; // epoch can be used to guard the vgHandle int32_t vgId; + char topicName[TSDB_TOPIC_FNAME_LEN]; SMqClientVg* vgHandle; SMqClientTopic* topicHandle; uint64_t reqId; @@ -178,8 +180,8 @@ typedef struct { } SMqPollRspWrapper; typedef struct { - int64_t refId; - int32_t epoch; +// int64_t refId; +// int32_t epoch; tsem_t rspSem; int32_t rspErr; } SMqSubscribeCbParam; @@ -194,8 +196,9 @@ typedef struct { typedef struct { int64_t refId; int32_t epoch; - SMqClientVg* pVg; - SMqClientTopic* pTopic; + char topicName[TSDB_TOPIC_FNAME_LEN]; +// SMqClientVg* pVg; +// SMqClientTopic* pTopic; int32_t vgId; uint64_t requestId; // request id for debug purpose } SMqPollCbParam; @@ -743,8 +746,8 @@ static void generateTimedTask(int64_t refId, int32_t type) { *pTaskType = type; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); + taosReleaseRef(tmqMgmt.rsetId, refId); } - taosReleaseRef(tmqMgmt.rsetId, refId); } void tmqAssignAskEpTask(void* param, void* tmrId) { @@ -1071,6 +1074,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->commitCb = conf->commitCb; pTmq->commitCbUserParam = conf->commitCbUserParam; pTmq->resetOffsetCfg = conf->resetOffset; + taosInitRWLatch(&pTmq->lock); pTmq->hbBgEnable = conf->hbBgEnable; @@ -1119,7 +1123,7 @@ _failed: } int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { - const int32_t MAX_RETRY_COUNT = 120 * 4; // let's wait for 4 mins at most + const int32_t MAX_RETRY_COUNT = 120 * 60; // let's wait for 2 mins at most const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); void* buf = NULL; @@ -1172,7 +1176,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { goto FAIL; } - SMqSubscribeCbParam param = { .rspErr = 0, .refId = tmq->refId, .epoch = tmq->epoch }; + SMqSubscribeCbParam param = { .rspErr = 0}; if (tsem_init(¶m.rspSem, 0, 0) != 0) { code = TSDB_CODE_TSC_INTERNAL_ERROR; goto FAIL; @@ -1206,8 +1210,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { int32_t retryCnt = 0; while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) { if (retryCnt++ > MAX_RETRY_COUNT) { - tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, max retry reached:%d", tmq->consumerId, retryCnt); - code = TSDB_CODE_TSC_INTERNAL_ERROR; + tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt); + code = TSDB_CODE_MND_CONSUMER_NOT_READY; goto FAIL; } @@ -1242,12 +1246,40 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para conf->commitCbUserParam = param; } -static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { +static SMqClientVg* getVgInfo(tmq_t* tmq, char* topicName, int32_t vgId){ + int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); + for(int i = 0; i < topicNumCur; i++){ + SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); + if(strcmp(pTopicCur->topicName, topicName) == 0){ + int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); + for (int32_t j = 0; j < vgNumCur; j++) { + SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); + if(pVgCur->vgId == vgId){ + return pVgCur; + } + } + } + } + return NULL; +} + +static SMqClientTopic* getTopicInfo(tmq_t* tmq, char* topicName){ + int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); + for(int i = 0; i < topicNumCur; i++){ + SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); + if(strcmp(pTopicCur->topicName, topicName) == 0){ + return pTopicCur; + } + } + return NULL; +} + +int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { SMqPollCbParam* pParam = (SMqPollCbParam*)param; int64_t refId = pParam->refId; - SMqClientVg* pVg = pParam->pVg; - SMqClientTopic* pTopic = pParam->pTopic; +// SMqClientVg* pVg = pParam->pVg; +// SMqClientTopic* pTopic = pParam->pTopic; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { @@ -1262,15 +1294,13 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t vgId = pParam->vgId; uint64_t requestId = pParam->requestId; - taosMemoryFree(pParam); - if (code != 0) { if (pMsg->pData) taosMemoryFree(pMsg->pData); if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet); // in case of consumer mismatch, wait for 500ms and retry if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) { - taosMsleep(500); +// taosMsleep(500); atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER); tscDebug("consumer:0x%" PRIx64 " wait for the re-balance, wait for 500ms and set status to be RECOVER", tmq->consumerId); @@ -1284,8 +1314,8 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { pRspWrapper->tmqRspType = TMQ_MSG_TYPE__END_RSP; taosWriteQitem(tmq->mqueue, pRspWrapper); - } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert - taosMsleep(500); +// } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert +// taosMsleep(5); } else{ tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId, vgId, epoch, tstrerror(code), requestId); @@ -1307,6 +1337,8 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); + taosMemoryFree(pParam); + return 0; } @@ -1328,11 +1360,12 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } pRspWrapper->tmqRspType = rspType; - pRspWrapper->vgHandle = pVg; - pRspWrapper->topicHandle = pTopic; +// pRspWrapper->vgHandle = pVg; +// pRspWrapper->topicHandle = pTopic; pRspWrapper->reqId = requestId; pRspWrapper->pEpset = pMsg->pEpSet; - pRspWrapper->vgId = pVg->vgId; + pRspWrapper->vgId = vgId; + strcpy(pRspWrapper->topicName, pParam->topicName); pMsg->pEpSet = NULL; if (rspType == TMQ_MSG_TYPE__POLL_RSP) { @@ -1371,16 +1404,23 @@ static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tsem_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); + taosMemoryFree(pParam); return 0; CREATE_MSG_FAIL: if (epoch == tmq->epoch) { - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = getVgInfo(tmq, pParam->topicName, vgId); + if(pVg){ + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + } + taosWUnLockLatch(&tmq->lock); } tsem_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); + taosMemoryFree(pParam); return -1; } @@ -1503,11 +1543,13 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) taosHashCleanup(pVgOffsetHashMap); + taosWLockLatch(&tmq->lock); // destroy current buffered existed topics info if (tmq->clientTopics) { taosArrayDestroyEx(tmq->clientTopics, freeClientVgInfo); } tmq->clientTopics = newTopics; + taosWUnLockLatch(&tmq->lock); int8_t flag = (topicNumGet == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY; atomic_store_8(&tmq->status, flag); @@ -1523,7 +1565,7 @@ int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { if (tmq == NULL) { terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; - pParam->pUserFn(tmq, terrno, NULL, pParam->pParam); +// pParam->pUserFn(tmq, terrno, NULL, pParam->pParam); taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -1682,8 +1724,9 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p pParam->refId = pTmq->refId; pParam->epoch = pTmq->epoch; - pParam->pVg = pVg; - pParam->pTopic = pTopic; +// pParam->pVg = pVg; // pVg may be released,fix it +// pParam->pTopic = pTopic; + strcpy(pParam->topicName, pTopic->topicName); pParam->vgId = pVg->vgId; pParam->requestId = req.reqId; @@ -1718,6 +1761,9 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p // broadcast the poll request to all related vnodes static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { + if(atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER){ + return 0; + } int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); tscDebug("consumer:0x%" PRIx64 " start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics); @@ -1802,8 +1848,14 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp; if (pDataRsp->head.epoch == consumerEpoch) { - SMqClientVg* pVg = pollRspWrapper->vgHandle; - + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + return NULL; + } // update the epset if (pollRspWrapper->pEpset != NULL) { SEp* pEp = GET_ACTIVE_EP(pollRspWrapper->pEpset); @@ -1862,8 +1914,16 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId); if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { - SMqClientVg* pVg = pollRspWrapper->vgHandle; - if (!pVg->seekUpdated) { + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + return NULL; + } + + if(pollRspWrapper->metaRsp.rspOffset.type != 0){ // if offset is validate pVg->offsetInfo.currentOffset = pollRspWrapper->metaRsp.rspOffset; } @@ -1883,8 +1943,16 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { int32_t consumerEpoch = atomic_load_32(&tmq->epoch); if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) { - SMqClientVg* pVg = pollRspWrapper->vgHandle; - if (!pVg->seekUpdated) { // if offset is validate + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + return NULL; + } + + if(pollRspWrapper->taosxRsp.rspOffset.type != 0){ // if offset is validate pVg->offsetInfo.currentOffset = pollRspWrapper->taosxRsp.rspOffset; } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index f30557e4ef..0b12177754 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2504,18 +2504,21 @@ _exit: return code; } -int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen) { int32_t code = 0; if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { ASSERT(pColData->type == pBind->buffer_type); } - + if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type for (int32_t i = 0; i < pBind->num; ++i) { if (pBind->is_null && pBind->is_null[i]) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); if (code) goto _exit; + } else if (pBind->length[i] > buffMaxLen) { + uError("var data length too big, len:%d, max:%d", pBind->length[i], buffMaxLen); + return TSDB_CODE_INVALID_PARA; } else { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( pColData, (uint8_t *)pBind->buffer + pBind->buffer_length * i, pBind->length[i]); @@ -3523,6 +3526,43 @@ static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum } } +static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(uint64_t *)sum = 0; + *(uint64_t *)max = 0; + *(uint64_t *)min = 0; + *numOfNull = 0; + + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: + case (HAS_NONE | HAS_NULL): + *numOfNull = pColData->nVal; + break; + case HAS_VALUE: + *numOfNull = 0; + break; + case (HAS_VALUE | HAS_NULL): + case (HAS_VALUE | HAS_NONE): + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + if (GET_BIT1(pColData->pBitMap, iVal) == 0) { + (*numOfNull)++; + } + } + break; + case (HAS_VALUE | HAS_NONE | HAS_NULL): + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + if (GET_BIT2(pColData->pBitMap, iVal) != 2) { + (*numOfNull)++; + } + } + break; + default: + ASSERT(0); + break; + } +} + void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull) = { NULL, tColDataCalcSMABool, // TSDB_DATA_TYPE_BOOL @@ -3532,14 +3572,14 @@ void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_ tColDataCalcSMABigInt, // TSDB_DATA_TYPE_BIGINT tColDataCalcSMAFloat, // TSDB_DATA_TYPE_FLOAT tColDataCalcSMADouble, // TSDB_DATA_TYPE_DOUBLE - NULL, // TSDB_DATA_TYPE_VARCHAR + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_VARCHAR tColDataCalcSMABigInt, // TSDB_DATA_TYPE_TIMESTAMP - NULL, // TSDB_DATA_TYPE_NCHAR + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_NCHAR tColDataCalcSMAUTinyInt, // TSDB_DATA_TYPE_UTINYINT tColDataCalcSMATinyUSmallInt, // TSDB_DATA_TYPE_USMALLINT tColDataCalcSMAUInt, // TSDB_DATA_TYPE_UINT tColDataCalcSMAUBigInt, // TSDB_DATA_TYPE_UBIGINT - NULL, // TSDB_DATA_TYPE_JSON + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_JSON NULL, // TSDB_DATA_TYPE_VARBINARY NULL, // TSDB_DATA_TYPE_DECIMAL NULL, // TSDB_DATA_TYPE_BLOB diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c5646d92d1..b18440bc56 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -142,8 +142,8 @@ int32_t tsCompressColData = -1; // count/hyperloglog function always return values in case of all NULL data or Empty data set. int32_t tsCountAlwaysReturnValue = 1; -// 10 ms for sliding time, the value will changed in case of time precision changed -int32_t tsMinSlidingTime = 10; +// 1 ms for sliding time, the value will changed in case of time precision changed +int32_t tsMinSlidingTime = 1; // the maxinum number of distict query result int32_t tsMaxNumOfDistinctResults = 1000 * 10000; @@ -151,8 +151,8 @@ int32_t tsMaxNumOfDistinctResults = 1000 * 10000; // 1 database precision unit for interval time range, changed accordingly int32_t tsMinIntervalTime = 1; -// maximum memory allowed to be allocated for a single csv load (in MB) -int32_t tsMaxMemUsedByInsert = 1024; +// maximum batch rows numbers imported from a single csv load +int32_t tsMaxInsertBatchRows = 1000000; float tsSelectivityRatio = 1.0; int32_t tsTagFilterResCacheSize = 1024 * 10; @@ -346,7 +346,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1; // if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; // if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1; - if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1; + if (cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, true) != 0) return -1; if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1; if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1; @@ -422,7 +422,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 10, 1000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1; @@ -776,7 +776,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { // tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; // tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32; - tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32; + tsMaxInsertBatchRows = cfgGetItem(pCfg, "maxInsertBatchRows")->i32; tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; @@ -1048,7 +1048,7 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { } else if (strcasecmp("maxNumOfDistinctRes", name) == 0) { tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; } else if (strcasecmp("maxMemUsedByInsert", name) == 0) { - tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32; + tsMaxInsertBatchRows = cfgGetItem(pCfg, "maxInsertBatchRows")->i32; } else if (strcasecmp("maxRetryWaitTime", name) == 0) { tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; } diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index dcd539bd91..d8c43747f7 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -82,6 +82,7 @@ static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_ static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); static char* forwardToTimeStringEnd(char* str); static bool checkTzPresent(const char* str, int32_t len); +static int32_t parseTimezone(char* str, int64_t* tzOffset); static int32_t (*parseLocaltimeFp[])(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) = { parseLocaltime, parseLocaltimeDst}; @@ -92,13 +93,13 @@ int32_t taosParseTime(const char* timestr, int64_t* utime, int32_t len, int32_t if (checkTzPresent(timestr, len)) { return parseTimeWithTz(timestr, utime, timePrec, 'T'); } else { - return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 'T'); + return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 'T'); } } else { if (checkTzPresent(timestr, len)) { return parseTimeWithTz(timestr, utime, timePrec, 0); } else { - return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 0); + return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 0); } } } @@ -713,16 +714,12 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { return t; } - if (unit != 'n' && unit != 'y') { + if (!IS_CALENDAR_TIME_DURATION(unit)) { return t + duration; } // The following code handles the y/n time duration - int64_t numOfMonth = duration; - if (unit == 'y') { - numOfMonth *= 12; - } - + int64_t numOfMonth = (unit == 'y')? duration*12:duration; int64_t fraction = t % TSDB_TICK_PER_SECOND(precision); struct tm tm; @@ -741,6 +738,7 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char ekey = skey; skey = tmp; } + if (unit != 'n' && unit != 'y') { return (int32_t)((ekey - skey) / interval); } @@ -764,13 +762,16 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char return (emon - smon) / (int32_t)interval; } -int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision) { +int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { if (pInterval->sliding == 0 && pInterval->interval == 0) { - return t; + return ts; } - int64_t start = t; - if (pInterval->slidingUnit == 'n' || pInterval->slidingUnit == 'y') { + int64_t start = ts; + int32_t precision = pInterval->precision; + + if (IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) { + start /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; time_t tt = (time_t)start; @@ -792,44 +793,72 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio start = (int64_t)(taosMktime(&tm) * TSDB_TICK_PER_SECOND(precision)); } else { - int64_t delta = t - pInterval->interval; - int32_t factor = (delta >= 0) ? 1 : -1; + if (IS_CALENDAR_TIME_DURATION(pInterval->intervalUnit)) { + int64_t news = (ts / pInterval->sliding) * pInterval->sliding; + ASSERT(news <= ts); - start = (delta / pInterval->sliding + factor) * pInterval->sliding; + if (news <= ts) { + int64_t prev = news; + int64_t newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; - if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { - /* - * here we revised the start time of day according to the local time zone, - * but in case of DST, the start time of one day need to be dynamically decided. - */ - // todo refactor to extract function that is available for Linux/Windows/Mac platform -#if defined(WINDOWS) && _MSC_VER >= 1900 - // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 - int64_t timezone = _timezone; - int32_t daylight = _daylight; - char** tzname = _tzname; -#endif + if (newe < ts) { // move towards the greater endpoint + while(newe < ts && news < ts) { + news += pInterval->sliding; + newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } - start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); - } - - int64_t end = 0; - - // not enough time range - if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { - end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - while (end < t) { // move forward to the correct time window - start += pInterval->sliding; - - if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { - end = start + pInterval->interval - 1; + prev = news; } else { - end = INT64_MAX; - break; + while (newe >= ts) { + prev = news; + news -= pInterval->sliding; + newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } } + + return prev; } } else { - end = INT64_MAX; + int64_t delta = ts - pInterval->interval; + int32_t factor = (delta >= 0) ? 1 : -1; + + start = (delta / pInterval->sliding + factor) * pInterval->sliding; + + if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { + /* + * here we revised the start time of day according to the local time zone, + * but in case of DST, the start time of one day need to be dynamically decided. + */ + // todo refactor to extract function that is available for Linux/Windows/Mac platform +#if defined(WINDOWS) && _MSC_VER >= 1900 + // see + // https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 + int64_t timezone = _timezone; + int32_t daylight = _daylight; + char** tzname = _tzname; +#endif + + start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); + } + + int64_t end = 0; + + // not enough time range + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { + end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; + while (end < ts) { // move forward to the correct time window + start += pInterval->sliding; + + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { + end = start + pInterval->interval - 1; + } else { + end = INT64_MAX; + break; + } + } + } else { + end = INT64_MAX; + } } } @@ -841,10 +870,10 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio // try to move current window to the left-hande-side, due to the offset effect. int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - int64_t newEnd = end; - while (newEnd >= t) { - end = newEnd; - newEnd = taosTimeAdd(newEnd, -pInterval->sliding, pInterval->slidingUnit, precision); + int64_t newe = end; + while (newe >= ts) { + end = newe; + newe = taosTimeAdd(newe, -pInterval->sliding, pInterval->slidingUnit, precision); } start = taosTimeAdd(end, -pInterval->interval, pInterval->intervalUnit, precision) + 1; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index a5f53c8703..e41c9e10d7 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -78,7 +78,7 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, pMsg); if (code != 0) { if (terrno != 0) code = terrno; - dGError("vgId:%d, msg:%p failed to query since %s", pVnode->vgId, pMsg, terrstr(code)); + dGError("vgId:%d, msg:%p failed to query since %s", pVnode->vgId, pMsg, tstrerror(code)); vmSendRsp(pMsg, code); } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index a451ae9df5..c6586e0396 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -929,7 +929,7 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) { cfgRsp->walRetentionSize = pDb->cfg.walRetentionSize; cfgRsp->walSegmentSize = pDb->cfg.walSegmentSize; cfgRsp->numOfRetensions = pDb->cfg.numOfRetensions; - cfgRsp->pRetensions = pDb->cfg.pRetensions; + cfgRsp->pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); cfgRsp->schemaless = pDb->cfg.schemaless; cfgRsp->sstTrigger = pDb->cfg.sstTrigger; } @@ -972,6 +972,8 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { _OVER: + tFreeSDbCfgRsp(&cfgRsp); + if (code != 0) { mError("db:%s, failed to get cfg since %s", cfgReq.db, terrstr()); } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index fbe2f4809c..def47187f2 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -149,7 +149,7 @@ int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int3 void tsdbReaderSetId(STsdbReader *pReader, const char *idstr); void tsdbReaderClose(STsdbReader *pReader); int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); -int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); +int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); void tsdbReleaseDataBlock(STsdbReader *pReader); SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index a7c2c6912e..4ba8d6d69f 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -98,18 +98,17 @@ typedef enum tq_handle_status { } tq_handle_status; typedef struct { - char subKey[TSDB_SUBSCRIBE_KEY_LEN]; - int64_t consumerId; - int32_t epoch; - int8_t fetchMeta; - int64_t snapshotVer; - SWalReader* pWalReader; - SWalRef* pRef; - // STqPushHandle pushHandle; // push - STqExecHandle execHandle; // exec - SRpcMsg* msg; - int32_t noDataPollCnt; - tq_handle_status status; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + int64_t consumerId; + int32_t epoch; + int8_t fetchMeta; + int64_t snapshotVer; + SWalReader* pWalReader; + SWalRef* pRef; +// STqPushHandle pushHandle; // push + STqExecHandle execHandle; // exec + SRpcMsg* msg; + tq_handle_status status; } STqHandle; struct STQ { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 726ed6a4c8..fa9eea5e29 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -317,42 +317,6 @@ _query: tDecoderClear(&dc); goto _exit; } - { // Traverse to find the previous qualified data - TBC *pCur; - tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); - STbDbKey key = {.version = sver, .uid = INT64_MAX}; - int c = 0; - tdbTbcMoveTo(pCur, &key, sizeof(key), &c); - if (c < 0) { - tdbTbcMoveToPrev(pCur); - } - - void *pKey = NULL; - void *pVal = NULL; - int vLen = 0, kLen = 0; - while (1) { - int32_t ret = tdbTbcPrev(pCur, &pKey, &kLen, &pVal, &vLen); - if (ret < 0) break; - - STbDbKey *tmp = (STbDbKey *)pKey; - if (tmp->uid != uid) { - continue; - } - SDecoder dcNew = {0}; - SMetaEntry meNew = {0}; - tDecoderInit(&dcNew, pVal, vLen); - metaDecodeEntry(&dcNew, &meNew); - pSchema = tCloneSSchemaWrapper(&meNew.stbEntry.schemaRow); - tDecoderClear(&dcNew); - tdbTbcClose(pCur); - tdbFree(pKey); - tdbFree(pVal); - goto _exit; - } - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCur); - } } else if (me.type == TSDB_CHILD_TABLE) { uid = me.ctbEntry.suid; tDecoderClear(&dc); @@ -377,7 +341,6 @@ _query: tDecoderClear(&dc); _exit: - tDecoderClear(&dc); if (lock) { metaULock(pMeta); } @@ -385,7 +348,6 @@ _exit: return pSchema; _err: - tDecoderClear(&dc); if (lock) { metaULock(pMeta); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7d75b3fb9a..b35672ecb6 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -23,8 +23,8 @@ static int32_t tqInitialize(STQ* pTq); static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_STATUS_EXEC == pHandle->status; } -static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_EXEC;} -static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_IDLE;} +static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_EXEC; } +static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_IDLE; } int32_t tqInit() { int8_t old; @@ -78,7 +78,7 @@ static void destroyTqHandle(void* data) { taosMemoryFreeClear(pData->execHandle.execTb.qmsg); nodesDestroyNode(pData->execHandle.execTb.node); } - if(pData->msg != NULL) { + if (pData->msg != NULL) { rpcFreeCont(pData->msg->pCont); taosMemoryFree(pData->msg); pData->msg = NULL; @@ -240,14 +240,15 @@ int32_t tqPushDataRsp(STqHandle* pHandle, int32_t vgId) { int64_t sver = 0, ever = 0; walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); - tqDoSendDataRsp(&pHandle->msg->info, &dataRsp, pHandle->epoch, pHandle->consumerId, TMQ_MSG_TYPE__POLL_RSP, sver, ever); + tqDoSendDataRsp(&pHandle->msg->info, &dataRsp, pHandle->epoch, pHandle->consumerId, TMQ_MSG_TYPE__POLL_RSP, sver, + ever); char buf1[80] = {0}; char buf2[80] = {0}; tFormatOffset(buf1, tListLen(buf1), &dataRsp.reqOffset); tFormatOffset(buf2, tListLen(buf2), &dataRsp.rspOffset); - tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) push rsp, block num: %d, req:%s, rsp:%s", - vgId, dataRsp.head.consumerId, dataRsp.head.epoch, dataRsp.blockNum, buf1, buf2); + tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) push rsp, block num: %d, req:%s, rsp:%s", vgId, + dataRsp.head.consumerId, dataRsp.head.epoch, dataRsp.blockNum, buf1, buf2); return 0; } @@ -263,8 +264,8 @@ int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* tFormatOffset(buf1, 80, &pRsp->reqOffset); tFormatOffset(buf2, 80, &pRsp->rspOffset); - tqDebug("vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64, - vgId, pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId); + tqDebug("vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64, vgId, + pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId); return 0; } @@ -336,8 +337,7 @@ int32_t tqProcessSeekReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) STqHandle* pHandle = taosHashGet(pTq->pHandle, pOffset->subKey, strlen(pOffset->subKey)); if (pHandle == NULL) { - tqError("tmq seek: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", vgOffset.consumerId, vgId, - pOffset->subKey); + tqError("tmq seek: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", vgOffset.consumerId, vgId, pOffset->subKey); terrno = TSDB_CODE_INVALID_MSG; return -1; } @@ -353,7 +353,7 @@ int32_t tqProcessSeekReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) } taosRUnLockLatch(&pTq->lock); - //3. check the offset info + // 3. check the offset info STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); if (pSavedOffset != NULL) { if (pSavedOffset->val.type != TMQ_OFFSET__LOG) { @@ -381,7 +381,7 @@ int32_t tqProcessSeekReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) tqDebug("vgId:%d sub:%s seek to:%" PRId64 " prev offset:%" PRId64, vgId, pOffset->subKey, pOffset->val.version, pSavedOffset->val.version); } else { - tqDebug("vgId:%d sub:%s seek to:%"PRId64" not saved yet", vgId, pOffset->subKey, pOffset->val.version); + tqDebug("vgId:%d sub:%s seek to:%" PRId64 " not saved yet", vgId, pOffset->subKey, pOffset->val.version); } if (tqOffsetWrite(pTq->pOffsetStore, pOffset) < 0) { @@ -423,6 +423,7 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { SMqPollReq req = {0}; + int code = 0; if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen); terrno = TSDB_CODE_INVALID_MSG; @@ -433,37 +434,49 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { int32_t reqEpoch = req.epoch; STqOffsetVal reqOffset = req.reqOffset; int32_t vgId = TD_VID(pTq->pVnode); + STqHandle* pHandle = NULL; - taosWLockLatch(&pTq->lock); - // 1. find handle - STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); - if (pHandle == NULL) { - tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); - terrno = TSDB_CODE_INVALID_MSG; + while (1) { + taosWLockLatch(&pTq->lock); + // 1. find handle + pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); + if (pHandle == NULL) { + tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_MSG; + taosWUnLockLatch(&pTq->lock); + return -1; + } + + // 2. check re-balance status + if (pHandle->consumerId != consumerId) { + tqError("ERROR tmq poll: consumer:0x%" PRIx64 + " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, + consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + taosWUnLockLatch(&pTq->lock); + return -1; + } + + bool exec = tqIsHandleExec(pHandle); + if (!exec) { + tqSetHandleExec(pHandle); + // qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, + req.subKey, pHandle); + taosWUnLockLatch(&pTq->lock); + break; + } taosWUnLockLatch(&pTq->lock); - return -1; - } - while (tqIsHandleExec(pHandle)) { - tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", consumerId, vgId, req.subKey); - taosMsleep(5); + tqDebug("tmq poll: consumer:0x%" PRIx64 + "vgId:%d, topic:%s, subscription is executing, wait for 10ms and retry, pHandle:%p", + consumerId, vgId, req.subKey, pHandle); + taosMsleep(10); } - // 2. check re-balance status - if (pHandle->consumerId != consumerId) { - tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, - consumerId, vgId, req.subKey, pHandle->consumerId); - terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; - taosWUnLockLatch(&pTq->lock); - return -1; - } - tqSetHandleExec(pHandle); - taosWUnLockLatch(&pTq->lock); - // 3. update the epoch value - int32_t savedEpoch = pHandle->epoch; - if (savedEpoch < reqEpoch) { - tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, + if (pHandle->epoch < reqEpoch) { + tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, pHandle->epoch, reqEpoch); pHandle->epoch = reqEpoch; } @@ -473,8 +486,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); - int code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); + code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); tqSetHandleIdle(pHandle); + + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, + req.subKey, pHandle); return code; } @@ -538,7 +554,7 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { if (reqOffset.type == TMQ_OFFSET__LOG) { int64_t currentVer = walReaderGetCurrentVer(pHandle->execHandle.pTqReader->pWalReader); - if (currentVer == -1) { // not start to read data from wal yet, return req offset directly + if (currentVer == -1) { // not start to read data from wal yet, return req offset directly dataRsp.rspOffset.version = reqOffset.version; } else { dataRsp.rspOffset.version = currentVer; // return current consume offset value @@ -562,7 +578,7 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; - int32_t vgId = TD_VID(pTq->pVnode); + int32_t vgId = TD_VID(pTq->pVnode); tqDebug("vgId:%d, tq process delete sub req %s", vgId, pReq->subKey); int32_t code = 0; @@ -571,8 +587,9 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); if (pHandle) { while (tqIsHandleExec(pHandle)) { - tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", vgId, pHandle->subKey); - taosMsleep(5); + tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 10ms and retry, pHandle:%p", vgId, + pHandle->subKey, pHandle); + taosMsleep(10); } if (pHandle->pRef) { @@ -631,9 +648,9 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t } int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - int ret = 0; + int ret = 0; SMqRebVgReq req = {0}; - SDecoder dc = {0}; + SDecoder dc = {0}; tDecoderInit(&dc, msg, msgLen); @@ -650,7 +667,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqDebug("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pVnode->config.vgId, req.subKey, req.oldConsumerId, req.newConsumerId); - taosWLockLatch(&pTq->lock); STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); if (pHandle == NULL) { if (req.oldConsumerId != -1) { @@ -689,7 +705,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->snapshotVer = ver; if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - pHandle->execHandle.execCol.qmsg = taosStrdup(req.qmsg);; + pHandle->execHandle.execCol.qmsg = taosStrdup(req.qmsg); + ; pHandle->execHandle.task = qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, vgId, &pHandle->execHandle.numOfCols, req.newConsumerId); @@ -711,7 +728,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->execHandle.execTb.suid = req.suid; pHandle->execHandle.execTb.qmsg = taosStrdup(req.qmsg); - if(strcmp(pHandle->execHandle.execTb.qmsg, "") != 0) { + if (strcmp(pHandle->execHandle.execTb.qmsg, "") != 0) { if (nodesStringToNode(pHandle->execHandle.execTb.qmsg, &pHandle->execHandle.execTb.node) != 0) { tqError("nodesStringToNode error in sub stable, since %s, vgId:%d, subkey:%s consumer:0x%" PRIx64, terrstr(), pVnode->config.vgId, req.subKey, pHandle->consumerId); @@ -725,61 +742,59 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg SArray* tbUidList = NULL; ret = qGetTableList(req.suid, pVnode, pHandle->execHandle.execTb.node, &tbUidList, pHandle->execHandle.task); - if(ret != TDB_CODE_SUCCESS) { - tqError("qGetTableList error:%d vgId:%d, subkey:%s consumer:0x%" PRIx64, ret, pVnode->config.vgId, req.subKey, pHandle->consumerId); + if (ret != TDB_CODE_SUCCESS) { + tqError("qGetTableList error:%d vgId:%d, subkey:%s consumer:0x%" PRIx64, ret, pVnode->config.vgId, req.subKey, + pHandle->consumerId); taosArrayDestroy(tbUidList); goto end; } - tqDebug("tq try to get ctb for stb subscribe, vgId:%d, subkey:%s consumer:0x%" PRIx64 " suid:%" PRId64, pVnode->config.vgId, req.subKey, pHandle->consumerId, req.suid); + tqDebug("tq try to get ctb for stb subscribe, vgId:%d, subkey:%s consumer:0x%" PRIx64 " suid:%" PRId64, + pVnode->config.vgId, req.subKey, pHandle->consumerId, req.suid); pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); } taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); - tqDebug("try to persist handle %s consumer:0x%" PRIx64, req.subKey, - pHandle->consumerId); + tqDebug("try to persist handle %s consumer:0x%" PRIx64, req.subKey, pHandle->consumerId); ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; } else { - while (tqIsHandleExec(pHandle)) { - tqDebug("sub req vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", vgId, pHandle->subKey); - taosMsleep(5); - } + taosWLockLatch(&pTq->lock); if (pHandle->consumerId == req.newConsumerId) { // do nothing - tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs", req.vgId, req.newConsumerId); - atomic_add_fetch_32(&pHandle->epoch, 1); - + tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs, should not reach here", req.vgId, + req.newConsumerId); } else { tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); - atomic_store_32(&pHandle->epoch, 0); } + // atomic_add_fetch_32(&pHandle->epoch, 1); + // kill executing task - qTaskInfo_t pTaskInfo = pHandle->execHandle.task; - if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); - } - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pTaskInfo); - } + // if(tqIsHandleExec(pHandle)) { + // qTaskInfo_t pTaskInfo = pHandle->execHandle.task; + // if (pTaskInfo != NULL) { + // qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); + // } + + // if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + // qStreamCloseTsdbReader(pTaskInfo); + // } + // } // remove if it has been register in the push manager, and return one empty block to consumer tqUnregisterPushHandle(pTq, pHandle); + taosWUnLockLatch(&pTq->lock); ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); - goto end; } end: - taosWUnLockLatch(&pTq->lock); - tDecoderClear(&dc); + taosMemoryFree(req.qmsg); return ret; } -void freePtr(void *ptr) { - taosMemoryFree(*(void**)ptr); -} +void freePtr(void* ptr) { taosMemoryFree(*(void**)ptr); } int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { int32_t vgId = TD_VID(pTq->pVnode); @@ -802,7 +817,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->chkInfo.currentVer = ver; // expand executor - pTask->status.taskStatus = (pTask->fillHistory)? TASK_STATUS__WAIT_DOWNSTREAM:TASK_STATUS__NORMAL; + pTask->status.taskStatus = (pTask->fillHistory) ? TASK_STATUS__WAIT_DOWNSTREAM : TASK_STATUS__NORMAL; if (pTask->taskLevel == TASK_LEVEL__SOURCE) { pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); @@ -868,8 +883,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { streamSetupTrigger(pTask); - tqInfo("vgId:%d expand stream task, s-task:%s, checkpoint ver:%" PRId64 " child id:%d, level:%d", vgId, pTask->id.idStr, - pTask->chkInfo.version, pTask->selfChildId, pTask->taskLevel); + tqInfo("vgId:%d expand stream task, s-task:%s, checkpoint ver:%" PRId64 " child id:%d, level:%d", vgId, + pTask->id.idStr, pTask->chkInfo.version, pTask->selfChildId, pTask->taskLevel); // next valid version will add one pTask->chkInfo.version += 1; @@ -982,7 +997,8 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("vgId:%d failed to create stream task due to out of memory, alloc size:%d", vgId, (int32_t) sizeof(SStreamTask)); + tqError("vgId:%d failed to create stream task due to out of memory, alloc size:%d", vgId, + (int32_t)sizeof(SStreamTask)); return -1; } @@ -1097,7 +1113,7 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t // do recovery step 2 int64_t st = taosGetTimestampMs(); - tqDebug("s-task:%s start step2 recover, ts:%"PRId64, pTask->id.idStr, st); + tqDebug("s-task:%s start step2 recover, ts:%" PRId64, pTask->id.idStr, st); code = streamSourceRecoverScanStep2(pTask, sversion); if (code < 0) { @@ -1105,7 +1121,7 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t return -1; } - qDebug("s-task:%s set the start wal offset to be:%"PRId64, pTask->id.idStr, sversion); + qDebug("s-task:%s set the start wal offset to be:%" PRId64, pTask->id.idStr, sversion); walReaderSeekVer(pTask->exec.pWalReader, sversion); pTask->chkInfo.currentVer = sversion; @@ -1129,7 +1145,7 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t return -1; } - double el = (taosGetTimestampMs() - st)/ 1000.0; + double el = (taosGetTimestampMs() - st) / 1000.0; tqDebug("s-task:%s step2 recover finished, el:%.2fs", pTask->id.idStr, el); // dispatch recover finish req to all related downstream task @@ -1245,8 +1261,8 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); if (pTask != NULL) { if (pTask->status.taskStatus == TASK_STATUS__NORMAL) { - tqDebug("vgId:%d s-task:%s start to process block from wal, last chk point:%" PRId64, vgId, - pTask->id.idStr, pTask->chkInfo.version); + tqDebug("vgId:%d s-task:%s start to process block from wal, last chk point:%" PRId64, vgId, pTask->id.idStr, + pTask->chkInfo.version); streamProcessRunReq(pTask); } else { if (streamTaskShouldPause(&pTask->status)) { @@ -1265,9 +1281,9 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { - char* msgStr = pMsg->pCont; - char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + char* msgStr = pMsg->pCont; + char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); SStreamDispatchReq req = {0}; @@ -1313,7 +1329,7 @@ int32_t tqProcessTaskDropReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgL int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)msg; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->taskId); + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->taskId); if (pTask) { tqDebug("vgId:%d s-task:%s set pause flag", pTq->pStreamMeta->vgId, pTask->id.idStr); atomic_store_8(&pTask->status.keepTaskStatus, pTask->status.taskStatus); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 239d65fdb7..f7cda69d2c 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -366,9 +366,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { taosArrayDestroy(tbUidList); } tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, vgId); - taosWLockLatch(&pTq->lock); taosHashPut(pTq->pHandle, pKey, kLen, &handle, sizeof(STqHandle)); - taosWUnLockLatch(&pTq->lock); } end: diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 377a5d1887..0a9905b544 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -78,7 +78,6 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { // todo remove this if (offset.val.type == TMQ_OFFSET__LOG) { - taosWLockLatch(&pStore->pTq->lock); STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey)); if (pHandle) { if (walSetRefVer(pHandle->pRef, offset.val.version) < 0) { @@ -86,7 +85,6 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { // offset.val.version); } } - taosWUnLockLatch(&pStore->pTq->lock); } taosMemoryFree(pMemBuf); diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 7f4fe48b8e..4c2e19dcfb 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -101,8 +101,11 @@ int32_t tqUnregisterPushHandle(STQ* pTq, void *handle) { STqHandle *pHandle = (STqHandle*)handle; int32_t vgId = TD_VID(pTq->pVnode); + if(taosHashGetSize(pTq->pPushMgr) <= 0) { + return 0; + } int32_t ret = taosHashRemove(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey)); - tqError("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); + tqDebug("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); if(pHandle->msg != NULL) { tqPushDataRsp(pHandle, vgId); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 6ef232995a..6cefe4d18f 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1059,7 +1059,6 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { int32_t vgId = TD_VID(pTq->pVnode); // update the table list for each consumer handle - taosWLockLatch(&pTq->lock); while (1) { pIter = taosHashIterate(pTq->pHandle, pIter); if (pIter == NULL) { @@ -1097,7 +1096,6 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } } } - taosWUnLockLatch(&pTq->lock); // update the table list handle for each stream scanner/wal reader taosWLockLatch(&pTq->pStreamMeta->lock); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 98304f9c18..1ff78c586f 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -84,8 +84,10 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs qStreamSetOpen(task); tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId); - if (qExecTask(task, &pDataBlock, &ts) != TSDB_CODE_SUCCESS) { - tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, terrstr()); + code = qExecTask(task, &pDataBlock, &ts); + if (code != TSDB_CODE_SUCCESS) { + tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, tstrerror(code)); + terrno = code; return -1; } @@ -128,8 +130,10 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; tqDebug("tmqsnap task start to execute"); - if (qExecTask(task, &pDataBlock, &ts) < 0) { - tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr()); + int code = qExecTask(task, &pDataBlock, &ts); + if (code != 0) { + tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, tstrerror(code)); + terrno = code; return -1; } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 6e02d5e21b..db1b5ed902 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -294,6 +294,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d char* ctbName = pDataBlock->info.parTbName; if (!ctbName[0]) { + memset(ctbName, 0, TSDB_TABLE_NAME_LEN); if (res == TSDB_CODE_SUCCESS) { memcpy(ctbName, pTableSinkInfo->tbName, strlen(pTableSinkInfo->tbName)); } else { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index b5308d8b98..a34e765e50 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -15,8 +15,7 @@ #include "tq.h" -#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) -#define NO_POLL_CNT 5 +#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, int32_t vgId); @@ -89,15 +88,13 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { return 0; } -static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, - SRpcMsg* pMsg, bool* pBlockReturned) { +static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, bool* pBlockReturned) { uint64_t consumerId = pRequest->consumerId; STqOffsetVal reqOffset = pRequest->reqOffset; STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); int32_t vgId = TD_VID(pTq->pVnode); *pBlockReturned = false; - // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value. if (pOffset != NULL) { *pOffsetVal = pOffset->val; @@ -121,21 +118,16 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand tqOffsetResetToData(pOffsetVal, 0, 0); } } else { - pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); - if (pHandle->pRef == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - // offset set to previous version when init + walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); } } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { + walRefLastVer(pTq->pVnode->pWal, pHandle->pRef); if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { SMqDataRsp dataRsp = {0}; tqInitDataRsp(&dataRsp, pRequest); - tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); + tqOffsetResetToLog(&dataRsp.rspOffset, pHandle->pRef->refVer); tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, pHandle->subKey, vgId, dataRsp.rspOffset.version); int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId); @@ -146,7 +138,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand } else { STaosxRsp taosxRsp = {0}; tqInitTaosxRsp(&taosxRsp, pRequest); - tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); + tqOffsetResetToLog(&taosxRsp.rspOffset, pHandle->pRef->refVer); int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); tDeleteSTaosxRsp(&taosxRsp); @@ -176,24 +168,18 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); code = tqScanData(pTq, pHandle, &dataRsp, pOffset); - if (code != 0) { + if(code != 0 && terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { goto end; } // till now, all data has been transferred to consumer, new data needs to push client once arrived. - if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && - dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { - if (pHandle->noDataPollCnt >= NO_POLL_CNT) { // send poll result to client if no data 5 times to avoid lost data - pHandle->noDataPollCnt = 0; - // lock - taosWLockLatch(&pTq->lock); - code = tqRegisterPushHandle(pTq, pHandle, pMsg); - taosWUnLockLatch(&pTq->lock); - tDeleteMqDataRsp(&dataRsp); - return code; - } else { - pHandle->noDataPollCnt++; - } + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.blockNum == 0) { + // lock + taosWLockLatch(&pTq->lock); + code = tqRegisterPushHandle(pTq, pHandle, pMsg); + taosWUnLockLatch(&pTq->lock); + tDeleteMqDataRsp(&dataRsp); + return code; } // NOTE: this pHandle->consumerId may have been changed already. diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 5df1ea0672..6138b1f7b4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -201,6 +201,9 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, void* tsdbCacherowsReaderClose(void* pReader) { SCacheRowsReader* p = pReader; + if (p == NULL) { + return NULL; + } if (p->pSchema != NULL) { for (int32_t i = 0; i < p->pSchema->numOfCols; ++i) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 7af0936bb5..9cae4ad5aa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -5030,7 +5030,8 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { return code; } -static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { +static bool doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { + bool hasNullSMA = false; // do fill all null column value SMA info int32_t i = 0, j = 0; int32_t size = (int32_t)taosArrayGetSize(pSup->pColAgg); @@ -5050,6 +5051,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ taosArrayInsert(pSup->pColAgg, i, &nullColAgg); i += 1; size++; + hasNullSMA = true; } j += 1; } @@ -5060,12 +5062,15 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; taosArrayInsert(pSup->pColAgg, i, &nullColAgg); i += 1; + hasNullSMA = true; } j++; } + + return hasNullSMA; } -int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave) { +int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool *hasNullSMA) { SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg; int32_t code = 0; @@ -5129,7 +5134,10 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, } // do fill all null column value SMA info - doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg); + if (doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg)) { + *hasNullSMA = true; + return TSDB_CODE_SUCCESS; + } size_t size = taosArrayGetSize(pSup->pColAgg); int32_t i = 0, j = 0; @@ -5296,6 +5304,9 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { } static int32_t getBucketIndex(int32_t startRow, int32_t bucketRange, int32_t numOfRows, int32_t numOfBucket) { + if (numOfRows < startRow) { + return 0; + } int32_t bucketIndex = ((numOfRows - startRow) / bucketRange); if (bucketIndex == numOfBucket) { bucketIndex -= 1; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 50fd9d7aa7..b25ab393da 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -523,7 +523,7 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type) || ((pColData->flag & HAS_VALUE) == 0)) continue; + if ((!pColData->smaOn) || ((pColData->flag & HAS_VALUE) == 0)) continue; SColumnDataAgg sma = {.colId = pColData->cid}; tColDataCalcSMA[pColData->type](pColData, &sma.sum, &sma.max, &sma.min, &sma.numOfNull); diff --git a/source/libs/cache/test/cacheTests.cpp b/source/libs/cache/test/cacheTests.cpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index cf864e8643..c83503dcfe 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1562,7 +1562,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { switch (pReq->type) { case AUTH_TYPE_READ: { - if (pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) { + if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) { req->singleType = AUTH_TYPE_READ; CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); if (pRes->pass || res->metaNotExists) { @@ -1578,7 +1578,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { break; } case AUTH_TYPE_WRITE: { - if (pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) { + if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) { req->singleType = AUTH_TYPE_WRITE; CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); if (pRes->pass || res->metaNotExists) { diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 966c5fa2e4..30911c6061 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -135,6 +135,10 @@ struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t i static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) { SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); + if (!bufPage) { + uFatal("failed to get the buffer page:%d since %s", pos->pageId, terrstr()); + return NULL; + } if (forUpdate) { setBufPageDirty(bufPage, true); } @@ -182,4 +186,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, void printDataBlock(SSDataBlock* pBlock, const char* flag); +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order); +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery); + #endif // TDENGINE_EXECUTIL_H diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index 78b3cd2f40..79837480d7 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -120,6 +120,8 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); +void taosFillUpdateStartTimestampInfo(SFillInfo* pFillInfo, int64_t ts); +bool taosFillNotStarted(const SFillInfo* pFillInfo); SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr, int32_t numOfNotFillCols, const struct SNodeListNode* val); bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 3752a56b92..331a2fa7ab 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -136,7 +136,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in size_t keyLen = 0; int32_t iter = 0; - int32_t bufLen = 0, offset = 0; + int64_t bufLen = 0, offset = 0; // todo move away and record this during create window while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { @@ -1761,12 +1761,12 @@ int32_t convertFillType(int32_t mode) { return type; } -static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) { +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) { if (ascQuery) { - *w = getAlignQueryTimeWindow(pInterval, pInterval->precision, ts); + *w = getAlignQueryTimeWindow(pInterval, ts); } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - *w = getAlignQueryTimeWindow(pInterval, pInterval->precision, ts); + *w = getAlignQueryTimeWindow(pInterval, ts); int64_t key = w->skey; while (key < ts) { // moving towards end @@ -1783,7 +1783,7 @@ static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindo static STimeWindow doCalculateTimeWindow(int64_t ts, SInterval* pInterval) { STimeWindow w = {0}; - w.skey = taosTimeTruncate(ts, pInterval, pInterval->precision); + w.skey = taosTimeTruncate(ts, pInterval); w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; return w; } @@ -1817,6 +1817,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI if (pRow) { w = pRow->win; } + // in case of typical time window, we can calculate time window directly. if (w.skey > ts || w.ekey < ts) { w = doCalculateTimeWindow(ts, pInterval); @@ -1831,6 +1832,34 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI return w; } +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); + if (!IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) { + tw->skey += pInterval->sliding * factor; + tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + return; + } + + // convert key to second + int64_t key = convertTimePrecision(tw->skey, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000; + + int64_t duration = pInterval->sliding; + if (pInterval->slidingUnit == 'y') { + duration *= 12; + } + + struct tm tm; + time_t t = (time_t) key; + taosLocalTime(&t, &tm, NULL); + + int mon = (int)(tm.tm_year * 12 + tm.tm_mon + duration * factor); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); + + tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; +} + bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo) { return (pLimitInfo->limit.limit != -1 || pLimitInfo->limit.offset != -1 || pLimitInfo->slimit.limit != -1 || pLimitInfo->slimit.offset != -1); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index ed8a4f3a3b..74af5c183f 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -186,6 +186,11 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { doSetTaskId(pTaskInfo->pRoot, &pTaskInfo->storageAPI); } +//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { +// SExecTaskInfo* pTaskInfo = tinfo; +// pTaskInfo->code = code; +//} + int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { return TSDB_CODE_APP_ERROR; diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index e18906ad18..4f1a0254e4 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -442,15 +442,15 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB } ///////////////////////////////////////////////////////////////////////////////////////////// -STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key) { +STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key) { STimeWindow win = {0}; - win.skey = taosTimeTruncate(key, pInterval, precision); + win.skey = taosTimeTruncate(key, pInterval); /* * if the realSkey > INT64_MAX - pInterval->interval, the query duration between * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. */ - win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; if (win.ekey < win.skey) { win.ekey = INT64_MAX; } @@ -726,7 +726,8 @@ void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultR pCtx[j].resultInfo->numOfRes = pRow->numOfRows; } } - + + blockDataEnsureCapacity(pBlock, pBlock->info.rows + pCtx[j].resultInfo->numOfRes); int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); if (TAOS_FAILED(code)) { qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); @@ -827,7 +828,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { - uint32_t newSize = pBlock->info.rows + pRow->numOfRows + (numOfRows - i) > 1 ? 1 : 0; + uint32_t newSize = pBlock->info.rows + pRow->numOfRows + ((numOfRows - i) > 1 ? 1 : 0); blockDataEnsureCapacity(pBlock, newSize); qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s", newSize, pBlock->info.capacity, GET_TASKID(pTaskInfo)); diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 519ddc1916..f9e8a32520 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -61,25 +61,28 @@ typedef struct SFillOperatorInfo { SExprSupp noFillExprSupp; } SFillOperatorInfo; +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order); static void destroyFillOperatorInfo(void* param); static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, - SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { + SResultInfo* pResultInfo, int32_t order) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; SSDataBlock* pResBlock = pInfo->pFinalRes; - int32_t order = TSDB_ORDER_ASC; +// int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; - getTableScanInfo(pOperator, &order, &scanFlag, false); - - int64_t ekey = pInfo->existNewGroupBlock->info.window.ekey; +// getTableScanInfo(pOperator, &order, &scanFlag, false); taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); blockDataCleanup(pInfo->pRes); doApplyScalarCalculation(pOperator, pInfo->existNewGroupBlock, order, scanFlag); - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ekey); + revisedFillStartKey(pInfo, pInfo->existNewGroupBlock, order); + + int64_t ts = (order == TSDB_ORDER_ASC)? pInfo->existNewGroupBlock->info.window.ekey:pInfo->existNewGroupBlock->info.window.skey; + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts); + taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); int32_t numOfResultRows = pResultInfo->capacity - pResBlock->info.rows; @@ -90,7 +93,7 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp } static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, - SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { + SResultInfo* pResultInfo, int32_t order) { if (taosFillHasMoreResults(pInfo->pFillInfo)) { int32_t numOfResultRows = pResultInfo->capacity - pInfo->pFinalRes->info.rows; taosFillResultDataBlock(pInfo->pFillInfo, pInfo->pFinalRes, numOfResultRows); @@ -100,7 +103,7 @@ static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOpera // handle the cached new group data block if (pInfo->existNewGroupBlock) { - doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, order); } } @@ -119,6 +122,53 @@ void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int pInfo->pRes->info.id.groupId = pBlock->info.id.groupId; } +// todo refactor: decide the start key according to the query time range. +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order) { + if (order == TSDB_ORDER_ASC) { + int64_t skey = pBlock->info.window.skey; + if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey); + } else if (pInfo->pFillInfo->start < skey) { + int64_t t = skey; + SInterval* pInterval = &pInfo->pFillInfo->interval; + + while(1) { + int64_t prev = taosTimeAdd(t, -pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + if (prev <= pInfo->pFillInfo->start) { + t = prev; + break; + } + t = prev; + } + + // todo time window chosen problem: t or prev value? + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t); + } + } else { + int64_t ekey = pBlock->info.window.ekey; + if (ekey > pInfo->pFillInfo->start) { + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, ekey); + } else if (ekey < pInfo->pFillInfo->start) { + int64_t t = ekey; + SInterval* pInterval = &pInfo->pFillInfo->interval; + + while(1) { + int64_t prev = taosTimeAdd(t, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + if (prev >= pInfo->pFillInfo->start) { + t = prev; + break; + } + t = prev; + } + + // todo time window chosen problem: t or prev value? + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t); + } + } +} + static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { SFillOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -132,13 +182,19 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { int32_t scanFlag = MAIN_SCAN; getTableScanInfo(pOperator, &order, &scanFlag, false); - doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo); + SOperatorInfo* pDownstream = pOperator->pDownstream[0]; + + // the scan order may be different from the output result order for agg interval operator. + if (pDownstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL) { + order = ((SIntervalAggOperatorInfo*) pDownstream->info)->resultTsOrder; + } + + doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, order); if (pResBlock->info.rows > 0) { pResBlock->info.id.groupId = pInfo->curGroupId; return pResBlock; } - SOperatorInfo* pDownstream = pOperator->pDownstream[0]; while (1) { SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); if (pBlock == NULL) { @@ -158,15 +214,16 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows); doApplyScalarCalculation(pOperator, pBlock, order, scanFlag); - if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.id.groupId) { + if (pInfo->curGroupId == 0 || (pInfo->curGroupId == pInfo->pRes->info.id.groupId)) { + if (pInfo->curGroupId == 0 && taosFillNotStarted(pInfo->pFillInfo)) { + revisedFillStartKey(pInfo, pBlock, order); + } + pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block pInfo->totalInputRows += pInfo->pRes->info.rows; - if (order == pInfo->pFillInfo->order) { - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.ekey); - } else { - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.skey); - } + int64_t ts = (order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey; + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts); taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); } else if (pInfo->curGroupId != pBlock->info.id.groupId) { // the new group data block pInfo->existNewGroupBlock = pBlock; @@ -190,7 +247,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { return pResBlock; } - doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, order); if (pResBlock->info.rows >= pOperator->resultInfo.threshold || pBlock == NULL) { pResBlock->info.id.groupId = pInfo->curGroupId; return pResBlock; @@ -198,7 +255,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { } else if (pInfo->existNewGroupBlock) { // try next group blockDataCleanup(pResBlock); - doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, order); if (pResBlock->info.rows > pResultInfo->threshold) { pResBlock->info.id.groupId = pInfo->curGroupId; return pResBlock; @@ -256,11 +313,11 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode); - int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; - STimeWindow w = getAlignQueryTimeWindow(pInterval, pInterval->precision, startKey); - w = getFirstQualifiedTimeWindow(startKey, &w, pInterval, order); + int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; - pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, +// STimeWindow w = {0}; +// getInitialStartTimeWindow(pInterval, startKey, &w, order == TSDB_ORDER_ASC); + pInfo->pFillInfo = taosCreateFillInfo(startKey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); if (order == TSDB_ORDER_ASC) { @@ -402,13 +459,13 @@ _error: TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) { STimeWindow win = {.skey = ts, .ekey = ts}; - getNextIntervalWindow(pInterval, &win, TSDB_ORDER_ASC); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); return win.skey; } TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { STimeWindow win = {.skey = ts, .ekey = ts}; - getNextIntervalWindow(pInterval, &win, TSDB_ORDER_DESC); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_DESC); return win.skey; } diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 754b5f4737..442f8162ed 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -30,11 +30,13 @@ typedef struct SJoinRowCtx { bool rowRemains; int64_t ts; SArray* leftRowLocations; - SArray* rightRowLocations; SArray* leftCreatedBlocks; SArray* rightCreatedBlocks; int32_t leftRowIdx; int32_t rightRowIdx; + + bool rightUseBuildTable; + SArray* rightRowLocations; } SJoinRowCtx; typedef struct SJoinOperatorInfo { @@ -50,7 +52,17 @@ typedef struct SJoinOperatorInfo { int32_t rightPos; SColumnInfo rightCol; SNode* pCondAfterMerge; + SNode* pColEqualOnConditions; + SArray* leftEqOnCondCols; + char* leftEqOnCondKeyBuf; + int32_t leftEqOnCondKeyLen; + + SArray* rightEqOnCondCols; + char* rightEqOnCondKeyBuf; + int32_t rightEqOnCondKeyLen; + + SSHashObj* rightBuildTable; SJoinRowCtx rowCtx; } SJoinOperatorInfo; @@ -92,6 +104,100 @@ static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDown setJoinColumnInfo(&pInfo->rightCol, rightTsCol); } +static void extractEqualOnCondColsFromOper(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstreams, SOperatorNode* pOperNode, + SColumn* pLeft, SColumn* pRight) { + SColumnNode* pLeftNode = (SColumnNode*)pOperNode->pLeft; + SColumnNode* pRightNode = (SColumnNode*)pOperNode->pRight; + if (pLeftNode->dataBlockId == pRightNode->dataBlockId || pLeftNode->dataBlockId == pDownstreams[0]->resultDataBlockId) { + *pLeft = extractColumnFromColumnNode((SColumnNode*)pOperNode->pLeft); + *pRight = extractColumnFromColumnNode((SColumnNode*)pOperNode->pRight); + } else { + *pLeft = extractColumnFromColumnNode((SColumnNode*)pOperNode->pRight); + *pRight = extractColumnFromColumnNode((SColumnNode*)pOperNode->pLeft); + } +} + +static void extractEqualOnCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownStream, SNode* pEqualOnCondNode, + SArray* leftTagEqCols, SArray* rightTagEqCols) { + SColumn left = {0}; + SColumn right = {0}; + if (nodeType(pEqualOnCondNode) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pEqualOnCondNode)->condType == LOGIC_COND_TYPE_AND) { + SNode* pNode = NULL; + FOREACH(pNode, ((SLogicConditionNode*)pEqualOnCondNode)->pParameterList) { + SOperatorNode* pOperNode = (SOperatorNode*)pNode; + extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + taosArrayPush(leftTagEqCols, &left); + taosArrayPush(rightTagEqCols, &right); + } + return; + } + + if (nodeType(pEqualOnCondNode) == QUERY_NODE_OPERATOR) { + SOperatorNode* pOperNode = (SOperatorNode*)pEqualOnCondNode; + extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + taosArrayPush(leftTagEqCols, &left); + taosArrayPush(rightTagEqCols, &right); + } +} + +static int32_t initTagColskeyBuf(int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { + int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = (SColumn*)taosArrayGet(pGroupColList, i); + (*keyLen) += pCol->bytes; // actual data + null_flag + } + + int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; + (*keyLen) += nullFlagSize; + + (*keyBuf) = taosMemoryCalloc(1, (*keyLen)); + if ((*keyBuf) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t fillKeyBufFromTagCols(SArray* pCols, SSDataBlock* pBlock, int32_t rowIndex, void* pKey) { + SColumnDataAgg* pColAgg = NULL; + size_t numOfGroupCols = taosArrayGetSize(pCols); + char* isNull = (char*)pKey; + char* pStart = (char*)pKey + sizeof(int8_t) * numOfGroupCols; + + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = (SColumn*) taosArrayGet(pCols, i); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); + + // valid range check. todo: return error code. + if (pCol->slotId > taosArrayGetSize(pBlock->pDataBlock)) { + continue; + } + + if (pBlock->pBlockAgg != NULL) { + pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? + } + + if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { + isNull[i] = 1; + } else { + isNull[i] = 0; + char* val = colDataGetData(pColInfoData, rowIndex); + if (pCol->type == TSDB_DATA_TYPE_JSON) { + int32_t dataLen = getJsonValueLen(val); + memcpy(pStart, val, dataLen); + pStart += dataLen; + } else if (IS_VAR_DATA_TYPE(pCol->type)) { + varDataCopy(pStart, val); + pStart += varDataTLen(val); + } else { + memcpy(pStart, val, pCol->bytes); + pStart += pCol->bytes; + } + } + } + return (int32_t)(pStart - (char*)pKey); +} + SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) { SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo)); @@ -153,6 +259,16 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } + pInfo->pColEqualOnConditions = pJoinNode->pColEqualOnConditions; + if (pInfo->pColEqualOnConditions != NULL) { + pInfo->leftEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->rightEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); + extractEqualOnCondCols(pInfo, pDownstream, pInfo->pColEqualOnConditions, pInfo->leftEqOnCondCols, pInfo->rightEqOnCondCols); + initTagColskeyBuf(&pInfo->leftEqOnCondKeyLen, &pInfo->leftEqOnCondKeyBuf, pInfo->leftEqOnCondCols); + initTagColskeyBuf(&pInfo->rightEqOnCondKeyLen, &pInfo->rightEqOnCondKeyBuf, pInfo->rightEqOnCondCols); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->rightBuildTable = tSimpleHashInit(256, hashFn); + } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { @@ -179,8 +295,28 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { pColumn->scale = pColumnNode->node.resType.scale; } +static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { + void* p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pBuildTable, p, &iter)) != NULL) { + SArray* rows = (*(SArray**)p); + taosArrayDestroy(rows); + } + + tSimpleHashCleanup(pBuildTable); +} + void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; + if (pJoinOperator->pColEqualOnConditions != NULL) { + mergeJoinDestoryBuildTable(pJoinOperator->rightBuildTable); + taosMemoryFreeClear(pJoinOperator->rightEqOnCondKeyBuf); + taosArrayDestroy(pJoinOperator->rightEqOnCondCols); + + taosMemoryFreeClear(pJoinOperator->leftEqOnCondKeyBuf); + taosArrayDestroy(pJoinOperator->leftEqOnCondCols); + } nodesDestroyNode(pJoinOperator->pCondAfterMerge); pJoinOperator->pRes = blockDataDestroy(pJoinOperator->pRes); @@ -300,21 +436,122 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator return 0; } +static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations) { + for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { + SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); + int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightEqOnCondCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightEqOnCondKeyBuf); + SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightEqOnCondKeyBuf, keyLen); + if (!ppRows) { + SArray* rows = taosArrayInit(4, sizeof(SRowLocation)); + taosArrayPush(rows, rightRow); + tSimpleHashPut(pInfo->rightBuildTable, pInfo->rightEqOnCondKeyBuf, keyLen, &rows, POINTER_BYTES); + } else { + taosArrayPush(*ppRows, rightRow); + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* pRes, int32_t* nRows, + const SArray* leftRowLocations, int32_t leftRowIdx, + int32_t rightRowIdx, bool useBuildTableTSRange, SArray* rightRowLocations, bool* pReachThreshold) { + *pReachThreshold = false; + uint32_t limitRowNum = pOperator->resultInfo.threshold; + SJoinOperatorInfo* pJoinInfo = pOperator->info; + size_t leftNumJoin = taosArrayGetSize(leftRowLocations); + + int32_t i,j; + + for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { + SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); + SArray* pRightRows = NULL; + if (useBuildTableTSRange) { + int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftEqOnCondCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftEqOnCondKeyBuf); + SArray** ppRightRows = tSimpleHashGet(pJoinInfo->rightBuildTable, pJoinInfo->leftEqOnCondKeyBuf, keyLen); + if (!ppRightRows) { + continue; + } + pRightRows = *ppRightRows; + } else { + pRightRows = rightRowLocations; + } + size_t rightRowsSize = taosArrayGetSize(pRightRows); + for (j = rightRowIdx; j < rightRowsSize; ++j) { + if (*nRows >= limitRowNum) { + *pReachThreshold = true; + break; + } + + SRowLocation* rightRow = taosArrayGet(pRightRows, j); + mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock, + rightRow->pos); + ++*nRows; + } + if (*pReachThreshold) { + break; + } + } + + if (*pReachThreshold) { + pJoinInfo->rowCtx.rowRemains = true; + pJoinInfo->rowCtx.leftRowIdx = i; + pJoinInfo->rowCtx.rightRowIdx = j; + } + return TSDB_CODE_SUCCESS; +} + +static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* leftRowLocations, SArray* leftCreatedBlocks, + SArray* rightCreatedBlocks, bool rightUseBuildTable, SArray* rightRowLocations) { + for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { + SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); + blockDataDestroy(pBlock); + } + taosArrayDestroy(rightCreatedBlocks); + for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) { + SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); + blockDataDestroy(pBlock); + } + if (rightRowLocations != NULL) { + taosArrayDestroy(rightRowLocations); + } + if (rightUseBuildTable) { + void* p = NULL; + int32_t iter = 0; + while ((p = tSimpleHashIterate(pJoinInfo->rightBuildTable, p, &iter)) != NULL) { + SArray* rows = (*(SArray**)p); + taosArrayDestroy(rows); + } + tSimpleHashClear(pJoinInfo->rightBuildTable); + } + + taosArrayDestroy(leftCreatedBlocks); + taosArrayDestroy(leftRowLocations); + + pJoinInfo->rowCtx.rowRemains = false; + pJoinInfo->rowCtx.leftRowLocations = NULL; + pJoinInfo->rowCtx.leftCreatedBlocks = NULL; + pJoinInfo->rowCtx.rightCreatedBlocks = NULL; + pJoinInfo->rowCtx.rightUseBuildTable = false; + pJoinInfo->rowCtx.rightRowLocations = NULL; +} + static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t timestamp, SSDataBlock* pRes, int32_t* nRows) { int32_t code = TSDB_CODE_SUCCESS; SJoinOperatorInfo* pJoinInfo = pOperator->info; SArray* leftRowLocations = NULL; - SArray* leftCreatedBlocks = NULL; SArray* rightRowLocations = NULL; + SArray* leftCreatedBlocks = NULL; SArray* rightCreatedBlocks = NULL; int32_t leftRowIdx = 0; int32_t rightRowIdx = 0; - int32_t i, j; - + SSHashObj* rightTableHash = NULL; + bool rightUseBuildTable = false; + if (pJoinInfo->rowCtx.rowRemains) { leftRowLocations = pJoinInfo->rowCtx.leftRowLocations; leftCreatedBlocks = pJoinInfo->rowCtx.leftCreatedBlocks; + rightUseBuildTable = pJoinInfo->rowCtx.rightUseBuildTable; rightRowLocations = pJoinInfo->rowCtx.rightRowLocations; rightCreatedBlocks = pJoinInfo->rowCtx.rightCreatedBlocks; leftRowIdx = pJoinInfo->rowCtx.leftRowIdx; @@ -330,78 +567,40 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); + if (pJoinInfo->pColEqualOnConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { + mergeJoinFillBuildTable(pJoinInfo, rightRowLocations); + rightUseBuildTable = true; + taosArrayDestroy(rightRowLocations); + rightRowLocations = NULL; + } } size_t leftNumJoin = taosArrayGetSize(leftRowLocations); - size_t rightNumJoin = taosArrayGetSize(rightRowLocations); - uint32_t maxRowNum = *nRows + (leftNumJoin - leftRowIdx - 1) * rightNumJoin + rightNumJoin - rightRowIdx; - uint32_t limitRowNum = maxRowNum; - if (maxRowNum > pOperator->resultInfo.threshold) { - limitRowNum = pOperator->resultInfo.threshold; - if (!pJoinInfo->rowCtx.rowRemains) { + code = blockDataEnsureCapacity(pRes, pOperator->resultInfo.threshold); + if (code != TSDB_CODE_SUCCESS) { + qError("%s can not ensure block capacity for join. left: %zu", GET_TASKID(pOperator->pTaskInfo), + leftNumJoin); + } + + bool reachThreshold = false; + + if (code == TSDB_CODE_SUCCESS) { + mergeJoinLeftRowsRightRows(pOperator, pRes, nRows, leftRowLocations, leftRowIdx, + rightRowIdx, rightUseBuildTable, rightRowLocations, &reachThreshold); + } + + if (!reachThreshold) { + mergeJoinDestroyTSRangeCtx(pJoinInfo, leftRowLocations, leftCreatedBlocks, rightCreatedBlocks, + rightUseBuildTable, rightRowLocations); + + } else { pJoinInfo->rowCtx.rowRemains = true; pJoinInfo->rowCtx.ts = timestamp; pJoinInfo->rowCtx.leftRowLocations = leftRowLocations; - pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; - } - } - - code = blockDataEnsureCapacity(pRes, limitRowNum); - if (code != TSDB_CODE_SUCCESS) { - qError("%s can not ensure block capacity for join. left: %zu, right: %zu", GET_TASKID(pOperator->pTaskInfo), - leftNumJoin, rightNumJoin); - } - - - if (code == TSDB_CODE_SUCCESS) { - bool done = false; - for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { - for (j = rightRowIdx; j < rightNumJoin; ++j) { - if (*nRows >= limitRowNum) { - done = true; - break; - } - - SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); - SRowLocation* rightRow = taosArrayGet(rightRowLocations, j); - mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock, - rightRow->pos); - ++*nRows; - } - if (done) { - break; - } - } - - if (maxRowNum > pOperator->resultInfo.threshold) { - pJoinInfo->rowCtx.leftRowIdx = i; - pJoinInfo->rowCtx.rightRowIdx = j; - } - } - - if (maxRowNum <= pOperator->resultInfo.threshold) { - for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { - SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); - blockDataDestroy(pBlock); - } - taosArrayDestroy(rightCreatedBlocks); - taosArrayDestroy(rightRowLocations); - for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) { - SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); - blockDataDestroy(pBlock); - } - taosArrayDestroy(leftCreatedBlocks); - taosArrayDestroy(leftRowLocations); - - if (pJoinInfo->rowCtx.rowRemains) { - pJoinInfo->rowCtx.rowRemains = false; - pJoinInfo->rowCtx.leftRowLocations = NULL; - pJoinInfo->rowCtx.rightRowLocations = NULL; - pJoinInfo->rowCtx.leftCreatedBlocks = NULL; - pJoinInfo->rowCtx.rightCreatedBlocks = NULL; - } + pJoinInfo->rowCtx.rightUseBuildTable = rightUseBuildTable; + pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 28abee4342..146552b058 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -88,39 +88,6 @@ static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } } -static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { - tw->skey += pInterval->sliding * factor; - tw->ekey = tw->skey + pInterval->interval - 1; - return; - } - - int64_t key = tw->skey, interval = pInterval->interval; - // convert key to second - key = convertTimePrecision(key, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pInterval->intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm, NULL); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); - - tw->ekey -= 1; -} - static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo, int32_t order) { STimeWindow w = {0}; @@ -130,7 +97,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } if (order == TSDB_ORDER_ASC) { - w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey); + w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.skey); ASSERT(w.ekey >= pBlockInfo->window.skey); if (w.ekey < pBlockInfo->window.ekey) { @@ -149,7 +116,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } } } else { - w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.ekey); + w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.ekey); ASSERT(w.skey <= pBlockInfo->window.ekey); if (w.skey > pBlockInfo->window.skey) { @@ -206,7 +173,7 @@ static int32_t insertTableToScanIgnoreList(STableScanInfo* pTableScanInfo, uint6 return TSDB_CODE_OUT_OF_MEMORY; } } - + taosHashPut(pTableScanInfo->pIgnoreTables, &uid, sizeof(uid), &pTableScanInfo->scanTimes, sizeof(pTableScanInfo->scanTimes)); return TSDB_CODE_SUCCESS; @@ -215,7 +182,7 @@ static int32_t insertTableToScanIgnoreList(STableScanInfo* pTableScanInfo, uint6 static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* pBlockInfo, uint32_t* status) { STableScanInfo* pTableScanInfo = pOperator->info; int32_t code = TSDB_CODE_SUCCESS; - + if (pTableScanInfo->base.pdInfo.pExprSup == NULL) { return TSDB_CODE_SUCCESS; } @@ -267,12 +234,13 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SStorageAPI* pAPI = &pTaskInfo->storageAPI; bool allColumnsHaveAgg = true; - int32_t code = pAPI->tsdReader.tsdReaderRetrieveBlockSMAInfo(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); + bool hasNullSMA = false; + int32_t code = pAPI->tsdReader.tsdReaderRetrieveBlockSMAInfo(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg, &hasNullSMA); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } - if (!allColumnsHaveAgg) { + if (!allColumnsHaveAgg || hasNullSMA) { return false; } return true; @@ -751,7 +719,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - + // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) { return NULL; @@ -1072,6 +1040,7 @@ static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t grou void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint64_t ver) { pTableScanInfo->base.cond.twindows = *pWin; + pTableScanInfo->base.cond.startVersion = 0; pTableScanInfo->base.cond.endVersion = ver; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; @@ -1194,6 +1163,7 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ } STableScanInfo* pTScanInfo = pInfo->pTableScanOp->info; + qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRIu64, win.skey, win.ekey, pInfo->pUpdateInfo->maxDataVersion); resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); pInfo->pTableScanOp->status = OP_OPENED; return true; @@ -1649,6 +1619,10 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { qDebug("start to exec queue scan, %s", id); + if (isTaskKilled(pTaskInfo)) { + return NULL; + } + if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { @@ -1661,7 +1635,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader); - + pTSInfo->base.dataReader = NULL; qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); if (pAPI->tqReaderFn.tqReaderSeek(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) { @@ -1889,6 +1863,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); } else { + pInfo->pUpdateInfo->maxDataVersion = pTaskInfo->streamInfo.fillHistoryVer2; doCheckUpdate(pInfo, pInfo->pRecoverRes->info.window.ekey, pInfo->pRecoverRes); } } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 6b49d235f4..7cc50a70ab 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -517,11 +517,16 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) return; } + // the endKey is now the aligned time window value. truncate time window isn't correct. pFillInfo->end = endKey; - if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->interval.precision); - pFillInfo->end = taosTimeAdd(pFillInfo->end, pFillInfo->interval.interval, pFillInfo->interval.intervalUnit,pFillInfo->interval.precision); + +#if 0 + if (pFillInfo->order == TSDB_ORDER_ASC) { + ASSERT(pFillInfo->start <= pFillInfo->end); + } else { + ASSERT(pFillInfo->start >= pFillInfo->end); } +#endif pFillInfo->index = 0; pFillInfo->numOfRows = numOfRows; @@ -531,6 +536,13 @@ void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) pFillInfo->pSrcBlock = (SSDataBlock*)pInput; } +void taosFillUpdateStartTimestampInfo(SFillInfo* pFillInfo, int64_t ts) { + pFillInfo->start = ts; + pFillInfo->currentKey = ts; +} + +bool taosFillNotStarted(const SFillInfo* pFillInfo) {return pFillInfo->start == pFillInfo->currentKey;} + bool taosFillHasMoreResults(SFillInfo* pFillInfo) { int32_t remain = taosNumOfRemainRows(pFillInfo); if (remain > 0) { @@ -562,9 +574,10 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma ASSERT(numOfRes >= numOfRows); } else { // reach the end of data if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) || - (ekey1 >= pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { + (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { return 0; } + numOfRes = taosTimeCountInterval(ekey1, pFillInfo->currentKey, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, pFillInfo->interval.precision); numOfRes += 1; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 2eb8d297ac..fc08f827a5 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -271,43 +271,6 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary return num; } -static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { - tw->skey += pInterval->sliding * factor; - tw->ekey = tw->skey + pInterval->interval - 1; - return; - } - - int64_t key = tw->skey, interval = pInterval->interval; - // convert key to second - key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pInterval->intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm, NULL); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision); - - tw->ekey -= 1; -} - -void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) { - getNextTimeWindow(pInterval, pInterval->precision, order, tw); -} - void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) { SqlFunctionCtx* pCtx = pSup->pCtx; @@ -449,7 +412,7 @@ static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SEx } bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd, EStreamType blockType) { - if (pInterval->interval != pInterval->sliding && + if (pInterval->interval != pInterval->sliding && ((pWin->ekey < calStart || pWin->skey > calEnd) || (blockType == STREAM_PULL_DATA && pWin->skey < calStart) )) { return false; } @@ -466,7 +429,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, bool ascQuery = (order == TSDB_ORDER_ASC); int32_t precision = pInterval->precision; - getNextTimeWindow(pInterval, precision, order, pNext); + getNextTimeWindow(pInterval, pNext, order); // next time window is not in current block if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) || @@ -511,7 +474,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, if (ascQuery && primaryKeys[startPos] > pNext->ekey) { TSKEY next = primaryKeys[startPos]; if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->skey = taosTimeTruncate(next, pInterval); pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; @@ -520,7 +483,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) { TSKEY next = primaryKeys[startPos]; if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->skey = taosTimeTruncate(next, pInterval); pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; @@ -1370,14 +1333,14 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa do { if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i], pBlock->info.type)) { - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); continue; } uint64_t winGpId = pGpDatas[i]; SWinKey winRes = {.ts = win.skey, .groupId = winGpId}; void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey)); if (chIds) { - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); continue; } bool res = doDeleteWindow(pOperator, win.skey, winGpId); @@ -1387,7 +1350,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa if (pUpdatedMap) { tSimpleHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); } - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); } while (win.ekey <= endTsCols[i]); } } diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp new file mode 100644 index 0000000000..2894c66587 --- /dev/null +++ b/source/libs/executor/test/timewindowTest.cpp @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "taos.h" +#include "thash.h" +#include "tsimplehash.h" +#include "executor.h" +#include "ttime.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +namespace { +SInterval createInterval(int64_t interval, int64_t sliding, int64_t offset, char intervalUnit, char slidingUnit, + char offsetUnit, int8_t precision) { + SInterval v = {0}; + v.interval = interval; + v.intervalUnit = intervalUnit; + v.sliding = sliding; + v.slidingUnit = slidingUnit; + v.offset = offset; + v.offsetUnit = offsetUnit; + v.precision = precision; + return v; +} + +void printTimeWindow(STimeWindow* pWindow, int8_t precision, int64_t ts) { + char buf[64] = {0}; + char bufs[64] = {0}; + char bufe[64] = {0}; + + taosFormatUtcTime(buf, tListLen(buf), ts, precision); + + taosFormatUtcTime(bufs, tListLen(bufs), pWindow->skey, precision); + taosFormatUtcTime(bufe, tListLen(bufe), pWindow->ekey, precision); + + printf("%s [%s - %s]\n", buf, bufs, bufe); +} +} // namespace + +TEST(testCase, timewindow_gen) { + // set correct time zone + osSetTimezone("UTC"); + int32_t precision = TSDB_TIME_PRECISION_MILLI; + + SInterval interval = + createInterval(10 * 86400 * 1000, 10 * 86400 * 1000, 0, 'd', 'd', 'd', precision); + + int64_t key = 1659312000L * 1000; // 2022-8-1 00:00:00 // UTC+8 (ms) + + STimeWindow w = {0}; + getInitialStartTimeWindow(&interval, key, &w, true); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&interval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, precision, key); + + printf("---------------------------------------------------\n"); + SInterval monthInterval = + createInterval(1, 1, 0, 'n', 'n', 'd', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&monthInterval, key, &w, true); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&monthInterval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, precision, key); + + printf("----------------------------------------------------------\n"); + SInterval slidingInterval = createInterval(1, 10*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&slidingInterval, key, &w, true); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + printf("-----------------calendar_interval_1n_sliding_1d-------\n"); + SInterval calendar_interval_1n = createInterval(1, 1*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + int64_t k1 = 1664409600 * 1000L; + getInitialStartTimeWindow(&calendar_interval_1n, k1, &w, true); + printTimeWindow(&w, precision, k1); + + printf("next\n"); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, key); + + printf("----------------interval_1d_clendar_sliding_1n---------\n"); + SInterval interval_1d_calendar_sliding_1n = createInterval(1*86400*1000L, 1, 0, 'd', 'n', 'd', TSDB_TIME_PRECISION_MILLI); + + k1 = 1664409600 * 1000L; + getInitialStartTimeWindow(&interval_1d_calendar_sliding_1n, k1, &w, true); + printTimeWindow(&w, precision, k1); + + printf("next time window:\n"); + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + printf("----------------interval_1d_sliding_1d_calendar_offset_1n---------\n"); + SInterval offset_1n = createInterval(10*86400*1000L, 10*86400*1000L, 1, 'd', 'd', 'n', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&offset_1n, k1, &w, true); + printTimeWindow(&w, precision, k1); + + +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5c47daca5b..1265c64c8c 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -499,7 +499,7 @@ static int64_t getNumOfElems(SqlFunctionCtx* pCtx) { */ SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows && !IS_VAR_DATA_TYPE(pInputCol->info.type)) { + if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows) { numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull; } else { if (pInputCol->hasNull) { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 2115044228..ce77336510 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -402,6 +402,7 @@ static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) { COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pMergeCondition); CLONE_NODE_FIELD(pOnConditions); + CLONE_NODE_FIELD(pColEqualOnConditions); COPY_SCALAR_FIELD(isSingleTableJoin); COPY_SCALAR_FIELD(inputTsOrder); return TSDB_CODE_SUCCESS; @@ -589,7 +590,7 @@ static int32_t physiSysTableScanCopy(const SSystemTableScanPhysiNode* pSrc, SSys return TSDB_CODE_SUCCESS; } -static int32_t physiWindowCopy(const SWinodwPhysiNode* pSrc, SWinodwPhysiNode* pDst) { +static int32_t physiWindowCopy(const SWindowPhysiNode* pSrc, SWindowPhysiNode* pDst) { COPY_BASE_OBJECT_FIELD(node, physiNodeCopy); CLONE_NODE_LIST_FIELD(pExprs); CLONE_NODE_LIST_FIELD(pFuncs); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index a6d808608d..6bf1ad01a8 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1428,6 +1428,7 @@ static int32_t jsonToLogicPlan(const SJson* pJson, void* pObj) { static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; static const char* jkJoinLogicPlanMergeCondition = "MergeConditions"; +static const char* jkJoinLogicPlanColEqualOnConditions = "ColumnEqualOnConditions"; static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj; @@ -1442,7 +1443,9 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinLogicPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions); + } return code; } @@ -1459,7 +1462,9 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions); } - + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinLogicPlanColEqualOnConditions, &pNode->pColEqualOnConditions); + } return code; } @@ -1890,6 +1895,7 @@ static const char* jkJoinPhysiPlanInputTsOrder = "InputTsOrder"; static const char* jkJoinPhysiPlanMergeCondition = "MergeCondition"; static const char* jkJoinPhysiPlanOnConditions = "OnConditions"; static const char* jkJoinPhysiPlanTargets = "Targets"; +static const char* jkJoinPhysiPlanColEqualOnConditions = "ColumnEqualOnConditions"; static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj; @@ -1910,7 +1916,9 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinPhysiPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions); + } return code; } @@ -1933,7 +1941,9 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets); } - + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinPhysiPlanColEqualOnConditions, &pNode->pColEqualOnConditions); + } return code; } @@ -2144,7 +2154,7 @@ static const char* jkWindowPhysiPlanOutputTsOrder = "outputTsOrder"; static const char* jkWindowPhysiPlanMergeDataBlock = "MergeDataBlock"; static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { - const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + const SWindowPhysiNode* pNode = (const SWindowPhysiNode*)pObj; int32_t code = physicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { @@ -2185,7 +2195,7 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { } static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) { - SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + SWindowPhysiNode* pNode = (SWindowPhysiNode*)pObj; int32_t code = jsonToPhysicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index eed70cfccf..f755b8cb8c 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -82,9 +82,7 @@ static bool columnNodeEqual(const SColumnNode* a, const SColumnNode* b) { COMPARE_STRING_FIELD(dbName); COMPARE_STRING_FIELD(tableName); COMPARE_STRING_FIELD(colName); - if (0 == a->tableId) { - COMPARE_STRING_FIELD(tableAlias); - } + COMPARE_STRING_FIELD(tableAlias); return true; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 4ffd38ee36..0631b91323 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2319,7 +2319,8 @@ enum { PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION, PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS, PHY_SORT_MERGE_JOIN_CODE_TARGETS, - PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER + PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, + PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS }; static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -2341,7 +2342,9 @@ static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); } - + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pColEqualOnConditions); + } return code; } @@ -2370,6 +2373,9 @@ static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) { case PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER: code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); break; + case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pColEqualOnConditions); + break; default: break; } @@ -2642,7 +2648,7 @@ enum { }; static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { - const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + const SWindowPhysiNode* pNode = (const SWindowPhysiNode*)pObj; int32_t code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); if (TSDB_CODE_SUCCESS == code) { @@ -2683,7 +2689,7 @@ static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { } static int32_t msgToPhysiWindowNode(STlvDecoder* pDecoder, void* pObj) { - SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + SWindowPhysiNode* pNode = (SWindowPhysiNode*)pObj; int32_t code = TSDB_CODE_SUCCESS; STlv* pTlv = NULL; diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index 0181da92a9..b57bba0cc9 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -120,9 +120,9 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { } if (colNode->tableAlias[0]) { - *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->colName); + *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->node.userAlias); } else { - *len += snprintf(buf + *len, bufSize - *len, "%s", colNode->colName); + *len += snprintf(buf + *len, bufSize - *len, "%s", colNode->node.userAlias); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 70fc45e07a..39e288f694 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -608,7 +608,7 @@ static void destroyPhysiNode(SPhysiNode* pNode) { nodesDestroyNode(pNode->pSlimit); } -static void destroyWinodwPhysiNode(SWinodwPhysiNode* pNode) { +static void destroyWinodwPhysiNode(SWindowPhysiNode* pNode) { destroyPhysiNode((SPhysiNode*)pNode); nodesDestroyList(pNode->pExprs); nodesDestroyList(pNode->pFuncs); @@ -1089,6 +1089,7 @@ void nodesDestroyNode(SNode* pNode) { destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyNode(pLogicNode->pMergeCondition); nodesDestroyNode(pLogicNode->pOnConditions); + nodesDestroyNode(pLogicNode->pColEqualOnConditions); break; } case QUERY_NODE_LOGIC_PLAN_AGG: { @@ -1221,6 +1222,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pMergeCondition); nodesDestroyNode(pPhyNode->pOnConditions); nodesDestroyList(pPhyNode->pTargets); + nodesDestroyNode(pPhyNode->pColEqualOnConditions); break; } case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { @@ -1258,7 +1260,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: - destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pNode); break; case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: { @@ -1274,19 +1276,19 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: - destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pNode); break; case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: { SStateWinodwPhysiNode* pPhyNode = (SStateWinodwPhysiNode*)pNode; - destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode); nodesDestroyNode(pPhyNode->pStateKey); break; } case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: { SEventWinodwPhysiNode* pPhyNode = (SEventWinodwPhysiNode*)pNode; - destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode); nodesDestroyNode(pPhyNode->pStartCond); nodesDestroyNode(pPhyNode->pEndCond); break; @@ -2074,7 +2076,7 @@ char* nodesGetNameFromColumnNode(SNode* pNode) { return "NULL"; } - return ((SColumnNode*)pNode)->colName; + return ((SColumnNode*)pNode)->node.userAlias; } int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots) { diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 7daaf8c69f..8af54e245b 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -259,8 +259,19 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { strcpy(pExpr->userAlias, ((SColumnNode*)pExpr)->colName); } else { int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); - strncpy(pExpr->aliasName, pRawExpr->p, len); - pExpr->aliasName[len] = '\0'; + + // See TS-3398. + // Len of pRawExpr->p could be larger than len of aliasName[TSDB_COL_NAME_LEN]. + // If aliasName is truncated, hash value of aliasName could be the same. + T_MD5_CTX ctx; + tMD5Init(&ctx); + tMD5Update(&ctx, (uint8_t*)pRawExpr->p, pRawExpr->n); + tMD5Final(&ctx); + char* p = pExpr->aliasName; + for (uint8_t i = 0; i < tListLen(ctx.digest); ++i) { + sprintf(p, "%02x", ctx.digest[i]); + p += 2; + } strncpy(pExpr->userAlias, pRawExpr->p, len); pExpr->userAlias[len] = '\0'; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index dea0c4bd50..801d43e2a4 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -586,7 +586,7 @@ static int32_t collectMetaKeyFromShowCreateTable(SCollectMetaKeyCxt* pCxt, SShow code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_READ, + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ, pCxt->pMetaCache); } return code; diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index 1586d8128b..251d3bd0cb 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -175,7 +175,7 @@ static int32_t authShowTables(SAuthCxt* pCxt, SShowStmt* pStmt) { static int32_t authShowCreateTable(SAuthCxt* pCxt, SShowCreateTableStmt* pStmt) { SNode* pTagCond = NULL; // todo check tag condition for subtable - return checkAuth(pCxt, pStmt->dbName, NULL, AUTH_TYPE_READ, &pTagCond); + return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ, &pTagCond); } static int32_t authCreateTable(SAuthCxt* pCxt, SCreateTableStmt* pStmt) { diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 43a73af3c3..f9b4e54318 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1605,7 +1605,7 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt (*pNumOfRows)++; } - if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxMemUsedByInsert * 1024 * 1024) { + if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxInsertBatchRows) { pStmt->fileProcessing = true; break; } @@ -1614,6 +1614,8 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt } taosMemoryFree(pLine); + parserDebug("0x%" PRIx64 " %d rows have been parsed", pCxt->pComCxt->requestId, *pNumOfRows); + if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) && (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) && !pStmt->fileProcessing) { code = buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index f654d2527f..bcbea6cd2e 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -267,7 +267,10 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in pBind = bind + c; } - tColDataAddValueByBind(pCol, pBind); + code = tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1); + if (code) { + goto _return; + } } qDebug("stmt all %d columns bind %d rows data", boundInfo->numOfBound, rowNum); @@ -310,7 +313,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu pBind = bind; } - tColDataAddValueByBind(pCol, pBind); + tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1); qDebug("stmt col %d bind %d rows data", colIdx, rowNum); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index f921094752..de7d154db6 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -272,6 +272,41 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat return code; } +static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) { + int32_t code = TSDB_CODE_SUCCESS; + SSubmitTbData* pTmp = taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (NULL == pTmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pTmp->flags = pSrc->flags; + pTmp->suid = pSrc->suid; + pTmp->uid = pSrc->uid; + pTmp->sver = pSrc->sver; + pTmp->pCreateTbReq = NULL; + if (pTmp->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + pTmp->aCol = taosArrayInit(128, sizeof(SColData)); + if (NULL == pTmp->aCol) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pTmp); + } + } else { + pTmp->aRowP = taosArrayInit(128, POINTER_BYTES); + if (NULL == pTmp->aRowP) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pTmp); + } + } + } + + taosMemoryFree(pSrc); + if (TSDB_CODE_SUCCESS == code) { + *pDst = pTmp; + } + + return code; +} + + static void resetColValues(SArray* pValues) { int32_t num = taosArrayGetSize(pValues); for (int32_t i = 0; i < num; ++i) { @@ -381,7 +416,7 @@ static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCx } } taosArrayPush(pVgCxt->pData->aSubmitTbData, pTableCxt->pData); - taosMemoryFreeClear(pTableCxt->pData); + rebuildTableData(pTableCxt->pData, &pTableCxt->pData); qDebug("add tableDataCxt uid:%" PRId64 " to vgId:%d", pTableCxt->pMeta->uid, pVgCxt->vgId); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c10ee5d988..9ebbecd148 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -834,7 +834,7 @@ static void setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SColum strcpy(pCol->node.aliasName, pCol->colName); } if ('\0' == pCol->node.userAlias[0]) { - strcpy(pCol->node.userAlias, pCol->colName); + strcpy(pCol->node.userAlias, pExpr->userAlias); } pCol->node.resType = pExpr->resType; } @@ -1770,6 +1770,7 @@ static int32_t rewriteFuncToValue(STranslateContext* pCxt, char* pLiteral, SNode return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pVal->node.aliasName, ((SExprNode*)*pNode)->aliasName); + strcpy(pVal->node.userAlias, ((SExprNode*)*pNode)->userAlias); pVal->node.resType = ((SExprNode*)*pNode)->resType; if (NULL == pLiteral) { pVal->isNull = true; @@ -2768,6 +2769,7 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { } else { len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->aliasName); strncpy(pFunc->node.aliasName, buf, TMIN(len, sizeof(pFunc->node.aliasName) - 1)); + len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->userAlias); strncpy(pFunc->node.userAlias, buf, TMIN(len, sizeof(pFunc->node.userAlias) - 1)); } @@ -3214,7 +3216,7 @@ static int32_t checkFill(STranslateContext* pCxt, SFillNode* pFill, SValueNode* int64_t timeRange = TABS(pFill->timeRange.skey - pFill->timeRange.ekey); int64_t intervalRange = 0; - if (TIME_IS_VAR_DURATION(pInterval->unit)) { + if (IS_CALENDAR_TIME_DURATION(pInterval->unit)) { int64_t f = 1; if (pInterval->unit == 'n') { f = 30LL * MILLISECOND_PER_DAY; @@ -3295,7 +3297,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; SValueNode* pInter = (SValueNode*)pInterval->pInterval; - bool valInter = TIME_IS_VAR_DURATION(pInter->unit); + bool valInter = IS_CALENDAR_TIME_DURATION(pInter->unit); if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime, getPrecisionStr(precision)); @@ -3309,7 +3311,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* if (pInter->unit == 'n' && pOffset->unit == 'y') { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT); } - bool fixed = !TIME_IS_VAR_DURATION(pOffset->unit) && !valInter; + bool fixed = !IS_CALENDAR_TIME_DURATION(pOffset->unit) && !valInter; if ((fixed && pOffset->datum.i >= pInter->datum.i) || (!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >= getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) { @@ -3325,7 +3327,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* const static int32_t INTERVAL_SLIDING_FACTOR = 100; SValueNode* pSliding = (SValueNode*)pInterval->pSliding; - if (TIME_IS_VAR_DURATION(pSliding->unit)) { + if (IS_CALENDAR_TIME_DURATION(pSliding->unit)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT); } if ((pSliding->datum.i < convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, precision)) || diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 4a51d3c31c..ba8392a48c 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -499,7 +499,7 @@ int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, char* pStr) { - return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL != pTable && '\0' == pTable[0]) ? NULL : pTable, + return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL == pTable || '\0' == pTable[0]) ? "``" : pTable, type); } @@ -525,6 +525,9 @@ static void getStringFromAuthStr(const char* pStart, char* pStr, char** pNext) { strncpy(pStr, pStart, p - pStart); *pNext = ++p; } + if (*pStart == '`' && *(pStart + 1) == '`') { + *pStr = 0; + } } static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUserAuth) { @@ -533,7 +536,11 @@ static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUser pUserAuth->tbName.acctId = getIntegerFromAuthStr(p, &p); getStringFromAuthStr(p, pUserAuth->tbName.dbname, &p); getStringFromAuthStr(p, pUserAuth->tbName.tname, &p); - pUserAuth->tbName.type = TSDB_TABLE_NAME_T; + if (pUserAuth->tbName.tname[0]) { + pUserAuth->tbName.type = TSDB_TABLE_NAME_T; + } else { + pUserAuth->tbName.type = TSDB_DB_NAME_T; + } pUserAuth->type = getIntegerFromAuthStr(p, &p); } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 2d8ce55b72..a770522921 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -472,6 +472,8 @@ TEST_F(ParserSelectTest, joinSemanticCheck) { run("SELECT * FROM (SELECT tag1, SUM(c1) s FROM st1 GROUP BY tag1) t1, st1 t2 where t1.tag1 = t2.tag1", TSDB_CODE_PAR_NOT_SUPPORT_JOIN); + + run("SELECT count(*) FROM t1 a join t1 b on a.ts=b.ts where a.ts=b.ts"); } } // namespace ParserTest diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 66b478004f..5bbc9acdad 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -37,19 +37,24 @@ typedef struct SRewriteExprCxt { int32_t errCode; SNodeList* pExprs; bool* pOutputs; + bool isPartitionBy; } SRewriteExprCxt; -static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol) { +static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol, bool isPartitionBy) { switch (pFunc->funcType) { case FUNCTION_TYPE_TBNAME: pCol->colType = COLUMN_TYPE_TBNAME; break; case FUNCTION_TYPE_WSTART: - pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + if (!isPartitionBy) { + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } pCol->colType = COLUMN_TYPE_WINDOW_START; break; case FUNCTION_TYPE_WEND: - pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + if (!isPartitionBy) { + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } pCol->colType = COLUMN_TYPE_WINDOW_END; break; case FUNCTION_TYPE_WDURATION: @@ -100,9 +105,10 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode); pCol->node.resType = pToBeRewrittenExpr->resType; strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); + strcpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias); strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { - setColumnInfo((SFunctionNode*)pExpr, pCol); + setColumnInfo((SFunctionNode*)pExpr, pCol, pCxt->isPartitionBy); } nodesDestroyNode(*pNode); *pNode = (SNode*)pCol; @@ -141,7 +147,8 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { static int32_t rewriteExprForSelect(SNode* pExpr, SSelectStmt* pSelect, ESqlClause clause) { nodesWalkExpr(pExpr, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL}; + bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL, .isPartitionBy = isPartitionBy}; cxt.errCode = nodesListMakeAppend(&cxt.pExprs, pExpr); if (TSDB_CODE_SUCCESS == cxt.errCode) { nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); @@ -169,7 +176,8 @@ static int32_t cloneRewriteExprs(SNodeList* pExprs, bool* pOutputs, SNodeList** static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause, SNodeList** pRewriteExprs) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = isPartitionBy}; if (NULL != pRewriteExprs) { cxt.pOutputs = taosMemoryCalloc(LIST_LENGTH(pExprs), sizeof(bool)); if (NULL == cxt.pOutputs) { @@ -186,14 +194,14 @@ static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ES static int32_t rewriteExpr(SNodeList* pExprs, SNode** pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = false}; nodesRewriteExpr(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = false}; nodesRewriteExprs(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } @@ -543,11 +551,16 @@ static SNode* createGroupingSetNode(SNode* pExpr) { return (SNode*)pGroupingSet; } -static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { +static EGroupAction getDistinctGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) ? GROUP_ACTION_KEEP : GROUP_ACTION_NONE; } +static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { + return ((pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) && !pSelect->isDistinct) ? GROUP_ACTION_KEEP + : GROUP_ACTION_NONE; +} + static EDataOrderLevel getRequireDataOrder(bool needTimeline, SSelectStmt* pSelect) { return needTimeline ? (NULL != pSelect->pPartitionByList ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL) : DATA_ORDER_LEVEL_NONE; @@ -1166,7 +1179,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe return TSDB_CODE_OUT_OF_MEMORY; } - pAgg->node.groupAction = GROUP_ACTION_CLEAR; + pAgg->node.groupAction = GROUP_ACTION_CLEAR;//getDistinctGroupAction(pCxt, pSelect); pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 59b7c417a0..8b75fe6b33 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -740,6 +740,85 @@ static int32_t pushDownCondOptJoinExtractMergeCond(SOptimizeContext* pCxt, SJoin return code; } +static bool pushDownCondOptIsTableColumn(SNode* pNode, SNodeList* pTableCols) { + if (QUERY_NODE_COLUMN != nodeType(pNode)) { + return false; + } + SColumnNode* pCol = (SColumnNode*)pNode; + return pushDownCondOptBelongThisTable(pNode, pTableCols); +} + +static bool pushDownCondOptIsColEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) { + if (QUERY_NODE_OPERATOR != nodeType(pCond)) { + return false; + } + SOperatorNode* pOper = (SOperatorNode*)pCond; + if (OP_TYPE_EQUAL != pOper->opType) { + return false; + } + if (QUERY_NODE_COLUMN != nodeType(pOper->pLeft) || QUERY_NODE_COLUMN != nodeType(pOper->pRight)) { + return false; + } + SColumnNode* pLeft = (SColumnNode*)(pOper->pLeft); + SColumnNode* pRight = (SColumnNode*)(pOper->pRight); + //TODO: add cast to operator and remove this restriction of optimization + if (pLeft->node.resType.type != pRight->node.resType.type || pLeft->node.resType.bytes != pRight->node.resType.bytes) { + return false; + } + SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; + SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; + if (pushDownCondOptIsTableColumn(pOper->pLeft, pLeftCols)) { + return pushDownCondOptIsTableColumn(pOper->pRight, pRightCols); + } else if (pushDownCondOptIsTableColumn(pOper->pLeft, pRightCols)) { + return pushDownCondOptIsTableColumn(pOper->pRight, pLeftCols); + } + return false; +} + +static int32_t pushDownCondOptJoinExtractColEqualOnLogicCond(SJoinLogicNode* pJoin) { + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions); + + int32_t code = TSDB_CODE_SUCCESS; + SNodeList* pEqualOnConds = NULL; + SNode* pCond = NULL; + FOREACH(pCond, pLogicCond->pParameterList) { + if (pushDownCondOptIsColEqualOnCond(pJoin, pCond)) { + code = nodesListMakeAppend(&pEqualOnConds, nodesCloneNode(pCond)); + } + } + + SNode* pTempTagEqCond = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = nodesMergeConds(&pTempTagEqCond, &pEqualOnConds); + } + + if (TSDB_CODE_SUCCESS == code) { + pJoin->pColEqualOnConditions = pTempTagEqCond; + return TSDB_CODE_SUCCESS; + } else { + nodesDestroyList(pEqualOnConds); + return TSDB_CODE_PLAN_INTERNAL_ERROR; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t pushDownCondOptJoinExtractColEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { + if (NULL == pJoin->pOnConditions) { + pJoin->pColEqualOnConditions = NULL; + return TSDB_CODE_SUCCESS; + } + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && + LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) { + return pushDownCondOptJoinExtractColEqualOnLogicCond(pJoin); + } + + if (pushDownCondOptIsColEqualOnCond(pJoin, pJoin->pOnConditions)) { + pJoin->pColEqualOnConditions = nodesCloneNode(pJoin->pOnConditions); + } + + return TSDB_CODE_SUCCESS; +} + static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) { return TSDB_CODE_SUCCESS; @@ -774,6 +853,10 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p code = pushDownCondOptJoinExtractMergeCond(pCxt, pJoin); } + if (TSDB_CODE_SUCCESS == code) { + code = pushDownCondOptJoinExtractColEqualOnCond(pCxt, pJoin); + } + if (TSDB_CODE_SUCCESS == code) { OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE); pCxt->optimized = true; @@ -1259,8 +1342,8 @@ static bool smaIndexOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pW .sliding = pIndex->sliding, .slidingUnit = pIndex->slidingUnit, .precision = pScan->node.precision}; - return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval, pScan->node.precision)) && - (pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval, pScan->node.precision)); + return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval)) && + (pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval)); } return true; } @@ -2234,7 +2317,7 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { if (QUERY_NODE_COLUMN == nodeType(pPar)) { SColumnNode* pCol = (SColumnNode*)pPar; if (pCol->colType != COLUMN_TYPE_COLUMN) { - return false; + return false; } } if (hasSelectFunc || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pAggFunc->pParameterList, 0))) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index be43bb008c..4f57193856 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -705,6 +705,9 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren pJoinLogicNode->pOnConditions, &pJoin->pOnConditions); } + if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pColEqualOnConditions) { + code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColEqualOnConditions, &pJoin->pColEqualOnConditions); + } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); } @@ -1147,7 +1150,7 @@ static int32_t createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNo } } -static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWinodwPhysiNode* pWindow, +static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowPhysiNode* pWindow, SWindowLogicNode* pWindowLogicNode) { pWindow->triggerType = pWindowLogicNode->triggerType; pWindow->watermark = pWindowLogicNode->watermark; @@ -1644,6 +1647,9 @@ static int32_t createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pChildren, (SNode*)pChild); } + if (TSDB_CODE_SUCCESS != code) { + break; + } } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/wal/src/walRef.c b/source/libs/wal/src/walRef.c index 6aba661926..eb36389f1d 100644 --- a/source/libs/wal/src/walRef.c +++ b/source/libs/wal/src/walRef.c @@ -63,21 +63,22 @@ int32_t walSetRefVer(SWalRef *pRef, int64_t ver) { return 0; } -SWalRef *walRefFirstVer(SWal *pWal, SWalRef *pRef) { - if (pRef == NULL) { - pRef = walOpenRef(pWal); - if (pRef == NULL) { - return NULL; - } - } +void walRefFirstVer(SWal *pWal, SWalRef *pRef) { taosThreadMutexLock(&pWal->mutex); int64_t ver = walGetFirstVer(pWal); pRef->refVer = ver; taosThreadMutexUnlock(&pWal->mutex); wDebug("vgId:%d, wal ref version %" PRId64 " for first", pWal->cfg.vgId, ver); +} - return pRef; +void walRefLastVer(SWal *pWal, SWalRef *pRef) { + taosThreadMutexLock(&pWal->mutex); + int64_t ver = walGetLastVer(pWal); + pRef->refVer = ver; + + taosThreadMutexUnlock(&pWal->mutex); + wDebug("vgId:%d, wal ref version %" PRId64 " for last", pWal->cfg.vgId, ver); } SWalRef *walRefCommittedVer(SWal *pWal) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index e28e67f83a..9abdeac877 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -382,7 +382,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, "Tag conditon too many TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, "Query not ready") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, "Multiple retrieval of this query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many time window in query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many groups/time window in query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit has reached") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR, "System error") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 711760e549..70588887a0 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -253,15 +253,15 @@ static void taosKeepOldLog(char *oldName) { (void)taosRenameFile(oldName, fileName); - if (tsLogKeepDays < 0) { - char compressFileName[LOG_FILE_NAME_LEN + 20]; - snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); - if (taosCompressFile(fileName, compressFileName) == 0) { - (void)taosRemoveFile(fileName); - } + char compressFileName[LOG_FILE_NAME_LEN + 20]; + snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); + if (taosCompressFile(fileName, compressFileName) == 0) { + (void)taosRemoveFile(fileName); } - taosRemoveOldFiles(tsLogDir, TABS(tsLogKeepDays)); + if (tsLogKeepDays > 0) { + taosRemoveOldFiles(tsLogDir, tsLogKeepDays); + } } static void *taosThreadToOpenNewFile(void *param) { diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index acbf0bb0d3..8f3a80ded4 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -482,6 +482,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { SPageInfo** pInfo = (SPageInfo**)((*pi)->pn->data); if (*pInfo != *pi) { + terrno = TSDB_CODE_APP_ERROR; uError("inconsistently data in paged buffer, pInfo:%p, pi:%p, %s", *pInfo, *pi, pBuf->id); return NULL; } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0b235d0dc8..c56852b88c 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -743,6 +743,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaBasic.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py @@ -1352,6 +1353,7 @@ ,,y,script,./test.sh -f tsim/query/multi_order_by.sim ,,y,script,./test.sh -f tsim/query/sys_tbname.sim ,,y,script,./test.sh -f tsim/query/groupby.sim +,,y,script,./test.sh -f tsim/query/groupby_distinct.sim ,,y,script,./test.sh -f tsim/query/event.sim ,,y,script,./test.sh -f tsim/query/forceFill.sim ,,y,script,./test.sh -f tsim/query/emptyTsRange.sim @@ -1360,6 +1362,7 @@ ,,y,script,./test.sh -f tsim/query/tableCount.sim ,,y,script,./test.sh -f tsim/query/tag_scan.sim ,,y,script,./test.sh -f tsim/query/nullColSma.sim +,,y,script,./test.sh -f tsim/query/bug3398.sim ,,y,script,./test.sh -f tsim/qnode/basic1.sim ,,y,script,./test.sh -f tsim/snode/basic1.sim ,,y,script,./test.sh -f tsim/mnode/basic1.sim diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index ec588659e9..5024f1e2fe 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1722,12 +1722,14 @@ class TaskCreateDb(StateTransitionTask): vg_nums = random.randint(1, 8) cache_model = Dice.choice(['none', 'last_row', 'last_value', 'both']) buffer = random.randint(3, 128) + walRetentionPeriod = random.randint(1, 10000) dbName = self._db.getName() - self.execWtSql(wt, "create database {} {} {} vgroups {} cachemodel '{}' buffer {} ".format(dbName, repStr, + self.execWtSql(wt, "create database {} {} {} vgroups {} cachemodel '{}' buffer {} wal_retention_period {} ".format(dbName, repStr, updatePostfix, vg_nums, cache_model, - buffer)) + buffer, + walRetentionPeriod)) if dbName == "db_0" and Config.getConfig().use_shadow_db: self.execWtSql(wt, "create database {} {} {} ".format("db_s", repStr, updatePostfix)) @@ -2041,18 +2043,19 @@ class TdSuperTable: for topic in current_topic_list: topic_list.append(topic) - consumer.subscribe(topic_list) - - # consumer with random work life - time_start = time.time() - while 1: - res = consumer.poll(1) - consumer.commit(res) - if time.time() - time_start > random.randint(5, 50): - break try: + consumer.subscribe(topic_list) + + # consumer with random work life + time_start = time.time() + while 1: + res = consumer.poll(1) + consumer.commit(res) + if time.time() - time_start > random.randint(5, 50): + break consumer.unsubscribe() - except TmqError as e: + consumer.close() + except TmqError as err: # topic deleted by other threads pass return diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index a8feefec1e..604d6ade89 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -16,8 +16,8 @@ int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; -int32_t optrIdxList[] = {0, 7}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_NCHAR}; +int32_t optrIdxList[] = {5, 11}; typedef struct { char* oper; @@ -123,6 +123,7 @@ int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos); int queryColumnTest(TAOS_STMT *stmt, TAOS *taos); int queryMiscTest(TAOS_STMT *stmt, TAOS *taos); int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos); +int insertVarLenErr(TAOS_STMT *stmt, TAOS *taos); enum { TTYPE_INSERT = 1, @@ -190,6 +191,7 @@ CaseCfg gCase[] = { {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, {"query:NG-TBNEXISTS",tListLen(fullColList), fullColList, TTYPE_INSERT_NG,0, false, false, insertNonExistsTb, 10, 10, 1, 3, 0, 0, 1, -1}, + {"query:NG-VARLENERR",tListLen(fullColList), fullColList, TTYPE_INSERT_NG,0, false, true, insertVarLenErr, 10, 10, 1, 3, 0, 0, 1, -1}, // {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, // {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, @@ -319,7 +321,7 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper #if 0 CaseCtrl gCaseCtrl = { // query case with specified col&oper - .bindNullNum = 1, + .bindNullNum = 0, .printCreateTblSql = true, .printQuerySql = true, .printStmtSql = true, @@ -329,18 +331,19 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindTagNum = 0, .bindRowNum = 0, .bindColTypeNum = 0, - .bindColTypeList = NULL, + .bindColTypeList = bindColTypeList, .optrIdxListNum = 0, - .optrIdxList = NULL, + .optrIdxList = optrIdxList, .checkParamNum = false, .printRes = true, .runTimes = 0, .caseRunIdx = -1, - //.optrIdxListNum = tListLen(optrIdxList), - //.optrIdxList = optrIdxList, - //.bindColTypeNum = tListLen(bindColTypeList), - //.bindColTypeList = bindColTypeList, - .caseIdx = 8, + .optrIdxListNum = tListLen(optrIdxList), + .optrIdxList = optrIdxList, + .bindColTypeNum = tListLen(bindColTypeList), + .bindColTypeList = bindColTypeList, + .caseRunIdx = -1, + .caseIdx = 24, .caseNum = 1, .caseRunNum = 1, }; @@ -1450,14 +1453,17 @@ void bpShowBindParam(TAOS_MULTI_BIND *bind, int32_t num) { } } -int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { +int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, bool expectFail) { static int32_t n = 0; - bpCheckColFields(stmt, bind); + if (!expectFail) { + bpCheckColFields(stmt, bind); + } if (gCurCase->bindRowNum > 1) { if (0 == (n++%2)) { if (taos_stmt_bind_param_batch(stmt, bind)) { + if (expectFail) return 0; printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); @@ -1465,6 +1471,7 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { } else { for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { if (taos_stmt_bind_single_param_batch(stmt, bind+i, i)) { + if (expectFail) continue; printf("!!!taos_stmt_bind_single_param_batch %d error:%s\n", taos_stmt_errstr(stmt), i); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); @@ -1474,12 +1481,14 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { } else { if (0 == (n++%2)) { if (taos_stmt_bind_param_batch(stmt, bind)) { + if (expectFail) return 0; printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); } } else { if (taos_stmt_bind_param(stmt, bind)) { + if (expectFail) return 0; printf("!!!taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); @@ -1542,7 +1551,7 @@ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1594,7 +1603,7 @@ int insertMBSETest2(TAOS_STMT *stmt, TAOS *taos) { } } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1652,7 +1661,7 @@ int insertMBMETest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1702,7 +1711,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1770,7 +1779,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { } } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1822,7 +1831,7 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos) { } } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1883,7 +1892,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1949,7 +1958,7 @@ int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -2016,7 +2025,7 @@ int insertAUTOTest2(TAOS_STMT *stmt, TAOS *taos) { if (gCaseCtrl.checkParamNum) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -2076,7 +2085,7 @@ int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -2130,7 +2139,7 @@ int queryColumnTest(TAOS_STMT *stmt, TAOS *taos) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum, false)) { exit(1); } @@ -2178,7 +2187,7 @@ int queryMiscTest(TAOS_STMT *stmt, TAOS *taos) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum, false)) { exit(1); } @@ -2245,6 +2254,42 @@ int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos) { return 0; } +void bpAddWrongVarBuffLen(TAOS_MULTI_BIND* pBind) { + for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { + if (pBind[i].buffer_type == TSDB_DATA_TYPE_BINARY || pBind[i].buffer_type == TSDB_DATA_TYPE_NCHAR) { + *pBind[i].length += 100; + } + } +} + +int insertVarLenErr(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + prepareInsertData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckIsInsert(stmt, 1); + + code = bpSetTableNameTags(&data, 0, "t0", stmt); + if (code != 0){ + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpAddWrongVarBuffLen(data.pBind); + + if (bpBindParam(stmt, data.pBind, true)) { + exit(1); + } + + destroyData(&data); + + return 0; +} int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { diff --git a/tests/script/tsim/compute/last_row.sim b/tests/script/tsim/compute/last_row.sim index 2e060dc285..8e62fbffb5 100644 --- a/tests/script/tsim/compute/last_row.sim +++ b/tests/script/tsim/compute/last_row.sim @@ -213,4 +213,54 @@ if $rows != 2 then return -1 endi +print =======================> regresss bug in last_row query +sql drop database if exists db; +sql create database if not exists db vgroups 1 cachemodel 'both'; +sql create table db.stb (ts timestamp, c0 bigint) tags(t1 int); +sql insert into db.stb_0 using db.stb tags(1) values ('2023-11-23 19:06:40.000', 491173569); +sql insert into db.stb_2 using db.stb tags(3) values ('2023-11-25 19:30:00.000', 2080726142); +sql insert into db.stb_3 using db.stb tags(4) values ('2023-11-26 06:48:20.000', 1907405128); +sql insert into db.stb_4 using db.stb tags(5) values ('2023-11-24 22:56:40.000', 220783803); + +sql create table db.stb_1 using db.stb tags(2); +sql insert into db.stb_1 (ts) values('2023-11-26 13:11:40.000'); +sql insert into db.stb_1 (ts, c0) values('2023-11-26 13:11:39.000', 11); + +sql select tbname,ts,last_row(c0) from db.stb; +if $rows != 1 then + return -1 +endi + +if $data00 != @stb_1@ then + return -1 +endi + +if $data01 != @23-11-26 13:11:40.000@ then + return -1 +endi + +if $data02 != NULL then + return -1 +endi + +sql alter database db cachemodel 'none'; +sql reset query cache; +sql select tbname,last_row(c0, ts) from db.stb; + +if $rows != 1 then + return -1 +endi + +if $data00 != @stb_1@ then + return -1 +endi + +if $data02 != @23-11-26 13:11:40.000@ then + return -1 +endi + +if $data01 != NULL then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/function.sim b/tests/script/tsim/parser/function.sim index 7f69aa2d02..120c4b8148 100644 --- a/tests/script/tsim/parser/function.sim +++ b/tests/script/tsim/parser/function.sim @@ -954,11 +954,14 @@ endi print =========================>TD-5190 sql select _wstart, stddev(f1) from st1 where ts>'2021-07-01 1:1:1' and ts<'2021-07-30 00:00:00' interval(1d) fill(NULL); if $rows != 29 then + print expect 29, actual: $rows return -1 endi + if $data00 != @21-07-01 00:00:00.000@ then return -1 endi + if $data01 != NULL then return -1 endi diff --git a/tests/script/tsim/parser/join_manyblocks.sim b/tests/script/tsim/parser/join_manyblocks.sim index a40a75f50c..7fd0df21b3 100644 --- a/tests/script/tsim/parser/join_manyblocks.sim +++ b/tests/script/tsim/parser/join_manyblocks.sim @@ -6,8 +6,8 @@ sql connect $dbPrefix = join_m_db $tbPrefix = join_tb $mtPrefix = join_mt -$tbNum = 3 -$rowNum = 2000 +$tbNum = 20 +$rowNum = 200 $totalNum = $tbNum * $rowNum print =============== join_manyBlocks.sim @@ -78,8 +78,8 @@ print ==============> td-3313 sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1; print $row -if $row != 6000 then - print expect 6000, actual: $row +if $row != 4000 then + print expect 4000, actual: $row return -1 endi diff --git a/tests/script/tsim/parser/sliding.sim b/tests/script/tsim/parser/sliding.sim index 1cb4cb5340..7aa69ce9a9 100644 --- a/tests/script/tsim/parser/sliding.sim +++ b/tests/script/tsim/parser/sliding.sim @@ -450,10 +450,11 @@ endi print ====================>check boundary check crash at client side sql select count(*) from sliding_mt0 where ts>now and ts < now-1h; +sql select sum(c1) from sliding_tb0 interval(1a) sliding(1a); + print ========================query on super table print ========================error case -sql_error select sum(c1) from sliding_tb0 interval(1a) sliding(1a); sql_error select sum(c1) from sliding_tb0 interval(10a) sliding(12a); sql_error select sum(c1) from sliding_tb0 sliding(1n) interval(1y); sql_error select sum(c1) from sliding_tb0 interval(-1y) sliding(1n); diff --git a/tests/script/tsim/query/bug3398.sim b/tests/script/tsim/query/bug3398.sim new file mode 100644 index 0000000000..3ca88cf459 --- /dev/null +++ b/tests/script/tsim/query/bug3398.sim @@ -0,0 +1,30 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== create database +sql create database test + +print =============== create super table and child table +sql use test + +sql CREATE STABLE st (day timestamp, c2 int) TAGS (vin binary(32)) + +sql insert into test.g using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) +sql insert into test.t using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) +sql insert into test.tg using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) + +sql select sum(case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end), sum(case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end) from test.t t, test.g g, test.tg tg where t.day = g.day and t.day = tg.day and t.day between '2021-05-03' and '2023-05-04' and t.vin = 'TAG1' and t.vin = g.vin and t.vin = tg.vin group by t.day; + +print $rows $data00 $data01 +if $rows != 1 then + return -1 +endi +if $data00 != 2.000000000 then + return -1 +endi + +if $data01 != 3.000000000 then + return -1 +endi diff --git a/tests/script/tsim/query/groupby_distinct.sim b/tests/script/tsim/query/groupby_distinct.sim new file mode 100644 index 0000000000..8b16bb1910 --- /dev/null +++ b/tests/script/tsim/query/groupby_distinct.sim @@ -0,0 +1,30 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + + +sql drop database if exists db1; +sql create database db1; +sql use db1; + +sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create table tba1 using sta tags(1, 1, 1); +sql insert into tba1 values ('2022-04-26 15:15:08', 1, "a"); +sql insert into tba1 values ('2022-04-26 15:15:07', 1, "b"); +sql insert into tba1 values ('2022-04-26 15:15:06', 1, "a"); +sql insert into tba1 values ('2022-04-26 15:15:05', 1, "b"); +sql insert into tba1 values ('2022-04-26 15:15:04', 1, "c"); +sql insert into tba1 values ('2022-04-26 15:15:03', 1, "c"); +sql insert into tba1 values ('2022-04-26 15:15:02', 1, "d"); +sql insert into tba1 values ('2022-04-26 15:15:01', 1, "d"); +sql select distinct avg(f1) as avgv from sta group by f2; +if $rows != 1 then + return -1 +endi +sql select distinct avg(f1) as avgv from sta group by f2 limit 1,10; +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/interval.sim b/tests/script/tsim/query/interval.sim index 833da4a8ba..e2b0d219cb 100644 --- a/tests/script/tsim/query/interval.sim +++ b/tests/script/tsim/query/interval.sim @@ -170,6 +170,42 @@ endi # return -1 #endi +print ================ step10 +print -------- create database and table +sql create database if not exists test +sql use test +sql create stable st (ts timestamp, c2 int) tags(tg int) +print -------- insert 300 rows data +$i = 0 +while $i < 300 + $t = 1577807983000 + $cc = $i * 1000 + $t = $t + $cc + sql select $i % 3 + if $data00 != 0.000000000 then + goto continue_while + endi + sql select $i % 4 + if $data00 == 0.000000000 then + goto continue_while + endi + sql insert into t1 using st tags(1) values ( $t , $i ) + continue_while: + $i = $i + 1 +endw + +$ms1 = 1577808120000 +$ms2 = 1577808000000 +sql select * from (select _wstart, last(ts) as ts, avg(c2) as av from t1 where ts <= $ms1 and ts >= $ms2 interval(10s) sliding(1s) fill(NULL)) order by ts asc +print ----> select asc rows: $rows +$asc_rows = $rows +sql select * from (select _wstart, last(ts) as ts, avg(c2) as av from t1 where ts <= $ms1 and ts >= $ms2 interval(10s) sliding(1s) fill(NULL)) order by ts desc +print ----> select desc rows: $rows +$desc_rows = $rows +if $desc_rows != $asc_rows then + return -1 +endi + print =============== clear #sql drop database $db #sql select * from information_schema.ins_databases diff --git a/tests/script/tsim/query/partitionby.sim b/tests/script/tsim/query/partitionby.sim index 4c221e02d3..76d4f87908 100644 --- a/tests/script/tsim/query/partitionby.sim +++ b/tests/script/tsim/query/partitionby.sim @@ -24,18 +24,85 @@ sql use $db sql create table $mt1 (ts timestamp, f1 int) TAGS(tag1 int, tag2 binary(500)) sql create table tb0 using $mt1 tags(0, 'a'); sql create table tb1 using $mt1 tags(1, 'b'); -sql create table tb2 using $mt1 tags(1, 'a'); -sql create table tb3 using $mt1 tags(1, 'a'); -sql create table tb4 using $mt1 tags(3, 'b'); -sql create table tb5 using $mt1 tags(3, 'a'); -sql create table tb6 using $mt1 tags(3, 'b'); -sql create table tb7 using $mt1 tags(3, 'b'); +sql create table tb2 using $mt1 tags(2, 'a'); +sql create table tb3 using $mt1 tags(3, 'a'); +sql create table tb4 using $mt1 tags(4, 'b'); +sql create table tb5 using $mt1 tags(5, 'a'); +sql create table tb6 using $mt1 tags(6, 'b'); +sql create table tb7 using $mt1 tags(7, 'b'); sql select * from $mt1 partition by tag1,tag2 limit 1; if $rows != 0 then return -1 endi +sql insert into tb0 values ('2022-04-26 15:15:08', 1); +sql insert into tb1 values ('2022-04-26 15:15:07', 2); +sql insert into tb2 values ('2022-04-26 15:15:06', 3); +sql insert into tb3 values ('2022-04-26 15:15:05', 4); +sql insert into tb4 values ('2022-04-26 15:15:04', 5); +sql insert into tb5 values ('2022-04-26 15:15:03', 6); +sql insert into tb6 values ('2022-04-26 15:15:02', 7); +sql insert into tb7 values ('2022-04-26 15:15:01', 8); + +sql select _wstart as ts, count(*) from $mt1 partition by tag1 interval(1s) order by _wstart; +if $rows != 8 then + return -1 +endi +if $data00 != @22-04-26 15:15:01.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data10 != @22-04-26 15:15:02.000@ then + return -1 +endi +if $data11 != 1 then + return -1 +endi +if $data20 != @22-04-26 15:15:03.000@ then + return -1 +endi +if $data21 != 1 then + return -1 +endi +if $data30 != @22-04-26 15:15:04.000@ then + return -1 +endi +if $data31 != 1 then + return -1 +endi +if $data40 != @22-04-26 15:15:05.000@ then + return -1 +endi +if $data41 != 1 then + return -1 +endi +if $data50 != @22-04-26 15:15:06.000@ then + return -1 +endi +if $data51 != 1 then + return -1 +endi +if $data60 != @22-04-26 15:15:07.000@ then + return -1 +endi +if $data61 != 1 then + return -1 +endi +if $data70 != @22-04-26 15:15:08.000@ then + return -1 +endi +if $data71 != 1 then + return -1 +endi +sql select * from (select _wstart as ts, count(*) from $mt1 partition by tag1 interval(1s) order by _wstart) order by ts; +sql select _wstart as ts, count(*) from $mt1 interval(1s) order by _wstart; +sql select * from (select _wstart as ts, count(*) from $mt1 interval(1s) order by _wstart) order by ts; +sql select diff(a) from (select _wstart as ts, count(*) a from $mt1 interval(1s) order by _wstart); +sql select diff(a) from (select _wstart as ts, count(*) a from $mt1 partition by tag1 interval(1s) order by _wstart); + sql insert into tb0 values (now, 0); sql insert into tb1 values (now, 1); sql insert into tb2 values (now, 2); @@ -54,7 +121,7 @@ sql select count(*) from (select ts from $mt1 where ts is not null partition by if $rows != 1 then return -1 endi -if $data00 != 2 then +if $data00 != 4 then return -1 endi @@ -62,7 +129,7 @@ sql select count(*) from (select ts from $mt1 where ts is not null partition by if $rows != 1 then return -1 endi -if $data00 != 8 then +if $data00 != 16 then return -1 endi diff --git a/tests/system-test/2-query/count_null.py b/tests/system-test/2-query/count_null.py new file mode 100644 index 0000000000..6d2c8db8d6 --- /dev/null +++ b/tests/system-test/2-query/count_null.py @@ -0,0 +1,144 @@ +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def check_results(self): + tdSql.query(f"select count(*) from tb1") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c2) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c3) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c4) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c5) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c6) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c7) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c8) from tb1") + tdSql.checkData(0, 0, 0) + + tdSql.query(f"select count(*) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c2) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c3) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c4) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c5) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c6) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c7) from tb2") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c8) from tb2") + tdSql.checkData(0, 0, 0) + + for i in range (3, 6): + tdSql.query(f"select count(*) from tb{i}") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c2) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c3) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c4) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c5) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c6) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c7) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c8) from tb{i}") + tdSql.checkData(0, 0, 10000) + + + def run(self): + dbname = 'db' + tbnames = ['tb1', 'tb2', 'tb3', 'tb4', 'tb5', 'tb6'] + num_rows = 20000 + num_tables = 6 + ts_base = 1685548800000 + + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + + for i in range (num_tables): + tdSql.execute( + f'''create table if not exists {dbname}.{tbnames[i]} + (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10)) + + ''' + ) + + + tdLog.printNoPrefix("==========step2:insert data") + + for i in range(num_rows): + tdSql.execute(f"insert into {dbname}.{tbnames[0]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + + for i in range(num_rows): + tdSql.execute(f"insert into {dbname}.{tbnames[1]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)") + + for i in range(num_rows): + if i % 2 == 0: + tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + for i in range(num_rows): + if i % 2 == 0: + tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, null, null, null, null, null, null, null, 'binary', 'nchar')") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)") + + for i in range(num_rows): + if i < num_rows / 2: + tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + for i in range(num_rows): + if i >= num_rows / 2: + tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + + tdLog.printNoPrefix("==========step3:check result in memory") + self.check_results(); + + tdLog.printNoPrefix("==========step3:check result from disk") + tdSql.execute(f"flush database db") + self.check_results(); + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/smaBasic.py b/tests/system-test/2-query/smaBasic.py new file mode 100644 index 0000000000..43c379ee53 --- /dev/null +++ b/tests/system-test/2-query/smaBasic.py @@ -0,0 +1,296 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + + value = f"({self.ts}, " + self.ts += 1 + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null)" + else: + value += f"{c2})" + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + + sql = "flush database db;" + tdLog.info(sql) + tdSql.execute(sql) + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1600000000000 + self.childCnt = 5 + self.childRow = 1000000 + self.batchSize = 5000 + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # create database db + sql = f"create database db vgroups 5 replica 3" + tdLog.info(sql) + tdSql.execute(sql) + sql = f"use db" + tdSql.execute(sql) + + # create super talbe st + sql = f"create table st(ts timestamp, c1 int, c2 bigint) tags(area int)" + tdLog.info(sql) + tdSql.execute(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table t{i} using st tags({i}) " + tdSql.execute(sql) + + # insert data + self.insertData() + + # query sql value + def queryValue(self, sql): + tdSql.query(sql) + return tdSql.getData(0, 0) + + # sum + def checkCorrentSum(self): + # query count + sql = "select sum(c1) from st" + val = self.queryValue(sql) + # c1Sum is equal c1Cnt + if val != self.c1Cnt: + tdLog.exit(f"Sum Not Expect. expect={self.c1Cnt} query={val} sql:{sql}") + return + + # not + sql1 = "select sum(c1) from st where c2 = 8764231" + val1 = self.queryValue(sql1) + sql2 = "select sum(c1) from st where c2 != 8764231" + val2 = self.queryValue(sql2) + sql3 = "select sum(c1) from st where c2 is null" + val3 = self.queryValue(sql3) + if val != val1 + val2 + val3: + tdLog.exit(f"Sum Not Equal. val != val1 + val2 + val3. val={val} val1={val1} val2={val2} val2={val3} sql1={sql1} sql2={sql2} sql2={sql3}") + return + + # over than + sql1 = "select sum(c1) from st where c2 > 8000" + val1 = self.queryValue(sql1) + sql2 = "select sum(c1) from st where c2 <= 8000" + val2 = self.queryValue(sql2) + sql3 = "select sum(c1) from st where c2 is null" + val3 = self.queryValue(sql3) + if val != val1 + val2 + val3: + tdLog.exit(f"Sum Not Equal. val != val1 + val2 + val3. val={val} val1={val1} val2={val2} val2={val3} sql1={sql1} sql2={sql2} sql2={sql3}") + return + + tdLog.info(f"check correct sum on c1 successfully.") + + # check result + def checkResult(self, fun, val, val1, val2, sql1, sql2): + if fun == "count": + if val != val1 + val2: + tdLog.exit(f"{fun} NOT SAME. val != val1 + val2. val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}") + return + elif fun == "max": + if val != max([val1, val2]): + tdLog.exit(f"{fun} NOT SAME . val != max(val1 ,val2) val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}") + return + elif fun == "min": + if val != min([val1, val2]): + tdLog.exit(f"{fun} NOT SAME . val != min(val1 ,val2) val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}") + return + + # sum + def checkCorrentFun(self, fun, expectVal): + # query + sql = f"select {fun}(c2) from st" + val = self.queryValue(sql) + if val != expectVal: + tdLog.exit(f"{fun} Not Expect. expect={expectVal} query={val} sql:{sql}") + return + + # not + sql1 = f"select {fun}(c2) from st where c2 = 8764231" + val1 = self.queryValue(sql1) + sql2 = f"select {fun}(c2) from st where c2 != 8764231" + val2 = self.queryValue(sql2) + self.checkResult(fun, val, val1, val2, sql1, sql2) + + # over than + sql1 = f"select {fun}(c2) from st where c2 > 8000" + val1 = self.queryValue(sql1) + sql2 = f"select {fun}(c2) from st where c2 <= 8000" + val2 = self.queryValue(sql2) + self.checkResult(fun, val, val1, val2, sql1, sql2) + + # successful + tdLog.info(f"check correct {fun} on c2 successfully.") + + # check query corrent + def checkCorrect(self): + # count + self.checkCorrentFun("count", self.c2Cnt) + # max + self.checkCorrentFun("max", self.c2Max) + # min + self.checkCorrentFun("min", self.c2Min) + # sum + self.checkCorrentSum() + + # c2 sum + sql = "select sum(c2) from st" + val = self.queryValue(sql) + # c1Sum is equal c1Cnt + if val != self.c2Sum: + tdLog.exit(f"c2 Sum Not Expect. expect={self.c2Sum} query={val} sql:{sql}") + return + + def checkPerformance(self): + # have sma caculate + sql1 = "select count(*) from st" + stime = time.time() + tdSql.execute(sql1, 1) + spend1 = time.time() - stime + + + # no sma caculate + sql2 = "select count(*) from st where c2 != 8764231 or c2 is null" + stime = time.time() + tdSql.execute(sql2, 1) + spend2 = time.time() - stime + + time1 = "%.2f"%(spend1*1000) + time2 = "%.2f"%(spend2*1000) + if spend2 < spend1 * 8: + tdLog.exit(f"performance not passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") + return + tdLog.info(f"performance passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") + + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.clock_gettime(time.CLOCK_REALTIME) + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # run + def run(self): + # prepare env + self.prepareEnv() + + # query + self.checkCorrect() + + # performance + self.checkPerformance() + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/eco-system/main.py b/tests/system-test/eco-system/main.py new file mode 100644 index 0000000000..a1f72147a0 --- /dev/null +++ b/tests/system-test/eco-system/main.py @@ -0,0 +1,37 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import re +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + + def run(self): + tdLog.info(" ------ eco-system main -------") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/eco-system/manager/drop_table.py b/tests/system-test/eco-system/manager/drop_table.py new file mode 100644 index 0000000000..d4cdf4a541 --- /dev/null +++ b/tests/system-test/eco-system/manager/drop_table.py @@ -0,0 +1,146 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + # describe table + def describe_table(self, tbname): + columns = [] + tags = [] + sql = f"describe {tbname}" + row_cnt = tdSql.query(sql) + for i in range(0, row_cnt): + col_name = tdSql.queryResult[i][0] + type_name = tdSql.queryResult[i][3] + if type_name == "TAG": + tags.append(col_name) + else: + columns.append(col_name) + + return columns,tags + + # show tables + def show_tables(self): + sql = "show tables;" + row_cnt = tdSql.query(sql) + tables = [] + for i in range(0, row_cnt): + tb_name = tdSql.queryResult[i][0] + tables.append(tb_name) + + + # execute sql + def execute(self, sql): + try: + tdSql.execute(sql, 3) + tdLog.info(f" exec ok. {sql}") + except: + tdLog.info(f" exe failed. {sql}") + traceback.print_exc() + + + # query + def query_table(self, columns, tags): + if len(columns) < 5 : + return + if len(tags) < 5: + return + + sel_cols = random.sample(columns, random.randint(1,int(len(columns)-1))) + sel_tags = random.sample(tags, random.randint(1, int(len(tags)-1))) + + field_cols = ",".join(sel_cols) + field_tags = ",".join(sel_tags) + + #sql = f"select {field_cols},{field_tags} from meters ;" + sql = f"select {field_cols},{field_tags} from meters" + try: + tdLog.info( " query sql:" + sql) + tdSql.query("select * from meters limit 1") + except: + tdLog.info( " query failed :" + sql) + traceback.print_exc() + + # change table schema + def drop_table(self, change_cnt): + # init + + tables = self.show_tables() + + + for i in range(change_cnt): + col_idx = random.randint(0, ncol - 1) + tag_idx = random.randint(0, ntag - 1) + + cols = list(self.column_dict.keys()) + tags = list(self.tag_dict.keys()) + + # column + key = cols[col_idx] + value = self.column_dict[key] + sql = f'alter table meters drop column {key}' + self.execute(sql) + sql = f'alter table meters add column {key} {value}' + self.execute(sql) + + + # column + key = tags[col_idx] + value = self.tag_dict[key] + sql = f'alter table meters drop tag {key}' + self.execute(sql) + sql = f'alter table meters add tag {key} {value}' + self.execute(sql) + + # drop and rename + if i % 5 == 0: + # update columns + #columns,tags = self.describe_table("meters") + tdLog.info(f" ======= describe table column count = {len(cols)} tags= {len(tags)}======") + self.query_table(cols, tags) + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + + # change meters + self.drop_table(1000000) + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/eco-system/manager/same_column.py b/tests/system-test/eco-system/manager/same_column.py new file mode 100644 index 0000000000..beaf4e449e --- /dev/null +++ b/tests/system-test/eco-system/manager/same_column.py @@ -0,0 +1,181 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.setsql = TDSetSql() + self.column_dict = { + 'bc': 'bool', + 'fc': 'float', + 'dc': 'double', + 'ti': 'tinyint', + 'si': 'smallint', + 'ic': 'int', + 'bi': 'bigint', + 'uit': 'tinyint unsigned', + 'usi': 'smallint unsigned', + 'ui': 'int unsigned', + 'ubi': 'bigint unsigned', + 'bin': 'binary(32)', + 'nch': 'nchar(64)' + } + self.tag_dict = { + 'groupid': 'tinyint', + 'location': 'binary(16)', + 'tfc': 'float', + 'tdc': 'double', + 'tti': 'tinyint', + 'tsi': 'smallint', + 'tic': 'int', + 'tbi': 'bigint', + 'tuit': 'tinyint unsigned', + 'tusi': 'smallint unsigned', + 'tui': 'int unsigned', + 'tubi': 'bigint unsigned', + 'tbin': 'binary(32)', + 'tnch': 'nchar(64)' + } + + # describe table + def describe_table(self, tbname): + columns = [] + tags = [] + sql = f"describe {tbname}" + row_cnt = tdSql.query(sql) + for i in range(20, row_cnt): + col_name = tdSql.queryResult[i][0] + type_name = tdSql.queryResult[i][3] + if type_name == "TAG": + tags.append(col_name) + else: + columns.append(col_name) + + return columns,tags + + def drop_tag(self, tags, cnt): + for i in range(cnt): + tag_cnt = len(tags) + sel = random.randint(1, tag_cnt-1) + sql = f"alter table meters drop tag `{tags[sel]}` " + try: + tdSql.execute(sql) + tdLog.info(sql) + del tags[sel] + except: + tdLog.info(f" drop tags failed. {sql}") + traceback.print_exc() + + # execute sql + def execute(self, sql): + try: + tdSql.execute(sql, 3) + tdLog.info(f" exec ok. {sql}") + except: + tdLog.info(f" exe failed. {sql}") + traceback.print_exc() + + + # query + def query_table(self, columns, tags): + if len(columns) < 5 : + return + if len(tags) < 5: + return + + sel_cols = random.sample(columns, random.randint(1,int(len(columns)-1))) + sel_tags = random.sample(tags, random.randint(1, int(len(tags)-1))) + + field_cols = ",".join(sel_cols) + field_tags = ",".join(sel_tags) + + #sql = f"select {field_cols},{field_tags} from meters ;" + sql = f"select {field_cols},{field_tags} from meters" + try: + tdLog.info( " query sql:" + sql) + tdSql.query("select * from meters limit 1") + except: + tdLog.info( " query failed :" + sql) + traceback.print_exc() + + # change table schema + def change_columns(self, change_cnt): + # init + + ncol = len(self.column_dict) + ntag = len(self.tag_dict) + + for i in range(change_cnt): + col_idx = random.randint(0, ncol - 1) + tag_idx = random.randint(0, ntag - 1) + + cols = list(self.column_dict.keys()) + tags = list(self.tag_dict.keys()) + + # column + key = cols[col_idx] + value = self.column_dict[key] + sql = f'alter table meters drop column {key}' + self.execute(sql) + sql = f'alter table meters add column {key} {value}' + self.execute(sql) + + + # column + key = tags[col_idx] + value = self.tag_dict[key] + sql = f'alter table meters drop tag {key}' + self.execute(sql) + sql = f'alter table meters add tag {key} {value}' + self.execute(sql) + + # drop and rename + if i % 5 == 0: + # update columns + #columns,tags = self.describe_table("meters") + tdLog.info(f" ======= describe table column count = {len(cols)} tags= {len(tags)}======") + self.query_table(cols, tags) + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + + # change meters + self.change_columns(1000000) + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/eco-system/manager/schema_change.py b/tests/system-test/eco-system/manager/schema_change.py new file mode 100644 index 0000000000..400d2b100b --- /dev/null +++ b/tests/system-test/eco-system/manager/schema_change.py @@ -0,0 +1,239 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.setsql = TDSetSql() + self.column_dict = { + 'col0': 'int', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': 'varchar(20)', + 'col13': 'nchar(20)' + } + self.tag_dict = { + 't1': 'tinyint', + 't2': 'smallint', + 't3': 'int', + 't4': 'bigint', + 't5': 'tinyint unsigned', + 't6': 'smallint unsigned', + 't7': 'int unsigned', + 't8': 'bigint unsigned', + 't9': 'float', + 't10': 'double', + 't11': 'bool', + 't12': 'varchar(20)', + 't13': 'nchar(20)', + 't14': 'timestamp' + } + + + # delete + def delete_col(self, columns, cnt, max_col): + # delte for random + for i in range(cnt): + col_cnt = len(columns) + if col_cnt == 0: + return + sel = random.randint(0, col_cnt - 1) + sql = f"alter table meters drop column `{columns[sel]}`" + try: + tdSql.execute(sql) + tdLog.info(f" drop cur col={len(columns)} max_col={max_col} {sql}") + del columns[sel] + except: + tdLog.info(f" drop column failed. {sql}") + traceback.print_exc() + + + # describe table + def describe_table(self, tbname): + columns = [] + tags = [] + sql = f"describe {tbname}" + row_cnt = tdSql.query(sql) + for i in range(20, row_cnt): + col_name = tdSql.queryResult[i][0] + type_name = tdSql.queryResult[i][3] + if type_name == "TAG": + tags.append(col_name) + else: + columns.append(col_name) + + return columns,tags + + def renames(self, tags, cnt): + col_cnt = len(tags) + if col_cnt < 10: + return + for i in range(cnt): + sel = random.randint(1, col_cnt-3) + new_name = tags[sel] + "n" + sql = f"alter table meters rename tag `{tags[sel]}` `{new_name}` " + try: + tdSql.execute(sql) + tdLog.info(sql) + tags[sel] = new_name + except: + tdLog.info(f" rename tag failed. {sql}") + traceback.print_exc() + + + def drop_tag(self, tags, cnt): + for i in range(cnt): + tag_cnt = len(tags) + sel = random.randint(1, tag_cnt-1) + sql = f"alter table meters drop tag `{tags[sel]}` " + try: + tdSql.execute(sql) + tdLog.info(sql) + del tags[sel] + except: + tdLog.info(f" drop tags failed. {sql}") + traceback.print_exc() + + # query + def query_table(self, columns, tags): + if len(columns) < 10 : + return + if len(tags) < 10: + return + + sel_cols = random.sample(columns, random.randint(2,int(len(columns)/10))) + sel_tags = random.sample(tags, random.randint(1,int(len(tags)/10))) + + field_cols = ",".join(sel_cols) + field_tags = ",".join(sel_tags) + + #sql = f"select {field_cols},{field_tags} from meters ;" + sql = f"select {field_cols},{field_tags} from meters" + try: + tdLog.info( " query sql:" + sql) + tdSql.query("select * from meters limit 1") + except: + tdLog.info( " query failed :" + sql) + traceback.print_exc() + + # change table schema + def change_schema(self, change_cnt): + # init + columns, tags = self.describe_table("meters") + max_col = random.randint(200, 2000) + tdLog.info(f" ----------- set max column = {max_col} -------------") + for i in range(change_cnt): + col_cnt = len(self.column_dict) + icol = random.randint(0, col_cnt-1) + key = f"col{icol}" + col_name = key + f"_{i}_{random.randint(1,100)}" + col_type = self.column_dict[key] + sql = f'alter table meters add column `{col_name}` {col_type}' + sql_tag = f'alter table meters add tag `t_{col_name}` {col_type}' + + try: + tdSql.execute(sql) + tdLog.info(f" add cur col={len(columns)} max_col={max_col} {sql}") + columns.append(col_name) + if random.randint(1, 4) == 2: + tdSql.execute(sql_tag) + tdLog.info(f" add tag tag_cnt={len(tags)} {sql_tag}") + + except: + tdLog.info(f" add column failed. {sql}") + traceback.print_exc() + + + col_cnt = len(columns) + # delete + if col_cnt > max_col + 100: + self.delete_col(columns, random.randint(1, 30), max_col) + elif col_cnt >= max_col + 30: + self.delete_col(columns, random.randint(1, 4), max_col) + max_col = random.randint(200, 2000) + tdLog.info(f" ----------- set max column = {max_col} -------------") + elif col_cnt > max_col: + self.delete_col(columns, random.randint(1, 3), max_col) + + + + if i % 50 == 0: + sql = f"flush database {self.dbname};" + tdSql.execute(sql) + tdLog.info(f" ***** {sql} *****") + + # query + if i % 70 == 0: + self.query_table(columns, tags) + + # drop and rename + if i % 10 == 0: + # update columns + columns,tags = self.describe_table("meters") + tdLog.info(f" ======= describe table column count = {len(columns)} tags= {len(tags)}======") + + if random.randint(1,3) == 2: + self.query_table(columns, tags) + + if len(tags) > 50: + self.drop_tag(tags, random.randint(1, 30)) + + self.renames(tags, random.randint(1, 10)) + + + # sleep + #time.sleep(0.3) + + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + + # change meters + self.change_schema(1000000) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/eco-system/schemaless/insert.py b/tests/system-test/eco-system/schemaless/insert.py new file mode 100644 index 0000000000..901196ebfd --- /dev/null +++ b/tests/system-test/eco-system/schemaless/insert.py @@ -0,0 +1,151 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback +import taos +import string +from taos import schemaless + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.setsql = TDSetSql() + self.conn = conn + self.schema = {} + + def random_string(self, count): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(count)) + + def genCol(self, col_name, isTag): + col_types = ["str","f64","f32","i8","u8","i16","u16","i32","u32","i64","u64"] + if self.schema.get(col_name) == None: + col_type = random.choice(col_types) + self.schema[col_name] = col_type + else: + col_type = self.schema[col_name] + + is_num = True + val = "" + if col_type == "str": + val = self.random_string(random.randint(1, 10)) + is_num = False + elif col_type == "f64": + val = random.randrange(-100000000000000, 1000000000000)/3*2.25678 + elif col_type == "f32": + val = random.randrange(-100000000, 1000000000)/3*1.2345 + elif col_type == "i8": + val = random.randint(-128, 127) + elif col_type == "u8": + val = random.randint(0, 256) + elif col_type == "i16": + val = random.randint(-32768, 32767) + elif col_type == "u16": + val = random.randint(0, 256*256) + elif col_type == "i32": + val = random.randint(-256*256*256*128, 256*256*256*128) + elif col_type == "u32": + val = random.randint(0, 256*256*256*256) + elif col_type == "i64": + val = random.randint(-256*256*256*256*256*256*256*128, 256*256*256*256*256*256*256*128) + elif col_type == "u64": + val = random.randint(0, 256*256*256*256*256*256*256*256) + else: + val = 100 + + if isTag: + col_val = val + elif is_num: + col_val = f'{val}{col_type}' + else: + col_val = '"' + val + '"' + + return f'{col_name}={col_val}' + + + # cols + def genCols(self, pre, max, index, isTag): + col_cnt = random.randint(1, max) + cols = [] + for i in range(col_cnt): + col_name = f'{pre}_{index}_{i}' + cols.append(self.genCol(col_name, isTag)) + + return ",".join(cols) + + + # execute sql + def insert(self,sql,i): + print("schema less insert") + try: + self.conn.schemaless_insert([sql], schemaless.SmlProtocol.LINE_PROTOCOL, schemaless.SmlPrecision.MILLI_SECONDS) + tdLog.info(f" exec ok i={i} {sql}") + except: + tdLog.info(f" exe failed. i={i} {sql}") + traceback.print_exc() + + def genTags(self, i): + tags = f"t1={i},t2=abc,t3=work" + return tags + + # change table schema + def schemaless_insert(self, change_cnt): + # init + ts = 1683194263000 + for i in range(change_cnt): + index = int(i/10000) % 600 + cols = self.genCols("c", 5, index, False) + tags = self.genTags(index) + sql = f'{self.stable},{tags} {cols} {ts + i}' + self.insert(sql, i) + + # run + + def run(self): + # seed + #random.seed(int(time.time())) + self.dbname = "eco_system" + self.stable = "sml_stb" + + # switch db + tdSql.execute(f"use {self.dbname};") + tdSql.execute(f"drop table if exists {self.stable};") + + + + # change meters + try: + self.schemaless_insert(1000000) + except: + traceback.print_exc() + + print(self.schema) + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase())