xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into enh/TD-22404-M

This commit is contained in:
kailixu 2023-03-18 14:25:34 +08:00
parent 43a029e3c4 cd498627bd
commit 4317564d5f
69 changed files with 1307 additions and 565 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG ad1a32b
GIT_TAG d11f210
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

2
include/libs/qworker/qworker.h
View File

@ -106,6 +106,8 @@ int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64
int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId,
void **pRsp, SArray *explainRes);
int32_t qWorkerDbgEnableDebug(char *option);
#ifdef __cplusplus
}
#endif

1
include/util/taoserror.h
View File

@ -688,6 +688,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_NOT_SUPPORT_JOIN TAOS_DEF_ERROR_CODE(0, 0x2664)
#define TSDB_CODE_PAR_INVALID_TAGS_PC TAOS_DEF_ERROR_CODE(0, 0x2665)
#define TSDB_CODE_PAR_INVALID_TIMELINE_QUERY TAOS_DEF_ERROR_CODE(0, 0x2666)
#define TSDB_CODE_PAR_INVALID_OPTR_USAGE TAOS_DEF_ERROR_CODE(0, 0x2667)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner

4
include/util/tdef.h
View File

@ -316,10 +316,10 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_KEEP_NS (365 * 292 * 1440) // data in db to be reserved.
#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years
#define TSDB_MIN_MINROWS_FBLOCK 10
#define TSDB_MAX_MINROWS_FBLOCK 1000
#define TSDB_MAX_MINROWS_FBLOCK 1000000
#define TSDB_DEFAULT_MINROWS_FBLOCK 100
#define TSDB_MIN_MAXROWS_FBLOCK 200
#define TSDB_MAX_MAXROWS_FBLOCK 10000
#define TSDB_MAX_MAXROWS_FBLOCK 10000000
#define TSDB_DEFAULT_MAXROWS_FBLOCK 4096
#define TSDB_MIN_FSYNC_PERIOD 0
#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond

31
source/client/inc/clientInt.h
View File

@ -287,22 +287,25 @@ static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
}
static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
SMqRspObj* msg = (SMqRspObj*)res;
msg->resIter++;
if (msg->resIter < msg->rsp.blockNum) {
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter);
if (msg->rsp.withSchema) {
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(msg->rsp.blockSchema, msg->resIter);
setResSchemaInfo(&msg->resInfo, pSW->pSchema, pSW->nCols);
taosMemoryFreeClear(msg->resInfo.row);
taosMemoryFreeClear(msg->resInfo.pCol);
taosMemoryFreeClear(msg->resInfo.length);
taosMemoryFreeClear(msg->resInfo.convertBuf);
taosMemoryFreeClear(msg->resInfo.convertJson);
SMqRspObj* pRspObj = (SMqRspObj*)res;
pRspObj->resIter++;
if (pRspObj->resIter < pRspObj->rsp.blockNum) {
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter);
if (pRspObj->rsp.withSchema) {
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter);
setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols);
taosMemoryFreeClear(pRspObj->resInfo.row);
taosMemoryFreeClear(pRspObj->resInfo.pCol);
taosMemoryFreeClear(pRspObj->resInfo.length);
taosMemoryFreeClear(pRspObj->resInfo.convertBuf);
taosMemoryFreeClear(pRspObj->resInfo.convertJson);
}
setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4, false);
return &msg->resInfo;
setQueryResultFromRsp(&pRspObj->resInfo, pRetrieve, convertUcs4, false);
return &pRspObj->resInfo;
}
return NULL;
}

9
source/client/src/clientImpl.c
View File

@ -1085,6 +1085,10 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId);
destorySqlCallbackWrapper(pWrapper);
if (TSDB_CODE_SUCCESS != code) {
pRequest->code = terrno;
}
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
}
@ -1132,11 +1136,6 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
pRequest->body.queryFp(pRequest->body.param, pRequest, -1);
break;
}
// TODO weired responding code?
if (TSDB_CODE_SUCCESS != code) {
pRequest->code = terrno;
}
}
int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) {

4
source/client/src/clientJniConnector.c
View File

@ -581,8 +581,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI
return JNI_RESULT_SET_NULL;
}
void *data;
int32_t numOfRows;
void *data = NULL;
int32_t numOfRows = 0;
int error_code = taos_fetch_raw_block(tres, &numOfRows, &data);
if (numOfRows == 0) {
if (error_code == JNI_SUCCESS) {

3
source/client/src/clientMain.c
View File

@ -611,6 +611,9 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
}
int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
*numOfRows = 0;
*pData = NULL;
if (res == NULL || TD_RES_TMQ_META(res)) {
return 0;
}

62
source/client/src/clientRawBlockWrite.c
View File

@ -776,12 +776,37 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
goto end;
}
SCatalog* pCatalog = NULL;
code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter,
.requestId = pRequest->requestId,
.requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
SName pName = {0};
toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName);
STableMeta *pTableMeta = NULL;
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){
code = TSDB_CODE_SUCCESS;
taosMemoryFreeClear(pTableMeta);
goto end;
}
if (code != TSDB_CODE_SUCCESS) {
uError("taosCreateTable:catalogGetTableMeta failed. table name: %s", req.name);
goto end;
}
pReq.suid = pTableMeta->uid;
taosMemoryFreeClear(pTableMeta);
// build drop stable
pReq.igNotExists = true;
pReq.source = TD_REQ_FROM_TAOX;
pReq.suid = processSuid(req.suid, pRequest->pDb);
// pReq.suid = processSuid(req.suid, pRequest->pDb);
uDebug("taosDropStb name:%s suid:%"PRId64" processSuid:%"PRId64, req.name, req.suid, pReq.suid);
uDebug("taosDropStb name:%s suid:%"PRId64" new suid:%"PRId64, req.name, req.suid, pReq.suid);
STscObj* pTscObj = pRequest->pTscObj;
SName tableName = {0};
tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name);
@ -806,7 +831,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
launchQueryImpl(pRequest, &pQuery, true, NULL);
if (pRequest->code == TSDB_CODE_SUCCESS) {
SCatalog* pCatalog = NULL;
// SCatalog* pCatalog = NULL;
catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
catalogRemoveTableMeta(pCatalog, &tableName);
}
@ -900,15 +925,15 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
STableMeta* pTableMeta = NULL;
SName sName = {0};
tb_uid_t oldSuid = pCreateReq->ctb.suid;
pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb);
uDebug("taosCreateTable name:%s sname:%s suid:%"PRId64" processSuid:%"PRId64, pCreateReq->name, pCreateReq->ctb.stbName, pCreateReq->ctb.suid, oldSuid);
// pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb);
toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName);
code = catalogGetTableMeta(pCatalog, &conn, &sName, &pTableMeta);
if (code != TSDB_CODE_SUCCESS) {
uError("taosCreateTable:catalogGetTableMeta failed. table name: %s", pCreateReq->ctb.stbName);
goto end;
}
pCreateReq->ctb.suid = pTableMeta->uid;
uDebug("taosCreateTable name:%s sname:%s suid:%"PRId64" new suid:%"PRId64, pCreateReq->name, pCreateReq->ctb.stbName, oldSuid, pCreateReq->ctb.suid);
for (int32_t i = 0; i < taosArrayGetSize(pCreateReq->ctb.tagName); i++) {
char* tName = taosArrayGet(pCreateReq->ctb.tagName, i);
@ -1041,8 +1066,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
pDropReq = req.pReqs + iReq;
pDropReq->igNotExists = true;
pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb);
uDebug("taosDropTable name:%s suid:%"PRId64" processSuid:%"PRId64, pDropReq->name, pDropReq->suid, pDropReq->suid);
// pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb);
SVgroupInfo pInfo = {0};
SName pName = {0};
@ -1052,6 +1076,22 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
goto end;
}
STableMeta *pTableMeta = NULL;
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){
code = TSDB_CODE_SUCCESS;
taosMemoryFreeClear(pTableMeta);
continue;
}
if (code != TSDB_CODE_SUCCESS) {
uError("taosDropTable:catalogGetTableMeta failed. table name: %s", pDropReq->name);
goto end;
}
tb_uid_t oldSuid = pDropReq->suid;
pDropReq->suid = pTableMeta->suid;
taosMemoryFreeClear(pTableMeta);
uDebug("taosDropTable name:%s suid:%"PRId64" new suid:%"PRId64, pDropReq->name, oldSuid, pDropReq->suid);
taosArrayPush(pRequest->tableList, &pName);
SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId));
if (pTableBatch == NULL) {
@ -1066,6 +1106,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
}
}
if (taosHashGetSize(pVgroupHashmap) == 0){
goto end;
}
SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap);
if (NULL == pBufArray) {
code = TSDB_CODE_OUT_OF_MEMORY;
@ -1673,7 +1716,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
}
if (strcmp(tbName, pCreateReq.name) == 0) {
cloneSVreateTbReq(&pCreateReq, &pCreateReqDst);
pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb);
// pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb);
tDecoderClear(&decoderTmp);
break;
}
@ -1706,6 +1749,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
if (pCreateReqDst) {
pTableMeta->vgId = vg.vgId;
pTableMeta->uid = pCreateReqDst->uid;
pCreateReqDst->ctb.suid = pTableMeta->suid;
}
void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId));
if (hData == NULL) {

337
source/client/src/clientTmq.c
View File

@ -24,6 +24,9 @@
#include "tref.h"
#include "ttimer.h"
#define EMPTY_BLOCK_POLL_IDLE_DURATION 100
#define DEFAULT_AUTO_COMMIT_INTERVAL 5000
struct SMqMgmt {
int8_t inited;
tmr_h timer;
@ -68,18 +71,16 @@ struct tmq_conf_t {
};
struct tmq_t {
int64_t refId;
// conf
char groupId[TSDB_CGROUP_LEN];
char clientId[256];
int8_t withTbName;
int8_t useSnapshot;
int8_t autoCommit;
int32_t autoCommitInterval;
int32_t resetOffsetCfg;
uint64_t consumerId;
bool hbBgEnable;
int64_t refId;
char groupId[TSDB_CGROUP_LEN];
char clientId[256];
int8_t withTbName;
int8_t useSnapshot;
int8_t autoCommit;
int32_t autoCommitInterval;
int32_t resetOffsetCfg;
uint64_t consumerId;
bool hbBgEnable;
tmq_commit_cb* commitCb;
void* commitCbUserParam;
@ -90,14 +91,15 @@ struct tmq_t {
int8_t epStatus;
int32_t epSkipCnt;
#endif
int64_t pollCnt;
// poll info
int64_t pollCnt;
int64_t totalRows;
// timer
tmr_h hbLiveTimer;
tmr_h epTimer;
tmr_h reportTimer;
tmr_h commitTimer;
tmr_h hbLiveTimer;
tmr_h epTimer;
tmr_h reportTimer;
tmr_h commitTimer;
STscObj* pTscObj; // connection
SArray* clientTopics; // SArray<SMqClientTopic>
STaosQueue* mqueue; // queue of rsp
@ -127,6 +129,7 @@ enum {
typedef struct {
int64_t pollCnt;
int64_t numOfRows;
STqOffsetVal committedOffset;
STqOffsetVal currentOffset;
int32_t vgId;
@ -149,6 +152,7 @@ typedef struct {
SMqClientVg* vgHandle;
SMqClientTopic* topicHandle;
uint64_t reqId;
SEpSet* pEpset;
union {
SMqDataRsp dataRsp;
SMqMetaRsp metaRsp;
@ -220,7 +224,7 @@ tmq_conf_t* tmq_conf_new() {
conf->withTbName = false;
conf->autoCommit = true;
conf->autoCommitInterval = 5000;
conf->autoCommitInterval = DEFAULT_AUTO_COMMIT_INTERVAL;
conf->resetOffset = TMQ_OFFSET__RESET_EARLIEAST;
conf->hbBgEnable = true;
@ -266,7 +270,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
}
if (strcasecmp(key, "auto.commit.interval.ms") == 0) {
conf->autoCommitInterval = atoi(value);
conf->autoCommitInterval = taosStr2int64(value);
return TMQ_CONF_OK;
}
@ -310,7 +314,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
}
if (strcasecmp(key, "experimental.snapshot.batch.size") == 0) {
conf->snapBatchSize = atoi(value);
conf->snapBatchSize = taosStr2int64(value);
return TMQ_CONF_OK;
}
@ -330,18 +334,22 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
conf->ip = taosStrdup(value);
return TMQ_CONF_OK;
}
if (strcasecmp(key, "td.connect.user") == 0) {
conf->user = taosStrdup(value);
return TMQ_CONF_OK;
}
if (strcasecmp(key, "td.connect.pass") == 0) {
conf->pass = taosStrdup(value);
return TMQ_CONF_OK;
}
if (strcasecmp(key, "td.connect.port") == 0) {
conf->port = atoi(value);
conf->port = taosStr2int64(value);
return TMQ_CONF_OK;
}
if (strcasecmp(key, "td.connect.db") == 0) {
return TMQ_CONF_OK;
}
@ -403,65 +411,42 @@ static SMqClientVg* foundClientVg(SArray* pTopicList, const char* pName, int32_t
return NULL;
}
// Two problems do not need to be addressed here
// 1. update to of epset. the response of poll request will automatically handle this problem
// 2. commit failure. This one needs to be resolved.
static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) {
SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params;
if (code != TSDB_CODE_SUCCESS) { // if commit offset failed, let's try again
taosThreadMutexLock(&pParam->pTmq->lock);
int32_t numOfVgroups, index;
SMqClientVg* pVg = foundClientVg(pParam->pTmq->clientTopics, pParam->topicName, pParam->vgId, &index, &numOfVgroups);
if (pVg == NULL) {
tscDebug("consumer:0x%" PRIx64
" subKey:%s vgId:%d commit failed, code:%s has been transferred to other consumer, no need retry ordinal:%d/%d",
pParam->pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, tstrerror(code), index + 1, numOfVgroups);
} else { // let's retry the commit
int32_t code1 = doSendCommitMsg(pParam->pTmq, pVg, pParam->topicName, pParamSet, index, numOfVgroups);
if (code1 != TSDB_CODE_SUCCESS) { // retry failed.
tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64
" retry failed, ignore this commit. code:%s ordinal:%d/%d",
pParam->pTmq->consumerId, pParam->topicName, pVg->vgId, pVg->committedOffset.version,
tstrerror(terrno), index + 1, numOfVgroups);
}
}
taosThreadMutexUnlock(&pParam->pTmq->lock);
taosMemoryFree(pParam->pOffset);
taosMemoryFree(pBuf->pData);
taosMemoryFree(pBuf->pEpSet);
tmqCommitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId);
return 0;
}
// todo replace the pTmq with refId
taosThreadMutexLock(&pParam->pTmq->lock);
tmq_t* pTmq = pParam->pTmq;
int32_t index = 0, numOfVgroups = 0;
SMqClientVg* pVg = foundClientVg(pTmq->clientTopics, pParam->topicName, pParam->vgId, &index, &numOfVgroups);
if (pVg == NULL) {
tscDebug("consumer:0x%" PRIx64 " subKey:%s vgId:%d has been transferred to other consumer, ordinal:%d/%d",
pParam->pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, index + 1, numOfVgroups);
} else { // update the epset if needed
if (pBuf->pEpSet != NULL) {
SEp* pEp = GET_ACTIVE_EP(pBuf->pEpSet);
SEp* pOld = GET_ACTIVE_EP(&(pVg->epSet));
tscDebug("consumer:0x%" PRIx64 " subKey:%s update the epset vgId:%d, ep:%s:%d, old ep:%s:%d, ordinal:%d/%d",
pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, pEp->fqdn, pEp->port, pOld->fqdn, pOld->port,
index + 1, numOfVgroups);
pVg->epSet = *pBuf->pEpSet;
}
tscDebug("consumer:0x%" PRIx64 " subKey:%s vgId:%d, commit offset success. ordinal:%d/%d", pTmq->consumerId,
pParam->pOffset->subKey, pParam->vgId, index + 1, numOfVgroups);
}
taosThreadMutexUnlock(&pParam->pTmq->lock);
// if (code != TSDB_CODE_SUCCESS) { // if commit offset failed, let's try again
// taosThreadMutexLock(&pParam->pTmq->lock);
// int32_t numOfVgroups, index;
// SMqClientVg* pVg = foundClientVg(pParam->pTmq->clientTopics, pParam->topicName, pParam->vgId, &index, &numOfVgroups);
// if (pVg == NULL) {
// tscDebug("consumer:0x%" PRIx64
// " subKey:%s vgId:%d commit failed, code:%s has been transferred to other consumer, no need retry ordinal:%d/%d",
// pParam->pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, tstrerror(code), index + 1, numOfVgroups);
// } else { // let's retry the commit
// int32_t code1 = doSendCommitMsg(pParam->pTmq, pVg, pParam->topicName, pParamSet, index, numOfVgroups);
// if (code1 != TSDB_CODE_SUCCESS) { // retry failed.
// tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64
// " retry failed, ignore this commit. code:%s ordinal:%d/%d",
// pParam->pTmq->consumerId, pParam->topicName, pVg->vgId, pVg->committedOffset.version,
// tstrerror(terrno), index + 1, numOfVgroups);
// }
// }
//
// taosThreadMutexUnlock(&pParam->pTmq->lock);
//
// taosMemoryFree(pParam->pOffset);
// taosMemoryFree(pBuf->pData);
// taosMemoryFree(pBuf->pEpSet);
//
// tmqCommitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId);
// return 0;
// }
//
// // todo replace the pTmq with refId
taosMemoryFree(pParam->pOffset);
taosMemoryFree(pBuf->pData);
@ -486,8 +471,8 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN
pOffset->subKey[groupLen] = TMQ_SEPARATOR;
strcpy(pOffset->subKey + groupLen + 1, pTopicName);
int32_t len;
int32_t code;
int32_t len = 0;
int32_t code = 0;
tEncodeSize(tEncodeSTqOffset, pOffset, len, code);
if (code < 0) {
return -1;
@ -647,8 +632,7 @@ FAIL:
return 0;
}
static int32_t tmqCommitConsumerImpl(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb,
void* userParam) {
static int32_t doAutoCommit(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, void* userParam) {
int32_t code = -1;
SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet));
@ -740,31 +724,30 @@ static int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic,
if (msg) { // user invoked commit
return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam);
} else { // this for auto commit
return tmqCommitConsumerImpl(tmq, automatic, async, userCb, userParam);
return doAutoCommit(tmq, automatic, async, userCb, userParam);
}
}
static void generateTimedTask(int64_t refId, int32_t type) {
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq != NULL) {
int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0);
*pTaskType = type;
taosWriteQitem(tmq->delayedTask, pTaskType);
tsem_post(&tmq->rspSem);
}
taosReleaseRef(tmqMgmt.rsetId, refId);
}
void tmqAssignAskEpTask(void* param, void* tmrId) {
int64_t refId = *(int64_t*)param;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq != NULL) {
int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0);
*pTaskType = TMQ_DELAYED_TASK__ASK_EP;
taosWriteQitem(tmq->delayedTask, pTaskType);
tsem_post(&tmq->rspSem);
}
generateTimedTask(refId, TMQ_DELAYED_TASK__ASK_EP);
taosMemoryFree(param);
}
void tmqAssignDelayedCommitTask(void* param, void* tmrId) {
int64_t refId = *(int64_t*)param;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq != NULL) {
int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0);
*pTaskType = TMQ_DELAYED_TASK__COMMIT;
taosWriteQitem(tmq->delayedTask, pTaskType);
tsem_post(&tmq->rspSem);
}
generateTimedTask(refId, TMQ_DELAYED_TASK__COMMIT);
taosMemoryFree(param);
}
@ -777,6 +760,8 @@ void tmqAssignDelayedReportTask(void* param, void* tmrId) {
taosWriteQitem(tmq->delayedTask, pTaskType);
tsem_post(&tmq->rspSem);
}
taosReleaseRef(tmqMgmt.rsetId, refId);
taosMemoryFree(param);
}
@ -790,6 +775,7 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
void tmqSendHbReq(void* param, void* tmrId) {
int64_t refId = *(int64_t*)param;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq == NULL) {
taosMemoryFree(param);
@ -803,17 +789,19 @@ void tmqSendHbReq(void* param, void* tmrId) {
int32_t tlen = tSerializeSMqHbReq(NULL, 0, &req);
if (tlen < 0) {
tscError("tSerializeSMqHbReq failed");
return;
goto OVER;
}
void* pReq = taosMemoryCalloc(1, tlen);
if (tlen < 0) {
tscError("failed to malloc MqHbReq msg, size:%d", tlen);
return;
goto OVER;
}
if (tSerializeSMqHbReq(pReq, tlen, &req) < 0) {
tscError("tSerializeSMqHbReq %d failed", tlen);
taosMemoryFree(pReq);
return;
goto OVER;
}
SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
@ -821,6 +809,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
taosMemoryFree(pReq);
goto OVER;
}
sendInfo->msgInfo = (SDataBuf){
.pData = pReq,
.len = tlen,
@ -840,6 +829,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
OVER:
taosTmrReset(tmqSendHbReq, 1000, param, tmqMgmt.timer, &tmq->hbLiveTimer);
taosReleaseRef(tmqMgmt.rsetId, refId);
}
int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
@ -892,15 +882,21 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) {
tDeleteSMqAskEpRsp(&pEpRspWrapper->msg);
} else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) {
SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper;
taosMemoryFreeClear(pRsp->pEpset);
taosArrayDestroyP(pRsp->dataRsp.blockData, taosMemoryFree);
taosArrayDestroy(pRsp->dataRsp.blockDataLen);
taosArrayDestroyP(pRsp->dataRsp.blockTbName, taosMemoryFree);
taosArrayDestroyP(pRsp->dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
} else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) {
SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper;
taosMemoryFreeClear(pRsp->pEpset);
taosMemoryFree(pRsp->metaRsp.metaRsp);
} else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) {
SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper;
taosMemoryFreeClear(pRsp->pEpset);
taosArrayDestroyP(pRsp->taosxRsp.blockData, taosMemoryFree);
taosArrayDestroy(pRsp->taosxRsp.blockDataLen);
taosArrayDestroyP(pRsp->taosxRsp.blockTbName, taosMemoryFree);
@ -974,8 +970,15 @@ int32_t tmq_unsubscribe(tmq_t* tmq) {
return rsp;
}
static void freeClientVgImpl(void* param) {
SMqClientTopic* pTopic = param;
taosMemoryFreeClear(pTopic->schema.pSchema);
taosArrayDestroy(pTopic->vgs);
}
void tmqFreeImpl(void* handle) {
tmq_t* tmq = (tmq_t*)handle;
tmq_t* tmq = (tmq_t*)handle;
int64_t id = tmq->consumerId;
// TODO stop timer
if (tmq->mqueue) {
@ -991,16 +994,11 @@ void tmqFreeImpl(void* handle) {
tsem_destroy(&tmq->rspSem);
taosThreadMutexDestroy(&tmq->lock);
int32_t sz = taosArrayGetSize(tmq->clientTopics);
for (int32_t i = 0; i < sz; i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
taosMemoryFreeClear(pTopic->schema.pSchema);
taosArrayDestroy(pTopic->vgs);
}
taosArrayDestroy(tmq->clientTopics);
taosArrayDestroyEx(tmq->clientTopics, freeClientVgImpl);
taos_close_internal(tmq->pTscObj);
taosMemoryFree(tmq);
tscDebug("consumer:0x%" PRIx64 " closed", id);
}
static void tmqMgmtInit(void) {
@ -1100,8 +1098,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
char buf[80] = {0};
STqOffsetVal offset = {.type = pTmq->resetOffsetCfg};
tFormatOffset(buf, tListLen(buf), &offset);
tscInfo("consumer:0x%" PRIx64 " is setup, groupId:%s, snapshot:%d, autoCommit:%d, commitInterval:%dms, offset:%s, backgroudHB:%d",
pTmq->consumerId, pTmq->groupId, pTmq->useSnapshot, pTmq->autoCommit, pTmq->autoCommitInterval, buf,
tscInfo("consumer:0x%" PRIx64 " is setup, refId:%"PRId64", groupId:%s, snapshot:%d, autoCommit:%d, commitInterval:%dms, offset:%s, backgroudHB:%d",
pTmq->consumerId, pTmq->refId, pTmq->groupId, pTmq->useSnapshot, pTmq->autoCommit, pTmq->autoCommitInterval, buf,
pTmq->hbBgEnable);
return pTmq;
@ -1242,10 +1240,12 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para
int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
SMqPollCbParam* pParam = (SMqPollCbParam*)param;
int64_t refId = pParam->refId;
SMqClientVg* pVg = pParam->pVg;
SMqClientTopic* pTopic = pParam->pTopic;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId);
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq == NULL) {
tsem_destroy(&pParam->rspSem);
taosMemoryFree(pParam);
@ -1296,6 +1296,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
tmq->consumerId, vgId, msgEpoch, tmqEpoch, requestId);
tsem_post(&tmq->rspSem);
taosReleaseRef(tmqMgmt.rsetId, refId);
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
return 0;
@ -1321,7 +1323,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
pRspWrapper->vgHandle = pVg;
pRspWrapper->topicHandle = pTopic;
pRspWrapper->reqId = requestId;
pRspWrapper->pEpset = pMsg->pEpSet;
pMsg->pEpSet = NULL;
if (rspType == TMQ_MSG_TYPE__POLL_RSP) {
SDecoder decoder;
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
@ -1350,13 +1354,14 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
}
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
taosWriteQitem(tmq->mqueue, pRspWrapper);
tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, reqId:0x%" PRIx64,
tmq->consumerId, rspType, vgId, tmq->mqueue->numOfItems, requestId);
tsem_post(&tmq->rspSem);
taosReleaseRef(tmqMgmt.rsetId, refId);
return 0;
CREATE_MSG_FAIL:
@ -1365,6 +1370,8 @@ CREATE_MSG_FAIL:
}
tsem_post(&tmq->rspSem);
taosReleaseRef(tmqMgmt.rsetId, refId);
return -1;
}
@ -1402,6 +1409,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
.vgStatus = TMQ_VG_STATUS__IDLE,
.vgSkipCnt = 0,
.emptyBlockReceiveTs = 0,
.numOfRows = 0,
};
taosArrayPush(pTopic->vgs, &clientVg);
@ -1516,6 +1524,14 @@ static int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) {
if (head->epoch <= epoch) {
tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, no need to update local ep",
tmq->consumerId, head->epoch, epoch);
if (tmq->status == TMQ_CONSUMER_STATUS__RECOVER) {
SMqAskEpRsp rsp;
tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
int8_t flag = (taosArrayGetSize(rsp.topics) == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY;
atomic_store_8(&tmq->status, flag);
tDeleteSMqAskEpRsp(&rsp);
}
goto END;
}
@ -1545,6 +1561,8 @@ static int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) {
}
END:
taosReleaseRef(tmqMgmt.rsetId, pParam->refId);
if (!async) {
tsem_post(&pParam->rspSem);
} else {
@ -1584,21 +1602,32 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
return pRspObj;
}
SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg) {
SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
pRspObj->resType = RES_TYPE__TMQ;
tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
pRspObj->vgId = pWrapper->vgHandle->vgId;
pRspObj->resIter = -1;
memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp));
pRspObj->resInfo.totalRows = 0;
pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
if (!pWrapper->dataRsp.withSchema) {
setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols);
}
// extract the rows in this data packet
for(int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) {
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, i);
int64_t rows = htobe64(pRetrieve->numOfRows);
pRspObj->resInfo.totalRows += rows;
pVg->numOfRows += rows;
}
return pRspObj;
}
@ -1702,13 +1731,13 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
for (int j = 0; j < numOfVg; j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < 100) { // less than 100ms
if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 100ms
tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 100ms before start next poll", tmq->consumerId, tmq->epoch,
pVg->vgId);
continue;
}
int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
if (vgStatus == TMQ_VG_STATUS__WAIT) {
int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1);
tscTrace("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch,
@ -1731,6 +1760,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
}
}
tscDebug("consumer:0x%" PRIx64 " end to poll data", tmq->consumerId);
return 0;
}
@ -1782,31 +1812,44 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
/*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/
int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) {
SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp;
if (pDataRsp->head.epoch == consumerEpoch) {
// todo fix it: race condition
SMqClientVg* pVg = pollRspWrapper->vgHandle;
pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset;
// update the epset
if (pollRspWrapper->pEpset != NULL) {
SEp* pEp = GET_ACTIVE_EP(pollRspWrapper->pEpset);
SEp* pOld = GET_ACTIVE_EP(&(pVg->epSet));
tscDebug("consumer:0x%" PRIx64 " update epset vgId:%d, ep:%s:%d, old ep:%s:%d", tmq->consumerId,
pVg->vgId, pEp->fqdn, pEp->port, pOld->fqdn, pOld->port);
pVg->epSet = *pollRspWrapper->pEpset;
}
pVg->currentOffset = pDataRsp->rspOffset;
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
if (pollRspWrapper->dataRsp.blockNum == 0) {
tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId,
pollRspWrapper->reqId);
char buf[80];
tFormatOffset(buf, 80, &pDataRsp->rspOffset);
if (pDataRsp->blockNum == 0) {
tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, reqId:0x%" PRIx64, tmq->consumerId,
pVg->vgId, buf, pollRspWrapper->reqId);
taosFreeQitem(pollRspWrapper);
rspWrapper = NULL;
continue;
} else { // build rsp
SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg);
tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%"PRId64" reqId:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, pRsp->resInfo.totalRows, pollRspWrapper->reqId);
tmq->totalRows += pRsp->resInfo.totalRows;
taosFreeQitem(pollRspWrapper);
return pRsp;
}
// build rsp
char buf[80];
tFormatOffset(buf, 80, &pVg->currentOffset);
SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper);
tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, reqId:0x%"PRIx64, tmq->consumerId,
pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, pollRspWrapper->reqId);
taosFreeQitem(pollRspWrapper);
return pRsp;
} else {
tscDebug("consumer:0x%" PRIx64 " msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
tmq->consumerId, pollRspWrapper->dataRsp.head.epoch, consumerEpoch);
tmq->consumerId, pDataRsp->head.epoch, consumerEpoch);
tmqFreeRspWrapper(rspWrapper);
taosFreeQitem(pollRspWrapper);
}
@ -1853,12 +1896,11 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
// build rsp
void* pRsp = NULL;
if (pollRspWrapper->taosxRsp.createTableNum == 0) {
pRsp = tmqBuildRspFromWrapper(pollRspWrapper);
pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg);
} else {
pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper);
}
char buf[80];
tFormatOffset(buf, 80, &pVg->currentOffset);
tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, reqId:0x%"PRIx64, tmq->consumerId, pVg->vgId,
@ -1938,7 +1980,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
return NULL;
}
if (timeout != -1) {
if (timeout >= 0) {
int64_t currentTime = taosGetTimestampMs();
int64_t elapsedTime = currentTime - startTime;
if (elapsedTime > timeout) {
@ -1946,9 +1988,6 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
tmq->consumerId, tmq->epoch, startTime, currentTime);
return NULL;
}
/*tscInfo("consumer:0x%" PRIx64 ", (epoch %d) wait, start time %" PRId64 ", current time %" PRId64*/
/*", left time %" PRId64,*/
/*tmq->consumerId, tmq->epoch, startTime, currentTime, (timeout - elapsedTime));*/
tsem_timewait(&tmq->rspSem, (timeout - elapsedTime));
} else {
// use tsem_timewait instead of tsem_wait to avoid unexpected stuck
@ -1964,6 +2003,24 @@ int32_t tmq_consumer_close(tmq_t* tmq) {
return rsp;
}
int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
tscDebug("consumer:0x%" PRIx64 " closing poll:%" PRId64 " rows:%" PRId64 " topics:%d, final epoch:%d",
tmq->consumerId, tmq->pollCnt, tmq->totalRows, numOfTopics, tmq->epoch);
tscDebug("consumer:0x%" PRIx64 " rows dist begin: ", tmq->consumerId);
for (int32_t i = 0; i < numOfTopics; ++i) {
SMqClientTopic* pTopics = taosArrayGet(tmq->clientTopics, i);
tscDebug("consumer:0x%" PRIx64 " topic:%d", tmq->consumerId, i);
int32_t numOfVgs = taosArrayGetSize(pTopics->vgs);
for (int32_t j = 0; j < numOfVgs; ++j) {
SMqClientVg* pVg = taosArrayGet(pTopics->vgs, j);
tscDebug("topic:%s, %d. vgId:%d rows:%" PRId64, pTopics->topicName, j, pVg->vgId, pVg->numOfRows);
}
}
tscDebug("consumer:0x%" PRIx64 " rows dist end", tmq->consumerId);
int32_t retryCnt = 0;
tmq_list_t* lst = tmq_list_new();
while (1) {
@ -2166,7 +2223,9 @@ int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg) {
}
int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) {
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParamSet->refId);
int64_t refId = pParamSet->refId;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq == NULL) {
if (!pParamSet->async) {
tsem_destroy(&pParamSet->rspSem);
@ -2194,6 +2253,8 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) {
taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree);
taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree);
#endif
taosReleaseRef(tmqMgmt.rsetId, refId);
return 0;
}

4
source/client/src/clientTmqConnector.c
View File

@ -361,8 +361,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(
TAOS_RES *tres = (TAOS_RES *)res;
void *data;
int32_t numOfRows;
void *data = NULL;
int32_t numOfRows = 0;
int error_code = taos_fetch_raw_block(tres, &numOfRows, &data);
if (numOfRows == 0) {
if (error_code == JNI_SUCCESS) {

10
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -77,7 +77,7 @@ void mndCleanupConsumer(SMnode *pMnode) {}
bool mndRebTryStart() {
int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1);
mInfo("tq timer, rebalance counter old val:%d", old);
mDebug("tq timer, rebalance counter old val:%d", old);
return old == 0;
}
@ -101,7 +101,7 @@ void mndRebCntDec() {
int32_t newVal = val - 1;
int32_t oldVal = atomic_val_compare_exchange_32(&mqRebInExecCnt, val, newVal);
if (oldVal == val) {
mInfo("rebalance trans end, rebalance counter:%d", newVal);
mDebug("rebalance trans end, rebalance counter:%d", newVal);
break;
}
}
@ -253,11 +253,11 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
SMqConsumerObj *pConsumer;
void *pIter = NULL;
mTrace("start to process mq timer");
mDebug("start to process mq timer");
// rebalance cannot be parallel
if (!mndRebTryStart()) {
mInfo("mq rebalance already in progress, do nothing");
mDebug("mq rebalance already in progress, do nothing");
return 0;
}
@ -356,7 +356,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
} else {
taosHashCleanup(pRebMsg->rebSubHash);
rpcFreeCont(pRebMsg);
mInfo("mq rebalance finished, no modification");
mDebug("mq rebalance finished, no modification");
mndRebEnd();
}
return 0;

20
source/dnode/mnode/impl/src/mndDb.c
View File

@ -279,6 +279,8 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
pOld->cfg.cacheLast = pNew->cfg.cacheLast;
pOld->cfg.replications = pNew->cfg.replications;
pOld->cfg.sstTrigger = pNew->cfg.sstTrigger;
pOld->cfg.minRows = pNew->cfg.minRows;
pOld->cfg.maxRows = pNew->cfg.maxRows;
pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize;
pOld->compactStartTime = pNew->compactStartTime;
taosWUnLockLatch(&pOld->lock);
@ -616,13 +618,8 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
}
} else {
if (terrno == TSDB_CODE_MND_DB_IN_CREATING) {
if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) {
mInfo("db:%s, is creating and createdb response after trans finished", createReq.db);
code = TSDB_CODE_ACTION_IN_PROGRESS;
goto _OVER;
} else {
goto _OVER;
}
code = terrno;
goto _OVER;
} else if (terrno == TSDB_CODE_MND_DB_IN_DROPPING) {
goto _OVER;
} else if (terrno == TSDB_CODE_MND_DB_NOT_EXIST) {
@ -1270,14 +1267,9 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) {
usedbRsp.errCode = terrno;
if (terrno == TSDB_CODE_MND_DB_IN_CREATING) {
if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, usedbReq.db) == 0) {
mInfo("db:%s, is creating and usedb response after trans finished", usedbReq.db);
code = TSDB_CODE_ACTION_IN_PROGRESS;
goto _OVER;
}
code = terrno;
goto _OVER;
}
mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr());
} else {
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) {
goto _OVER;

7
source/dnode/mnode/impl/src/mndStb.c
View File

@ -3114,6 +3114,7 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(typeName, "SUPER_TABLE");
while (numOfRows < rows) {
void *prevIter = pShow->pIter;
pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb);
if (pShow->pIter == NULL) break;
@ -3122,6 +3123,12 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
continue;
}
if ((numOfRows + pStb->numOfColumns) > rows) {
pShow->pIter = prevIter;
sdbRelease(pSdb, pStb);
break;
}
SName name = {0};
char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
mndExtractTbNameFromStbFullName(pStb->name, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN);

81
source/dnode/vnode/src/meta/metaCache.c
View File

@ -14,6 +14,7 @@
*/
#include "meta.h"
#define TAG_FILTER_RES_KEY_LEN 32
#define META_CACHE_BASE_BUCKET 1024
#define META_CACHE_STATS_BUCKET 16
@ -34,7 +35,6 @@ typedef struct SMetaStbStatsEntry {
typedef struct STagFilterResEntry {
SList list; // the linked list of md5 digest, extracted from the serialized tag query condition
uint32_t hitTimes; // queried times for current super table
uint32_t accTime;
} STagFilterResEntry;
struct SMetaCache {
@ -455,26 +455,37 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv
return 0;
}
static FORCE_INLINE void setMD5DigestInKey(uint64_t* pBuf, const char* key, int32_t keyLen) {
// ASSERT(keyLen == sizeof(int64_t) * 2);
memcpy(&pBuf[2], key, keyLen);
}
// the format of key:
// hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes)
static void initCacheKey(uint64_t* buf, const SHashObj* pHashMap, uint64_t suid, const char* key, int32_t keyLen) {
buf[0] = (uint64_t) pHashMap;
buf[1] = suid;
setMD5DigestInKey(buf, key, keyLen);
ASSERT(keyLen == sizeof(uint64_t) * 2);
}
int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1,
bool* acquireRes) {
int32_t vgId = TD_VID(pMeta->pVnode);
// generate the composed key for LRU cache
SLRUCache* pCache = pMeta->pCache->sTagFilterResCache.pUidResCache;
SHashObj* pTableMap = pMeta->pCache->sTagFilterResCache.pTableEntry;
TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock;
uint64_t buf[4];
*acquireRes = 0;
buf[0] = (uint64_t)pTableMap;
buf[1] = suid;
memcpy(&buf[2], pKey, keyLen);
uint64_t key[4];
initCacheKey(key, pTableMap, suid, (const char*)pKey, keyLen);
taosThreadMutexLock(pLock);
pMeta->pCache->sTagFilterResCache.accTimes += 1;
int32_t len = keyLen + sizeof(uint64_t) * 2;
LRUHandle* pHandle = taosLRUCacheLookup(pCache, buf, len);
LRUHandle* pHandle = taosLRUCacheLookup(pCache, key, TAG_FILTER_RES_KEY_LEN);
if (pHandle == NULL) {
taosThreadMutexUnlock(pLock);
return TSDB_CODE_SUCCESS;
@ -499,7 +510,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK
uint32_t acc = pMeta->pCache->sTagFilterResCache.accTimes;
if ((*pEntry)->hitTimes % 5000 == 0 && (*pEntry)->hitTimes > 0) {
metaInfo("cache hit:%d, total acc:%d, rate:%.2f", (*pEntry)->hitTimes, acc, ((double)(*pEntry)->hitTimes) / acc);
metaInfo("vgId:%d cache hit:%d, total acc:%d, rate:%.2f", vgId, (*pEntry)->hitTimes, acc, ((double)(*pEntry)->hitTimes) / acc);
}
taosLRUCacheRelease(pCache, pHandle, false);
@ -563,10 +574,13 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL
// check both the payload size and selectivity ratio
int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
int32_t payloadLen, double selectivityRatio) {
int32_t code = 0;
int32_t vgId = TD_VID(pMeta->pVnode);
if (selectivityRatio > tsSelectivityRatio) {
metaDebug("vgId:%d, suid:%" PRIu64
" failed to add to uid list cache, due to selectivity ratio %.2f less than threshold %.2f",
TD_VID(pMeta->pVnode), suid, selectivityRatio, tsSelectivityRatio);
vgId, suid, selectivityRatio, tsSelectivityRatio);
taosMemoryFree(pPayload);
return TSDB_CODE_SUCCESS;
}
@ -574,7 +588,7 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int
if (payloadLen > tsTagFilterResCacheSize) {
metaDebug("vgId:%d, suid:%" PRIu64
" failed to add to uid list cache, due to payload length %d greater than threshold %d",
TD_VID(pMeta->pVnode), suid, payloadLen, tsTagFilterResCacheSize);
vgId, suid, payloadLen, tsTagFilterResCacheSize);
taosMemoryFree(pPayload);
return TSDB_CODE_SUCCESS;
}
@ -583,26 +597,17 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int
SHashObj* pTableEntry = pMeta->pCache->sTagFilterResCache.pTableEntry;
TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock;
// the format of key:
// hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes)
uint64_t key[4] = {0};
initCacheKey(key, pTableEntry, suid, pKey, keyLen);
uint64_t buf[4] = {0};
buf[0] = (uint64_t)pTableEntry;
buf[1] = suid;
memcpy(&buf[2], pKey, keyLen);
ASSERT(keyLen == 16);
int32_t code = 0;
taosThreadMutexLock(pLock);
STagFilterResEntry** pEntry = taosHashGet(pTableEntry, &suid, sizeof(uint64_t));
if (pEntry == NULL) {
code = addNewEntry(pTableEntry, pKey, keyLen, suid);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
}
} else {
// check if it exists or not
} else { // check if it exists or not
size_t size = listNEles(&(*pEntry)->list);
if (size == 0) {
tdListAppend(&(*pEntry)->list, pKey);
@ -620,12 +625,11 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int
}
// add to cache.
taosLRUCacheInsert(pCache, buf, sizeof(uint64_t) * 2 + keyLen, pPayload, payloadLen, freePayload, NULL,
taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freePayload, NULL,
TAOS_LRU_PRIORITY_LOW);
_end:
taosThreadMutexUnlock(pLock);
metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", TD_VID(pMeta->pVnode), suid,
metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", vgId, suid,
(int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry));
return code;
@ -633,33 +637,36 @@ _end:
// remove the lru cache that are expired due to the tags value update, or creating, or dropping, of child tables
int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) {
int32_t keyLen = sizeof(uint64_t) * 3;
uint64_t p[4] = {0};
uint64_t p[4] = {0};
int32_t vgId = TD_VID(pMeta->pVnode);
SHashObj* pEntryHashMap = pMeta->pCache->sTagFilterResCache.pTableEntry;
p[0] = (uint64_t)pMeta->pCache->sTagFilterResCache.pTableEntry;
p[1] = suid;
uint64_t dummy[2] = {0};
initCacheKey(p, pEntryHashMap, suid, (char*) &dummy[0], 16);
TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock;
taosThreadMutexLock(pLock);
STagFilterResEntry** pEntry = taosHashGet(pMeta->pCache->sTagFilterResCache.pTableEntry, &suid, sizeof(uint64_t));
STagFilterResEntry** pEntry = taosHashGet(pEntryHashMap, &suid, sizeof(uint64_t));
if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) {
taosThreadMutexUnlock(pLock);
return TSDB_CODE_SUCCESS;
}
(*pEntry)->hitTimes = 0;
SListIter iter = {0};
tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD);
SListNode* pNode = NULL;
while ((pNode = tdListNext(&iter)) != NULL) {
memcpy(&p[2], pNode->data, 16);
taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, keyLen);
setMD5DigestInKey(p, pNode->data, 2 * sizeof(uint64_t));
taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, TAG_FILTER_RES_KEY_LEN);
}
(*pEntry)->hitTimes = 0;
tdListEmpty(&(*pEntry)->list);
taosThreadMutexUnlock(pLock);
metaDebug("vgId:%d suid:%"PRId64" cached related tag filter uid list cleared", vgId, suid);
return TSDB_CODE_SUCCESS;
}

2
source/dnode/vnode/src/tq/tqRead.c
View File

@ -296,7 +296,7 @@ int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) {
// todo set the correct vgId
tqDebug("tmq poll: wal seek to version:%"PRId64" %s", ver, id);
if (walReadSeekVer(pReader->pWalReader, ver) < 0) {
tqError("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id);
tqDebug("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id);
return -1;
} else {
tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id);

1
source/dnode/vnode/src/tq/tqSink.c
View File

@ -619,6 +619,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry.suid);
metaReaderClear(&mr);
taosMemoryFree(ctbName);
continue;
}
tbData.uid = mr.me.uid;

210
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -596,9 +596,11 @@ typedef struct {
int64_t lastTs;
} SFSLastNextRowIter;
static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) {
static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
int nCols) {
SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter;
int32_t code = 0;
bool checkRemainingRow = true;
switch (state->state) {
case SFSLASTNEXTROW_FS:
@ -631,6 +633,8 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
if (code) goto _err;
}
state->pLoadInfo->colIds = aCols;
state->pLoadInfo->numOfCols = nCols;
tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid,
&(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX},
&(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true);
@ -646,22 +650,71 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
goto _next_fileset;
}
state->state = SFSLASTNEXTROW_BLOCKROW;
checkRemainingRow = false;
}
case SFSLASTNEXTROW_BLOCKROW: {
bool hasVal = false;
state->row = tMergeTreeGetRow(&state->mergeTree);
*ppRow = &state->row;
hasVal = tMergeTreeNext(&state->mergeTree);
if (TSDBROW_TS(&state->row) <= state->lastTs) {
*pIgnoreEarlierTs = true;
*ppRow = NULL;
return code;
}
bool skipRow = false;
do {
bool hasVal = false;
state->row = tMergeTreeGetRow(&state->mergeTree);
*ppRow = &state->row;
hasVal = tMergeTreeNext(&state->mergeTree);
if (TSDBROW_TS(&state->row) <= state->lastTs) {
*pIgnoreEarlierTs = true;
*ppRow = NULL;
return code;
}
*pIgnoreEarlierTs = false;
if (!hasVal) {
state->state = SFSLASTNEXTROW_FILESET;
}
*pIgnoreEarlierTs = false;
if (!hasVal) {
state->state = SFSLASTNEXTROW_FILESET;
break;
}
if (checkRemainingRow) {
bool skipBlock = true;
SBlockData *pBlockData = state->row.pBlockData;
for (int inputColIndex = 0; inputColIndex < nCols; ++inputColIndex) {
for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
SColData *pColData = &pBlockData->aColData[colIndex];
int16_t cid = pColData->cid;
if (cid == aCols[inputColIndex]) {
if (isLast && (pColData->flag & HAS_VALUE)) {
skipBlock = false;
break;
} else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
skipBlock = false;
break;
}
}
}
}
/*
for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
SColData *pColData = &pBlockData->aColData[colIndex];
int16_t cid = pColData->cid;
if (inputColIndex < nCols && cid == aCols[inputColIndex]) {
if (isLast && (pColData->flag & HAS_VALUE)) {
skipBlock = false;
break;
} else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
skipBlock = false;
break;
}
++inputColIndex;
}
}
*/
if (skipBlock) {
skipRow = true;
}
}
} while (skipRow);
return code;
}
@ -740,9 +793,11 @@ typedef struct SFSNextRowIter {
int64_t lastTs;
} SFSNextRowIter;
static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) {
static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
int nCols) {
SFSNextRowIter *state = (SFSNextRowIter *)iter;
int32_t code = 0;
bool checkRemainingRow = true;
switch (state->state) {
case SFSNEXTROW_FS:
@ -828,8 +883,11 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
}
}
case SFSNEXTROW_BLOCKDATA:
_next_datablock:
if (state->iBlock >= 0) {
SDataBlk block = {0};
bool skipBlock = true;
int inputColIndex = 0;
tDataBlkReset(&block);
tBlockDataReset(state->pBlockData);
@ -848,18 +906,94 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
*pIgnoreEarlierTs = false;
tBlockDataReset(state->pBlockData);
TABLEID tid = {.suid = state->suid, .uid = state->uid};
code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, NULL, 0);
code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, aCols, nCols);
if (code) goto _err;
code = tsdbReadDataBlock(*state->pDataFReader, &block, state->pBlockData);
if (code) goto _err;
for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) {
SColData *pColData = &state->pBlockData->aColData[colIndex];
if (isLast && (pColData->flag & HAS_VALUE)) {
skipBlock = false;
break;
} else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
skipBlock = false;
break;
}
}
if (skipBlock) {
if (--state->iBlock < 0) {
tsdbDataFReaderClose(state->pDataFReader);
*state->pDataFReader = NULL;
// resetLastBlockLoadInfo(state->pLoadInfo);
if (state->aBlockIdx) {
// taosArrayDestroy(state->aBlockIdx);
tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle);
state->aBlockIdxHandle = NULL;
state->aBlockIdx = NULL;
}
state->state = SFSNEXTROW_FILESET;
goto _next_fileset;
} else {
goto _next_datablock;
}
}
state->nRow = state->blockData.nRow;
state->iRow = state->nRow - 1;
state->state = SFSNEXTROW_BLOCKROW;
checkRemainingRow = false;
}
case SFSNEXTROW_BLOCKROW:
case SFSNEXTROW_BLOCKROW: {
if (checkRemainingRow) {
bool skipBlock = true;
int inputColIndex = 0;
for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) {
SColData *pColData = &state->pBlockData->aColData[colIndex];
int16_t cid = pColData->cid;
if (inputColIndex < nCols && cid == aCols[inputColIndex]) {
if (isLast && (pColData->flag & HAS_VALUE)) {
skipBlock = false;
break;
} else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
skipBlock = false;
break;
}
++inputColIndex;
}
}
if (skipBlock) {
if (--state->iBlock < 0) {
tsdbDataFReaderClose(state->pDataFReader);
*state->pDataFReader = NULL;
// resetLastBlockLoadInfo(state->pLoadInfo);
if (state->aBlockIdx) {
// taosArrayDestroy(state->aBlockIdx);
tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle);
state->aBlockIdxHandle = NULL;
state->aBlockIdx = NULL;
}
state->state = SFSNEXTROW_FILESET;
goto _next_fileset;
} else {
goto _next_datablock;
}
}
}
if (state->iRow >= 0) {
state->row = tsdbRowFromBlockData(state->pBlockData, state->iRow);
*ppRow = &state->row;
@ -885,6 +1019,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
}
return code;
}
default:
ASSERT(0);
break;
@ -960,7 +1095,8 @@ typedef struct SMemNextRowIter {
// TSDBROW *curRow;
} SMemNextRowIter;
static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) {
static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
int nCols) {
SMemNextRowIter *state = (SMemNextRowIter *)iter;
int32_t code = 0;
*pIgnoreEarlierTs = false;
@ -1072,7 +1208,8 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) {
return deleted;
}
typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs);
typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
int nCols);
typedef int32_t (*_next_row_clear_fn_t)(void *iter);
typedef struct {
@ -1222,12 +1359,14 @@ _err:
}
// iterate next row non deleted backward ts, version (from high to low)
static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) {
static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast,
int16_t *aCols, int nCols) {
int code = 0;
for (;;) {
for (int i = 0; i < 4; ++i) {
if (pIter->input[i].next && !pIter->input[i].stop) {
code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, &pIter->input[i].ignoreEarlierTs);
code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, &pIter->input[i].ignoreEarlierTs,
isLast, aCols, nCols);
if (code) goto _err;
if (pIter->input[i].pRow == NULL) {
@ -1358,7 +1497,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo
do {
TSDBROW *pRow = NULL;
nextRowIterGet(&iter, &pRow, &ignoreEarlierTs);
nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, NULL, 0);
if (!pRow) {
break;
@ -1488,11 +1627,21 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
bool ignoreEarlierTs = false;
SArray *pColArray = NULL;
SColVal *pColVal = &(SColVal){0};
int16_t nCols = nLastCol;
int32_t code = initLastColArray(pTSchema, &pColArray);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
SArray *aColArray = taosArrayInit(nCols, sizeof(int16_t));
if (NULL == aColArray) {
taosArrayDestroy(pColArray);
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int i = 1; i < pTSchema->numOfCols; ++i) {
taosArrayPush(aColArray, &pTSchema->columns[i].colId);
}
TSKEY lastRowTs = TSKEY_MAX;
@ -1502,7 +1651,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
do {
TSDBROW *pRow = NULL;
nextRowIterGet(&iter, &pRow, &ignoreEarlierTs);
nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray));
if (!pRow) {
break;
@ -1547,9 +1696,14 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
}
if (!COL_VAL_IS_VALUE(pColVal) && !setNoneCol) {
noneCol = iCol;
setNoneCol = true;
if (!COL_VAL_IS_VALUE(pColVal)) {
if (!setNoneCol) {
noneCol = iCol;
setNoneCol = true;
}
} else {
int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ);
taosArrayRemove(aColArray, aColIndex);
}
}
if (!setNoneCol) {
@ -1590,6 +1744,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
}
taosArraySet(pColArray, iCol, &lastCol);
int32_t aColIndex = taosArraySearchIdx(aColArray, &lastCol.colVal.cid, compareInt16Val, TD_EQ);
taosArrayRemove(aColArray, aColIndex);
} else if (!COL_VAL_IS_VALUE(tColVal) && !COL_VAL_IS_VALUE(pColVal) && !setNoneCol) {
noneCol = iCol;
setNoneCol = true;
@ -1613,6 +1769,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
//}
nextRowIterClose(&iter);
taosArrayDestroy(aColArray);
// taosMemoryFreeClear(pTSchema);
return code;
@ -1621,6 +1778,7 @@ _err:
// taosMemoryFreeClear(pTSchema);
*ppLastArray = NULL;
taosArrayDestroy(pColArray);
taosArrayDestroy(aColArray);
return code;
}

1
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -118,7 +118,6 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id
if (suid != 0) {
p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, suid, -1, 1);
if (p->pSchema == NULL) {
taosMemoryFree(p);
tsdbWarn("stable:%" PRIu64 " has been dropped, failed to retrieve cached rows, %s", suid, idstr);
return TSDB_CODE_PAR_TABLE_NOT_EXIST;
}

9
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -390,8 +390,10 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf
pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));
if (pUidList->tableUidList == NULL) {
taosHashCleanup(pTableMap);
return NULL;
}
pUidList->currentIndex = 0;
for (int32_t j = 0; j < numOfTables; ++j) {
@ -4548,6 +4550,8 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock,
return TSDB_CODE_SUCCESS;
}
int64_t st = taosGetTimestampUs();
SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
if (tDataBlkHasSma(pBlock)) {
code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg);
@ -4609,6 +4613,9 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock,
*pBlockSMA = pResBlock->pBlockAgg;
pReader->cost.smaDataLoad += 1;
double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;
pReader->cost.smaLoadTime += elapsedTime;
tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr);
return code;
}
@ -4763,7 +4770,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa
pTableBlockInfo->defMinRows = pc->minRows;
pTableBlockInfo->defMaxRows = pc->maxRows;
int32_t bucketRange = ceil((pc->maxRows - pc->minRows) / numOfBucket);
int32_t bucketRange = ceil(((double)(pc->maxRows - pc->minRows)) / numOfBucket);
pTableBlockInfo->numOfFiles += 1;

2
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -510,7 +510,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
vTrace("vgId:%d, msg:%p in fetch queue is processing", pVnode->config.vgId, pMsg);
if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG ||
pMsg->msgType == TDMT_VND_BATCH_META) &&
pMsg->msgType == TDMT_VND_BATCH_META || pMsg->msgType == TDMT_VND_TMQ_CONSUME) &&
!syncIsReadyForRead(pVnode->sync)) {
vnodeRedirectRpcMsg(pVnode, pMsg, terrno);
return 0;

3
source/libs/catalog/src/ctgDbg.c
View File

@ -21,6 +21,8 @@
extern SCatalogMgmt gCtgMgmt;
SCtgDebug gCTGDebug = {0};
#if 0
void ctgdUserCallback(SMetaData *pResult, void *param, int32_t code) {
taosMemoryFree(param);
@ -224,6 +226,7 @@ _return:
CTG_RET(code);
}
#endif
int32_t ctgdEnableDebug(char *option, bool enable) {
if (0 == strcasecmp(option, "lock")) {

3
source/libs/catalog/src/ctgUtil.c
View File

@ -1330,6 +1330,7 @@ static void* ctgCloneDnodeList(void* pSrc) { return taosArrayDup((const SArray*)
static void ctgFreeDnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); }
#if 0
static int32_t ctgCloneMetaDataArray(SArray* pSrc, __array_item_dup_fn_t copyFunc, SArray** pDst) {
if (NULL == pSrc) {
return TSDB_CODE_SUCCESS;
@ -1421,3 +1422,5 @@ void catalogFreeMetaData(SMetaData* pData) {
taosMemoryFreeClear(pData->pSvrVer);
taosMemoryFree(pData);
}
#endif

1
source/libs/executor/src/executil.c
View File

@ -820,6 +820,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa
int32_t code = blockDataEnsureCapacity(pResBlock, numOfTables);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
taosMemoryFree(pResBlock);
return NULL;
}

1
source/libs/executor/src/executorimpl.c
View File

@ -2344,6 +2344,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pOptr = createEventwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo);
} else {
terrno = TSDB_CODE_INVALID_PARA;
taosMemoryFree(ops);
return NULL;
}

13
source/libs/executor/src/groupoperator.c
View File

@ -173,9 +173,14 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData
size_t numOfGroupCols = taosArrayGetSize(pGroupCols);
for (int32_t i = 0; i < numOfGroupCols; ++i) {
SColumn* pCol = taosArrayGet(pGroupCols, i);
SColumn* pCol = (SColumn*) taosArrayGet(pGroupCols, i);
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId);
// valid range check. todo: return error code.
if (pCol->slotId > taosArrayGetSize(pBlock->pDataBlock)) {
continue;
}
if (pBlock->pBlockAgg != NULL) {
pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched?
}
@ -999,14 +1004,14 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp*
memset(tbName, 0, TSDB_TABLE_NAME_LEN);
int32_t len = 0;
if (colDataIsNull_s(pTbCol, pDestBlock->info.rows - 1)) {
len = TMIN(sizeof(TSDB_DATA_NULL_STR), TSDB_TABLE_NAME_LEN - 1);
memcpy(tbName, TSDB_DATA_NULL_STR, len);
len = 1;
tbName[0] = 0;
} else {
void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1);
len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1);
memcpy(tbName, varDataVal(pData), len);
streamStatePutParName(pState, groupId, tbName);
}
streamStatePutParName(pState, groupId, tbName);
memcpy(pTmpBlock->info.parTbName, tbName, len);
pDestBlock->info.rows--;
} else {

32
source/libs/executor/src/scanoperator.c
View File

@ -3189,7 +3189,7 @@ static SSDataBlock* buildSysDbTableCount(SOperatorInfo* pOperator, STableCountSc
size_t perfdbTableNum;
getPerfDbMeta(NULL, &perfdbTableNum);
if (pSupp->groupByDbName) {
if (pSupp->groupByDbName || pSupp->groupByStbName) {
buildSysDbGroupedTableCount(pOperator, pInfo, pSupp, pRes, infodbTableNum, perfdbTableNum);
return (pRes->info.rows > 0) ? pRes : NULL;
} else {
@ -3214,11 +3214,23 @@ static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountSca
STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum,
size_t perfdbTableNum) {
if (pInfo->currGrpIdx == 0) {
uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB));
uint64_t groupId = 0;
if (pSupp->groupByDbName) {
groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB));
} else {
groupId = calcGroupId("", 0);
}
pRes->info.id.groupId = groupId;
fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes);
} else if (pInfo->currGrpIdx == 1) {
uint64_t groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB));
uint64_t groupId = 0;
if (pSupp->groupByDbName) {
groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB));
} else {
groupId = calcGroupId("", 0);
}
pRes->info.id.groupId = groupId;
fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes);
} else {
@ -3256,7 +3268,7 @@ static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCount
tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB);
tNameGetDbName(&sn, dbName);
if (pSupp->groupByDbName) {
if (pSupp->groupByDbName || pSupp->groupByStbName) {
buildVnodeGroupedTableCount(pOperator, pInfo, pSupp, pRes, vgId, dbName);
} else {
buildVnodeFilteredTbCount(pOperator, pInfo, pSupp, pRes, dbName);
@ -3317,7 +3329,10 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO
static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
SSDataBlock* pRes, char* dbName) {
char fullStbName[TSDB_TABLE_FNAME_LEN] = {0};
snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, "");
if (pSupp->groupByDbName) {
snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, "");
}
uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
pRes->info.id.groupId = groupId;
int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta);
@ -3332,7 +3347,12 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S
metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName);
char fullStbName[TSDB_TABLE_FNAME_LEN] = {0};
snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName);
if (pSupp->groupByDbName) {
snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName);
} else {
snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s", stbName);
}
uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
pRes->info.id.groupId = groupId;

8
source/libs/executor/src/sysscanoperator.c
View File

@ -562,15 +562,17 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
continue;
}
sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName);
if (numOfRows >= pOperator->resultInfo.capacity) {
if ((numOfRows + schemaRow->nCols) > pOperator->resultInfo.capacity) {
relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
numOfRows = 0;
metaTbCursorPrev(pInfo->pCur);
if (pInfo->pRes->info.rows > 0) {
break;
}
} else {
sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName);
}
}

2
source/libs/executor/src/timewindowoperator.c
View File

@ -3911,7 +3911,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
blockDataEnsureCapacity(pAggSup->pScanBlock, rows);
SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
for (int32_t i = 0; i < rows; i += winRows) {
if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup) || colDataIsNull_s(pKeyColInfo, i)) {
i++;
continue;
}

1
source/libs/executor/src/tsort.c
View File

@ -214,7 +214,6 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
if (pPage == NULL) {
taosArrayDestroy(pPageIdList);
blockDataDestroy(p);
taosArrayDestroy(pPageIdList);
return terrno;
}

32
source/libs/parser/src/parTranslater.c
View File

@ -1331,6 +1331,32 @@ static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount)
return code;
}
static bool isCountNotNullValue(SFunctionNode* pFunc) {
if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) {
return false;
}
SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
return (QUERY_NODE_VALUE == nodeType(pPara) && !((SValueNode*)pPara)->isNull);
}
// count(1) is rewritten as count(ts) for scannning optimization
static int32_t rewriteCountNotNullValue(STranslateContext* pCxt, SFunctionNode* pCount) {
SValueNode* pValue = (SValueNode*)nodesListGetNode(pCount->pParameterList, 0);
STableNode* pTable = NULL;
int32_t code = findTable(pCxt, NULL, &pTable);
if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) {
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
code = TSDB_CODE_OUT_OF_MEMORY;
} else {
setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol);
NODES_DESTORY_LIST(pCount->pParameterList);
code = nodesListMakeAppend(&pCount->pParameterList, (SNode*)pCol);
}
}
return code;
}
static bool isCountTbname(SFunctionNode* pFunc) {
if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) {
return false;
@ -2041,7 +2067,7 @@ static int32_t getGroupByErrorCode(STranslateContext* pCxt) {
if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) {
return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION;
}
return TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN;
return TSDB_CODE_PAR_INVALID_OPTR_USAGE;
}
static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode) {
@ -2114,13 +2140,13 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) {
}
if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc) {
return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt));
return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias);
} else {
return rewriteColToSelectValFunc(pCxt, pNode);
}
}
if (isVectorFunc(*pNode) && isDistinctOrderBy(pCxt)) {
return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt));
return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias);
}
return DEAL_RES_CONTINUE;
}

2
source/libs/parser/src/parUtil.c
View File

@ -174,6 +174,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "Invalid usage of RANGE clause, EVERY clause or FILL clause";
case TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN:
return "No valid function in window query";
case TSDB_CODE_PAR_INVALID_OPTR_USAGE:
return "Invalid usage of expr: %s";
case TSDB_CODE_OUT_OF_MEMORY:
return "Out of memory";
default:

2
source/libs/parser/test/parSelectTest.cpp
View File

@ -286,7 +286,7 @@ TEST_F(ParserSelectTest, interval) {
TEST_F(ParserSelectTest, intervalSemanticCheck) {
useDb("root", "test");
run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN);
run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_INVALID_OPTR_USAGE);
run("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 3 INTERVAL(1d) FILL(NEXT)", TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE);
run("SELECT HISTOGRAM(c1, 'log_bin', '{\"start\": -33,\"factor\": 55,\"count\": 5,\"infinity\": false}', 1) FROM t1 "
"WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' INTERVAL(10s) FILL(NULL)",

2
source/libs/qworker/src/qwDbg.c
View File

@ -271,7 +271,7 @@ void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped) {
}
}
int32_t qwDbgEnableDebug(char *option) {
int32_t qWorkerDbgEnableDebug(char *option) {
if (0 == strcasecmp(option, "lock")) {
gQWDebug.lockEnable = true;
qError("qw lock debug enabled");

2
source/libs/stream/src/streamExec.c
View File

@ -20,7 +20,7 @@
static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) {
int32_t code;
void* exec = pTask->exec.executor;
while(atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
while(pTask->taskLevel == TASK_LEVEL__SOURCE && atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
qError("stream task wait for the end of fill history");
taosMsleep(2);
continue;

22
source/libs/stream/src/streamState.c
View File

@ -130,21 +130,25 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int
char cfgPath[1030];
sprintf(cfgPath, "%s/cfg", statePath);
szPage = szPage < 0 ? 4096 : szPage;
pages = pages < 0 ? 256 : pages;
char cfg[1024];
memset(cfg, 0, 1024);
TdFilePtr pCfgFile = taosOpenFile(cfgPath, TD_FILE_READ);
if (pCfgFile != NULL) {
int64_t size;
int64_t size = 0;
taosFStatFile(pCfgFile, &size, NULL);
taosReadFile(pCfgFile, cfg, size);
sscanf(cfg, "%d\n%d\n", &szPage, &pages);
if (size > 0) {
taosReadFile(pCfgFile, cfg, size);
sscanf(cfg, "%d\n%d\n", &szPage, &pages);
}
} else {
taosMulModeMkDir(statePath, 0755);
pCfgFile = taosOpenFile(cfgPath, TD_FILE_WRITE | TD_FILE_CREATE);
szPage = szPage < 0 ? 4096 : szPage;
pages = pages < 0 ? 256 : pages;
sprintf(cfg, "%d\n%d\n", szPage, pages);
taosWriteFile(pCfgFile, cfg, strlen(cfg));
int32_t code = taosMulModeMkDir(statePath, 0755);
if (code == 0) {
pCfgFile = taosOpenFile(cfgPath, TD_FILE_WRITE | TD_FILE_CREATE);
sprintf(cfg, "%d\n%d\n", szPage, pages);
taosWriteFile(pCfgFile, cfg, strlen(cfg));
}
}
taosCloseFile(&pCfgFile);

3
tests/parallel_test/cases.task
View File

@ -864,6 +864,7 @@
,,y,script,./test.sh -f tsim/query/forceFill.sim
,,y,script,./test.sh -f tsim/query/emptyTsRange.sim
,,y,script,./test.sh -f tsim/query/partitionby.sim
,,y,script,./test.sh -f tsim/query/tableCount.sim
,,y,script,./test.sh -f tsim/qnode/basic1.sim
,,y,script,./test.sh -f tsim/snode/basic1.sim
,,y,script,./test.sh -f tsim/mnode/basic1.sim
@ -1096,7 +1097,7 @@
,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py
#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py
,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R
,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R
#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R
#docs-examples test
,,n,docs-examples-test,bash python.sh

98
tests/pytest/util/sql.py
View File

@ -23,6 +23,29 @@ import pandas as pd
from util.log import *
from util.constant import *
# from datetime import timezone
from tzlocal import get_localzone
import pytz
import time
def _locaTzTimeStamp(utctimestamp):
tz = get_localzone()
temptz = str(tz)
localtz = pytz.timezone(temptz)
defUtctimestamp=1035640800
local_dt = datetime.datetime(2002, 10, 27, 6, 0, 0, tzinfo=localtz)
defLocaltimestamp = time.mktime(local_dt.timetuple())
deltaTzTime = int(defLocaltimestamp)-int(defUtctimestamp)
temp = int(str(utctimestamp)[0:10])
tempOther = str(utctimestamp)[10-len(str(utctimestamp)):]
localtemp = deltaTzTime + temp
localAll = str(localtemp) + str(tempOther)
localtimestamp = int(localAll)
print(f"local timezone is {localtimestamp}, Deltel time is {deltaTzTime}s")
return localtimestamp
def _parse_datetime(timestr):
try:
return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f')
@ -227,28 +250,80 @@ class TDSql:
self.checkRowCol(row, col)
return self.cursor.istype(col, dataType)
def checkData(self, row, col, data):
if row >= self.queryRows:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row+1, self.queryRows)
tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args)
if col >= self.queryCols:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, col+1, self.queryCols)
tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args)
self.checkRowCol(row, col)
if self.queryResult[row][col] != data:
if self.cursor.istype(col, "TIMESTAMP"):
# suppose user want to check nanosecond timestamp if a longer data passed
if (len(data) >= 28):
if pd.to_datetime(self.queryResult[row][col]) == pd.to_datetime(data):
tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
# suppose user want to check nanosecond timestamp if a longer data passed``
if isinstance(data,str) :
if (len(data) >= 28):
resultData = _locaTzTimeStamp(self.queryResult[row][col])
if pd.to_datetime(resultData) == pd.to_datetime(data):
# tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{pd.to_datetime(resultData)} == expect:{data}")
tdLog.info("check successfully")
else:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
else:
if self.queryResult[row][col] == _parse_datetime(data):
# tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
tdLog.info("check successfully")
else:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
return
elif isinstance(data,int) :
if len(str(data)) == 16 :
unitTime = 'us'
elif len(str(data)) == 13 :
unitTime = 'ms'
elif len(str(data)) == 19 :
unitTime = 'ns'
else:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
resultData = pd.to_datetime(_locaTzTimeStamp(data),unit=unitTime)
if resultData == self.queryResult[row][col] :
# tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{resultData}")
tdLog.info("check successfully")
else:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
return
else:
if self.queryResult[row][col] == _parse_datetime(data):
tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
return
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
if str(self.queryResult[row][col]) == str(data):
tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
# tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
tdLog.info("check successfully")
return
elif isinstance(data, float):
if abs(data) >= 1 and abs((self.queryResult[row][col] - data) / data) <= 0.000001:
tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
# tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
tdLog.info("check successfully")
elif abs(data) < 1 and abs(self.queryResult[row][col] - data) <= 0.000001:
tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
# tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
tdLog.info("check successfully")
else:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
@ -258,8 +333,7 @@ class TDSql:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
tdLog.info("check successfully")
# return true or false replace exit, no print out
def checkRowColNoExit(self, row, col):

1
tests/requirements.txt
View File

@ -8,3 +8,4 @@ distro
requests
pexpect
faker
pyopenssl

8
tests/script/tsim/db/alter_option.sim
View File

@ -285,12 +285,12 @@ sql_error alter database db keep -1
print ============== modify minrows
sql_error alter database db minrows 8
sql_error alter database db minrows 8000
sql_error alter database db minrows 8001
sql_error alter database db minrows 8000000
sql_error alter database db minrows 8001000
print ============== modify maxrows
sql_error alter database db maxrows 1000
sql_error alter database db maxrows 2000
sql_error alter database db maxrows 10000001
sql_error alter database db maxrows 20000000
sql_error alter database db maxrows 11 # equal minrows
sql_error alter database db maxrows 10 # little than minrows

4
tests/script/tsim/db/create_all_options.sim
View File

@ -298,11 +298,11 @@ sql drop database db
sql_error create database db MAXROWS -1
sql_error create database db MAXROWS 0
sql_error create database db MAXROWS 199
sql_error create database db MAXROWS 10001
sql_error create database db MAXROWS 10000001
sql_error create database db MINROWS -1
sql_error create database db MINROWS 0
sql_error create database db MINROWS 9
sql_error create database db MINROWS 1001
sql_error create database db MINROWS 1000001
sql_error create database db MAXROWS 500 MINROWS 1000
print ====> PRECISION ['ms' | 'us' | 'ns', default: ms]

3
tests/script/tsim/parser/col_arithmetic_operation.sim
View File

@ -9,7 +9,7 @@ $dbPrefix = ca_db
$tbPrefix = ca_tb
$stbPrefix = ca_stb
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
$ts0 = 1537146000000
$delta = 600000
@ -18,6 +18,7 @@ $i = 0
$db = $dbPrefix . $i
$stb = $stbPrefix . $i
print drop database $db -x step1
sql drop database $db -x step1
step1:
sql create database $db

33
tests/script/tsim/parser/col_arithmetic_query.sim
View File

@ -4,7 +4,7 @@
$dbPrefix = ca_db
$tbPrefix = ca_tb
$stbPrefix = ca_stb
$rowNum = 10000
$rowNum = 1000
$i = 0
$db = $dbPrefix . $i
@ -33,7 +33,7 @@ endi
# asc/desc order [d.2] ======================================================
sql select c1 *( 2 / 3 ), c1/c1 from $tb order by ts asc;
if $rows != 10000 then
if $rows != 1000 then
return -1
endi
if $data00 != 0.000000000 then
@ -57,7 +57,7 @@ if $data91 != 1.000000000 then
endi
sql select (c1 * 2) % 7.9, c1*1, c1*1*1, c1*c1, c1*c1*c1 from $tb order by ts desc;
if $rows != 10000 then
if $rows != 1000 then
return -1
endi
if $data00 != 2.200000000 then
@ -151,7 +151,7 @@ sql select top(c1,1) - 88 from $tb
# all data types [d.6] ================================================================
sql select c2-c1*1.1, c3/c2, c4*c3, c5%c4, (c6+c4)%22, c2-c2 from $tb
if $rows != 10000 then
if $rows != 1000 then
return -1
endi
if $data00 != 0.000000000 then
@ -221,7 +221,7 @@ if $data90 != 76.000000000 then
return -1
endi
sql select c4 / 99.123 from $tb limit 10 offset 9999;
sql select c4 / 99.123 from $tb limit 10 offset 999;
if $rows != 1 then
return -1
endi
@ -237,7 +237,7 @@ sql_error select c2-c2, c3-c4, c5%c3 from $tb fill(value, 12);
# constant column. [d.13]==============================================================
sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb
if $rows != 10000 then
if $rows != 1000 then
return -1
endi
if $data00 != 0 then
@ -261,7 +261,7 @@ endi
# column value filter [d.14]===========================================================
sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb where c1<2
if $rows != 2000 then
if $rows != 200 then
return -1
endi
if $data00 != 0 then
@ -338,13 +338,14 @@ sql select (count(c1) * 2) % 7.9, (count(c1) * 2), ( count(1)*2) from $stb
if $rows != 1 then
return -1
endi
if $data00 != 1.800000000 then
print $data00
if $data00 != 6.500000000 then
return -1
endi
if $data01 != 100000.000000000 then
if $data01 != 10000.000000000 then
return -1
endi
if $data02 != 200000.000000000 then
if $data02 != 20000.000000000 then
return -1
endi
@ -371,7 +372,7 @@ endi
if $data00 != 0.000000000 then
return -1
endi
if $data01 != 225000.000000000 then
if $data01 != 22500.000000000 then
return -1
endi
if $data02 != 8.077777778 then
@ -383,7 +384,7 @@ endi
if $data04 != 0.444444444 then
return -1
endi
if $data05 != 450000.000000000 then
if $data05 != 45000.000000000 then
return -1
endi
@ -484,10 +485,10 @@ endi
if $data10 != 0.000000000 then
return -1
endi
if $data20 != 0.997600000 then
if $data20 != 0.976000000 then
return -1
endi
if $data90 != 7.980800000 then
if $data90 != 7.808000000 then
return -1
endi
@ -496,7 +497,7 @@ sql select first(c6) - sum(c6) + 12 from $stb limit 12 offset 0;
if $rows != 1 then
return -1
endi
if $data00 != -449988.000000000 then
if $data00 != -44988.000000000 then
return -1
endi
@ -546,7 +547,7 @@ endi
# interval query [d.17]===============================================================
sql select avg(c2)*count(c2), sum(c3)-first(c3), last(c4)+9 from ca_stb0 interval(1s)
if $rows != 10000 then
if $rows != 1000 then
return -1
endi

2
tests/script/tsim/parser/commit.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = sc_db
$tbPrefix = sc_tb
$stbPrefix = sc_stb
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
$loops = 5
$log = 1

80
tests/script/tsim/parser/groupby.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = group_db
$tbPrefix = group_tb
$mtPrefix = group_mt
$tbNum = 8
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
print =============== groupby.sim
@ -88,7 +88,7 @@ if $row != 20 then
return -1
endi
if $data00 != 100 then
if $data00 != 10 then
return -1
endi
@ -96,7 +96,7 @@ if $data01 != 0 then
return -1
endi
if $data10 != 100 then
if $data10 != 10 then
return -1
endi
@ -186,7 +186,7 @@ if $data04 != 0.00000 then
return -1
endi
if $data10 != 100 then
if $data10 != 10 then
return -1
endi
@ -227,7 +227,7 @@ if $row != 20 then
return -1
endi
if $data00 != 800 then
if $data00 != 80 then
return -1
endi
@ -235,7 +235,7 @@ if $data01 != 0 then
return -1
endi
if $data10 != 800 then
if $data10 != 80 then
return -1
endi
@ -243,7 +243,7 @@ if $data11 != 1 then
return -1
endi
if $data90 != 800 then
if $data90 != 80 then
return -1
endi
@ -297,7 +297,7 @@ if $data00 != $data03 then
return -1
endi
if $data01 != @70-01-01 08:01:49.900@ then
if $data01 != @70-01-01 08:01:40.900@ then
return -1
endi
@ -305,7 +305,7 @@ if $data02 != @70-01-01 08:01:40.000@ then
return -1
endi
if $data07 != 800 then
if $data07 != 80 then
return -1
endi
@ -313,7 +313,7 @@ if $data10 != $data13 then
return -1
endi
if $data11 != @70-01-01 08:01:49.901@ then
if $data11 != @70-01-01 08:01:40.901@ then
return -1
endi
@ -321,7 +321,7 @@ if $data12 != @70-01-01 08:01:40.001@ then
return -1
endi
if $data17 != 800 then
if $data17 != 80 then
return -1
endi
@ -329,7 +329,7 @@ if $data90 != $data93 then
return -1
endi
if $data91 != @70-01-01 08:01:49.909@ then
if $data91 != @70-01-01 08:01:40.909@ then
return -1
endi
@ -337,11 +337,11 @@ if $data92 != @70-01-01 08:01:40.009@ then
return -1
endi
if $data97 != 800 then
if $data97 != 80 then
return -1
endi
if $data95 != 7200 then
if $data95 != 720 then
return -1
endi
@ -358,7 +358,7 @@ if $data00 != 0 then
return -1
endi
if $data11 != 800 then
if $data11 != 80 then
return -1
endi
@ -372,7 +372,7 @@ if $data00 != 0 then
return -1
endi
if $data01 != @70-01-01 08:01:49.900@ then
if $data01 != @70-01-01 08:01:40.900@ then
return -1
endi
@ -389,13 +389,13 @@ if $data04 != 0 then
return -1
endi
if $data06 != 100 then
if $data06 != 10 then
return -1
endi
sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 order by c4;
if $rows != 10000 then
if $rows != 1000 then
return -1
endi
@ -420,13 +420,13 @@ if $rows != 1 then
return -1
endi
sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 slimit 20 soffset 9990;
sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 slimit 20 soffset 990;
if $rows != 10 then
return -1
endi
sql select count(*),first(ts),last(ts),min(c3),max(c3),sum(c3),avg(c3),sum(c4)/count(c4) from group_tb1 group by c4;
if $rows != 10000 then
if $rows != 1000 then
return -1
endi
@ -441,32 +441,32 @@ if $rows != 100 then
return -1
endi
if $data00 != 100 then
if $data00 != 10 then
return -1
endi
if $data01 != 495000 then
if $data01 != 4500 then
return -1
endi
if $data02 != 100 then
if $data02 != 10 then
return -1
endi
if $data03 != 4950.000000000 then
print expect 4950.000000000 , acutal $data03
if $data03 != 450.000000000 then
print expect 450.000000000 , acutal $data03
return -1
endi
if $data10 != 100 then
if $data10 != 10 then
return -1
endi
if $data11 != 495100 then
if $data11 != 4510 then
return -1
endi
if $data13 != 4951.000000000 then
if $data13 != 451.000000000 then
return -1
endi
@ -481,19 +481,19 @@ if $rows != 1 then
return -1
endi
#if $data00 != 79200.000000000 then
#if $data00 != 2160.000000000 then
# return -1
#endi
#if $data01 != @binary99@ then
#if $data01 != @binary27@ then
# return -1
#endi
#if $data02 != 99.000000000 then
#if $data02 != 27.000000000 then
# return -1
#endi
#if $data03 != 99.000000000 then
#if $data03 != 27.000000000 then
# return -1
#endi
@ -503,7 +503,7 @@ if $rows != 100 then
return -1
endi
if $data00 != 4851.000000000 then
if $data00 != 441.000000000 then
return -1
endi
@ -511,19 +511,19 @@ if $data01 != 0 then
return -1
endi
if $data02 != 9900 then
if $data02 != 900 then
return -1
endi
if $data03 != 4950.000000000 then
if $data03 != 450.000000000 then
return -1
endi
if $data04 != 2886.607004772 then
if $data04 != 287.228132327 then
return -1
endi
if $data10 != 4852.000000000 then
if $data10 != 442.000000000 then
return -1
endi
@ -531,15 +531,15 @@ if $data11 != 1 then
return -1
endi
if $data12 != 9901 then
if $data12 != 901 then
return -1
endi
if $data13 != 4951.000000000 then
if $data13 != 451.000000000 then
return -1
endi
if $data14 != 2886.607004772 then
if $data14 != 287.228132327 then
return -1
endi

2
tests/script/tsim/parser/interp.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = intp_db
$tbPrefix = intp_tb
$stbPrefix = intp_stb
$tbNum = 4
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
$ts0 = 1537146000000
$delta = 600000

6
tests/script/tsim/parser/join_manyblocks.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = join_m_db
$tbPrefix = join_tb
$mtPrefix = join_mt
$tbNum = 3
$rowNum = 20000
$rowNum = 2000
$totalNum = $tbNum * $rowNum
print =============== join_manyBlocks.sim
@ -78,8 +78,8 @@ print ==============> td-3313
sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1;
print $row
if $row != 60000 then
print expect 60000, actual: $row
if $row != 6000 then
print expect 6000, actual: $row
return -1
endi

2
tests/script/tsim/parser/limit1.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = lm1_db
$tbPrefix = lm1_tb
$stbPrefix = lm1_stb
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
$ts0 = 1537146000000
$delta = 600000

5
tests/script/tsim/parser/limit1_stb.sim
View File

@ -4,7 +4,7 @@ $dbPrefix = lm1_db
$tbPrefix = lm1_tb
$stbPrefix = lm1_stb
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
$ts0 = 1537146000000
$delta = 600000
@ -453,13 +453,14 @@ endi
### [TBASE-361]
$offset = $rowNum / 2
$offset = $offset + 1
print === select _wstart, max(c1), min(c2), avg(c3), count(c4), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 8 interval(5m) limit $offset offset $offset
sql select _wstart, max(c1), min(c2), avg(c3), count(c4), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 8 interval(5m) limit $offset offset $offset
$val = $rowNum - $offset
if $rows != $val then
print expect $val, actual:$rows
return -1
endi
if $data00 != @18-10-22 02:30:00.000@ then
if $data00 != @18-09-20 20:30:00.000@ then
return -1
endi
if $data01 != 1 then

36
tests/script/tsim/parser/limit1_tb.sim
View File

@ -4,7 +4,7 @@ $dbPrefix = lm1_db
$tbPrefix = lm1_tb
$stbPrefix = lm1_stb
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
$ts0 = 1537146000000
$delta = 600000
@ -13,7 +13,7 @@ $i = 0
$db = $dbPrefix . $i
$stb = $stbPrefix . $i
print ====== use db
print ====== use $db
sql use $db
##### select from table
@ -664,9 +664,9 @@ endi
if $data21 != 4.027681991 then
return -1
endi
print select _wstart, count(c1), count(c2), count(c3), count(c4), count(c5), count(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(27m)
sql select _wstart, count(c1), count(c2), count(c3), count(c4), count(c5), count(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(27m)
if $rows != 3704 then
if $rows != 371 then
return -1
endi
if $data01 != 2 then
@ -994,11 +994,14 @@ endi
$offset = $rowNum / 10
$offset = $offset * 3
$offset = $offset - 1
print === select * from $tb where c1 < 5 and c1 > 1 order by ts asc limit 3 offset $offset
sql select * from $tb where c1 < 5 and c1 > 1 order by ts asc limit 3 offset $offset
if $rows != 1 then
return -1
endi
if $data00 != @18-11-25 18:40:00.000@ then
print $data00
if $data00 != @18-09-24 06:40:00.000@ then
return -1
endi
if $data01 != 4 then
@ -1012,19 +1015,19 @@ sql select * from $tb order by ts desc limit 5
if $rows != 5 then
return -1
endi
if $data00 != @18-11-25 19:30:00.000@ then
if $data00 != @18-09-24 07:30:00.000@ then
return -1
endi
if $data01 != 9 then
return -1
endi
if $data10 != @18-11-25 19:20:00.000@ then
if $data10 != @18-09-24 07:20:00.000@ then
return -1
endi
if $data12 != 8 then
return -1
endi
if $data20 != @18-11-25 19:10:00.000@ then
if $data20 != @18-09-24 07:10:00.000@ then
return -1
endi
if $data23 != 7.00000 then
@ -1048,7 +1051,7 @@ endi
if $data39 != nchar6 then
return -1
endi
if $data40 != @18-11-25 18:50:00.000@ then
if $data40 != @18-09-24 06:50:00.000@ then
return -1
endi
@ -1056,19 +1059,19 @@ sql select * from $tb order by ts desc limit 5 offset 1
if $rows != 5 then
return -1
endi
if $data00 != @18-11-25 19:20:00.000@ then
if $data00 != @18-09-24 07:20:00.000@ then
return -1
endi
if $data01 != 8 then
return -1
endi
if $data10 != @18-11-25 19:10:00.000@ then
if $data10 != @18-09-24 07:10:00.000@ then
return -1
endi
if $data12 != 7 then
return -1
endi
if $data20 != @18-11-25 19:00:00.000@ then
if $data20 != @18-09-24 07:00:00.000@ then
return -1
endi
if $data23 != 6.00000 then
@ -1092,12 +1095,13 @@ endi
if $data39 != nchar5 then
return -1
endi
if $data40 != @18-11-25 18:40:00.000@ then
if $data40 != @18-09-24 06:40:00.000@ then
return -1
endi
$offset = $rowNum
$offset = $offset - 2
print ==== select * from $tb order by ts desc limit 5 offset $offset
sql select * from $tb order by ts desc limit 5 offset $offset
if $rows != 2 then
return -1
@ -1119,16 +1123,16 @@ sql select * from $tb where c1 < 8 order by ts desc limit 3 offset 2
if $rows != 3 then
return -1
endi
if $data00 != @18-11-25 18:50:00.000@ then
if $data00 != @18-09-24 06:50:00.000@ then
return -1
endi
if $data01 != 5 then
return -1
endi
if $data10 != @18-11-25 18:40:00.000@ then
if $data10 != @18-09-24 06:40:00.000@ then
return -1
endi
if $data20 != @18-11-25 18:30:00.000@ then
if $data20 != @18-09-24 06:30:00.000@ then
return -1
endi
if $data12 != 4 then

10
tests/script/tsim/parser/nestquery.sim
View File

@ -8,7 +8,7 @@ print ======================== dnode1 start
$dbPrefix = nest_db
$tbPrefix = nest_tb
$mtPrefix = nest_mt
$tbNum = 10
$tbNum = 3
$rowNum = 10000
$totalNum = $tbNum * $rowNum
@ -23,7 +23,7 @@ sql create database if not exists $db
sql use $db
sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int)
$half = $tbNum / 2
$half = 2
$i = 0
while $i < $half
@ -69,7 +69,7 @@ sql select count(*) from (select count(*) from nest_mt0 group by tbname)
if $rows != 1 then
return -1
endi
if $data00 != 10 then
if $data00 != 4 then
return -1
endi
@ -77,7 +77,7 @@ sql select count(*) from (select count(*) from nest_mt0 partition by tbname inte
if $rows != 1 then
return -1
endi
if $data00 != 170 then
if $data00 != 68 then
return -1
endi
@ -85,7 +85,7 @@ sql select sum(a) from (select count(*) a from nest_mt0 partition by tbname inte
if $rows != 1 then
return -1
endi
if $data00 != 100000 then
if $data00 != 40000 then
return -1
endi

84
tests/script/tsim/parser/projection_limit_offset.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = group_db
$tbPrefix = group_tb
$mtPrefix = group_mt
$tbNum = 8
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
print =============== projection_limit_offset.sim
@ -81,199 +81,199 @@ $ts2 = $tb2 . .ts
sql select ts from group_mt0
print $rows
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 0;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 0;
if $rows != 4008 then
print expect 4008, actual:$rows
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 1;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 1;
if $rows != 4007 then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 101;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 101;
print $rows
if $rows != 3907 then
return -1
endi
if $data00 != @70-01-01 08:01:43.101@ then
if $data00 != @70-01-01 08:01:40.101@ then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 902;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 902;
if $rows != 3106 then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 400;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 400;
if $rows != 3608 then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 4007;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 4007;
if $rows != 1 then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 2000 offset 4008;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 2000 offset 4008;
if $rows != 0 then
return -1
endi
#==================================order by desc, multi vnode, limit/offset===================================
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 0;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 0;
if $rows != 4008 then
return -1
endi
if $data00 != @70-01-01 08:01:43.500@ then
if $data00 != @70-01-01 08:01:40.500@ then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 1;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 1;
if $rows != 4007 then
return -1
endi
if $data00 != @70-01-01 08:01:43.500@ then
if $data00 != @70-01-01 08:01:40.500@ then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 101;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 101;
print $rows
if $rows != 3907 then
return -1
endi
if $data00 != @70-01-01 08:01:43.488@ then
if $data00 != @70-01-01 08:01:40.488@ then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 902;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 902;
if $rows != 3106 then
return -1
endi
if $data00 != @70-01-01 08:01:43.388@ then
if $data00 != @70-01-01 08:01:40.388@ then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 400;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 400;
if $rows != 3608 then
return -1
endi
if $data00 != @70-01-01 08:01:43.450@ then
if $data00 != @70-01-01 08:01:40.450@ then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 4007;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 4007;
if $rows != 1 then
return -1
endi
if $data00 != @70-01-01 08:01:43.000@ then
if $data00 != @70-01-01 08:01:40.000@ then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 2000 offset 4008;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 2000 offset 4008;
if $rows != 0 then
return -1
endi
#=================================single value filter======================================
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 0;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 0;
if $row != 8 then
return -1
endi
if $data00 != @70-01-01 08:01:43.000@ then
if $data00 != @70-01-01 08:01:40.000@ then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 1;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 1;
if $row != 7 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 2;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 2;
if $row != 6 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 4;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 4;
if $row != 4 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 7;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 7;
if $row != 1 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 8;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 8;
if $row != 0 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 9;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 9;
if $row != 0 then
return -1
endi
#===============================single value filter, order by desc============================
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 0;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 0;
if $row != 8 then
return -1
endi
if $data00 != @70-01-01 08:01:43.000@ then
if $data00 != @70-01-01 08:01:40.000@ then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 1;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 1;
if $row != 7 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 2;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 2;
if $row != 6 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 4;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 4;
if $row != 4 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 7;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 7;
if $row != 1 then
return -1
endi
if $data00 != @70-01-01 08:01:43.000@ then
if $data00 != @70-01-01 08:01:40.000@ then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 8;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 8;
if $row != 0 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 9;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 9;
if $row != 0 then
return -1
endi
#[tbase-695]
sql select ts,tbname from group_mt0 where ts>='1970-01-01 8:1:40' and ts<'1970-1-1 8:1:45' and c1<99999999 limit 100000 offset 5000
sql select ts,tbname from group_mt0 where ts>='1970-01-01 8:1:40' and ts<'1970-1-1 8:1:40.500' and c1<99999999 limit 10000 offset 500
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $row != 35000 then
if $row != 3500 then
return -1
endi
@ -301,7 +301,7 @@ print tbase-722
sql select spread(ts) from group_tb0;
print $data00
if $data00 != 9999.000000000 then
if $data00 != 999.000000000 then
return -1
endi

4
tests/script/tsim/parser/selectResNum.sim
View File

@ -7,10 +7,10 @@ $dbPrefix = sc_db
$tbPrefix = sc_tb
$stbPrefix = sc_stb
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
$loops = 200000
$log = 10000
$log = 1000
$ts0 = 1537146000000
$delta = 600000
print ========== selectResNum.sim

102
tests/script/tsim/parser/sliding.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = sliding_db
$tbPrefix = sliding_tb
$mtPrefix = sliding_mt
$tbNum = 8
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
print =============== sliding.sim
@ -88,25 +88,25 @@ $ts1 = $tb1 . .ts
$ts2 = $tb2 . .ts
print ===============================interval_sliding query
sql select _wstart, count(*) from sliding_tb0 interval(30s) sliding(30s);
sql select _wstart, count(*) from sliding_tb0 interval(3s) sliding(3s);
if $row != 10 then
return -1
endi
if $data00 != @00-01-01 00:00:00.000@ then
return -1
endi
if $data01 != 1000 then
if $data01 != 100 then
return -1
endi
if $data10 != @00-01-01 00:00:30.000@ then
if $data10 != @00-01-01 00:00:03.000@ then
return -1
endi
if $data11 != 1000 then
if $data11 != 100 then
return -1
endi
sql select _wstart, stddev(c1) from sliding_tb0 interval(10a) sliding(10a);
if $row != 10000 then
if $row != 1000 then
return -1
endi
if $data00 != @00-01-01 00:00:00.000@ then
@ -123,10 +123,10 @@ if $data91 != 0.000000000 then
endi
sql select _wstart, stddev(c1),count(c2),first(c3),last(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc;
if $row != 10000 then
if $row != 1000 then
return -1
endi
if $data00 != @00-01-01 00:04:59.970@ then
if $data00 != @00-01-01 00:00:29.970@ then
return -1
endi
if $data01 != 0.000000000 then
@ -141,7 +141,7 @@ endi
if $data04 != 99 then
return -1
endi
if $data90 != @00-01-01 00:04:59.700@ then
if $data90 != @00-01-01 00:00:29.700@ then
return -1
endi
if $data91 != 0.000000000 then
@ -157,41 +157,41 @@ if $data94 != 90 then
return -1
endi
sql select _wstart, count(c2),last(c4) from sliding_tb0 interval(30s) sliding(10s) order by _wstart asc;
sql select _wstart, count(c2),last(c4) from sliding_tb0 interval(3s) sliding(1s) order by _wstart asc;
if $row != 32 then
return -1
endi
if $data00 != @99-12-31 23:59:40.000@ then
print expect 12-31 23:59:40.000, actual: $data00
if $data00 != @99-12-31 23:59:58.000@ then
print expect 12-31 23:59:58.000, actual: $data00
return -1
endi
if $data01 != 334 then
if $data01 != 34 then
return -1
endi
if $data02 != 33 then
return -1
endi
sql select _wstart, count(c2),stddev(c3),first(c4),last(c4) from sliding_tb0 where ts>'2000-01-01 0:0:0' and ts<'2000-1-1 0:0:31' interval(30s) sliding(30s) order by _wstart asc;
sql select _wstart, count(c2),stddev(c3),first(c4),last(c4) from sliding_tb0 where ts>'2000-01-01 0:0:0' and ts<'2000-1-1 0:0:4' interval(3s) sliding(3s) order by _wstart asc;
if $row != 2 then
return -1
endi
if $data04 != 99 then
return -1
endi
if $data01 != 999 then
if $data01 != 99 then
return -1
endi
if $data02 != 28.837977152 then
if $data02 != 28.577380332 then
return -1
endi
#interval offset + limit
sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc limit 10 offset 990;
sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc limit 10 offset 90;
if $row != 10 then
return -1
endi
if $data00 != @00-01-01 00:04:30.270@ then
if $data00 != @00-01-01 00:00:27.270@ then
return -1
endi
if $data01 != 1 then
@ -203,7 +203,7 @@ endi
if $data03 != 0.000000000 then
return -1
endi
if $data90 != @00-01-01 00:04:30.000@ then
if $data90 != @00-01-01 00:00:27.000@ then
return -1
endi
if $data91 != 1 then
@ -217,43 +217,43 @@ if $data93 != 0.000000000 then
endi
#interval offset test
sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(30s) order by _wstart asc limit 1000 offset 1;
sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(3s) order by _wstart asc limit 100 offset 1;
if $row != 9 then
return -1
endi
if $data00 != @00-01-01 00:00:30.000@ then
if $data00 != @00-01-01 00:00:03.000@ then
return -1
endi
if $data01 != 1000 then
if $data01 != 100 then
return -1
endi
if $data02 != 99 then
return -1
endi
if $data80 != @00-01-01 00:04:30.000@ then
if $data80 != @00-01-01 00:00:27.000@ then
return -1
endi
if $data81 != 1000 then
if $data81 != 100 then
return -1
endi
sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 where ts>'2000-1-1 0:0:0' and ts<'2000-1-1 0:0:31' interval(30s) sliding(30s) order by _wstart asc limit 1000 offset 0;
sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 where ts>'2000-1-1 0:0:0' and ts<'2000-1-1 0:0:4' interval(3s) sliding(3s) order by _wstart asc limit 100 offset 0;
if $row != 2 then
return -1
endi
if $data00 != @00-01-01 00:00:00.000@ then
return -1
endi
if $data01 != 999 then
if $data01 != 99 then
return -1
endi
if $data02 != 99 then
return -1
endi
if $data03 != 28.837977152 then
if $data03 != 28.577380332 then
return -1
endi
if $data10 != @00-01-01 00:00:30.000@ then
if $data10 != @00-01-01 00:00:03.000@ then
return -1
endi
if $data11 != 34 then
@ -266,14 +266,14 @@ if $data13 != 9.810708435 then
return -1
endi
sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 1;
sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 1;
if $row != 15 then
return -1
endi
if $data00 != @00-01-01 00:00:00.000@ then
return -1
endi
if $data01 != 1000 then
if $data01 != 100 then
return -1
endi
if $data02 != 99 then
@ -282,85 +282,85 @@ endi
if $data03 != 28.866070048 then
return -1
endi
if $data90 != @00-01-01 00:03:00.000@ then
if $data90 != @00-01-01 00:00:18.000@ then
return -1
endi
if $data91 != 1000 then
if $data91 != 100 then
return -1
endi
if $data92 != 99 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 5;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 5;
if $row != 11 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 6;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 6;
if $row != 10 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 7;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 7;
if $row != 9 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 8;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 8;
if $row != 8 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 9;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 9;
if $row != 7 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 10;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 10;
if $row != 6 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 11;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 11;
if $row != 5 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 12;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 12;
if $row != 4 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 13;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 13;
if $row != 3 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 14;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 14;
if $row != 2 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 15;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 15;
if $row != 1 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 16;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 16;
if $row != 0 then
return -1
endi
sql select _wstart, count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) order by _wstart desc;
sql select _wstart, count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(3s) order by _wstart desc;
if $row != 10 then
return -1
endi
#00-01-01 00:04:30.000| 10| 0| 0.000000000| 0.000000000|
if $data00 != @00-01-01 00:04:30.000@ then
#00-01-01 00:00:27.000 | 1 | 0 | 0.000000000 | 0.000000000 |
if $data00 != @00-01-01 00:00:27.000@ then
return -1
endi
if $data01 != 10 then
if $data01 != 1 then
return -1
endi
if $data02 != 0 then
@ -370,18 +370,18 @@ if $data03 != 0.000000000 then
return -1
endi
sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by _wstart desc limit 1 offset 15;
sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(3s) sliding(2s) order by _wstart desc limit 1 offset 14;
if $row != 1 then
return -1
endi
sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by _wstart desc limit 1 offset 16;
sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(3s) sliding(2s) order by _wstart desc limit 1 offset 15;
if $row != 0 then
return -1
endi
sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) order by _wstart desc limit 10 offset 2;
if $data00 != @00-01-01 00:04:59.910@ then
if $data00 != @00-01-01 00:00:29.910@ then
return -1
endi

22
tests/script/tsim/parser/union.sim
View File

@ -8,7 +8,7 @@ $tbPrefix = union_tb
$tbPrefix1 = union_tb_
$mtPrefix = union_mt
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
print =============== union.sim
@ -65,7 +65,7 @@ sql create table $mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c
$j = 0
$t = 1578203484000
$rowNum = 1000
$rowNum = 100
$tbNum = 5
$i = 0
@ -117,7 +117,7 @@ sql_error (select c1 from union_tb0 limit 1 union all select c1 from union_tb1 l
# sql with parenthese
sql (select c1 from union_tb0)
if $rows != 10000 then
if $rows != 1000 then
return -1
endi
if $data00 != 0 then
@ -187,7 +187,7 @@ endi
if $data10 != 1 then
return -1
endi
if $data20 != 4950000 then
if $data20 != 495000 then
return -1
endi
@ -202,7 +202,7 @@ endi
if $data10 != 1 then
return -1
endi
if $data20 != 495000 then
if $data20 != 49500 then
return -1
endi
@ -211,7 +211,7 @@ sql (select count(*) as c from union_tb0, union_tb1 where union_tb0.ts=union_tb1
if $rows != 11 then
return -1
endi
if $data00 != 10000 then
if $data00 != 1000 then
return -1
endi
if $data10 != 9 then
@ -227,7 +227,7 @@ endi
print ===========================================tags union
# two super table tag union, limit is not active during retrieve tags query
sql (select t1 from union_mt0) union all (select t1 from union_mt0)
if $rows != 200000 then
if $rows != 20000 then
return -1
endi
@ -248,7 +248,7 @@ sql (select count(*) as c from union_tb0 where ts > now + 3650d) union all (sele
if $rows != 2 then
return -1
endi
#if $data00 != 495000 then
#if $data00 != 49500 then
# return -1
#endi
@ -272,7 +272,7 @@ endi
# multi-vnode projection query
sql (select c1 from union_mt0) union all select c1 from union_mt0;
if $rows != 200000 then
if $rows != 20000 then
return -1
endi
@ -330,10 +330,10 @@ sql (select sum(c1) as a from union_tb0 limit 1) union all (select sum(c3) as a
if $rows != 2 then
return -1
endi
if $data00 != 495000 then
if $data00 != 49500 then
return -1
endi
if $data10 != 495000 then
if $data10 != 49500 then
return -1
endi

8
tests/script/tsim/parser/where.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = wh_db
$tbPrefix = wh_tb
$mtPrefix = wh_mt
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
print =============== where.sim
@ -62,7 +62,7 @@ sql select count(*) from $tb where c1<10 and c1<>2
if $rows != 1 then
return -1
endi
if $data00 != 900 then
if $data00 != 90 then
return -1
endi
@ -308,9 +308,9 @@ if $row != 0 then
endi
sql select * from wh_mt0 where c3 = 1;
print $rows -> 1000
print $rows -> 100
print $data00 $data01 $data02
if $row != 1000 then
if $row != 100 then
return -1
endi

18
tests/script/tsim/query/explain.sim
View File

@ -4,7 +4,7 @@ system sh/exec.sh -n dnode1 -s start
sql connect
print ======== step1
sql create database db1 vgroups 3;
sql create database db1 vgroups 3 cachesize 10 cachemodel 'both';
sql use db1;
sql select * from information_schema.ins_databases;
sql create stable st1 (ts timestamp, f1 int, f2 binary(200)) tags(t1 int);
@ -60,6 +60,15 @@ sql explain verbose true select * from information_schema.ins_stables where db_n
sql explain verbose true select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0;
sql explain verbose true insert into t1(ts) select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0;
sql explain verbose true insert into t1(ts, t1) select _wstart, count(*) from st1 interval(10s);
sql explain verbose true select distinct tbname, table_name from information_schema.ins_tables;
sql explain verbose true select diff(f1) as f11 from tb1 order by f11;
sql explain verbose true select count(*) from st1 where ts > now - 3m and ts < now interval(10s) fill(linear);
sql explain verbose true select count(*) from st1 partition by tbname;
sql explain verbose true select count(*) from information_schema.ins_tables group by stable_name;
sql explain verbose true select last(*) from st1;
sql explain verbose true select last_row(*) from st1;
sql explain verbose true select interp(f1) from tb1 where ts > now - 3m and ts < now range(now-3m,now) every(1m) fill(prev);
sql explain verbose true select _wstart, _wend, count(*) from tb1 EVENT_WINDOW start with f1 > 0 end with f1 < 10;
print ======== step4
sql explain analyze select ts from st1 where -2;
@ -96,6 +105,13 @@ sql explain analyze verbose true select * from information_schema.ins_stables wh
sql explain analyze verbose true select * from (select min(f1),count(*) a from st1 where f1 > 0) where a < 0;
sql explain analyze verbose true select count(f1) from st1 group by tbname;
sql explain analyze verbose true select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0;
sql explain analyze verbose true select diff(f1) as f11 from tb1 order by f11;
sql explain analyze verbose true select count(*) from st1 where ts > now - 3m and ts < now interval(10s) fill(linear);
sql explain analyze verbose true select count(*) from information_schema.ins_tables group by stable_name;
sql explain analyze verbose true select last(*) from st1;
sql explain analyze verbose true select last_row(*) from st1;
sql explain analyze verbose true select interp(f1) from tb1 where ts > now - 3m and ts < now range(now-3m,now) every(1m) fill(prev);
sql explain analyze verbose true select _wstart, _wend, count(*) from tb1 EVENT_WINDOW start with f1 > 0 end with f1 < 10;
#not pass case
#sql explain verbose true select count(*),sum(f1) as aa from tb1 where (f1 > 0 or f1 < -1) and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00' order by aa;

107
tests/script/tsim/query/tableCount.sim Normal file
View File

@ -0,0 +1,107 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
sql drop database if exists db1;
sql create database db1 vgroups 3;
sql create database db1;
sql use db1;
sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create table tba1 using sta tags(1, 1, 1);
sql create table tba2 using sta tags(2, 2, 2);
sql create table tba3 using sta tags(3, 3, 3);
sql create table tba4 using sta tags(3, 3, 3);
sql create table tba5 using sta tags(3, 3, 3);
sql create table tba6 using sta tags(3, 3, 3);
sql create table tba7 using sta tags(3, 3, 3);
sql create table tba8 using sta tags(3, 3, 3);
sql create table tbb1 using stb tags(4, 4, 4);
sql create table tbb2 using stb tags(5, 5, 5);
sql create table tbb3 using stb tags(6, 6, 6);
sql create table tbb4 using stb tags(4, 4, 4);
sql create table tbb5 using stb tags(5, 5, 5);
sql create table tbb6 using stb tags(6, 6, 6);
sql create table tbb7 using stb tags(7, 7, 7);
sql create table tbb8 using stb tags(8, 8, 8);
sql create table tbn1 (ts timestamp, f1 int);
sql create database db2 vgroups 3;
sql create database db2;
sql use db2;
sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create table tba1 using sta tags(1, 1, 1);
sql create table tba2 using sta tags(2, 2, 2);
sql create table tbb1 using stb tags(4, 4, 4);
sql create table tbb2 using stb tags(5, 5, 5);
sql create table tbb3 using stb tags(6, 6, 6);
sql select count(table_name) from information_schema.ins_tables group by stable_name;
if $rows != 3 then
return -1
endi
sql select count(table_name) from information_schema.ins_tables group by db_name;
if $rows != 4 then
return -1
endi
sql select count(table_name) from information_schema.ins_tables group by db_name, stable_name;
if $rows != 7 then
return -1
endi
sql select stable_name,count(table_name) from information_schema.ins_tables group by stable_name order by stable_name;
if $rows != 3 then
return -1
endi
if $data01 != 30 then
return -1
endi
if $data11 != 10 then
return -1
endi
if $data21 != 11 then
return -1
endi
sql select db_name,count(table_name) from information_schema.ins_tables group by db_name order by db_name;
if $rows != 4 then
return -1
endi
if $data01 != 17 then
return -1
endi
if $data11 != 5 then
return -1
endi
if $data21 != 24 then
return -1
endi
if $data31 != 5 then
return -1
endi
sql select db_name,stable_name,count(table_name) from information_schema.ins_tables group by db_name, stable_name order by db_name, stable_name;
if $rows != 7 then
return -1
endi
if $data02 != 1 then
return -1
endi
if $data12 != 8 then
return -1
endi
if $data22 != 8 then
return -1
endi
if $data32 != 2 then
return -1
endi
if $data42 != 3 then
return -1
endi
if $data52 != 24 then
return -1
endi
if $data62 != 5 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT

10
tests/script/tsim/show/basic.sim
View File

@ -244,5 +244,15 @@ if $rows <= 0 then
return -1
endi
sql show cluster alive;
if $rows <= 0 then
return -1
endi
sql show db.alive;
if $rows <= 0 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT

9
tests/script/tsim/stream/deleteState.sim
View File

@ -51,13 +51,8 @@ if $loop_count == 10 then
return -1
endi
if $data01 != 1 then
print =====data01=$data01
goto loop1
endi
if $data02 != NULL then
print =====data02=$data02
if $rows != 0 then
print =====rows=$rows
goto loop1
endi

9
tests/script/tsim/stream/partitionbyColumnState.sim
View File

@ -27,13 +27,8 @@ if $loop_count == 10 then
return -1
endi
if $data01 != 1 then
print =====data01=$data01
goto loop0
endi
if $data02 != NULL then
print =====data02=$data02
if $rows != 0 then
print =====rows=$rows
goto loop0
endi

101
tests/script/tsim/stream/state1.sim Normal file
View File

@ -0,0 +1,101 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print =============== create database
sql create database test vgroups 4;
sql select * from information_schema.ins_databases;
if $rows != 3 then
return -1
endi
print $data00 $data01 $data02
sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double, id int);
print create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart, count(*) c1 from t1 state_window(a);
sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart, count(*) c1 from t1 state_window(a);
sql insert into t1(ts) values(1648791213000);
$loop_count = 0
loop0:
sleep 300
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
sql select * from streamt1;
print data00 data01
print data10 data11
if $rows != 0 then
print =====rows=$rows
goto loop0
endi
sql insert into t1 values(1648791214000,1,2,3,1.0,3);
$loop_count = 0
loop1:
sleep 300
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
sql select * from streamt1;
print data00 data01
print data10 data11
if $rows != 1 then
print =====rows=$rows
goto loop1
endi
sql insert into t1 values(1648791215000,2,2,3,1.0,4);
$loop_count = 0
loop2:
sleep 300
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
sql select * from streamt1;
if $rows != 2 then
print =====rows=$rows
goto loop2
endi
sql insert into t1(ts) values(1648791216000);
$loop_count = 0
loop3:
sleep 300
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
sql select * from streamt1;
if $rows != 2 then
print =====rows=$rows
goto loop2
endi
print state1 end
system sh/stop_dnodes.sh

12
tests/script/tsim/valgrind/checkError7.sim
View File

@ -8,8 +8,9 @@ sql create database d1
sql use d1
$x = 0
while $x < 128
while $x < 5
$tb = d1.s . $x
print create table $tb (ts timestamp, i int) tags (j int)
sql create table $tb (ts timestamp, i int) tags (j int)
$x = $x + 1
endw
@ -17,7 +18,7 @@ endw
print ======================== describe stables
# TODO : create stable error
$m = 0
while $m < 128
while $m < 5
$tb = s . $m
$filter = ' . $tb
$filter = $filter . '
@ -36,15 +37,16 @@ print ======================== show stables
sql show d1.stables
print num of stables is $rows
if $rows != 128 then
if $rows != 5 then
return -1
endi
print ======================== create table
$x = 0
while $x < 424
while $x < 42
$tb = d1.t . $x
print create table $tb using d1.s0 tags( $x )
sql create table $tb using d1.s0 tags( $x )
$x = $x + 1
endw
@ -54,7 +56,7 @@ print ======================== show stables
sql show d1.tables
print num of tables is $rows
if $rows != 424 then
if $rows != 42 then
return -1
endi

10
tests/system-test/0-others/information_schema.py
View File

@ -101,10 +101,18 @@ class TDTestCase:
tdSql.checkEqual(i[1],len(self.perf_list))
elif i[0].lower() == self.dbname:
tdSql.checkEqual(i[1],self.tbnum+1)
def ins_columns_check(self):
tdSql.execute('create database db2 vgroups 2 replica 1')
tdSql.execute('create table db2.stb2 (ts timestamp,c0 int,c1 int, c2 double, c3 float, c4 binary(1000), c5 nchar(100),c7 bigint, c8 bool, c9 smallint) tags(t0 int)')
for i in range(2000):
tdSql.execute("create table db2.ctb%d using db2.stb2 tags(%d)" %(i,i))
tdSql.query(f'select * from information_schema.ins_columns where db_name="db2" and table_type="CHILD_TABLE"')
tdSql.checkEqual(20000,len(tdSql.queryResult))
print("number of ins_columns of child table in db2 is %s" % len(tdSql.queryResult))
def run(self):
self.prepare_data()
self.count_check()
self.ins_columns_check()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)

15
tests/system-test/2-query/Today.py
View File

@ -16,6 +16,7 @@ class TDTestCase:
tdSql.init(conn.cursor())
self.today_date = datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d")
self.today_ts = datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d").timestamp()
self.today_ts_ns = 0
self.time_unit = ['b','u','a','s','m','h','d','w']
self.error_param = ['1.5','abc','!@#','"abc"','today()']
self.arithmetic_operators = ['+','-','*','/']
@ -74,8 +75,10 @@ class TDTestCase:
num_same = 0
if v.lower() == 'timestamp':
tdSql.query(f'select {k} from {tbname}')
for i in tdSql.queryResult:
if precision == 'ms':
self.today_ts_trans = int(self.today_ts)*1000
if int(i[0].timestamp())*1000 > int(self.today_ts)*1000:
num_up += 1
elif int(i[0].timestamp())*1000 == int(self.today_ts)*1000:
@ -83,6 +86,7 @@ class TDTestCase:
elif int(i[0].timestamp())*1000 < int(self.today_ts)*1000:
num_down += 1
elif precision == 'us':
self.today_ts_trans = int(self.today_ts)*1000000
if int(i[0].timestamp())*1000000 > int(self.today_ts)*1000000:
num_up += 1
elif int(i[0].timestamp())*1000000 == int(self.today_ts)*1000000:
@ -90,6 +94,7 @@ class TDTestCase:
elif int(i[0].timestamp())*1000000 < int(self.today_ts)*1000000:
num_down += 1
elif precision == 'ns':
self.today_ts_trans = int(self.today_ts)*1000000000
if i[0] > int(self.today_ts)*1000000000:
num_up += 1
elif i[0] == int(self.today_ts)*1000000000:
@ -97,8 +102,9 @@ class TDTestCase:
elif i[0] < int(self.today_ts)*1000000000:
num_down += 1
tdSql.query(f"select today() from {tbname}")
tdSql.checkRows(len(values_list)*tb_num)
tdSql.checkData(0, 0, str(self.today_date))
tdSql.checkRows(len(values_list)*tb_num)
print(self.today_ts_trans,self.today_ts,precision,num_up,num_down,i[0])
tdSql.checkData(0, 0, self.today_ts_trans)
tdSql.query(f"select * from {tbname} where {k}=today()")
if tb == 'tb':
tdSql.checkRows(num_same*tb_num)
@ -149,11 +155,12 @@ class TDTestCase:
if tb == 'tb':
tdSql.checkRows(num_same*tb_num)
for i in range(num_same*tb_num):
tdSql.checkData(i, 0, str(self.today_date))
print(self.today_ts_trans,precision,num_up,num_down)
tdSql.checkData(i, 0, self.today_ts_trans)
elif tb == 'stb':
tdSql.checkRows(num_same)
for i in range(num_same):
tdSql.checkData(i, 0, str(self.today_date))
tdSql.checkData(i, 0, self.today_ts_trans)
def today_check_ntb(self):
for time_unit in self.db_percision:

25
tests/system-test/2-query/cast.py
View File

@ -65,7 +65,6 @@ class TDTestCase:
data_ct4_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
tdSql.query(f"select c1 from {self.dbname}.t1")
data_t1_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
tdLog.printNoPrefix("==========step2: cast int to bigint, expect no changes")
tdSql.query(f"select cast(c1 as bigint) as b from {self.dbname}.ct4")
@ -108,12 +107,12 @@ class TDTestCase:
tdSql.query(f"select cast(c1 as timestamp) as b from {self.dbname}.t1")
for i in range(len(data_t1_c1)):
if data_ct4_c1[i] is None:
if data_t1_c1[i] is None:
tdSql.checkData( i, 0 , None )
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c1[i]/1000)
date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c1[i]/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
@ -225,12 +224,12 @@ class TDTestCase:
tdSql.query(f"select cast(c3 as timestamp) as b from {self.dbname}.t1")
for i in range(len(data_t1_c3)):
if data_ct4_c3[i] is None:
if data_t1_c3[i] is None:
tdSql.checkData( i, 0 , None )
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c3[i]/1000)
date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c3[i]/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
@ -282,12 +281,12 @@ class TDTestCase:
tdSql.query(f"select cast(c4 as timestamp) as b from {self.dbname}.t1")
for i in range(len(data_t1_c4)):
if data_ct4_c4[i] is None:
if data_t1_c4[i] is None:
tdSql.checkData( i, 0 , None )
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c4[i]/1000)
date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c4[i]/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
@ -329,7 +328,7 @@ class TDTestCase:
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c5[i]/1000))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c5[i])/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
tdSql.query(f"select cast(c5 as timestamp) as b from {self.dbname}.t1")
@ -339,7 +338,7 @@ class TDTestCase:
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c5[i]/1000))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c5[i])/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
@ -385,7 +384,7 @@ class TDTestCase:
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c6[i]/1000))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c6[i])/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
@ -398,7 +397,7 @@ class TDTestCase:
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c6[i]/1000))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c6[i])/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
@ -439,7 +438,7 @@ class TDTestCase:
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c7[i]/1000))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c7[i])/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
tdSql.query(f"select cast(c7 as timestamp) as b from {self.dbname}.t1")
@ -449,7 +448,7 @@ class TDTestCase:
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c7[i]/1000))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c7[i])/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)

2
tests/system-test/2-query/json_tag.py
View File

@ -301,7 +301,7 @@ class TDTestCase:
# where json value is not exist
tdSql.query(f"select * from {dbname}.jsons1 where jtag->'tag1' is null")
tdSql.checkData(0, 0, 'jsons1_9')
# tdSql.checkData(0, 0, 'jsons1_9')
tdSql.checkRows(2)
tdSql.query(f"select * from {dbname}.jsons1 where jtag->'tag4' is null")
tdSql.checkRows(9)

44
tests/system-test/2-query/limit.py
View File

@ -140,7 +140,7 @@ class TDTestCase:
val2 = paraDict["ctbNum"] - 1
# select count(*), t1, t2, t3, t4, t5, t6 from $stb where t1 > $val1 and t1 < $val2 group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0
sqlStr = f"select count(*), t1, t2, t3, t4, t5, t6 from %s where t1 > %d and t1 < %d group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0"%(paraDict["stbName"], val1, val2)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(1)
tdSql.checkData(0, 0, paraDict["rowsPerTbl"])
@ -152,7 +152,7 @@ class TDTestCase:
# select count(*), t3, t4 from $stb where t2 like '%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 2 offset 0
sqlStr = f"select count(*), t3, t4 from %s where t2 like '%%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 2 offset 0"%(paraDict["stbName"])
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(2)
tdSql.checkData(0, 0, paraDict["rowsPerTbl"])
@ -164,7 +164,7 @@ class TDTestCase:
# select count(*) from $stb where t2 like '%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 1 offset 1
sqlStr = f"select count(*) from %s where t2 like '%%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 1 offset 1"%(paraDict["stbName"])
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(1)
@ -179,18 +179,18 @@ class TDTestCase:
tb = paraDict["ctbPrefix"] + '0'
# select _wstart, max(c1) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1) limit 10 offset 1
sqlStr = f"select _wstart, max(c1) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1) limit 10 offset 1"%(tb, ts0, tsu)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(10)
tdSql.checkData(0, 0, "18-09-17 09:05:00.000")
tdSql.checkData(0, 0, "2018-09-17 09:05:00.000")
tdSql.checkData(0, 1, -1)
tdSql.checkData(1, 1, 1)
tdSql.checkData(9, 0, "18-09-17 09:50:00.000")
tdSql.checkData(9, 0, "2018-09-17 09:50:00.000")
tdSql.checkData(9, 1, 5)
tb5 = paraDict["ctbPrefix"] + '5'
sqlStr = f"select max(c1), min(c2) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1, -2) limit 10 offset 1"%(tb5, ts0, tsu)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(10)
tdSql.checkData(0, 0, -1)
@ -206,22 +206,22 @@ class TDTestCase:
limit = paraDict["rowsPerTbl"]
offset = limit / 2
sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb, ts0, tsu, limit, offset)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(limit)
tdSql.checkData(0, 1, 0)
sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 8200"
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(8200)
sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 100000;"
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10 offset 8190;"
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(10)
tdSql.checkData(0, 0, 5)
@ -231,7 +231,7 @@ class TDTestCase:
sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10 offset 10001;"
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(10)
tdSql.checkData(0, 0, -1000)
@ -240,13 +240,13 @@ class TDTestCase:
tdSql.checkData(3, 0, 2)
sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10000 offset 10001;"
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(9998)
sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 100 offset 20001;"
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(0)
@ -254,7 +254,7 @@ class TDTestCase:
limit = paraDict["rowsPerTbl"]
offset = limit / 2
sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(linear) limit %d offset %d"%(tb,ts0,tsu,limit, offset)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(limit)
tdSql.checkData(0, 1, 0)
@ -272,7 +272,7 @@ class TDTestCase:
limit = paraDict["rowsPerTbl"]
offset = limit / 2
sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(prev) limit %d offset %d"%(tb,ts0,tsu,limit, offset)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(limit)
@ -280,7 +280,7 @@ class TDTestCase:
limit = paraDict["rowsPerTbl"]
offset = limit / 2 + 10
sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d and c1 = 5 interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb,ts0,tsu,limit, offset)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(limit)
tdSql.checkData(0, 1, 5)
@ -298,7 +298,7 @@ class TDTestCase:
limit = paraDict["rowsPerTbl"]
offset = limit * 2 - 11
sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d and c1 = 5 interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb,ts0,tsu,limit, offset)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(10)
tdSql.checkData(0, 1, -1)
@ -314,7 +314,7 @@ class TDTestCase:
### [TBASE-350]
## stb + interval + fill + group by + limit offset
sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') limit 2 offset 10"
print("====sql:%s"%(sqlStr))
tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(2)
@ -322,7 +322,7 @@ class TDTestCase:
offset = paraDict["rowsPerTbl"] * 2
offset = offset - 2
sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') order by t1 limit %d offset %d"%(limit, offset)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(1)
tdSql.checkData(0, 0, 9)
@ -339,7 +339,7 @@ class TDTestCase:
tdLog.printNoPrefix("======== test case 1 end ...... ")
def run(self):
tdSql.prepare()
# tdSql.prepare()
self.prepareTestEnv()
self.tmqCase1()

69
tests/system-test/2-query/timeerror.py Normal file
View File

@ -0,0 +1,69 @@
import taos
import sys
from time import sleep
from util.log import *
from util.sql import *
from util.cases import *
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
def run(self):
rows = 10
tdSql.execute("create database dbus PRECISION 'us' ")
tdSql.execute("create database dbns PRECISION 'ns' ")
tdSql.execute("create database dbms PRECISION 'ms' ")
dball = ["dbns","dbus","dbms"]
for i in range(len(dball)):
dbname = dball[i]
stb = f"{dbname}.stb1"
if i == 0 :
qts = 1678883666951061471
qts1 = 1678883666951061471
qtime = "2023-03-15 20:34:26.951061471"
elif i == 1 :
qts = 1678883666951061
qts1 = 1678883666951061
qtime = "2023-03-15 20:34:26.951061"
else:
qts = 1678883666951
qts1 = 1678883666953
qtime = "2023-03-15 20:34:26.951"
tdSql.execute(f"use {dbname}")
tdLog.printNoPrefix("==========step1:create table")
tdSql.execute(
f'''create table if not exists {stb}
(ts timestamp, c1 int, c2 float, c3 bigint, c4 double, c5 smallint, c6 tinyint)
tags(location binary(64), type int, isused bool , family nchar(64))'''
)
tdSql.execute(f"create table {dbname}.t1 using {stb} tags('beijing', 1, 1, 'nchar1')")
tdSql.execute(f"create table {dbname}.t2 using {stb} tags('shanghai', 2, 0, 'nchar2')")
tdSql.execute(f"create table {dbname}.t3 using {stb} tags('shanghai', 3, 0, 'nchar3')")
tdLog.printNoPrefix("==========step2:insert data")
tdSql.execute(
f"insert into {dbname}.t3 values ({qts}, null , {-3.4*10**38}, null , {-1.7*10**308}, null , null)"
)
tdLog.printNoPrefix("==========step3:query timestamp type")
tdSql.query(f"select ts from {dbname}.t3 limit 1")
tdSql.checkData(0,0,qtime)
tdSql.checkData(0,0,qts)
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())