xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into lihui3.0/test

This commit is contained in:
plum-lihui 2023-03-18 13:52:38 +08:00
parent e81d485d9a 50178cd5c9
commit 16778d83d7
88 changed files with 2646 additions and 1183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG ad1a32b
GIT_TAG d11f210
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

7
include/common/tcommon.h
View File

@ -25,13 +25,6 @@
extern "C" {
#endif
// TODO remove it
enum {
TMQ_CONF__RESET_OFFSET__NONE = -3,
TMQ_CONF__RESET_OFFSET__EARLIEAST = -2,
TMQ_CONF__RESET_OFFSET__LATEST = -1,
};
// clang-format off
#define IS_META_MSG(x) ( \
x == TDMT_VND_CREATE_STB \

1
include/common/tmsg.h
View File

@ -3186,6 +3186,7 @@ typedef struct {
SArray* blockData;
SArray* blockTbName;
SArray* blockSchema;
// the following attributes are extended from SMqDataRsp
int32_t createTableNum;
SArray* createTableLen;
SArray* createTableReq;

2
include/libs/qworker/qworker.h
View File

@ -106,6 +106,8 @@ int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64
int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId,
void **pRsp, SArray *explainRes);
int32_t qWorkerDbgEnableDebug(char *option);
#ifdef __cplusplus
}
#endif

1
include/util/taoserror.h
View File

@ -688,6 +688,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_NOT_SUPPORT_JOIN TAOS_DEF_ERROR_CODE(0, 0x2664)
#define TSDB_CODE_PAR_INVALID_TAGS_PC TAOS_DEF_ERROR_CODE(0, 0x2665)
#define TSDB_CODE_PAR_INVALID_TIMELINE_QUERY TAOS_DEF_ERROR_CODE(0, 0x2666)
#define TSDB_CODE_PAR_INVALID_OPTR_USAGE TAOS_DEF_ERROR_CODE(0, 0x2667)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner

4
include/util/tdef.h
View File

@ -316,10 +316,10 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_KEEP_NS (365 * 292 * 1440) // data in db to be reserved.
#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years
#define TSDB_MIN_MINROWS_FBLOCK 10
#define TSDB_MAX_MINROWS_FBLOCK 1000
#define TSDB_MAX_MINROWS_FBLOCK 1000000
#define TSDB_DEFAULT_MINROWS_FBLOCK 100
#define TSDB_MIN_MAXROWS_FBLOCK 200
#define TSDB_MAX_MAXROWS_FBLOCK 10000
#define TSDB_MAX_MAXROWS_FBLOCK 10000000
#define TSDB_DEFAULT_MAXROWS_FBLOCK 4096
#define TSDB_MIN_FSYNC_PERIOD 0
#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond

31
source/client/inc/clientInt.h
View File

@ -287,22 +287,25 @@ static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
}
static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
SMqRspObj* msg = (SMqRspObj*)res;
msg->resIter++;
if (msg->resIter < msg->rsp.blockNum) {
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter);
if (msg->rsp.withSchema) {
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(msg->rsp.blockSchema, msg->resIter);
setResSchemaInfo(&msg->resInfo, pSW->pSchema, pSW->nCols);
taosMemoryFreeClear(msg->resInfo.row);
taosMemoryFreeClear(msg->resInfo.pCol);
taosMemoryFreeClear(msg->resInfo.length);
taosMemoryFreeClear(msg->resInfo.convertBuf);
taosMemoryFreeClear(msg->resInfo.convertJson);
SMqRspObj* pRspObj = (SMqRspObj*)res;
pRspObj->resIter++;
if (pRspObj->resIter < pRspObj->rsp.blockNum) {
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter);
if (pRspObj->rsp.withSchema) {
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter);
setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols);
taosMemoryFreeClear(pRspObj->resInfo.row);
taosMemoryFreeClear(pRspObj->resInfo.pCol);
taosMemoryFreeClear(pRspObj->resInfo.length);
taosMemoryFreeClear(pRspObj->resInfo.convertBuf);
taosMemoryFreeClear(pRspObj->resInfo.convertJson);
}
setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4, false);
return &msg->resInfo;
setQueryResultFromRsp(&pRspObj->resInfo, pRetrieve, convertUcs4, false);
return &pRspObj->resInfo;
}
return NULL;
}

9
source/client/src/clientImpl.c
View File

@ -1085,6 +1085,10 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId);
destorySqlCallbackWrapper(pWrapper);
if (TSDB_CODE_SUCCESS != code) {
pRequest->code = terrno;
}
pRequest->body.queryFp(pRequest->body.param, pRequest, code);
}
@ -1132,11 +1136,6 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
pRequest->body.queryFp(pRequest->body.param, pRequest, -1);
break;
}
// TODO weired responding code?
if (TSDB_CODE_SUCCESS != code) {
pRequest->code = terrno;
}
}
int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) {

4
source/client/src/clientJniConnector.c
View File

@ -581,8 +581,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI
return JNI_RESULT_SET_NULL;
}
void *data;
int32_t numOfRows;
void *data = NULL;
int32_t numOfRows = 0;
int error_code = taos_fetch_raw_block(tres, &numOfRows, &data);
if (numOfRows == 0) {
if (error_code == JNI_SUCCESS) {

3
source/client/src/clientMain.c
View File

@ -611,6 +611,9 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
}
int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
*numOfRows = 0;
*pData = NULL;
if (res == NULL || TD_RES_TMQ_META(res)) {
return 0;
}

62
source/client/src/clientRawBlockWrite.c
View File

@ -776,12 +776,37 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
goto end;
}
SCatalog* pCatalog = NULL;
code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter,
.requestId = pRequest->requestId,
.requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
SName pName = {0};
toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName);
STableMeta *pTableMeta = NULL;
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){
code = TSDB_CODE_SUCCESS;
taosMemoryFreeClear(pTableMeta);
goto end;
}
if (code != TSDB_CODE_SUCCESS) {
uError("taosCreateTable:catalogGetTableMeta failed. table name: %s", req.name);
goto end;
}
pReq.suid = pTableMeta->uid;
taosMemoryFreeClear(pTableMeta);
// build drop stable
pReq.igNotExists = true;
pReq.source = TD_REQ_FROM_TAOX;
pReq.suid = processSuid(req.suid, pRequest->pDb);
// pReq.suid = processSuid(req.suid, pRequest->pDb);
uDebug("taosDropStb name:%s suid:%"PRId64" processSuid:%"PRId64, req.name, req.suid, pReq.suid);
uDebug("taosDropStb name:%s suid:%"PRId64" new suid:%"PRId64, req.name, req.suid, pReq.suid);
STscObj* pTscObj = pRequest->pTscObj;
SName tableName = {0};
tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name);
@ -806,7 +831,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
launchQueryImpl(pRequest, &pQuery, true, NULL);
if (pRequest->code == TSDB_CODE_SUCCESS) {
SCatalog* pCatalog = NULL;
// SCatalog* pCatalog = NULL;
catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
catalogRemoveTableMeta(pCatalog, &tableName);
}
@ -900,15 +925,15 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
STableMeta* pTableMeta = NULL;
SName sName = {0};
tb_uid_t oldSuid = pCreateReq->ctb.suid;
pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb);
uDebug("taosCreateTable name:%s sname:%s suid:%"PRId64" processSuid:%"PRId64, pCreateReq->name, pCreateReq->ctb.stbName, pCreateReq->ctb.suid, oldSuid);
// pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb);
toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName);
code = catalogGetTableMeta(pCatalog, &conn, &sName, &pTableMeta);
if (code != TSDB_CODE_SUCCESS) {
uError("taosCreateTable:catalogGetTableMeta failed. table name: %s", pCreateReq->ctb.stbName);
goto end;
}
pCreateReq->ctb.suid = pTableMeta->uid;
uDebug("taosCreateTable name:%s sname:%s suid:%"PRId64" new suid:%"PRId64, pCreateReq->name, pCreateReq->ctb.stbName, oldSuid, pCreateReq->ctb.suid);
for (int32_t i = 0; i < taosArrayGetSize(pCreateReq->ctb.tagName); i++) {
char* tName = taosArrayGet(pCreateReq->ctb.tagName, i);
@ -1041,8 +1066,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
pDropReq = req.pReqs + iReq;
pDropReq->igNotExists = true;
pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb);
uDebug("taosDropTable name:%s suid:%"PRId64" processSuid:%"PRId64, pDropReq->name, pDropReq->suid, pDropReq->suid);
// pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb);
SVgroupInfo pInfo = {0};
SName pName = {0};
@ -1052,6 +1076,22 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
goto end;
}
STableMeta *pTableMeta = NULL;
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){
code = TSDB_CODE_SUCCESS;
taosMemoryFreeClear(pTableMeta);
continue;
}
if (code != TSDB_CODE_SUCCESS) {
uError("taosDropTable:catalogGetTableMeta failed. table name: %s", pDropReq->name);
goto end;
}
tb_uid_t oldSuid = pDropReq->suid;
pDropReq->suid = pTableMeta->suid;
taosMemoryFreeClear(pTableMeta);
uDebug("taosDropTable name:%s suid:%"PRId64" new suid:%"PRId64, pDropReq->name, oldSuid, pDropReq->suid);
taosArrayPush(pRequest->tableList, &pName);
SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId));
if (pTableBatch == NULL) {
@ -1066,6 +1106,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
}
}
if (taosHashGetSize(pVgroupHashmap) == 0){
goto end;
}
SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap);
if (NULL == pBufArray) {
code = TSDB_CODE_OUT_OF_MEMORY;
@ -1673,7 +1716,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
}
if (strcmp(tbName, pCreateReq.name) == 0) {
cloneSVreateTbReq(&pCreateReq, &pCreateReqDst);
pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb);
// pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb);
tDecoderClear(&decoderTmp);
break;
}
@ -1706,6 +1749,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
if (pCreateReqDst) {
pTableMeta->vgId = vg.vgId;
pTableMeta->uid = pCreateReqDst->uid;
pCreateReqDst->ctb.suid = pTableMeta->suid;
}
void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId));
if (hData == NULL) {

1005
source/client/src/clientTmq.c
View File

File diff suppressed because it is too large Load Diff

4
source/client/src/clientTmqConnector.c
View File

@ -361,8 +361,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(
TAOS_RES *tres = (TAOS_RES *)res;
void *data;
int32_t numOfRows;
void *data = NULL;
int32_t numOfRows = 0;
int error_code = taos_fetch_raw_block(tres, &numOfRows, &data);
if (numOfRows == 0) {
if (error_code == JNI_SUCCESS) {

87
source/client/test/clientTests.cpp
View File

@ -898,7 +898,7 @@ TEST(clientCase, update_test) {
}
}
TEST(clientCase, subscription_test) {
TEST(clientCase, sub_db_test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
@ -912,7 +912,7 @@ TEST(clientCase, subscription_test) {
tmq_conf_t* conf = tmq_conf_new();
tmq_conf_set(conf, "enable.auto.commit", "true");
tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
tmq_conf_set(conf, "group.id", "cgrpName");
tmq_conf_set(conf, "group.id", "cgrpNamedb");
tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "auto.offset.reset", "earliest");
@ -925,7 +925,7 @@ TEST(clientCase, subscription_test) {
// 创建订阅 topics 列表
tmq_list_t* topicList = tmq_list_new();
// tmq_list_append(topicList, "topic_t1");
tmq_list_append(topicList, "topic_db1");
// 启动订阅
tmq_subscribe(tmq, topicList);
@ -954,7 +954,86 @@ TEST(clientCase, subscription_test) {
printf("db: %s\n", dbName);
printf("vgroup id: %d\n", vgroupId);
if (count ++ > 20) {
if (count ++ > 200) {
tmq_unsubscribe(tmq);
break;
}
while (1) {
TAOS_ROW row = taos_fetch_row(pRes);
if (row == NULL) break;
fields = taos_fetch_fields(pRes);
numOfFields = taos_field_count(pRes);
precision = taos_result_precision(pRes);
rows++;
taos_print_row(buf, row, fields, numOfFields);
printf("precision: %d, row content: %s\n", precision, buf);
}
}
// return rows;
}
fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows);
}
TEST(clientCase, sub_tb_test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
// TAOS_RES* pRes = taos_query(pConn, "create topic topic_t1 as select * from t1");
// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
// printf("failed to create topic, code:%s", taos_errstr(pRes));
// taos_free_result(pRes);
// return;
// }
tmq_conf_t* conf = tmq_conf_new();
tmq_conf_set(conf, "enable.auto.commit", "true");
tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
tmq_conf_set(conf, "group.id", "cgrpName");
tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "auto.offset.reset", "earliest");
tmq_conf_set(conf, "experimental.snapshot.enable", "true");
tmq_conf_set(conf, "msg.with.table.name", "true");
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
tmq_conf_destroy(conf);
// 创建订阅 topics 列表
tmq_list_t* topicList = tmq_list_new();
tmq_list_append(topicList, "topic_t1");
// 启动订阅
tmq_subscribe(tmq, topicList);
tmq_list_destroy(topicList);
TAOS_FIELD* fields = NULL;
int32_t numOfFields = 0;
int32_t precision = 0;
int32_t totalRows = 0;
int32_t msgCnt = 0;
int32_t timeout = 5000;
int32_t count = 0;
while (1) {
TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout);
if (pRes) {
char buf[1024];
int32_t rows = 0;
const char* topicName = tmq_get_topic_name(pRes);
const char* dbName = tmq_get_db_name(pRes);
int32_t vgroupId = tmq_get_vgroup_id(pRes);
printf("topic: %s\n", topicName);
printf("db: %s\n", dbName);
printf("vgroup id: %d\n", vgroupId);
if (count ++ > 200) {
tmq_unsubscribe(tmq);
break;
}

6
source/common/src/tmsg.c
View File

@ -6639,8 +6639,9 @@ int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) {
} else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA || pVal->type == TMQ_OFFSET__SNAPSHOT_META) {
snprintf(buf, maxLen, "offset(snapshot) uid:%" PRId64 " ts:%" PRId64, pVal->uid, pVal->ts);
} else {
ASSERT(0);
return TSDB_CODE_INVALID_PARA;
}
return 0;
}
@ -6823,8 +6824,7 @@ int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
}
void tDeleteSMqDataRsp(SMqDataRsp *pRsp) {
taosArrayDestroy(pRsp->blockDataLen);
pRsp->blockDataLen = NULL;
pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen);;
taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree);
pRsp->blockData = NULL;
taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper);

4
source/dnode/mnode/impl/inc/mndConsumer.h
View File

@ -24,10 +24,10 @@ extern "C" {
enum {
MQ_CONSUMER_STATUS__MODIFY = 1,
MQ_CONSUMER_STATUS__MODIFY_IN_REB,
MQ_CONSUMER_STATUS__MODIFY_IN_REB, // this value is not used anymore
MQ_CONSUMER_STATUS__READY,
MQ_CONSUMER_STATUS__LOST,
MQ_CONSUMER_STATUS__LOST_IN_REB,
MQ_CONSUMER_STATUS__LOST_IN_REB, // this value is not used anymore
MQ_CONSUMER_STATUS__LOST_REBD,
MQ_CONSUMER_STATUS__REMOVED,
};

12
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -77,7 +77,7 @@ void mndCleanupConsumer(SMnode *pMnode) {}
bool mndRebTryStart() {
int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1);
mInfo("tq timer, rebalance counter old val:%d", old);
mDebug("tq timer, rebalance counter old val:%d", old);
return old == 0;
}
@ -101,7 +101,7 @@ void mndRebCntDec() {
int32_t newVal = val - 1;
int32_t oldVal = atomic_val_compare_exchange_32(&mqRebInExecCnt, val, newVal);
if (oldVal == val) {
mInfo("rebalance trans end, rebalance counter:%d", newVal);
mDebug("rebalance trans end, rebalance counter:%d", newVal);
break;
}
}
@ -253,11 +253,11 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
SMqConsumerObj *pConsumer;
void *pIter = NULL;
mTrace("start to process mq timer");
mDebug("start to process mq timer");
// rebalance cannot be parallel
if (!mndRebTryStart()) {
mInfo("mq rebalance already in progress, do nothing");
mDebug("mq rebalance already in progress, do nothing");
return 0;
}
@ -317,7 +317,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId);
}
taosRUnLockLatch(&pConsumer->lock);
} else if (status == MQ_CONSUMER_STATUS__MODIFY) {
} else if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
taosRLockLatch(&pConsumer->lock);
int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics);
@ -356,7 +356,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
} else {
taosHashCleanup(pRebMsg->rebSubHash);
rpcFreeCont(pRebMsg);
mInfo("mq rebalance finished, no modification");
mDebug("mq rebalance finished, no modification");
mndRebEnd();
}
return 0;

20
source/dnode/mnode/impl/src/mndDb.c
View File

@ -279,6 +279,8 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
pOld->cfg.cacheLast = pNew->cfg.cacheLast;
pOld->cfg.replications = pNew->cfg.replications;
pOld->cfg.sstTrigger = pNew->cfg.sstTrigger;
pOld->cfg.minRows = pNew->cfg.minRows;
pOld->cfg.maxRows = pNew->cfg.maxRows;
pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize;
pOld->compactStartTime = pNew->compactStartTime;
taosWUnLockLatch(&pOld->lock);
@ -616,13 +618,8 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
}
} else {
if (terrno == TSDB_CODE_MND_DB_IN_CREATING) {
if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) {
mInfo("db:%s, is creating and createdb response after trans finished", createReq.db);
code = TSDB_CODE_ACTION_IN_PROGRESS;
goto _OVER;
} else {
goto _OVER;
}
code = terrno;
goto _OVER;
} else if (terrno == TSDB_CODE_MND_DB_IN_DROPPING) {
goto _OVER;
} else if (terrno == TSDB_CODE_MND_DB_NOT_EXIST) {
@ -1270,14 +1267,9 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) {
usedbRsp.errCode = terrno;
if (terrno == TSDB_CODE_MND_DB_IN_CREATING) {
if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, usedbReq.db) == 0) {
mInfo("db:%s, is creating and usedb response after trans finished", usedbReq.db);
code = TSDB_CODE_ACTION_IN_PROGRESS;
goto _OVER;
}
code = terrno;
goto _OVER;
}
mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr());
} else {
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) {
goto _OVER;

7
source/dnode/mnode/impl/src/mndStb.c
View File

@ -3114,6 +3114,7 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(typeName, "SUPER_TABLE");
while (numOfRows < rows) {
void *prevIter = pShow->pIter;
pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb);
if (pShow->pIter == NULL) break;
@ -3122,6 +3123,12 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
continue;
}
if ((numOfRows + pStb->numOfColumns) > rows) {
pShow->pIter = prevIter;
sdbRelease(pSdb, pStb);
break;
}
SName name = {0};
char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
mndExtractTbNameFromStbFullName(pStb->name, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN);

12
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -224,7 +224,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
.pVgEp = pVgEp,
};
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
mInfo("sub:%s mq re-balance remove vgId:%d from consumer:%" PRIx64, sub, pVgEp->vgId, consumerId);
mInfo("sub:%s mq re-balance remove vgId:%d from consumer:0x%" PRIx64, sub, pVgEp->vgId, consumerId);
}
taosArrayDestroy(pConsumerEp->vgs);
taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
@ -329,7 +329,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
taosArrayPush(pOutput->newConsumers, &consumerId);
mInfo("sub:%s mq rebalance add new consumer:%" PRIx64, sub, consumerId);
mInfo("sub:%s mq rebalance add new consumer:0x%" PRIx64, sub, consumerId);
}
}
@ -357,7 +357,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
pRebVg->newConsumerId = pConsumerEp->consumerId;
taosArrayPush(pOutput->rebVgs, pRebVg);
mInfo("mq rebalance: add vgId:%d to consumer:%" PRIx64 " (second scan) (not enough)", pRebVg->pVgEp->vgId,
mInfo("mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " (second scan) (not enough)", pRebVg->pVgEp->vgId,
pConsumerEp->consumerId);
}
}
@ -387,12 +387,12 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
pRebVg->newConsumerId = pConsumerEp->consumerId;
if (pRebVg->newConsumerId == pRebVg->oldConsumerId) {
mInfo("mq rebalance: skip vg %d for same consumer:%" PRIx64 " (second scan)", pRebVg->pVgEp->vgId,
mInfo("mq rebalance: skip vg %d for same consumer:0x%" PRIx64 " (second scan)", pRebVg->pVgEp->vgId,
pConsumerEp->consumerId);
continue;
}
taosArrayPush(pOutput->rebVgs, pRebVg);
mInfo("mq rebalance: add vgId:%d to consumer:%" PRIx64 " (second scan) (unassigned)", pRebVg->pVgEp->vgId,
mInfo("mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " (second scan) (unassigned)", pRebVg->pVgEp->vgId,
pConsumerEp->consumerId);
}
} else {
@ -1019,7 +1019,7 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumerEp->consumerId, false);
mDebug("mnd show subscriptions: topic %s, consumer:%" PRIx64 " cgroup %s vgid %d", varDataVal(topic),
mDebug("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic),
pConsumerEp->consumerId, varDataVal(cgroup), pVgEp->vgId);
// offset

44
source/dnode/vnode/src/inc/tq.h
View File

@ -79,43 +79,33 @@ typedef struct {
} STqExecDb;
typedef struct {
int8_t subType;
STqReader* pExecReader;
qTaskInfo_t task;
int8_t subType;
STqReader* pExecReader;
qTaskInfo_t task;
union {
STqExecCol execCol;
STqExecTb execTb;
STqExecDb execDb;
};
int32_t numOfCols; // number of out pout column, temporarily used
int32_t numOfCols; // number of out pout column, temporarily used
} STqExecHandle;
typedef struct {
// info
char subKey[TSDB_SUBSCRIBE_KEY_LEN];
int64_t consumerId;
int32_t epoch;
int8_t fetchMeta;
int64_t snapshotVer;
SWalReader* pWalReader;
SWalRef* pRef;
// push
STqPushHandle pushHandle;
// exec
STqExecHandle execHandle;
char subKey[TSDB_SUBSCRIBE_KEY_LEN];
int64_t consumerId;
int32_t epoch;
int8_t fetchMeta;
int64_t snapshotVer;
SWalReader* pWalReader;
SWalRef* pRef;
STqPushHandle pushHandle; // push
STqExecHandle execHandle; // exec
} STqHandle;
typedef struct {
SMqDataRsp dataRsp;
SMqDataRsp* pDataRsp;
char subKey[TSDB_SUBSCRIBE_KEY_LEN];
SRpcHandleInfo pInfo;
SRpcHandleInfo info;
} STqPushEntry;
struct STQ {
@ -151,13 +141,13 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle);
// tqRead
int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset);
int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset);
int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum);
int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum);
// tqExec
int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp);
// int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp);
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision);
int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp);
int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type);
int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry);
// tqMeta

3
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -192,6 +192,9 @@ void tqCleanUp();
STQ* tqOpen(const char* path, SVnode* pVnode);
void tqClose(STQ*);
int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
int tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, int32_t type);
int tqRemovePushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer);
int tqCommit(STQ*);
int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);
int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId);

81
source/dnode/vnode/src/meta/metaCache.c
View File

@ -14,6 +14,7 @@
*/
#include "meta.h"
#define TAG_FILTER_RES_KEY_LEN 32
#define META_CACHE_BASE_BUCKET 1024
#define META_CACHE_STATS_BUCKET 16
@ -34,7 +35,6 @@ typedef struct SMetaStbStatsEntry {
typedef struct STagFilterResEntry {
SList list; // the linked list of md5 digest, extracted from the serialized tag query condition
uint32_t hitTimes; // queried times for current super table
uint32_t accTime;
} STagFilterResEntry;
struct SMetaCache {
@ -455,26 +455,37 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv
return 0;
}
static FORCE_INLINE void setMD5DigestInKey(uint64_t* pBuf, const char* key, int32_t keyLen) {
// ASSERT(keyLen == sizeof(int64_t) * 2);
memcpy(&pBuf[2], key, keyLen);
}
// the format of key:
// hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes)
static void initCacheKey(uint64_t* buf, const SHashObj* pHashMap, uint64_t suid, const char* key, int32_t keyLen) {
buf[0] = (uint64_t) pHashMap;
buf[1] = suid;
setMD5DigestInKey(buf, key, keyLen);
ASSERT(keyLen == sizeof(uint64_t) * 2);
}
int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1,
bool* acquireRes) {
int32_t vgId = TD_VID(pMeta->pVnode);
// generate the composed key for LRU cache
SLRUCache* pCache = pMeta->pCache->sTagFilterResCache.pUidResCache;
SHashObj* pTableMap = pMeta->pCache->sTagFilterResCache.pTableEntry;
TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock;
uint64_t buf[4];
*acquireRes = 0;
buf[0] = (uint64_t)pTableMap;
buf[1] = suid;
memcpy(&buf[2], pKey, keyLen);
uint64_t key[4];
initCacheKey(key, pTableMap, suid, (const char*)pKey, keyLen);
taosThreadMutexLock(pLock);
pMeta->pCache->sTagFilterResCache.accTimes += 1;
int32_t len = keyLen + sizeof(uint64_t) * 2;
LRUHandle* pHandle = taosLRUCacheLookup(pCache, buf, len);
LRUHandle* pHandle = taosLRUCacheLookup(pCache, key, TAG_FILTER_RES_KEY_LEN);
if (pHandle == NULL) {
taosThreadMutexUnlock(pLock);
return TSDB_CODE_SUCCESS;
@ -499,7 +510,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK
uint32_t acc = pMeta->pCache->sTagFilterResCache.accTimes;
if ((*pEntry)->hitTimes % 5000 == 0 && (*pEntry)->hitTimes > 0) {
metaInfo("cache hit:%d, total acc:%d, rate:%.2f", (*pEntry)->hitTimes, acc, ((double)(*pEntry)->hitTimes) / acc);
metaInfo("vgId:%d cache hit:%d, total acc:%d, rate:%.2f", vgId, (*pEntry)->hitTimes, acc, ((double)(*pEntry)->hitTimes) / acc);
}
taosLRUCacheRelease(pCache, pHandle, false);
@ -563,10 +574,13 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL
// check both the payload size and selectivity ratio
int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
int32_t payloadLen, double selectivityRatio) {
int32_t code = 0;
int32_t vgId = TD_VID(pMeta->pVnode);
if (selectivityRatio > tsSelectivityRatio) {
metaDebug("vgId:%d, suid:%" PRIu64
" failed to add to uid list cache, due to selectivity ratio %.2f less than threshold %.2f",
TD_VID(pMeta->pVnode), suid, selectivityRatio, tsSelectivityRatio);
vgId, suid, selectivityRatio, tsSelectivityRatio);
taosMemoryFree(pPayload);
return TSDB_CODE_SUCCESS;
}
@ -574,7 +588,7 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int
if (payloadLen > tsTagFilterResCacheSize) {
metaDebug("vgId:%d, suid:%" PRIu64
" failed to add to uid list cache, due to payload length %d greater than threshold %d",
TD_VID(pMeta->pVnode), suid, payloadLen, tsTagFilterResCacheSize);
vgId, suid, payloadLen, tsTagFilterResCacheSize);
taosMemoryFree(pPayload);
return TSDB_CODE_SUCCESS;
}
@ -583,26 +597,17 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int
SHashObj* pTableEntry = pMeta->pCache->sTagFilterResCache.pTableEntry;
TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock;
// the format of key:
// hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes)
uint64_t key[4] = {0};
initCacheKey(key, pTableEntry, suid, pKey, keyLen);
uint64_t buf[4] = {0};
buf[0] = (uint64_t)pTableEntry;
buf[1] = suid;
memcpy(&buf[2], pKey, keyLen);
ASSERT(keyLen == 16);
int32_t code = 0;
taosThreadMutexLock(pLock);
STagFilterResEntry** pEntry = taosHashGet(pTableEntry, &suid, sizeof(uint64_t));
if (pEntry == NULL) {
code = addNewEntry(pTableEntry, pKey, keyLen, suid);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
}
} else {
// check if it exists or not
} else { // check if it exists or not
size_t size = listNEles(&(*pEntry)->list);
if (size == 0) {
tdListAppend(&(*pEntry)->list, pKey);
@ -620,12 +625,11 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int
}
// add to cache.
taosLRUCacheInsert(pCache, buf, sizeof(uint64_t) * 2 + keyLen, pPayload, payloadLen, freePayload, NULL,
taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freePayload, NULL,
TAOS_LRU_PRIORITY_LOW);
_end:
taosThreadMutexUnlock(pLock);
metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", TD_VID(pMeta->pVnode), suid,
metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", vgId, suid,
(int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry));
return code;
@ -633,33 +637,36 @@ _end:
// remove the lru cache that are expired due to the tags value update, or creating, or dropping, of child tables
int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) {
int32_t keyLen = sizeof(uint64_t) * 3;
uint64_t p[4] = {0};
uint64_t p[4] = {0};
int32_t vgId = TD_VID(pMeta->pVnode);
SHashObj* pEntryHashMap = pMeta->pCache->sTagFilterResCache.pTableEntry;
p[0] = (uint64_t)pMeta->pCache->sTagFilterResCache.pTableEntry;
p[1] = suid;
uint64_t dummy[2] = {0};
initCacheKey(p, pEntryHashMap, suid, (char*) &dummy[0], 16);
TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock;
taosThreadMutexLock(pLock);
STagFilterResEntry** pEntry = taosHashGet(pMeta->pCache->sTagFilterResCache.pTableEntry, &suid, sizeof(uint64_t));
STagFilterResEntry** pEntry = taosHashGet(pEntryHashMap, &suid, sizeof(uint64_t));
if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) {
taosThreadMutexUnlock(pLock);
return TSDB_CODE_SUCCESS;
}
(*pEntry)->hitTimes = 0;
SListIter iter = {0};
tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD);
SListNode* pNode = NULL;
while ((pNode = tdListNext(&iter)) != NULL) {
memcpy(&p[2], pNode->data, 16);
taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, keyLen);
setMD5DigestInKey(p, pNode->data, 2 * sizeof(uint64_t));
taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, TAG_FILTER_RES_KEY_LEN);
}
(*pEntry)->hitTimes = 0;
tdListEmpty(&(*pEntry)->list);
taosThreadMutexUnlock(pLock);
metaDebug("vgId:%d suid:%"PRId64" cached related tag filter uid list cleared", vgId, suid);
return TSDB_CODE_SUCCESS;
}

697
source/dnode/vnode/src/tq/tq.c
View File

@ -68,7 +68,13 @@ static void destroySTqHandle(void* data) {
static void tqPushEntryFree(void* data) {
STqPushEntry* p = *(void**)data;
tDeleteSMqDataRsp(&p->dataRsp);
if (p->pDataRsp->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) {
tDeleteSMqDataRsp(p->pDataRsp);
} else if (p->pDataRsp->head.mqMsgType == TMQ_MSG_TYPE__TAOSX_RSP) {
tDeleteSTaosxRsp((STaosxRsp*)p->pDataRsp);
}
taosMemoryFree(p->pDataRsp);
taosMemoryFree(p);
}
@ -166,12 +172,16 @@ int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq,
return 0;
}
int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) {
SMqDataRsp* pRsp = &pPushEntry->dataRsp;
static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch,
int64_t consumerId, int32_t type) {
int32_t len = 0;
int32_t code = 0;
tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code);
if (type == TMQ_MSG_TYPE__POLL_RSP) {
tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code);
} else if (type == TMQ_MSG_TYPE__TAOSX_RSP) {
tEncodeSize(tEncodeSTaosxRsp, (STaosxRsp*)pRsp, len, code);
}
if (code < 0) {
return -1;
@ -183,23 +193,83 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) {
return -1;
}
memcpy(buf, &pPushEntry->dataRsp.head, sizeof(SMqRspHead));
((SMqRspHead*)buf)->mqMsgType = type;
((SMqRspHead*)buf)->epoch = epoch;
((SMqRspHead*)buf)->consumerId = consumerId;
void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
SEncoder encoder = {0};
tEncoderInit(&encoder, abuf, len);
tEncodeSMqDataRsp(&encoder, pRsp);
if (type == TMQ_MSG_TYPE__POLL_RSP) {
tEncodeSMqDataRsp(&encoder, pRsp);
} else if (type == TMQ_MSG_TYPE__TAOSX_RSP) {
tEncodeSTaosxRsp(&encoder, (STaosxRsp*) pRsp);
}
tEncoderClear(&encoder);
SRpcMsg rsp = {
.info = pPushEntry->pInfo,
.info = *pRpcHandleInfo,
.pCont = buf,
.contLen = tlen,
.code = 0,
};
tmsgSendRsp(&rsp);
return 0;
}
int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) {
SMqDataRsp* pRsp = pPushEntry->pDataRsp;
#if 0
A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum);
A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum);
A(!pRsp->withSchema);
A(taosArrayGetSize(pRsp->blockSchema) == 0);
if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
A(pRsp->rspOffset.version > pRsp->reqOffset.version);
}
#endif
// int32_t len = 0;
// int32_t code = 0;
// tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code);
// if (code < 0) {
// return -1;
// }
//
// int32_t tlen = sizeof(SMqRspHead) + len;
// void* buf = rpcMallocCont(tlen);
// if (buf == NULL) {
// return -1;
// }
//
// memcpy(buf, &pPushEntry->dataRsp.head, sizeof(SMqRspHead));
//
// void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
//
// SEncoder encoder = {0};
// tEncoderInit(&encoder, abuf, len);
// tEncodeSMqDataRsp(&encoder, pRsp);
// tEncoderClear(&encoder);
//
// SRpcMsg rsp = {
// .info = pPushEntry->pInfo,
// .pCont = buf,
// .contLen = tlen,
// .code = 0,
// };
//
// tmsgSendRsp(&rsp);
//
SMqRspHead* pHeader = &pPushEntry->pDataRsp->head;
doSendDataRsp(&pPushEntry->info, pRsp, pHeader->epoch, pHeader->consumerId, pHeader->mqMsgType);
char buf1[80] = {0};
char buf2[80] = {0};
@ -207,93 +277,146 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) {
tFormatOffset(buf2, tListLen(buf2), &pRsp->rspOffset);
tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) push rsp, block num: %d, req:%s, rsp:%s",
TD_VID(pTq->pVnode), pRsp->head.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2);
return 0;
}
int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp) {
int32_t len = 0;
int32_t code = 0;
tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code);
if (code < 0) {
return -1;
}
int32_t tlen = sizeof(SMqRspHead) + len;
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
return -1;
int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type) {
#if 0
A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum);
A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum);
A(!pRsp->withSchema);
A(taosArrayGetSize(pRsp->blockSchema) == 0);
if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
if (pRsp->blockNum > 0) {
A(pRsp->rspOffset.version > pRsp->reqOffset.version);
} else {
A(pRsp->rspOffset.version >= pRsp->reqOffset.version);
}
}
#endif
((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
((SMqRspHead*)buf)->epoch = pReq->epoch;
((SMqRspHead*)buf)->consumerId = pReq->consumerId;
void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
SEncoder encoder = {0};
tEncoderInit(&encoder, abuf, len);
tEncodeSMqDataRsp(&encoder, pRsp);
tEncoderClear(&encoder);
SRpcMsg rsp = {
.info = pMsg->info,
.pCont = buf,
.contLen = tlen,
.code = 0,
};
tmsgSendRsp(&rsp);
// int32_t len = 0;
// int32_t code = 0;
//
// if (type == TMQ_MSG_TYPE__POLL_RSP) {
// tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code);
// } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) {
// tEncodeSize(tEncodeSTaosxRsp, (STaosxRsp*)pRsp, len, code);
// }
//
// if (code < 0) {
// return -1;
// }
//
// int32_t tlen = sizeof(SMqRspHead) + len;
// void* buf = rpcMallocCont(tlen);
// if (buf == NULL) {
// return -1;
// }
//
// ((SMqRspHead*)buf)->mqMsgType = type;
// ((SMqRspHead*)buf)->epoch = pReq->epoch;
// ((SMqRspHead*)buf)->consumerId = pReq->consumerId;
//
// void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
//
// SEncoder encoder = {0};
// tEncoderInit(&encoder, abuf, len);
//
// if (type == TMQ_MSG_TYPE__POLL_RSP) {
// tEncodeSMqDataRsp(&encoder, pRsp);
// } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) {
// tEncodeSTaosxRsp(&encoder, (STaosxRsp*) pRsp);
// }
//
// tEncoderClear(&encoder);
//
// SRpcMsg rsp = {
// .info = pMsg->info,
// .pCont = buf,
// .contLen = tlen,
// .code = 0,
// };
//
// tmsgSendRsp(&rsp);
doSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type);
char buf1[80] = {0};
char buf2[80] = {0};
tFormatOffset(buf1, 80, &pRsp->reqOffset);
tFormatOffset(buf2, 80, &pRsp->rspOffset);
tqDebug("vgId:%d consumer:0x%" PRIx64 " (epoch %d), block num:%d, req:%s, rsp:%s",
tqDebug("vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s",
TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2);
return 0;
}
int32_t tqSendTaosxRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const STaosxRsp* pRsp) {
int32_t len = 0;
int32_t code = 0;
tEncodeSize(tEncodeSTaosxRsp, pRsp, len, code);
if (code < 0) {
return -1;
}
int32_t tlen = sizeof(SMqRspHead) + len;
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
return -1;
}
((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__TAOSX_RSP;
((SMqRspHead*)buf)->epoch = pReq->epoch;
((SMqRspHead*)buf)->consumerId = pReq->consumerId;
void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
SEncoder encoder = {0};
tEncoderInit(&encoder, abuf, len);
tEncodeSTaosxRsp(&encoder, pRsp);
tEncoderClear(&encoder);
SRpcMsg rsp = {
.info = pMsg->info,
.pCont = buf,
.contLen = tlen,
.code = 0,
};
tmsgSendRsp(&rsp);
char buf1[80] = {0};
char buf2[80] = {0};
tFormatOffset(buf1, 80, &pRsp->reqOffset);
tFormatOffset(buf2, 80, &pRsp->rspOffset);
tqDebug("taosx rsp, vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, numOfBlks:%d, req:%s, rsp:%s",
TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2);
return 0;
}
//int32_t tqSendTaosxRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const STaosxRsp* pRsp) {
//#if 0
// A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum);
// A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum);
//
// if (pRsp->withSchema) {
// A(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum);
// } else {
// A(taosArrayGetSize(pRsp->blockSchema) == 0);
// }
//
// if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
// if (pRsp->blockNum > 0) {
// A(pRsp->rspOffset.version > pRsp->reqOffset.version);
// } else {
// A(pRsp->rspOffset.version >= pRsp->reqOffset.version);
// }
// }
//#endif
//
// int32_t len = 0;
// int32_t code = 0;
// tEncodeSize(tEncodeSTaosxRsp, pRsp, len, code);
// if (code < 0) {
// return -1;
// }
//
// int32_t tlen = sizeof(SMqRspHead) + len;
// void* buf = rpcMallocCont(tlen);
// if (buf == NULL) {
// terrno = TSDB_CODE_OUT_OF_MEMORY;
// return -1;
// }
//
// ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__TAOSX_RSP;
// ((SMqRspHead*)buf)->epoch = pReq->epoch;
// ((SMqRspHead*)buf)->consumerId = pReq->consumerId;
//
// void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
//
// SEncoder encoder = {0};
// tEncoderInit(&encoder, abuf, len);
// tEncodeSTaosxRsp(&encoder, pRsp);
// tEncoderClear(&encoder);
//
// SRpcMsg rsp = {
// .info = pMsg->info,
// .pCont = buf,
// .contLen = tlen,
// .code = 0,
// };
//
// tmsgSendRsp(&rsp);
//
// char buf1[80] = {0};
// char buf2[80] = {0};
// tFormatOffset(buf1, 80, &pRsp->reqOffset);
// tFormatOffset(buf2, 80, &pRsp->rspOffset);
//
// tqDebug("taosx rsp, vgId:%d, consumer:0x%" PRIx64 " (epoch %d) send rsp, numOfBlks:%d, req:%s, rsp:%s",
// TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2);
// return 0;
//}
static FORCE_INLINE bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) {
return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG &&
@ -382,8 +505,18 @@ static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t su
}
pRsp->withTbName = 0;
pRsp->withSchema = false;
#if 0
pRsp->withTbName = pReq->withTbName;
if (pRsp->withTbName) {
pRsp->blockTbName = taosArrayInit(0, sizeof(void*));
if (pRsp->blockTbName == NULL) {
// TODO free
return -1;
}
}
#endif
pRsp->withSchema = false;
return 0;
}
@ -404,170 +537,147 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) {
return 0;
}
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
SMqPollReq req = {0};
int32_t code = 0;
STqOffsetVal fetchOffsetNew;
SWalCkHead* pCkHead = NULL;
static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
SRpcMsg* pMsg, bool* pBlockReturned) {
uint64_t consumerId = pRequest->consumerId;
STqOffsetVal reqOffset = pRequest->reqOffset;
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey);
*pBlockReturned = false;
if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen);
return -1;
}
// In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value.
if (pOffset != NULL) {
*pOffsetVal = pOffset->val;
int64_t consumerId = req.consumerId;
int32_t reqEpoch = req.epoch;
STqOffsetVal reqOffset = req.reqOffset;
// 1. find handle
STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
if (pHandle == NULL) {
tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s not found", consumerId, TD_VID(pTq->pVnode),
req.subKey);
return -1;
}
// 2. check rebalance
if (pHandle->consumerId != consumerId) {
tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64,
consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId);
terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH;
return -1;
}
// update epoch if need
int32_t savedEpoch = atomic_load_32(&pHandle->epoch);
while (savedEpoch < reqEpoch) {
tqDebug("tmq poll: consumer:0x%"PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, reqEpoch);
savedEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, savedEpoch, reqEpoch);
}
char buf[80];
tFormatOffset(buf, 80, &reqOffset);
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s", consumerId,
req.epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf);
// 2.reset offset if needed
if (reqOffset.type > 0) {
fetchOffsetNew = reqOffset;
char formatBuf[80];
tFormatOffset(formatBuf, 80, pOffsetVal);
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, prev offset found, offset reset to %s and continue.",
consumerId, pHandle->subKey, TD_VID(pTq->pVnode), formatBuf);
return 0;
} else {
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey);
if (pOffset != NULL) {
fetchOffsetNew = pOffset->val;
char formatBuf[80];
tFormatOffset(formatBuf, 80, &fetchOffsetNew);
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vg %d, offset reset to %s", consumerId, pHandle->subKey,
TD_VID(pTq->pVnode), formatBuf);
} else {
if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
if (req.useSnapshot) {
if (pHandle->fetchMeta) {
tqOffsetResetToMeta(&fetchOffsetNew, 0);
} else {
tqOffsetResetToData(&fetchOffsetNew, 0, 0);
}
// no poll occurs in this vnode for this topic, let's seek to the right offset value.
if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
if (pRequest->useSnapshot) {
if (pHandle->fetchMeta) {
tqOffsetResetToMeta(pOffsetVal, 0);
} else {
pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef);
if (pHandle->pRef == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
tqOffsetResetToLog(&fetchOffsetNew, pHandle->pRef->refVer - 1);
tqOffsetResetToData(pOffsetVal, 0, 0);
}
} else {
pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef);
if (pHandle->pRef == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
} else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
SMqDataRsp dataRsp = {0};
tqInitDataRsp(&dataRsp, &req, pHandle->execHandle.subType);
tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, offset reset to %" PRId64, consumerId,
pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.rspOffset.version);
if (tqSendDataRsp(pTq, pMsg, &req, &dataRsp) < 0) {
code = -1;
}
tDeleteSMqDataRsp(&dataRsp);
return code;
} else {
STaosxRsp taosxRsp = {0};
tqInitTaosxRsp(&taosxRsp, &req);
tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) {
code = -1;
}
tDeleteSTaosxRsp(&taosxRsp);
return code;
}
} else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
tqError("tmq poll: subkey %s, no offset committed for consumer:0x%" PRIx64
" in vg %d, subkey %s, reset none failed",
pHandle->subKey, consumerId, TD_VID(pTq->pVnode), req.subKey);
terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
return -1;
tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1);
}
}
}
} else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
SMqDataRsp dataRsp = {0};
tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType);
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
SMqDataRsp dataRsp = {0};
tqInitDataRsp(&dataRsp, &req, pHandle->execHandle.subType);
// lock
taosWLockLatch(&pTq->pushLock);
if (tqScanData(pTq, pHandle, &dataRsp, &fetchOffsetNew) < 0) {
tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, offset reset to %" PRId64, consumerId,
pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.rspOffset.version);
int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP);
tDeleteSMqDataRsp(&dataRsp);
*pBlockReturned = true;
return code;
} else {
STaosxRsp taosxRsp = {0};
tqInitTaosxRsp(&taosxRsp, pRequest);
tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
// int32_t code = tqSendTaosxRsp(pTq, pMsg, pRequest, &taosxRsp);
tDeleteSTaosxRsp(&taosxRsp);
*pBlockReturned = true;
return code;
}
} else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
tqError("tmq poll: subkey %s, no offset committed for consumer:0x%" PRIx64
" in vg %d, subkey %s, reset none failed",
pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pRequest->subKey);
terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
return -1;
}
}
// till now, all data has been rsp to consumer, new data needs to push client once arrived.
return 0;
}
static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) {
int32_t code = -1;
STqOffsetVal offset = {0};
SWalCkHead* pCkHead = NULL;
int32_t vgId = TD_VID(pTq->pVnode);
STqOffsetVal reqOffset = pRequest->reqOffset;
uint64_t consumerId = pRequest->consumerId;
// 1. reset the offset if needed
if (reqOffset.type > 0) {
offset = reqOffset;
} else { // handle the reset offset cases, according to the consumer's choice.
bool blockReturned = false;
code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned);
if (code != 0) {
return code;
}
// empty block returned, quit
if (blockReturned) {
return 0;
}
}
// this is a normal subscription requirement
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
SMqDataRsp dataRsp = {0};
tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType);
// lock
taosWLockLatch(&pTq->pushLock);
code = tqScanData(pTq, pHandle, &dataRsp, &offset);
// till now, all data has been transferred to consumer, new data needs to push client once arrived.
if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG &&
dataRsp.reqOffset.version == dataRsp.rspOffset.version) {
STqPushEntry* pPushEntry = taosMemoryCalloc(1, sizeof(STqPushEntry));
if (pPushEntry != NULL) {
pPushEntry->pInfo = pMsg->info;
memcpy(pPushEntry->subKey, pHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN);
dataRsp.withTbName = 0;
memcpy(&pPushEntry->dataRsp, &dataRsp, sizeof(SMqDataRsp));
pPushEntry->dataRsp.head.consumerId = consumerId;
pPushEntry->dataRsp.head.epoch = reqEpoch;
pPushEntry->dataRsp.head.mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey) + 1, &pPushEntry, sizeof(void*));
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s offset:%" PRId64 ", vgId:%d save handle to push mgr",
consumerId, pHandle->subKey, dataRsp.reqOffset.version, TD_VID(pTq->pVnode));
// unlock
taosWUnLockLatch(&pTq->pushLock);
return 0;
}
dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) {
code = tqRegisterPushEntry(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP);
taosWUnLockLatch(&pTq->pushLock);
return code;
}
taosWUnLockLatch(&pTq->pushLock);
code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP);
if (tqSendDataRsp(pTq, pMsg, &req, &dataRsp) < 0) {
code = -1;
}
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, offset type:%d, uid/version:%" PRId64 ", ts:%" PRId64 "",
consumerId, pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.blockNum, dataRsp.rspOffset.type,
dataRsp.rspOffset.uid, dataRsp.rspOffset.ts);
// NOTE: this pHandle->consumerId may have been changed already.
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, offset type:%d, uid/version:%" PRId64
", ts:%" PRId64,
consumerId, pHandle->subKey, vgId, dataRsp.blockNum, dataRsp.rspOffset.type, dataRsp.rspOffset.uid,
dataRsp.rspOffset.ts);
tDeleteSMqDataRsp(&dataRsp);
return code;
}
// todo handle the case where re-balance occurs.
// for taosx
SMqMetaRsp metaRsp = {0};
STaosxRsp taosxRsp = {0};
tqInitTaosxRsp(&taosxRsp, &req);
tqInitTaosxRsp(&taosxRsp, pRequest);
if (fetchOffsetNew.type != TMQ_OFFSET__LOG) {
if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew) < 0) {
if (offset.type != TMQ_OFFSET__LOG) {
if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &offset) < 0) {
return -1;
}
if (metaRsp.metaRspLen > 0) {
if (tqSendMetaPollRsp(pTq, pMsg, &req, &metaRsp) < 0) {
code = -1;
}
tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey %s, vg %d, send meta offset type:%d,uid:%" PRId64
code = tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp);
tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64
",version:%" PRId64,
consumerId, pHandle->subKey, TD_VID(pTq->pVnode), metaRsp.rspOffset.type, metaRsp.rspOffset.uid,
consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid,
metaRsp.rspOffset.version);
taosMemoryFree(metaRsp.metaRsp);
tDeleteSTaosxRsp(&taosxRsp);
@ -575,54 +685,56 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
}
if (taosxRsp.blockNum > 0) {
if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) {
code = -1;
}
code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
tDeleteSTaosxRsp(&taosxRsp);
return code;
} else {
fetchOffsetNew = taosxRsp.rspOffset;
offset = taosxRsp.rspOffset;
}
tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey %s, vg %d, send data blockNum:%d, offset type:%d,uid:%" PRId64
tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64
",version:%" PRId64,
consumerId, pHandle->subKey, TD_VID(pTq->pVnode), taosxRsp.blockNum, taosxRsp.rspOffset.type,
taosxRsp.rspOffset.uid, taosxRsp.rspOffset.version);
}
consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid,
taosxRsp.rspOffset.version);
} else {
if (fetchOffsetNew.type == TMQ_OFFSET__LOG) {
int64_t fetchVer = fetchOffsetNew.version + 1;
// if (offset.type == TMQ_OFFSET__LOG) {
int64_t fetchVer = offset.version + 1;
pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
if (pCkHead == NULL) {
tDeleteSTaosxRsp(&taosxRsp);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
walSetReaderCapacity(pHandle->pWalReader, 2048);
while (1) {
savedEpoch = atomic_load_32(&pHandle->epoch);
if (savedEpoch > reqEpoch) {
tqWarn("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, vg %d offset %" PRId64
// todo refactor: this is not correct.
int32_t savedEpoch = atomic_load_32(&pHandle->epoch);
if (savedEpoch > pRequest->epoch) {
tqWarn("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey:%s vgId:%d offset %" PRId64
", found new consumer epoch %d, discard req epoch %d",
consumerId, req.epoch, pHandle->subKey, TD_VID(pTq->pVnode), fetchVer, savedEpoch, reqEpoch);
consumerId, pRequest->epoch, pHandle->subKey, vgId, fetchVer, savedEpoch, pRequest->epoch);
break;
}
if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead) < 0) {
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) {
code = -1;
}
tDeleteSTaosxRsp(&taosxRsp);
taosMemoryFreeClear(pCkHead);
return code;
// if (terrno == 0) { // failed to seek to given ver, but no errors happen.
// code = tqRegisterPushEntry(pTq, pHandle, pRequest, pMsg, (SMqDataRsp*) &taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
// return code;
// } else { // error happens, return to consumers
code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
tDeleteSTaosxRsp(&taosxRsp);
taosMemoryFreeClear(pCkHead);
return code;
// }
}
SWalCont* pHead = &pCkHead->head;
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", consumerId,
req.epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType);
pRequest->epoch, vgId, fetchVer, pHead->msgType);
if (pHead->msgType == TDMT_VND_SUBMIT) {
SPackedData submit = {
@ -630,16 +742,16 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
.msgLen = pHead->bodyLen - sizeof(SSubmitReq2Msg),
.ver = pHead->version,
};
if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp) < 0) {
tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", consumerId, TD_VID(pTq->pVnode),
req.subKey);
tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", consumerId, vgId,
pRequest->subKey);
return -1;
}
if (taosxRsp.blockNum > 0 /* threshold */) {
if (taosxRsp.blockNum > 0) {
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) {
code = -1;
}
code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP);
tDeleteSTaosxRsp(&taosxRsp);
taosMemoryFreeClear(pCkHead);
return code;
@ -655,7 +767,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
metaRsp.resMsgType = pHead->msgType;
metaRsp.metaRspLen = pHead->bodyLen;
metaRsp.metaRsp = pHead->body;
if (tqSendMetaPollRsp(pTq, pMsg, &req, &metaRsp) < 0) {
if (tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp) < 0) {
code = -1;
taosMemoryFreeClear(pCkHead);
tDeleteSTaosxRsp(&taosxRsp);
@ -674,6 +786,55 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
return 0;
}
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
SMqPollReq req = {0};
if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen);
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
int64_t consumerId = req.consumerId;
int32_t reqEpoch = req.epoch;
STqOffsetVal reqOffset = req.reqOffset;
int32_t vgId = TD_VID(pTq->pVnode);
// 1. find handle
STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
if (pHandle == NULL) {
tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey);
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
// 2. check re-balance status
taosRLockLatch(&pTq->pushLock);
if (pHandle->consumerId != consumerId) {
tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64,
consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId);
terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH;
taosRUnLockLatch(&pTq->pushLock);
return -1;
}
taosRUnLockLatch(&pTq->pushLock);
taosWLockLatch(&pTq->pushLock);
// 3. update the epoch value
int32_t savedEpoch = pHandle->epoch;
if (savedEpoch < reqEpoch) {
tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, reqEpoch);
pHandle->epoch = reqEpoch;
}
taosWUnLockLatch(&pTq->pushLock);
char buf[80];
tFormatOffset(buf, 80, &reqOffset);
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64,
consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId);
return extractDataForMq(pTq, pHandle, &req, pMsg);
}
int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg;
@ -744,9 +905,9 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t
int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
SMqRebVgReq req = {0};
tDecodeSMqRebVgReq(msg, &req);
// todo lock
tqDebug("vgId:%d, tq process sub req %s", pTq->pVnode->config.vgId, req.subKey);
tqDebug("vgId:%d, tq process sub req %s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pTq->pVnode->config.vgId, req.subKey,
req.oldConsumerId, req.newConsumerId);
STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
if (pHandle == NULL) {
@ -754,11 +915,13 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId is %" PRId64 "",
req.vgId, req.subKey, req.newConsumerId, req.oldConsumerId);
}
if (req.newConsumerId == -1) {
tqError("vgId:%d, tq invalid rebalance request, new consumerId %" PRId64 "", req.vgId, req.newConsumerId);
taosMemoryFree(req.qmsg);
return 0;
}
STqHandle tqHandle = {0};
pHandle = &tqHandle;
/*taosInitRWLatch(&pExec->lock);*/
@ -774,8 +937,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
// TODO version should be assigned and refed during preprocess
SWalRef* pRef = walRefCommittedVer(pTq->pVnode->pWal);
if (pRef == NULL) {
taosMemoryFree(req.qmsg);
return -1;
}
int64_t ver = pRef->refVer;
pHandle->pRef = pRef;
@ -786,6 +951,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
.initTqReader = true,
.version = ver,
};
pHandle->snapshotVer = ver;
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
@ -800,6 +966,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) {
pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
pHandle->execHandle.pExecReader = tqOpenReader(pTq->pVnode);
pHandle->execHandle.execDb.pFilterOutTbUid =
taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta,
@ -808,7 +975,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, NULL, NULL);
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
pHandle->execHandle.execTb.suid = req.suid;
SArray* tbUidList = taosArrayInit(0, sizeof(int64_t));
@ -828,28 +994,45 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
}
taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle));
tqDebug("try to persist handle %s consumer:0x%" PRIx64" , old consumer:0x%"PRIx64, req.subKey, pHandle->consumerId,
oldConsumerId);
tqDebug("try to persist handle %s consumer:0x%" PRIx64 " , old consumer:0x%" PRIx64, req.subKey,
pHandle->consumerId, oldConsumerId);
if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
taosMemoryFree(req.qmsg);
return -1;
}
} else {
// TODO handle qmsg and exec modification
tqInfo("update the consumer info, old consumer id:0x%"PRIx64", new Id:0x%"PRIx64, pHandle->consumerId, req.newConsumerId);
if (pHandle->consumerId == req.newConsumerId) { // do nothing
tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs", req.vgId, req.newConsumerId);
atomic_store_32(&pHandle->epoch, -1);
atomic_add_fetch_32(&pHandle->epoch, 1);
taosMemoryFree(req.qmsg);
return tqMetaSaveHandle(pTq, req.subKey, pHandle);
}
tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId,
req.newConsumerId);
taosWLockLatch(&pTq->pushLock);
atomic_store_32(&pHandle->epoch, -1);
// remove if it has been register in the push manager, and return one empty block to consumer
tqRemovePushEntry(pTq, req.subKey, (int32_t) strlen(req.subKey), pHandle->consumerId, true);
atomic_store_64(&pHandle->consumerId, req.newConsumerId);
atomic_add_fetch_32(&pHandle->epoch, 1);
taosMemoryFree(req.qmsg);
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
qStreamCloseTsdbReader(pHandle->execHandle.task);
}
taosWUnLockLatch(&pTq->pushLock);
if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
taosMemoryFree(req.qmsg);
return -1;
}
// close handle
}
taosMemoryFree(req.qmsg);
return 0;
}
@ -924,15 +1107,15 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
pTask->tbSink.vnode = pTq->pVnode;
pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline2;
int32_t version = 1;
int32_t ver1 = 1;
SMetaInfo info = {0};
int32_t code = metaGetInfo(pTq->pVnode->pMeta, pTask->tbSink.stbUid, &info, NULL);
if (code == TSDB_CODE_SUCCESS) {
version = info.skmVer;
ver1 = info.skmVer;
}
pTask->tbSink.pTSchema =
tBuildTSchema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, version);
tBuildTSchema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, ver1);
if(pTask->tbSink.pTSchema == NULL) {
return -1;
}

9
source/dnode/vnode/src/tq/tqExec.c
View File

@ -46,11 +46,13 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp
static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, int32_t n) {
SMetaReader mr = {0};
metaReaderInit(&mr, pTq->pVnode->pMeta, 0);
// TODO add reference to gurantee success
if (metaGetTableEntryByUidCache(&mr, uid) < 0) {
metaReaderClear(&mr);
return -1;
}
for (int32_t i = 0; i < n; i++) {
char* tbName = taosStrdup(mr.me.name);
taosArrayPush(pRsp->blockTbName, &tbName);
@ -83,13 +85,16 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
while (1) {
SSDataBlock* pDataBlock = NULL;
uint64_t ts = 0;
tqDebug("vgId:%d, tmq task start to execute", pTq->pVnode->config.vgId);
if (qExecTask(task, &pDataBlock, &ts) < 0) {
tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr());
return -1;
}
tqDebug("vgId:%d, tmq task executed, get %p", pTq->pVnode->config.vgId, pDataBlock);
// current scan should be stopped asap, since the rebalance occurs.
if (pDataBlock == NULL) {
break;
}
@ -99,7 +104,9 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
rowCnt += pDataBlock->info.rows;
if (rowCnt >= 4096) break;
if (rowCnt >= 4096) {
break;
}
}
}

2
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -283,7 +283,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
tdbTbcMoveToFirst(pCur);
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
STqHandle handle;
STqHandle handle = {0};
tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
tDecodeSTqHandle(&decoder, &handle);
tDecoderClear(&decoder);

84
source/dnode/vnode/src/tq/tqPush.c
View File

@ -209,6 +209,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
void* pReq = POINTER_SHIFT(msg, sizeof(SSubmitReq2Msg));
int32_t len = msgLen - sizeof(SSubmitReq2Msg);
int32_t vgId = TD_VID(pTq->pVnode);
if (msgType == TDMT_VND_SUBMIT) {
// lock push mgr to avoid potential msg lost
@ -217,7 +218,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
int32_t numOfRegisteredPush = taosHashGetSize(pTq->pPushMgr);
if (numOfRegisteredPush > 0) {
tqDebug("vgId:%d tq push msg version:%" PRId64 " type:%s, head:%p, body:%p len:%d, numOfPushed consumers:%d",
pTq->pVnode->config.vgId, ver, TMSG_INFO(msgType), msg, pReq, len, numOfRegisteredPush);
vgId, ver, TMSG_INFO(msgType), msg, pReq, len, numOfRegisteredPush);
SArray* cachedKeys = taosArrayInit(0, sizeof(void*));
SArray* cachedKeyLens = taosArrayInit(0, sizeof(size_t));
@ -239,7 +240,10 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
void* pIter = NULL;
while (1) {
pIter = taosHashIterate(pTq->pPushMgr, pIter);
if (pIter == NULL) break;
if (pIter == NULL) {
break;
}
STqPushEntry* pPushEntry = *(STqPushEntry**)pIter;
STqHandle* pHandle = taosHashGet(pTq->pHandle, pPushEntry->subKey, strlen(pPushEntry->subKey));
@ -248,17 +252,16 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
continue;
}
if (pPushEntry->dataRsp.reqOffset.version >= ver) {
tqDebug("vgId:%d, push entry req version %" PRId64 ", while push version %" PRId64 ", skip",
pTq->pVnode->config.vgId, pPushEntry->dataRsp.reqOffset.version, ver);
SMqDataRsp* pRsp = pPushEntry->pDataRsp;
if (pRsp->reqOffset.version >= ver) {
tqDebug("vgId:%d, push entry req version %" PRId64 ", while push version %" PRId64 ", skip", vgId,
pRsp->reqOffset.version, ver);
continue;
}
STqExecHandle* pExec = &pHandle->execHandle;
qTaskInfo_t task = pExec->task;
SMqDataRsp* pRsp = &pPushEntry->dataRsp;
// prepare scan mem data
SPackedData submit = {
.msgStr = data,
@ -274,7 +277,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
SSDataBlock* pDataBlock = NULL;
uint64_t ts = 0;
if (qExecTask(task, &pDataBlock, &ts) < 0) {
tqDebug("vgId:%d, tq exec error since %s", pTq->pVnode->config.vgId, terrstr());
tqDebug("vgId:%d, tq exec error since %s", vgId, terrstr());
}
if (pDataBlock == NULL) {
@ -285,11 +288,11 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
pRsp->blockNum++;
}
tqDebug("vgId:%d, tq handle push, subkey:%s, block num:%d", pTq->pVnode->config.vgId, pPushEntry->subKey,
pRsp->blockNum);
tqDebug("vgId:%d, tq handle push, subkey:%s, block num:%d", vgId, pPushEntry->subKey, pRsp->blockNum);
if (pRsp->blockNum > 0) {
// set offset
tqOffsetResetToLog(&pRsp->rspOffset, ver);
// remove from hash
size_t kLen;
void* key = taosHashGetKey(pIter, &kLen);
@ -311,6 +314,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
tqError("vgId:%d, tq push hash remove key error, key: %s", pTq->pVnode->config.vgId, (char*)key);
}
}
taosArrayDestroyP(cachedKeys, (FDelete)taosMemoryFree);
taosArrayDestroy(cachedKeyLens);
taosMemoryFree(data);
@ -336,9 +340,9 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
};
tqDebug("tq copy write msg %p %d %" PRId64 " from %p", data, len, ver, pReq);
tqProcessSubmitReq(pTq, submit);
}
if (msgType == TDMT_VND_DELETE) {
tqProcessDelReq(pTq, POINTER_SHIFT(msg, sizeof(SMsgHead)), msgLen - sizeof(SMsgHead), ver);
}
@ -346,3 +350,61 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
return 0;
}
int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg,
SMqDataRsp* pDataRsp, int32_t type) {
uint64_t consumerId = pRequest->consumerId;
int32_t vgId = TD_VID(pTq->pVnode);
STqHandle* pTqHandle = pHandle;
STqPushEntry* pPushEntry = taosMemoryCalloc(1, sizeof(STqPushEntry));
if (pPushEntry == NULL) {
tqDebug("tmq poll: consumer:0x%" PRIx64 ", vgId:%d failed to malloc, size:%d", consumerId, vgId,
(int32_t)sizeof(STqPushEntry));
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pPushEntry->info = pRpcMsg->info;
memcpy(pPushEntry->subKey, pTqHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN);
if (type == TMQ_MSG_TYPE__TAOSX_RSP) {
pPushEntry->pDataRsp = taosMemoryCalloc(1, sizeof(STaosxRsp));
memcpy(pPushEntry->pDataRsp, pDataRsp, sizeof(STaosxRsp));
} else if (type == TMQ_MSG_TYPE__POLL_RSP) {
pPushEntry->pDataRsp = taosMemoryCalloc(1, sizeof(SMqDataRsp));
memcpy(pPushEntry->pDataRsp, pDataRsp, sizeof(SMqDataRsp));
}
SMqRspHead* pHead = &pPushEntry->pDataRsp->head;
pHead->consumerId = consumerId;
pHead->epoch = pRequest->epoch;
pHead->mqMsgType = type;
taosHashPut(pTq->pPushMgr, pTqHandle->subKey, strlen(pTqHandle->subKey), &pPushEntry, sizeof(void*));
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s offset:%" PRId64 ", vgId:%d save handle to push mgr, total:%d", consumerId,
pTqHandle->subKey, pDataRsp->reqOffset.version, vgId, taosHashGetSize(pTq->pPushMgr));
return 0;
}
int32_t tqRemovePushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) {
int32_t vgId = TD_VID(pTq->pVnode);
STqPushEntry** pEntry = taosHashGet(pTq->pPushMgr, pKey, keyLen);
if (pEntry != NULL) {
uint64_t cId = (*pEntry)->pDataRsp->head.consumerId;
ASSERT(consumerId == cId);
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s vgId:%d remove from push mgr, remains:%d", consumerId,
(*pEntry)->subKey, vgId, taosHashGetSize(pTq->pPushMgr) - 1);
if (rspConsumer) { // rsp the old consumer with empty block.
tqPushDataRsp(pTq, *pEntry);
}
taosHashRemove(pTq->pPushMgr, pKey, keyLen);
}
return 0;
}

8
source/dnode/vnode/src/tq/tqRead.c
View File

@ -183,14 +183,15 @@ end:
return tbSuid == realTbSuid;
}
int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) {
int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) {
int32_t code = 0;
taosThreadMutexLock(&pHandle->pWalReader->mutex);
int64_t offset = *fetchOffset;
while (1) {
if (walFetchHead(pHandle->pWalReader, offset, *ppCkHead) < 0) {
tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return",
tqDebug("tmq poll: consumer:0x%" PRIx64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return",
pHandle->consumerId, pHandle->epoch, TD_VID(pTq->pVnode), offset);
*fetchOffset = offset - 1;
code = -1;
@ -241,6 +242,7 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea
offset++;
}
}
END:
taosThreadMutexUnlock(&pHandle->pWalReader->mutex);
return code;
@ -294,7 +296,7 @@ int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) {
// todo set the correct vgId
tqDebug("tmq poll: wal seek to version:%"PRId64" %s", ver, id);
if (walReadSeekVer(pReader->pWalReader, ver) < 0) {
tqError("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id);
tqDebug("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id);
return -1;
} else {
tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id);

1
source/dnode/vnode/src/tq/tqSink.c
View File

@ -619,6 +619,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry.suid);
metaReaderClear(&mr);
taosMemoryFree(ctbName);
continue;
}
tbData.uid = mr.me.uid;

210
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -596,9 +596,11 @@ typedef struct {
int64_t lastTs;
} SFSLastNextRowIter;
static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) {
static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
int nCols) {
SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter;
int32_t code = 0;
bool checkRemainingRow = true;
switch (state->state) {
case SFSLASTNEXTROW_FS:
@ -631,6 +633,8 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
if (code) goto _err;
}
state->pLoadInfo->colIds = aCols;
state->pLoadInfo->numOfCols = nCols;
tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid,
&(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX},
&(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true);
@ -646,22 +650,71 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
goto _next_fileset;
}
state->state = SFSLASTNEXTROW_BLOCKROW;
checkRemainingRow = false;
}
case SFSLASTNEXTROW_BLOCKROW: {
bool hasVal = false;
state->row = tMergeTreeGetRow(&state->mergeTree);
*ppRow = &state->row;
hasVal = tMergeTreeNext(&state->mergeTree);
if (TSDBROW_TS(&state->row) <= state->lastTs) {
*pIgnoreEarlierTs = true;
*ppRow = NULL;
return code;
}
bool skipRow = false;
do {
bool hasVal = false;
state->row = tMergeTreeGetRow(&state->mergeTree);
*ppRow = &state->row;
hasVal = tMergeTreeNext(&state->mergeTree);
if (TSDBROW_TS(&state->row) <= state->lastTs) {
*pIgnoreEarlierTs = true;
*ppRow = NULL;
return code;
}
*pIgnoreEarlierTs = false;
if (!hasVal) {
state->state = SFSLASTNEXTROW_FILESET;
}
*pIgnoreEarlierTs = false;
if (!hasVal) {
state->state = SFSLASTNEXTROW_FILESET;
break;
}
if (checkRemainingRow) {
bool skipBlock = true;
SBlockData *pBlockData = state->row.pBlockData;
for (int inputColIndex = 0; inputColIndex < nCols; ++inputColIndex) {
for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
SColData *pColData = &pBlockData->aColData[colIndex];
int16_t cid = pColData->cid;
if (cid == aCols[inputColIndex]) {
if (isLast && (pColData->flag & HAS_VALUE)) {
skipBlock = false;
break;
} else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
skipBlock = false;
break;
}
}
}
}
/*
for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
SColData *pColData = &pBlockData->aColData[colIndex];
int16_t cid = pColData->cid;
if (inputColIndex < nCols && cid == aCols[inputColIndex]) {
if (isLast && (pColData->flag & HAS_VALUE)) {
skipBlock = false;
break;
} else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
skipBlock = false;
break;
}
++inputColIndex;
}
}
*/
if (skipBlock) {
skipRow = true;
}
}
} while (skipRow);
return code;
}
@ -740,9 +793,11 @@ typedef struct SFSNextRowIter {
int64_t lastTs;
} SFSNextRowIter;
static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) {
static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
int nCols) {
SFSNextRowIter *state = (SFSNextRowIter *)iter;
int32_t code = 0;
bool checkRemainingRow = true;
switch (state->state) {
case SFSNEXTROW_FS:
@ -828,8 +883,11 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
}
}
case SFSNEXTROW_BLOCKDATA:
_next_datablock:
if (state->iBlock >= 0) {
SDataBlk block = {0};
bool skipBlock = true;
int inputColIndex = 0;
tDataBlkReset(&block);
tBlockDataReset(state->pBlockData);
@ -848,18 +906,94 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
*pIgnoreEarlierTs = false;
tBlockDataReset(state->pBlockData);
TABLEID tid = {.suid = state->suid, .uid = state->uid};
code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, NULL, 0);
code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, aCols, nCols);
if (code) goto _err;
code = tsdbReadDataBlock(*state->pDataFReader, &block, state->pBlockData);
if (code) goto _err;
for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) {
SColData *pColData = &state->pBlockData->aColData[colIndex];
if (isLast && (pColData->flag & HAS_VALUE)) {
skipBlock = false;
break;
} else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
skipBlock = false;
break;
}
}
if (skipBlock) {
if (--state->iBlock < 0) {
tsdbDataFReaderClose(state->pDataFReader);
*state->pDataFReader = NULL;
// resetLastBlockLoadInfo(state->pLoadInfo);
if (state->aBlockIdx) {
// taosArrayDestroy(state->aBlockIdx);
tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle);
state->aBlockIdxHandle = NULL;
state->aBlockIdx = NULL;
}
state->state = SFSNEXTROW_FILESET;
goto _next_fileset;
} else {
goto _next_datablock;
}
}
state->nRow = state->blockData.nRow;
state->iRow = state->nRow - 1;
state->state = SFSNEXTROW_BLOCKROW;
checkRemainingRow = false;
}
case SFSNEXTROW_BLOCKROW:
case SFSNEXTROW_BLOCKROW: {
if (checkRemainingRow) {
bool skipBlock = true;
int inputColIndex = 0;
for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) {
SColData *pColData = &state->pBlockData->aColData[colIndex];
int16_t cid = pColData->cid;
if (inputColIndex < nCols && cid == aCols[inputColIndex]) {
if (isLast && (pColData->flag & HAS_VALUE)) {
skipBlock = false;
break;
} else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
skipBlock = false;
break;
}
++inputColIndex;
}
}
if (skipBlock) {
if (--state->iBlock < 0) {
tsdbDataFReaderClose(state->pDataFReader);
*state->pDataFReader = NULL;
// resetLastBlockLoadInfo(state->pLoadInfo);
if (state->aBlockIdx) {
// taosArrayDestroy(state->aBlockIdx);
tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle);
state->aBlockIdxHandle = NULL;
state->aBlockIdx = NULL;
}
state->state = SFSNEXTROW_FILESET;
goto _next_fileset;
} else {
goto _next_datablock;
}
}
}
if (state->iRow >= 0) {
state->row = tsdbRowFromBlockData(state->pBlockData, state->iRow);
*ppRow = &state->row;
@ -885,6 +1019,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
}
return code;
}
default:
ASSERT(0);
break;
@ -960,7 +1095,8 @@ typedef struct SMemNextRowIter {
// TSDBROW *curRow;
} SMemNextRowIter;
static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) {
static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
int nCols) {
SMemNextRowIter *state = (SMemNextRowIter *)iter;
int32_t code = 0;
*pIgnoreEarlierTs = false;
@ -1072,7 +1208,8 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) {
return deleted;
}
typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs);
typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols,
int nCols);
typedef int32_t (*_next_row_clear_fn_t)(void *iter);
typedef struct {
@ -1222,12 +1359,14 @@ _err:
}
// iterate next row non deleted backward ts, version (from high to low)
static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pIgnoreEarlierTs) {
static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast,
int16_t *aCols, int nCols) {
int code = 0;
for (;;) {
for (int i = 0; i < 4; ++i) {
if (pIter->input[i].next && !pIter->input[i].stop) {
code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, &pIter->input[i].ignoreEarlierTs);
code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, &pIter->input[i].ignoreEarlierTs,
isLast, aCols, nCols);
if (code) goto _err;
if (pIter->input[i].pRow == NULL) {
@ -1358,7 +1497,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo
do {
TSDBROW *pRow = NULL;
nextRowIterGet(&iter, &pRow, &ignoreEarlierTs);
nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, NULL, 0);
if (!pRow) {
break;
@ -1488,11 +1627,21 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
bool ignoreEarlierTs = false;
SArray *pColArray = NULL;
SColVal *pColVal = &(SColVal){0};
int16_t nCols = nLastCol;
int32_t code = initLastColArray(pTSchema, &pColArray);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
SArray *aColArray = taosArrayInit(nCols, sizeof(int16_t));
if (NULL == aColArray) {
taosArrayDestroy(pColArray);
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int i = 1; i < pTSchema->numOfCols; ++i) {
taosArrayPush(aColArray, &pTSchema->columns[i].colId);
}
TSKEY lastRowTs = TSKEY_MAX;
@ -1502,7 +1651,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
do {
TSDBROW *pRow = NULL;
nextRowIterGet(&iter, &pRow, &ignoreEarlierTs);
nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray));
if (!pRow) {
break;
@ -1547,9 +1696,14 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
}
if (!COL_VAL_IS_VALUE(pColVal) && !setNoneCol) {
noneCol = iCol;
setNoneCol = true;
if (!COL_VAL_IS_VALUE(pColVal)) {
if (!setNoneCol) {
noneCol = iCol;
setNoneCol = true;
}
} else {
int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ);
taosArrayRemove(aColArray, aColIndex);
}
}
if (!setNoneCol) {
@ -1590,6 +1744,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
}
taosArraySet(pColArray, iCol, &lastCol);
int32_t aColIndex = taosArraySearchIdx(aColArray, &lastCol.colVal.cid, compareInt16Val, TD_EQ);
taosArrayRemove(aColArray, aColIndex);
} else if (!COL_VAL_IS_VALUE(tColVal) && !COL_VAL_IS_VALUE(pColVal) && !setNoneCol) {
noneCol = iCol;
setNoneCol = true;
@ -1613,6 +1769,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
//}
nextRowIterClose(&iter);
taosArrayDestroy(aColArray);
// taosMemoryFreeClear(pTSchema);
return code;
@ -1621,6 +1778,7 @@ _err:
// taosMemoryFreeClear(pTSchema);
*ppLastArray = NULL;
taosArrayDestroy(pColArray);
taosArrayDestroy(aColArray);
return code;
}

1
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -118,7 +118,6 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id
if (suid != 0) {
p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, suid, -1, 1);
if (p->pSchema == NULL) {
taosMemoryFree(p);
tsdbWarn("stable:%" PRIu64 " has been dropped, failed to retrieve cached rows, %s", suid, idstr);
return TSDB_CODE_PAR_TABLE_NOT_EXIST;
}

9
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -390,8 +390,10 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf
pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));
if (pUidList->tableUidList == NULL) {
taosHashCleanup(pTableMap);
return NULL;
}
pUidList->currentIndex = 0;
for (int32_t j = 0; j < numOfTables; ++j) {
@ -4548,6 +4550,8 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock,
return TSDB_CODE_SUCCESS;
}
int64_t st = taosGetTimestampUs();
SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
if (tDataBlkHasSma(pBlock)) {
code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg);
@ -4609,6 +4613,9 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock,
*pBlockSMA = pResBlock->pBlockAgg;
pReader->cost.smaDataLoad += 1;
double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;
pReader->cost.smaLoadTime += elapsedTime;
tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr);
return code;
}
@ -4763,7 +4770,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa
pTableBlockInfo->defMinRows = pc->minRows;
pTableBlockInfo->defMaxRows = pc->maxRows;
int32_t bucketRange = ceil((pc->maxRows - pc->minRows) / numOfBucket);
int32_t bucketRange = ceil(((double)(pc->maxRows - pc->minRows)) / numOfBucket);
pTableBlockInfo->numOfFiles += 1;

2
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -510,7 +510,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
vTrace("vgId:%d, msg:%p in fetch queue is processing", pVnode->config.vgId, pMsg);
if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG ||
pMsg->msgType == TDMT_VND_BATCH_META) &&
pMsg->msgType == TDMT_VND_BATCH_META || pMsg->msgType == TDMT_VND_TMQ_CONSUME) &&
!syncIsReadyForRead(pVnode->sync)) {
vnodeRedirectRpcMsg(pVnode, pMsg, terrno);
return 0;

3
source/libs/catalog/src/ctgDbg.c
View File

@ -21,6 +21,8 @@
extern SCatalogMgmt gCtgMgmt;
SCtgDebug gCTGDebug = {0};
#if 0
void ctgdUserCallback(SMetaData *pResult, void *param, int32_t code) {
taosMemoryFree(param);
@ -224,6 +226,7 @@ _return:
CTG_RET(code);
}
#endif
int32_t ctgdEnableDebug(char *option, bool enable) {
if (0 == strcasecmp(option, "lock")) {

3
source/libs/catalog/src/ctgUtil.c
View File

@ -1330,6 +1330,7 @@ static void* ctgCloneDnodeList(void* pSrc) { return taosArrayDup((const SArray*)
static void ctgFreeDnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); }
#if 0
static int32_t ctgCloneMetaDataArray(SArray* pSrc, __array_item_dup_fn_t copyFunc, SArray** pDst) {
if (NULL == pSrc) {
return TSDB_CODE_SUCCESS;
@ -1421,3 +1422,5 @@ void catalogFreeMetaData(SMetaData* pData) {
taosMemoryFreeClear(pData->pSvrVer);
taosMemoryFree(pData);
}
#endif

2
source/libs/command/src/command.c
View File

@ -307,7 +307,7 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
bool existLeaderRole(TAOS_ROW row, TAOS_FIELD* fields, int nFields) {
// vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | v4_dnode |
// v4_status | cacheload | tsma |
if (nFields != 13) {
if (nFields != 14) {
return false;
}

1
source/libs/executor/src/executil.c
View File

@ -820,6 +820,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa
int32_t code = blockDataEnsureCapacity(pResBlock, numOfTables);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
taosMemoryFree(pResBlock);
return NULL;
}

4
source/libs/executor/src/executor.c
View File

@ -221,12 +221,12 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks,
qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema) {
if (msg == NULL) {
// create raw scan
SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo));
if (NULL == pTaskInfo) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
pTaskInfo->cost.created = taosGetTimestampUs();
@ -715,7 +715,6 @@ void qStopTaskOperators(SExecTaskInfo* pTaskInfo) {
int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo;
if (pTaskInfo == NULL) {
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
@ -723,7 +722,6 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) {
qDebug("%s execTask async killed", GET_TASKID(pTaskInfo));
setTaskKilled(pTaskInfo, rspCode);
qStopTaskOperators(pTaskInfo);
return TSDB_CODE_SUCCESS;

22
source/libs/executor/src/executorimpl.c
View File

@ -2344,6 +2344,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pOptr = createEventwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo);
} else {
terrno = TSDB_CODE_INVALID_PARA;
taosMemoryFree(ops);
return NULL;
}
@ -2774,11 +2775,17 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
}
void qStreamCloseTsdbReader(void* task) {
if (task == NULL) return;
if (task == NULL) {
return;
}
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)task;
SOperatorInfo* pOp = pTaskInfo->pRoot;
qDebug("stream close tsdb reader, reset status uid %" PRId64 " ts %" PRId64, pTaskInfo->streamInfo.lastStatus.uid,
qDebug("stream close tsdb reader, reset status uid:%" PRId64 " ts:%" PRId64, pTaskInfo->streamInfo.lastStatus.uid,
pTaskInfo->streamInfo.lastStatus.ts);
// todo refactor, other thread may already use this read to extract data.
pTaskInfo->streamInfo.lastStatus = (STqOffsetVal){0};
while (pOp->numOfDownstream == 1 && pOp->pDownstream[0]) {
SOperatorInfo* pDownstreamOp = pOp->pDownstream[0];
@ -2786,8 +2793,19 @@ void qStreamCloseTsdbReader(void* task) {
SStreamScanInfo* pInfo = pDownstreamOp->info;
if (pInfo->pTableScanOp) {
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
setOperatorCompleted(pInfo->pTableScanOp);
while(pTaskInfo->owner != 0) {
taosMsleep(100);
qDebug("wait for the reader stopping");
}
tsdbReaderClose(pTSInfo->base.dataReader);
pTSInfo->base.dataReader = NULL;
// restore the status, todo refactor.
pInfo->pTableScanOp->status = OP_OPENED;
pTaskInfo->status = TASK_NOT_COMPLETED;
return;
}
}

13
source/libs/executor/src/groupoperator.c
View File

@ -173,9 +173,14 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData
size_t numOfGroupCols = taosArrayGetSize(pGroupCols);
for (int32_t i = 0; i < numOfGroupCols; ++i) {
SColumn* pCol = taosArrayGet(pGroupCols, i);
SColumn* pCol = (SColumn*) taosArrayGet(pGroupCols, i);
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId);
// valid range check. todo: return error code.
if (pCol->slotId > taosArrayGetSize(pBlock->pDataBlock)) {
continue;
}
if (pBlock->pBlockAgg != NULL) {
pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched?
}
@ -999,14 +1004,14 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp*
memset(tbName, 0, TSDB_TABLE_NAME_LEN);
int32_t len = 0;
if (colDataIsNull_s(pTbCol, pDestBlock->info.rows - 1)) {
len = TMIN(sizeof(TSDB_DATA_NULL_STR), TSDB_TABLE_NAME_LEN - 1);
memcpy(tbName, TSDB_DATA_NULL_STR, len);
len = 1;
tbName[0] = 0;
} else {
void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1);
len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1);
memcpy(tbName, varDataVal(pData), len);
streamStatePutParName(pState, groupId, tbName);
}
streamStatePutParName(pState, groupId, tbName);
memcpy(pTmpBlock->info.parTbName, tbName, len);
pDestBlock->info.rows--;
} else {

43
source/libs/executor/src/scanoperator.c
View File

@ -751,7 +751,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
while (1) {
SSDataBlock* result = doGroupedTableScan(pOperator);
if (result) {
if (result || (pOperator->status == OP_EXEC_DONE)) {
return result;
}
@ -985,6 +985,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) {
pTableScanInfo->scanTimes = 0;
pTableScanInfo->currentGroupId = -1;
tsdbReaderClose(pTableScanInfo->base.dataReader);
qDebug("1");
pTableScanInfo->base.dataReader = NULL;
}
@ -1143,6 +1144,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTableScanInfo->base.dataReader);
qDebug("2");
pTableScanInfo->base.dataReader = NULL;
return NULL;
}
@ -1616,6 +1618,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
if (!pTaskInfo->streamInfo.returned) {
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTSInfo->base.dataReader);
qDebug("3");
pTSInfo->base.dataReader = NULL;
tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer);
qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1);
@ -1767,6 +1770,8 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
/*resetTableScanInfo(pTSInfo, pWin);*/
tsdbReaderClose(pTSInfo->base.dataReader);
qDebug("4");
pTSInfo->base.dataReader = NULL;
pInfo->pTableScanOp->status = OP_OPENED;
@ -1837,6 +1842,8 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE;
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTSInfo->base.dataReader);
qDebug("5");
pTSInfo->base.dataReader = NULL;
pTSInfo->base.cond.startVersion = -1;
@ -2635,6 +2642,8 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
return pBlock;
}
qDebug("8");
tsdbReaderClose(pInfo->base.dataReader);
pInfo->base.dataReader = NULL;
return NULL;
@ -3180,7 +3189,7 @@ static SSDataBlock* buildSysDbTableCount(SOperatorInfo* pOperator, STableCountSc
size_t perfdbTableNum;
getPerfDbMeta(NULL, &perfdbTableNum);
if (pSupp->groupByDbName) {
if (pSupp->groupByDbName || pSupp->groupByStbName) {
buildSysDbGroupedTableCount(pOperator, pInfo, pSupp, pRes, infodbTableNum, perfdbTableNum);
return (pRes->info.rows > 0) ? pRes : NULL;
} else {
@ -3205,11 +3214,23 @@ static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountSca
STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum,
size_t perfdbTableNum) {
if (pInfo->currGrpIdx == 0) {
uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB));
uint64_t groupId = 0;
if (pSupp->groupByDbName) {
groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB));
} else {
groupId = calcGroupId("", 0);
}
pRes->info.id.groupId = groupId;
fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes);
} else if (pInfo->currGrpIdx == 1) {
uint64_t groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB));
uint64_t groupId = 0;
if (pSupp->groupByDbName) {
groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB));
} else {
groupId = calcGroupId("", 0);
}
pRes->info.id.groupId = groupId;
fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes);
} else {
@ -3247,7 +3268,7 @@ static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCount
tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB);
tNameGetDbName(&sn, dbName);
if (pSupp->groupByDbName) {
if (pSupp->groupByDbName || pSupp->groupByStbName) {
buildVnodeGroupedTableCount(pOperator, pInfo, pSupp, pRes, vgId, dbName);
} else {
buildVnodeFilteredTbCount(pOperator, pInfo, pSupp, pRes, dbName);
@ -3308,7 +3329,10 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO
static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
SSDataBlock* pRes, char* dbName) {
char fullStbName[TSDB_TABLE_FNAME_LEN] = {0};
snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, "");
if (pSupp->groupByDbName) {
snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, "");
}
uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
pRes->info.id.groupId = groupId;
int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta);
@ -3323,7 +3347,12 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S
metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName);
char fullStbName[TSDB_TABLE_FNAME_LEN] = {0};
snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName);
if (pSupp->groupByDbName) {
snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName);
} else {
snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s", stbName);
}
uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
pRes->info.id.groupId = groupId;

8
source/libs/executor/src/sysscanoperator.c
View File

@ -562,15 +562,17 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
continue;
}
sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName);
if (numOfRows >= pOperator->resultInfo.capacity) {
if ((numOfRows + schemaRow->nCols) > pOperator->resultInfo.capacity) {
relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
numOfRows = 0;
metaTbCursorPrev(pInfo->pCur);
if (pInfo->pRes->info.rows > 0) {
break;
}
} else {
sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName);
}
}

2
source/libs/executor/src/timewindowoperator.c
View File

@ -3911,7 +3911,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
blockDataEnsureCapacity(pAggSup->pScanBlock, rows);
SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
for (int32_t i = 0; i < rows; i += winRows) {
if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup) || colDataIsNull_s(pKeyColInfo, i)) {
i++;
continue;
}

1
source/libs/executor/src/tsort.c
View File

@ -214,7 +214,6 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
if (pPage == NULL) {
taosArrayDestroy(pPageIdList);
blockDataDestroy(p);
taosArrayDestroy(pPageIdList);
return terrno;
}

32
source/libs/parser/src/parTranslater.c
View File

@ -1331,6 +1331,32 @@ static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount)
return code;
}
static bool isCountNotNullValue(SFunctionNode* pFunc) {
if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) {
return false;
}
SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
return (QUERY_NODE_VALUE == nodeType(pPara) && !((SValueNode*)pPara)->isNull);
}
// count(1) is rewritten as count(ts) for scannning optimization
static int32_t rewriteCountNotNullValue(STranslateContext* pCxt, SFunctionNode* pCount) {
SValueNode* pValue = (SValueNode*)nodesListGetNode(pCount->pParameterList, 0);
STableNode* pTable = NULL;
int32_t code = findTable(pCxt, NULL, &pTable);
if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) {
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
code = TSDB_CODE_OUT_OF_MEMORY;
} else {
setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol);
NODES_DESTORY_LIST(pCount->pParameterList);
code = nodesListMakeAppend(&pCount->pParameterList, (SNode*)pCol);
}
}
return code;
}
static bool isCountTbname(SFunctionNode* pFunc) {
if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) {
return false;
@ -2041,7 +2067,7 @@ static int32_t getGroupByErrorCode(STranslateContext* pCxt) {
if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) {
return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION;
}
return TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN;
return TSDB_CODE_PAR_INVALID_OPTR_USAGE;
}
static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode) {
@ -2114,13 +2140,13 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) {
}
if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc) {
return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt));
return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias);
} else {
return rewriteColToSelectValFunc(pCxt, pNode);
}
}
if (isVectorFunc(*pNode) && isDistinctOrderBy(pCxt)) {
return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt));
return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias);
}
return DEAL_RES_CONTINUE;
}

2
source/libs/parser/src/parUtil.c
View File

@ -174,6 +174,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "Invalid usage of RANGE clause, EVERY clause or FILL clause";
case TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN:
return "No valid function in window query";
case TSDB_CODE_PAR_INVALID_OPTR_USAGE:
return "Invalid usage of expr: %s";
case TSDB_CODE_OUT_OF_MEMORY:
return "Out of memory";
default:

2
source/libs/parser/test/parSelectTest.cpp
View File

@ -286,7 +286,7 @@ TEST_F(ParserSelectTest, interval) {
TEST_F(ParserSelectTest, intervalSemanticCheck) {
useDb("root", "test");
run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN);
run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_INVALID_OPTR_USAGE);
run("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 3 INTERVAL(1d) FILL(NEXT)", TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE);
run("SELECT HISTOGRAM(c1, 'log_bin', '{\"start\": -33,\"factor\": 55,\"count\": 5,\"infinity\": false}', 1) FROM t1 "
"WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' INTERVAL(10s) FILL(NULL)",

2
source/libs/qworker/src/qwDbg.c
View File

@ -271,7 +271,7 @@ void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped) {
}
}
int32_t qwDbgEnableDebug(char *option) {
int32_t qWorkerDbgEnableDebug(char *option) {
if (0 == strcasecmp(option, "lock")) {
gQWDebug.lockEnable = true;
qError("qw lock debug enabled");

2
source/libs/stream/src/streamExec.c
View File

@ -20,7 +20,7 @@
static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) {
int32_t code;
void* exec = pTask->exec.executor;
while(atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
while(pTask->taskLevel == TASK_LEVEL__SOURCE && atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
qError("stream task wait for the end of fill history");
taosMsleep(2);
continue;

22
source/libs/stream/src/streamState.c
View File

@ -130,21 +130,25 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int
char cfgPath[1030];
sprintf(cfgPath, "%s/cfg", statePath);
szPage = szPage < 0 ? 4096 : szPage;
pages = pages < 0 ? 256 : pages;
char cfg[1024];
memset(cfg, 0, 1024);
TdFilePtr pCfgFile = taosOpenFile(cfgPath, TD_FILE_READ);
if (pCfgFile != NULL) {
int64_t size;
int64_t size = 0;
taosFStatFile(pCfgFile, &size, NULL);
taosReadFile(pCfgFile, cfg, size);
sscanf(cfg, "%d\n%d\n", &szPage, &pages);
if (size > 0) {
taosReadFile(pCfgFile, cfg, size);
sscanf(cfg, "%d\n%d\n", &szPage, &pages);
}
} else {
taosMulModeMkDir(statePath, 0755);
pCfgFile = taosOpenFile(cfgPath, TD_FILE_WRITE | TD_FILE_CREATE);
szPage = szPage < 0 ? 4096 : szPage;
pages = pages < 0 ? 256 : pages;
sprintf(cfg, "%d\n%d\n", szPage, pages);
taosWriteFile(pCfgFile, cfg, strlen(cfg));
int32_t code = taosMulModeMkDir(statePath, 0755);
if (code == 0) {
pCfgFile = taosOpenFile(cfgPath, TD_FILE_WRITE | TD_FILE_CREATE);
sprintf(cfg, "%d\n%d\n", szPage, pages);
taosWriteFile(pCfgFile, cfg, strlen(cfg));
}
}
taosCloseFile(&pCfgFile);

4
tests/parallel_test/cases.task
View File

@ -140,6 +140,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data_muti_rows.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/db_tb_name_check.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/InsertFuturets.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_wide_column.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py
@ -864,6 +865,7 @@
,,y,script,./test.sh -f tsim/query/forceFill.sim
,,y,script,./test.sh -f tsim/query/emptyTsRange.sim
,,y,script,./test.sh -f tsim/query/partitionby.sim
,,y,script,./test.sh -f tsim/query/tableCount.sim
,,y,script,./test.sh -f tsim/qnode/basic1.sim
,,y,script,./test.sh -f tsim/snode/basic1.sim
,,y,script,./test.sh -f tsim/mnode/basic1.sim
@ -1096,7 +1098,7 @@
,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py
#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py
,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R
,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R
#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R
#docs-examples test
,,n,docs-examples-test,bash python.sh

168
tests/pytest/util/autogen.py Normal file
View File

@ -0,0 +1,168 @@
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
import threading
import random
import string
import time
#
# Auto Gen class
#
class AutoGen:
def __init__(self):
self.ts = 1600000000000
self.batch_size = 100
seed = time.clock_gettime(time.CLOCK_REALTIME)
random.seed(seed)
# set start ts
def set_start_ts(self, ts):
self.ts = ts
# set batch size
def set_batch_size(self, batch_size):
self.batch_size = batch_size
# _columns_sql
def gen_columns_sql(self, pre, cnt, binary_len, nchar_len):
types = [
'timestamp',
'tinyint',
'smallint',
'tinyint unsigned',
'smallint unsigned',
'int',
'bigint',
'int unsigned',
'bigint unsigned',
'float',
'double',
'bool',
f'varchar({binary_len})',
f'nchar({nchar_len})'
]
sqls = ""
metas = []
for i in range(cnt):
colname = f"{pre}{i}"
if i < len(types):
sel = i
else:
sel = random.randint(0, len(types)-1)
coltype = types[sel]
sql = f"{colname} {coltype}"
if sqls != "":
sqls += ","
sqls += sql
metas.append(sel)
return metas, sqls;
# gen tags data
def gen_data(self, i, marr):
datas = ""
for c in marr:
data = ""
if c == 0 : # timestamp
data = "%d" % (self.ts + i)
elif c <= 4 : # small
data = "%d"%(i%128)
elif c <= 8 : # int
data = f"{i}"
elif c <= 10 : # float
data = "%f"%(i+i/1000)
elif c <= 11 : # bool
data = "%d"%(i%2)
elif c == 12 : # binary
data = '"' + self.random_string(self.bin_len) + '"'
elif c == 13 : # binary
data = '"' + self.random_string(self.nch_len) + '"'
if datas != "":
datas += ","
datas += data
return datas
# generate specail wide random string
def random_string(self, count):
letters = string.ascii_letters
return ''.join(random.choice(letters) for i in range(count))
# create db
def create_db(self, dbname):
self.dbname = dbname
tdSql.execute(f'create database {dbname}')
tdSql.execute(f'use {dbname}')
# create table or stable
def create_stable(self, stbname, tag_cnt, column_cnt, binary_len, nchar_len):
self.bin_len = binary_len
self.nch_len = nchar_len
self.stbname = stbname
self.mtags, tags = self.gen_columns_sql("t", tag_cnt, binary_len, nchar_len)
self.mcols, cols = self.gen_columns_sql("c", column_cnt - 1, binary_len, nchar_len)
sql = f"create table {stbname} (ts timestamp, {cols}) tags({tags})"
tdSql.execute(sql)
# create child table
def create_child(self, stbname, prename, cnt):
self.child_cnt = cnt
self.child_name = prename
for i in range(cnt):
tags_data = self.gen_data(i, self.mtags)
sql = f"create table {prename}{i} using {stbname} tags({tags_data})"
tdSql.execute(sql)
tdLog.info(f"create child tables {cnt} ok")
def insert_data_child(self, child_name, cnt, batch_size, step):
values = ""
print("insert child data")
ts = self.ts
# loop do
for i in range(cnt):
value = self.gen_data(i, self.mcols)
ts += step
values += f"({ts},{value}) "
if batch_size == 1 or (i > 0 and i % batch_size == 0) :
sql = f"insert into {child_name} values {values}"
tdSql.execute(sql)
if batch_size > 40:
tdLog.info(f" insert data i={i}")
values = ""
# end batch
if values != "":
sql = f"insert into {child_name} values {values}"
tdSql.execute(sql)
tdLog.info(f" insert data i={i}")
values = ""
tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}")
# insert data
def insert_data(self, cnt):
for i in range(self.child_cnt):
name = f"{self.child_name}{i}"
self.insert_data_child(name, cnt, self.batch_size, 1)
tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}")
# insert same timestamp to all childs
def insert_samets(self, cnt):
for i in range(self.child_cnt):
name = f"{self.child_name}{i}"
self.insert_data_child(name, cnt, self.batch_size, 0)
tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}")

98
tests/pytest/util/sql.py
View File

@ -23,6 +23,29 @@ import pandas as pd
from util.log import *
from util.constant import *
# from datetime import timezone
from tzlocal import get_localzone
import pytz
import time
def _locaTzTimeStamp(utctimestamp):
tz = get_localzone()
temptz = str(tz)
localtz = pytz.timezone(temptz)
defUtctimestamp=1035640800
local_dt = datetime.datetime(2002, 10, 27, 6, 0, 0, tzinfo=localtz)
defLocaltimestamp = time.mktime(local_dt.timetuple())
deltaTzTime = int(defLocaltimestamp)-int(defUtctimestamp)
temp = int(str(utctimestamp)[0:10])
tempOther = str(utctimestamp)[10-len(str(utctimestamp)):]
localtemp = deltaTzTime + temp
localAll = str(localtemp) + str(tempOther)
localtimestamp = int(localAll)
print(f"local timezone is {localtimestamp}, Deltel time is {deltaTzTime}s")
return localtimestamp
def _parse_datetime(timestr):
try:
return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f')
@ -227,28 +250,80 @@ class TDSql:
self.checkRowCol(row, col)
return self.cursor.istype(col, dataType)
def checkData(self, row, col, data):
if row >= self.queryRows:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row+1, self.queryRows)
tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args)
if col >= self.queryCols:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, col+1, self.queryCols)
tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args)
self.checkRowCol(row, col)
if self.queryResult[row][col] != data:
if self.cursor.istype(col, "TIMESTAMP"):
# suppose user want to check nanosecond timestamp if a longer data passed
if (len(data) >= 28):
if pd.to_datetime(self.queryResult[row][col]) == pd.to_datetime(data):
tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
# suppose user want to check nanosecond timestamp if a longer data passed``
if isinstance(data,str) :
if (len(data) >= 28):
resultData = _locaTzTimeStamp(self.queryResult[row][col])
if pd.to_datetime(resultData) == pd.to_datetime(data):
# tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{pd.to_datetime(resultData)} == expect:{data}")
tdLog.info("check successfully")
else:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
else:
if self.queryResult[row][col] == _parse_datetime(data):
# tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
tdLog.info("check successfully")
else:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
return
elif isinstance(data,int) :
if len(str(data)) == 16 :
unitTime = 'us'
elif len(str(data)) == 13 :
unitTime = 'ms'
elif len(str(data)) == 19 :
unitTime = 'ns'
else:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
resultData = pd.to_datetime(_locaTzTimeStamp(data),unit=unitTime)
if resultData == self.queryResult[row][col] :
# tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{resultData}")
tdLog.info("check successfully")
else:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
return
else:
if self.queryResult[row][col] == _parse_datetime(data):
tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
return
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
if str(self.queryResult[row][col]) == str(data):
tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
# tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
tdLog.info("check successfully")
return
elif isinstance(data, float):
if abs(data) >= 1 and abs((self.queryResult[row][col] - data) / data) <= 0.000001:
tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
# tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
tdLog.info("check successfully")
elif abs(data) < 1 and abs(self.queryResult[row][col] - data) <= 0.000001:
tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
# tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
tdLog.info("check successfully")
else:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
@ -258,8 +333,7 @@ class TDSql:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}")
tdLog.info("check successfully")
# return true or false replace exit, no print out
def checkRowColNoExit(self, row, col):

1
tests/requirements.txt
View File

@ -8,3 +8,4 @@ distro
requests
pexpect
faker
pyopenssl

8
tests/script/tsim/db/alter_option.sim
View File

@ -285,12 +285,12 @@ sql_error alter database db keep -1
print ============== modify minrows
sql_error alter database db minrows 8
sql_error alter database db minrows 8000
sql_error alter database db minrows 8001
sql_error alter database db minrows 8000000
sql_error alter database db minrows 8001000
print ============== modify maxrows
sql_error alter database db maxrows 1000
sql_error alter database db maxrows 2000
sql_error alter database db maxrows 10000001
sql_error alter database db maxrows 20000000
sql_error alter database db maxrows 11 # equal minrows
sql_error alter database db maxrows 10 # little than minrows

4
tests/script/tsim/db/create_all_options.sim
View File

@ -298,11 +298,11 @@ sql drop database db
sql_error create database db MAXROWS -1
sql_error create database db MAXROWS 0
sql_error create database db MAXROWS 199
sql_error create database db MAXROWS 10001
sql_error create database db MAXROWS 10000001
sql_error create database db MINROWS -1
sql_error create database db MINROWS 0
sql_error create database db MINROWS 9
sql_error create database db MINROWS 1001
sql_error create database db MINROWS 1000001
sql_error create database db MAXROWS 500 MINROWS 1000
print ====> PRECISION ['ms' | 'us' | 'ns', default: ms]

3
tests/script/tsim/parser/col_arithmetic_operation.sim
View File

@ -9,7 +9,7 @@ $dbPrefix = ca_db
$tbPrefix = ca_tb
$stbPrefix = ca_stb
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
$ts0 = 1537146000000
$delta = 600000
@ -18,6 +18,7 @@ $i = 0
$db = $dbPrefix . $i
$stb = $stbPrefix . $i
print drop database $db -x step1
sql drop database $db -x step1
step1:
sql create database $db

33
tests/script/tsim/parser/col_arithmetic_query.sim
View File

@ -4,7 +4,7 @@
$dbPrefix = ca_db
$tbPrefix = ca_tb
$stbPrefix = ca_stb
$rowNum = 10000
$rowNum = 1000
$i = 0
$db = $dbPrefix . $i
@ -33,7 +33,7 @@ endi
# asc/desc order [d.2] ======================================================
sql select c1 *( 2 / 3 ), c1/c1 from $tb order by ts asc;
if $rows != 10000 then
if $rows != 1000 then
return -1
endi
if $data00 != 0.000000000 then
@ -57,7 +57,7 @@ if $data91 != 1.000000000 then
endi
sql select (c1 * 2) % 7.9, c1*1, c1*1*1, c1*c1, c1*c1*c1 from $tb order by ts desc;
if $rows != 10000 then
if $rows != 1000 then
return -1
endi
if $data00 != 2.200000000 then
@ -151,7 +151,7 @@ sql select top(c1,1) - 88 from $tb
# all data types [d.6] ================================================================
sql select c2-c1*1.1, c3/c2, c4*c3, c5%c4, (c6+c4)%22, c2-c2 from $tb
if $rows != 10000 then
if $rows != 1000 then
return -1
endi
if $data00 != 0.000000000 then
@ -221,7 +221,7 @@ if $data90 != 76.000000000 then
return -1
endi
sql select c4 / 99.123 from $tb limit 10 offset 9999;
sql select c4 / 99.123 from $tb limit 10 offset 999;
if $rows != 1 then
return -1
endi
@ -237,7 +237,7 @@ sql_error select c2-c2, c3-c4, c5%c3 from $tb fill(value, 12);
# constant column. [d.13]==============================================================
sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb
if $rows != 10000 then
if $rows != 1000 then
return -1
endi
if $data00 != 0 then
@ -261,7 +261,7 @@ endi
# column value filter [d.14]===========================================================
sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb where c1<2
if $rows != 2000 then
if $rows != 200 then
return -1
endi
if $data00 != 0 then
@ -338,13 +338,14 @@ sql select (count(c1) * 2) % 7.9, (count(c1) * 2), ( count(1)*2) from $stb
if $rows != 1 then
return -1
endi
if $data00 != 1.800000000 then
print $data00
if $data00 != 6.500000000 then
return -1
endi
if $data01 != 100000.000000000 then
if $data01 != 10000.000000000 then
return -1
endi
if $data02 != 200000.000000000 then
if $data02 != 20000.000000000 then
return -1
endi
@ -371,7 +372,7 @@ endi
if $data00 != 0.000000000 then
return -1
endi
if $data01 != 225000.000000000 then
if $data01 != 22500.000000000 then
return -1
endi
if $data02 != 8.077777778 then
@ -383,7 +384,7 @@ endi
if $data04 != 0.444444444 then
return -1
endi
if $data05 != 450000.000000000 then
if $data05 != 45000.000000000 then
return -1
endi
@ -484,10 +485,10 @@ endi
if $data10 != 0.000000000 then
return -1
endi
if $data20 != 0.997600000 then
if $data20 != 0.976000000 then
return -1
endi
if $data90 != 7.980800000 then
if $data90 != 7.808000000 then
return -1
endi
@ -496,7 +497,7 @@ sql select first(c6) - sum(c6) + 12 from $stb limit 12 offset 0;
if $rows != 1 then
return -1
endi
if $data00 != -449988.000000000 then
if $data00 != -44988.000000000 then
return -1
endi
@ -546,7 +547,7 @@ endi
# interval query [d.17]===============================================================
sql select avg(c2)*count(c2), sum(c3)-first(c3), last(c4)+9 from ca_stb0 interval(1s)
if $rows != 10000 then
if $rows != 1000 then
return -1
endi

2
tests/script/tsim/parser/commit.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = sc_db
$tbPrefix = sc_tb
$stbPrefix = sc_stb
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
$loops = 5
$log = 1

80
tests/script/tsim/parser/groupby.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = group_db
$tbPrefix = group_tb
$mtPrefix = group_mt
$tbNum = 8
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
print =============== groupby.sim
@ -88,7 +88,7 @@ if $row != 20 then
return -1
endi
if $data00 != 100 then
if $data00 != 10 then
return -1
endi
@ -96,7 +96,7 @@ if $data01 != 0 then
return -1
endi
if $data10 != 100 then
if $data10 != 10 then
return -1
endi
@ -186,7 +186,7 @@ if $data04 != 0.00000 then
return -1
endi
if $data10 != 100 then
if $data10 != 10 then
return -1
endi
@ -227,7 +227,7 @@ if $row != 20 then
return -1
endi
if $data00 != 800 then
if $data00 != 80 then
return -1
endi
@ -235,7 +235,7 @@ if $data01 != 0 then
return -1
endi
if $data10 != 800 then
if $data10 != 80 then
return -1
endi
@ -243,7 +243,7 @@ if $data11 != 1 then
return -1
endi
if $data90 != 800 then
if $data90 != 80 then
return -1
endi
@ -297,7 +297,7 @@ if $data00 != $data03 then
return -1
endi
if $data01 != @70-01-01 08:01:49.900@ then
if $data01 != @70-01-01 08:01:40.900@ then
return -1
endi
@ -305,7 +305,7 @@ if $data02 != @70-01-01 08:01:40.000@ then
return -1
endi
if $data07 != 800 then
if $data07 != 80 then
return -1
endi
@ -313,7 +313,7 @@ if $data10 != $data13 then
return -1
endi
if $data11 != @70-01-01 08:01:49.901@ then
if $data11 != @70-01-01 08:01:40.901@ then
return -1
endi
@ -321,7 +321,7 @@ if $data12 != @70-01-01 08:01:40.001@ then
return -1
endi
if $data17 != 800 then
if $data17 != 80 then
return -1
endi
@ -329,7 +329,7 @@ if $data90 != $data93 then
return -1
endi
if $data91 != @70-01-01 08:01:49.909@ then
if $data91 != @70-01-01 08:01:40.909@ then
return -1
endi
@ -337,11 +337,11 @@ if $data92 != @70-01-01 08:01:40.009@ then
return -1
endi
if $data97 != 800 then
if $data97 != 80 then
return -1
endi
if $data95 != 7200 then
if $data95 != 720 then
return -1
endi
@ -358,7 +358,7 @@ if $data00 != 0 then
return -1
endi
if $data11 != 800 then
if $data11 != 80 then
return -1
endi
@ -372,7 +372,7 @@ if $data00 != 0 then
return -1
endi
if $data01 != @70-01-01 08:01:49.900@ then
if $data01 != @70-01-01 08:01:40.900@ then
return -1
endi
@ -389,13 +389,13 @@ if $data04 != 0 then
return -1
endi
if $data06 != 100 then
if $data06 != 10 then
return -1
endi
sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 order by c4;
if $rows != 10000 then
if $rows != 1000 then
return -1
endi
@ -420,13 +420,13 @@ if $rows != 1 then
return -1
endi
sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 slimit 20 soffset 9990;
sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 slimit 20 soffset 990;
if $rows != 10 then
return -1
endi
sql select count(*),first(ts),last(ts),min(c3),max(c3),sum(c3),avg(c3),sum(c4)/count(c4) from group_tb1 group by c4;
if $rows != 10000 then
if $rows != 1000 then
return -1
endi
@ -441,32 +441,32 @@ if $rows != 100 then
return -1
endi
if $data00 != 100 then
if $data00 != 10 then
return -1
endi
if $data01 != 495000 then
if $data01 != 4500 then
return -1
endi
if $data02 != 100 then
if $data02 != 10 then
return -1
endi
if $data03 != 4950.000000000 then
print expect 4950.000000000 , acutal $data03
if $data03 != 450.000000000 then
print expect 450.000000000 , acutal $data03
return -1
endi
if $data10 != 100 then
if $data10 != 10 then
return -1
endi
if $data11 != 495100 then
if $data11 != 4510 then
return -1
endi
if $data13 != 4951.000000000 then
if $data13 != 451.000000000 then
return -1
endi
@ -481,19 +481,19 @@ if $rows != 1 then
return -1
endi
#if $data00 != 79200.000000000 then
#if $data00 != 2160.000000000 then
# return -1
#endi
#if $data01 != @binary99@ then
#if $data01 != @binary27@ then
# return -1
#endi
#if $data02 != 99.000000000 then
#if $data02 != 27.000000000 then
# return -1
#endi
#if $data03 != 99.000000000 then
#if $data03 != 27.000000000 then
# return -1
#endi
@ -503,7 +503,7 @@ if $rows != 100 then
return -1
endi
if $data00 != 4851.000000000 then
if $data00 != 441.000000000 then
return -1
endi
@ -511,19 +511,19 @@ if $data01 != 0 then
return -1
endi
if $data02 != 9900 then
if $data02 != 900 then
return -1
endi
if $data03 != 4950.000000000 then
if $data03 != 450.000000000 then
return -1
endi
if $data04 != 2886.607004772 then
if $data04 != 287.228132327 then
return -1
endi
if $data10 != 4852.000000000 then
if $data10 != 442.000000000 then
return -1
endi
@ -531,15 +531,15 @@ if $data11 != 1 then
return -1
endi
if $data12 != 9901 then
if $data12 != 901 then
return -1
endi
if $data13 != 4951.000000000 then
if $data13 != 451.000000000 then
return -1
endi
if $data14 != 2886.607004772 then
if $data14 != 287.228132327 then
return -1
endi

2
tests/script/tsim/parser/interp.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = intp_db
$tbPrefix = intp_tb
$stbPrefix = intp_stb
$tbNum = 4
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
$ts0 = 1537146000000
$delta = 600000

6
tests/script/tsim/parser/join_manyblocks.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = join_m_db
$tbPrefix = join_tb
$mtPrefix = join_mt
$tbNum = 3
$rowNum = 20000
$rowNum = 2000
$totalNum = $tbNum * $rowNum
print =============== join_manyBlocks.sim
@ -78,8 +78,8 @@ print ==============> td-3313
sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1;
print $row
if $row != 60000 then
print expect 60000, actual: $row
if $row != 6000 then
print expect 6000, actual: $row
return -1
endi

2
tests/script/tsim/parser/limit1.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = lm1_db
$tbPrefix = lm1_tb
$stbPrefix = lm1_stb
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
$ts0 = 1537146000000
$delta = 600000

5
tests/script/tsim/parser/limit1_stb.sim
View File

@ -4,7 +4,7 @@ $dbPrefix = lm1_db
$tbPrefix = lm1_tb
$stbPrefix = lm1_stb
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
$ts0 = 1537146000000
$delta = 600000
@ -453,13 +453,14 @@ endi
### [TBASE-361]
$offset = $rowNum / 2
$offset = $offset + 1
print === select _wstart, max(c1), min(c2), avg(c3), count(c4), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 8 interval(5m) limit $offset offset $offset
sql select _wstart, max(c1), min(c2), avg(c3), count(c4), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 8 interval(5m) limit $offset offset $offset
$val = $rowNum - $offset
if $rows != $val then
print expect $val, actual:$rows
return -1
endi
if $data00 != @18-10-22 02:30:00.000@ then
if $data00 != @18-09-20 20:30:00.000@ then
return -1
endi
if $data01 != 1 then

36
tests/script/tsim/parser/limit1_tb.sim
View File

@ -4,7 +4,7 @@ $dbPrefix = lm1_db
$tbPrefix = lm1_tb
$stbPrefix = lm1_stb
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
$ts0 = 1537146000000
$delta = 600000
@ -13,7 +13,7 @@ $i = 0
$db = $dbPrefix . $i
$stb = $stbPrefix . $i
print ====== use db
print ====== use $db
sql use $db
##### select from table
@ -664,9 +664,9 @@ endi
if $data21 != 4.027681991 then
return -1
endi
print select _wstart, count(c1), count(c2), count(c3), count(c4), count(c5), count(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(27m)
sql select _wstart, count(c1), count(c2), count(c3), count(c4), count(c5), count(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(27m)
if $rows != 3704 then
if $rows != 371 then
return -1
endi
if $data01 != 2 then
@ -994,11 +994,14 @@ endi
$offset = $rowNum / 10
$offset = $offset * 3
$offset = $offset - 1
print === select * from $tb where c1 < 5 and c1 > 1 order by ts asc limit 3 offset $offset
sql select * from $tb where c1 < 5 and c1 > 1 order by ts asc limit 3 offset $offset
if $rows != 1 then
return -1
endi
if $data00 != @18-11-25 18:40:00.000@ then
print $data00
if $data00 != @18-09-24 06:40:00.000@ then
return -1
endi
if $data01 != 4 then
@ -1012,19 +1015,19 @@ sql select * from $tb order by ts desc limit 5
if $rows != 5 then
return -1
endi
if $data00 != @18-11-25 19:30:00.000@ then
if $data00 != @18-09-24 07:30:00.000@ then
return -1
endi
if $data01 != 9 then
return -1
endi
if $data10 != @18-11-25 19:20:00.000@ then
if $data10 != @18-09-24 07:20:00.000@ then
return -1
endi
if $data12 != 8 then
return -1
endi
if $data20 != @18-11-25 19:10:00.000@ then
if $data20 != @18-09-24 07:10:00.000@ then
return -1
endi
if $data23 != 7.00000 then
@ -1048,7 +1051,7 @@ endi
if $data39 != nchar6 then
return -1
endi
if $data40 != @18-11-25 18:50:00.000@ then
if $data40 != @18-09-24 06:50:00.000@ then
return -1
endi
@ -1056,19 +1059,19 @@ sql select * from $tb order by ts desc limit 5 offset 1
if $rows != 5 then
return -1
endi
if $data00 != @18-11-25 19:20:00.000@ then
if $data00 != @18-09-24 07:20:00.000@ then
return -1
endi
if $data01 != 8 then
return -1
endi
if $data10 != @18-11-25 19:10:00.000@ then
if $data10 != @18-09-24 07:10:00.000@ then
return -1
endi
if $data12 != 7 then
return -1
endi
if $data20 != @18-11-25 19:00:00.000@ then
if $data20 != @18-09-24 07:00:00.000@ then
return -1
endi
if $data23 != 6.00000 then
@ -1092,12 +1095,13 @@ endi
if $data39 != nchar5 then
return -1
endi
if $data40 != @18-11-25 18:40:00.000@ then
if $data40 != @18-09-24 06:40:00.000@ then
return -1
endi
$offset = $rowNum
$offset = $offset - 2
print ==== select * from $tb order by ts desc limit 5 offset $offset
sql select * from $tb order by ts desc limit 5 offset $offset
if $rows != 2 then
return -1
@ -1119,16 +1123,16 @@ sql select * from $tb where c1 < 8 order by ts desc limit 3 offset 2
if $rows != 3 then
return -1
endi
if $data00 != @18-11-25 18:50:00.000@ then
if $data00 != @18-09-24 06:50:00.000@ then
return -1
endi
if $data01 != 5 then
return -1
endi
if $data10 != @18-11-25 18:40:00.000@ then
if $data10 != @18-09-24 06:40:00.000@ then
return -1
endi
if $data20 != @18-11-25 18:30:00.000@ then
if $data20 != @18-09-24 06:30:00.000@ then
return -1
endi
if $data12 != 4 then

10
tests/script/tsim/parser/nestquery.sim
View File

@ -8,7 +8,7 @@ print ======================== dnode1 start
$dbPrefix = nest_db
$tbPrefix = nest_tb
$mtPrefix = nest_mt
$tbNum = 10
$tbNum = 3
$rowNum = 10000
$totalNum = $tbNum * $rowNum
@ -23,7 +23,7 @@ sql create database if not exists $db
sql use $db
sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int)
$half = $tbNum / 2
$half = 2
$i = 0
while $i < $half
@ -69,7 +69,7 @@ sql select count(*) from (select count(*) from nest_mt0 group by tbname)
if $rows != 1 then
return -1
endi
if $data00 != 10 then
if $data00 != 4 then
return -1
endi
@ -77,7 +77,7 @@ sql select count(*) from (select count(*) from nest_mt0 partition by tbname inte
if $rows != 1 then
return -1
endi
if $data00 != 170 then
if $data00 != 68 then
return -1
endi
@ -85,7 +85,7 @@ sql select sum(a) from (select count(*) a from nest_mt0 partition by tbname inte
if $rows != 1 then
return -1
endi
if $data00 != 100000 then
if $data00 != 40000 then
return -1
endi

84
tests/script/tsim/parser/projection_limit_offset.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = group_db
$tbPrefix = group_tb
$mtPrefix = group_mt
$tbNum = 8
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
print =============== projection_limit_offset.sim
@ -81,199 +81,199 @@ $ts2 = $tb2 . .ts
sql select ts from group_mt0
print $rows
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 0;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 0;
if $rows != 4008 then
print expect 4008, actual:$rows
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 1;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 1;
if $rows != 4007 then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 101;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 101;
print $rows
if $rows != 3907 then
return -1
endi
if $data00 != @70-01-01 08:01:43.101@ then
if $data00 != @70-01-01 08:01:40.101@ then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 902;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 902;
if $rows != 3106 then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 400;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 400;
if $rows != 3608 then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 4007;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 4007;
if $rows != 1 then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 2000 offset 4008;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 2000 offset 4008;
if $rows != 0 then
return -1
endi
#==================================order by desc, multi vnode, limit/offset===================================
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 0;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 0;
if $rows != 4008 then
return -1
endi
if $data00 != @70-01-01 08:01:43.500@ then
if $data00 != @70-01-01 08:01:40.500@ then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 1;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 1;
if $rows != 4007 then
return -1
endi
if $data00 != @70-01-01 08:01:43.500@ then
if $data00 != @70-01-01 08:01:40.500@ then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 101;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 101;
print $rows
if $rows != 3907 then
return -1
endi
if $data00 != @70-01-01 08:01:43.488@ then
if $data00 != @70-01-01 08:01:40.488@ then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 902;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 902;
if $rows != 3106 then
return -1
endi
if $data00 != @70-01-01 08:01:43.388@ then
if $data00 != @70-01-01 08:01:40.388@ then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 400;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 400;
if $rows != 3608 then
return -1
endi
if $data00 != @70-01-01 08:01:43.450@ then
if $data00 != @70-01-01 08:01:40.450@ then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 4007;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 4007;
if $rows != 1 then
return -1
endi
if $data00 != @70-01-01 08:01:43.000@ then
if $data00 != @70-01-01 08:01:40.000@ then
return -1
endi
sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 2000 offset 4008;
sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 2000 offset 4008;
if $rows != 0 then
return -1
endi
#=================================single value filter======================================
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 0;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 0;
if $row != 8 then
return -1
endi
if $data00 != @70-01-01 08:01:43.000@ then
if $data00 != @70-01-01 08:01:40.000@ then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 1;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 1;
if $row != 7 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 2;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 2;
if $row != 6 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 4;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 4;
if $row != 4 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 7;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 7;
if $row != 1 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 8;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 8;
if $row != 0 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 9;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 9;
if $row != 0 then
return -1
endi
#===============================single value filter, order by desc============================
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 0;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 0;
if $row != 8 then
return -1
endi
if $data00 != @70-01-01 08:01:43.000@ then
if $data00 != @70-01-01 08:01:40.000@ then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 1;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 1;
if $row != 7 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 2;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 2;
if $row != 6 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 4;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 4;
if $row != 4 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 7;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 7;
if $row != 1 then
return -1
endi
if $data00 != @70-01-01 08:01:43.000@ then
if $data00 != @70-01-01 08:01:40.000@ then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 8;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 8;
if $row != 0 then
return -1
endi
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 9;
sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 9;
if $row != 0 then
return -1
endi
#[tbase-695]
sql select ts,tbname from group_mt0 where ts>='1970-01-01 8:1:40' and ts<'1970-1-1 8:1:45' and c1<99999999 limit 100000 offset 5000
sql select ts,tbname from group_mt0 where ts>='1970-01-01 8:1:40' and ts<'1970-1-1 8:1:40.500' and c1<99999999 limit 10000 offset 500
print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
if $row != 35000 then
if $row != 3500 then
return -1
endi
@ -301,7 +301,7 @@ print tbase-722
sql select spread(ts) from group_tb0;
print $data00
if $data00 != 9999.000000000 then
if $data00 != 999.000000000 then
return -1
endi

4
tests/script/tsim/parser/selectResNum.sim
View File

@ -7,10 +7,10 @@ $dbPrefix = sc_db
$tbPrefix = sc_tb
$stbPrefix = sc_stb
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
$loops = 200000
$log = 10000
$log = 1000
$ts0 = 1537146000000
$delta = 600000
print ========== selectResNum.sim

102
tests/script/tsim/parser/sliding.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = sliding_db
$tbPrefix = sliding_tb
$mtPrefix = sliding_mt
$tbNum = 8
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
print =============== sliding.sim
@ -88,25 +88,25 @@ $ts1 = $tb1 . .ts
$ts2 = $tb2 . .ts
print ===============================interval_sliding query
sql select _wstart, count(*) from sliding_tb0 interval(30s) sliding(30s);
sql select _wstart, count(*) from sliding_tb0 interval(3s) sliding(3s);
if $row != 10 then
return -1
endi
if $data00 != @00-01-01 00:00:00.000@ then
return -1
endi
if $data01 != 1000 then
if $data01 != 100 then
return -1
endi
if $data10 != @00-01-01 00:00:30.000@ then
if $data10 != @00-01-01 00:00:03.000@ then
return -1
endi
if $data11 != 1000 then
if $data11 != 100 then
return -1
endi
sql select _wstart, stddev(c1) from sliding_tb0 interval(10a) sliding(10a);
if $row != 10000 then
if $row != 1000 then
return -1
endi
if $data00 != @00-01-01 00:00:00.000@ then
@ -123,10 +123,10 @@ if $data91 != 0.000000000 then
endi
sql select _wstart, stddev(c1),count(c2),first(c3),last(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc;
if $row != 10000 then
if $row != 1000 then
return -1
endi
if $data00 != @00-01-01 00:04:59.970@ then
if $data00 != @00-01-01 00:00:29.970@ then
return -1
endi
if $data01 != 0.000000000 then
@ -141,7 +141,7 @@ endi
if $data04 != 99 then
return -1
endi
if $data90 != @00-01-01 00:04:59.700@ then
if $data90 != @00-01-01 00:00:29.700@ then
return -1
endi
if $data91 != 0.000000000 then
@ -157,41 +157,41 @@ if $data94 != 90 then
return -1
endi
sql select _wstart, count(c2),last(c4) from sliding_tb0 interval(30s) sliding(10s) order by _wstart asc;
sql select _wstart, count(c2),last(c4) from sliding_tb0 interval(3s) sliding(1s) order by _wstart asc;
if $row != 32 then
return -1
endi
if $data00 != @99-12-31 23:59:40.000@ then
print expect 12-31 23:59:40.000, actual: $data00
if $data00 != @99-12-31 23:59:58.000@ then
print expect 12-31 23:59:58.000, actual: $data00
return -1
endi
if $data01 != 334 then
if $data01 != 34 then
return -1
endi
if $data02 != 33 then
return -1
endi
sql select _wstart, count(c2),stddev(c3),first(c4),last(c4) from sliding_tb0 where ts>'2000-01-01 0:0:0' and ts<'2000-1-1 0:0:31' interval(30s) sliding(30s) order by _wstart asc;
sql select _wstart, count(c2),stddev(c3),first(c4),last(c4) from sliding_tb0 where ts>'2000-01-01 0:0:0' and ts<'2000-1-1 0:0:4' interval(3s) sliding(3s) order by _wstart asc;
if $row != 2 then
return -1
endi
if $data04 != 99 then
return -1
endi
if $data01 != 999 then
if $data01 != 99 then
return -1
endi
if $data02 != 28.837977152 then
if $data02 != 28.577380332 then
return -1
endi
#interval offset + limit
sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc limit 10 offset 990;
sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc limit 10 offset 90;
if $row != 10 then
return -1
endi
if $data00 != @00-01-01 00:04:30.270@ then
if $data00 != @00-01-01 00:00:27.270@ then
return -1
endi
if $data01 != 1 then
@ -203,7 +203,7 @@ endi
if $data03 != 0.000000000 then
return -1
endi
if $data90 != @00-01-01 00:04:30.000@ then
if $data90 != @00-01-01 00:00:27.000@ then
return -1
endi
if $data91 != 1 then
@ -217,43 +217,43 @@ if $data93 != 0.000000000 then
endi
#interval offset test
sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(30s) order by _wstart asc limit 1000 offset 1;
sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(3s) order by _wstart asc limit 100 offset 1;
if $row != 9 then
return -1
endi
if $data00 != @00-01-01 00:00:30.000@ then
if $data00 != @00-01-01 00:00:03.000@ then
return -1
endi
if $data01 != 1000 then
if $data01 != 100 then
return -1
endi
if $data02 != 99 then
return -1
endi
if $data80 != @00-01-01 00:04:30.000@ then
if $data80 != @00-01-01 00:00:27.000@ then
return -1
endi
if $data81 != 1000 then
if $data81 != 100 then
return -1
endi
sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 where ts>'2000-1-1 0:0:0' and ts<'2000-1-1 0:0:31' interval(30s) sliding(30s) order by _wstart asc limit 1000 offset 0;
sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 where ts>'2000-1-1 0:0:0' and ts<'2000-1-1 0:0:4' interval(3s) sliding(3s) order by _wstart asc limit 100 offset 0;
if $row != 2 then
return -1
endi
if $data00 != @00-01-01 00:00:00.000@ then
return -1
endi
if $data01 != 999 then
if $data01 != 99 then
return -1
endi
if $data02 != 99 then
return -1
endi
if $data03 != 28.837977152 then
if $data03 != 28.577380332 then
return -1
endi
if $data10 != @00-01-01 00:00:30.000@ then
if $data10 != @00-01-01 00:00:03.000@ then
return -1
endi
if $data11 != 34 then
@ -266,14 +266,14 @@ if $data13 != 9.810708435 then
return -1
endi
sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 1;
sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 1;
if $row != 15 then
return -1
endi
if $data00 != @00-01-01 00:00:00.000@ then
return -1
endi
if $data01 != 1000 then
if $data01 != 100 then
return -1
endi
if $data02 != 99 then
@ -282,85 +282,85 @@ endi
if $data03 != 28.866070048 then
return -1
endi
if $data90 != @00-01-01 00:03:00.000@ then
if $data90 != @00-01-01 00:00:18.000@ then
return -1
endi
if $data91 != 1000 then
if $data91 != 100 then
return -1
endi
if $data92 != 99 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 5;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 5;
if $row != 11 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 6;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 6;
if $row != 10 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 7;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 7;
if $row != 9 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 8;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 8;
if $row != 8 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 9;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 9;
if $row != 7 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 10;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 10;
if $row != 6 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 11;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 11;
if $row != 5 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 12;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 12;
if $row != 4 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 13;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 13;
if $row != 3 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 14;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 14;
if $row != 2 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 15;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 15;
if $row != 1 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 16;
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 16;
if $row != 0 then
return -1
endi
sql select _wstart, count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) order by _wstart desc;
sql select _wstart, count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(3s) order by _wstart desc;
if $row != 10 then
return -1
endi
#00-01-01 00:04:30.000| 10| 0| 0.000000000| 0.000000000|
if $data00 != @00-01-01 00:04:30.000@ then
#00-01-01 00:00:27.000 | 1 | 0 | 0.000000000 | 0.000000000 |
if $data00 != @00-01-01 00:00:27.000@ then
return -1
endi
if $data01 != 10 then
if $data01 != 1 then
return -1
endi
if $data02 != 0 then
@ -370,18 +370,18 @@ if $data03 != 0.000000000 then
return -1
endi
sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by _wstart desc limit 1 offset 15;
sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(3s) sliding(2s) order by _wstart desc limit 1 offset 14;
if $row != 1 then
return -1
endi
sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by _wstart desc limit 1 offset 16;
sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(3s) sliding(2s) order by _wstart desc limit 1 offset 15;
if $row != 0 then
return -1
endi
sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) order by _wstart desc limit 10 offset 2;
if $data00 != @00-01-01 00:04:59.910@ then
if $data00 != @00-01-01 00:00:29.910@ then
return -1
endi

22
tests/script/tsim/parser/union.sim
View File

@ -8,7 +8,7 @@ $tbPrefix = union_tb
$tbPrefix1 = union_tb_
$mtPrefix = union_mt
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
print =============== union.sim
@ -65,7 +65,7 @@ sql create table $mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c
$j = 0
$t = 1578203484000
$rowNum = 1000
$rowNum = 100
$tbNum = 5
$i = 0
@ -117,7 +117,7 @@ sql_error (select c1 from union_tb0 limit 1 union all select c1 from union_tb1 l
# sql with parenthese
sql (select c1 from union_tb0)
if $rows != 10000 then
if $rows != 1000 then
return -1
endi
if $data00 != 0 then
@ -187,7 +187,7 @@ endi
if $data10 != 1 then
return -1
endi
if $data20 != 4950000 then
if $data20 != 495000 then
return -1
endi
@ -202,7 +202,7 @@ endi
if $data10 != 1 then
return -1
endi
if $data20 != 495000 then
if $data20 != 49500 then
return -1
endi
@ -211,7 +211,7 @@ sql (select count(*) as c from union_tb0, union_tb1 where union_tb0.ts=union_tb1
if $rows != 11 then
return -1
endi
if $data00 != 10000 then
if $data00 != 1000 then
return -1
endi
if $data10 != 9 then
@ -227,7 +227,7 @@ endi
print ===========================================tags union
# two super table tag union, limit is not active during retrieve tags query
sql (select t1 from union_mt0) union all (select t1 from union_mt0)
if $rows != 200000 then
if $rows != 20000 then
return -1
endi
@ -248,7 +248,7 @@ sql (select count(*) as c from union_tb0 where ts > now + 3650d) union all (sele
if $rows != 2 then
return -1
endi
#if $data00 != 495000 then
#if $data00 != 49500 then
# return -1
#endi
@ -272,7 +272,7 @@ endi
# multi-vnode projection query
sql (select c1 from union_mt0) union all select c1 from union_mt0;
if $rows != 200000 then
if $rows != 20000 then
return -1
endi
@ -330,10 +330,10 @@ sql (select sum(c1) as a from union_tb0 limit 1) union all (select sum(c3) as a
if $rows != 2 then
return -1
endi
if $data00 != 495000 then
if $data00 != 49500 then
return -1
endi
if $data10 != 495000 then
if $data10 != 49500 then
return -1
endi

8
tests/script/tsim/parser/where.sim
View File

@ -7,7 +7,7 @@ $dbPrefix = wh_db
$tbPrefix = wh_tb
$mtPrefix = wh_mt
$tbNum = 10
$rowNum = 10000
$rowNum = 1000
$totalNum = $tbNum * $rowNum
print =============== where.sim
@ -62,7 +62,7 @@ sql select count(*) from $tb where c1<10 and c1<>2
if $rows != 1 then
return -1
endi
if $data00 != 900 then
if $data00 != 90 then
return -1
endi
@ -308,9 +308,9 @@ if $row != 0 then
endi
sql select * from wh_mt0 where c3 = 1;
print $rows -> 1000
print $rows -> 100
print $data00 $data01 $data02
if $row != 1000 then
if $row != 100 then
return -1
endi

18
tests/script/tsim/query/explain.sim
View File

@ -4,7 +4,7 @@ system sh/exec.sh -n dnode1 -s start
sql connect
print ======== step1
sql create database db1 vgroups 3;
sql create database db1 vgroups 3 cachesize 10 cachemodel 'both';
sql use db1;
sql select * from information_schema.ins_databases;
sql create stable st1 (ts timestamp, f1 int, f2 binary(200)) tags(t1 int);
@ -60,6 +60,15 @@ sql explain verbose true select * from information_schema.ins_stables where db_n
sql explain verbose true select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0;
sql explain verbose true insert into t1(ts) select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0;
sql explain verbose true insert into t1(ts, t1) select _wstart, count(*) from st1 interval(10s);
sql explain verbose true select distinct tbname, table_name from information_schema.ins_tables;
sql explain verbose true select diff(f1) as f11 from tb1 order by f11;
sql explain verbose true select count(*) from st1 where ts > now - 3m and ts < now interval(10s) fill(linear);
sql explain verbose true select count(*) from st1 partition by tbname;
sql explain verbose true select count(*) from information_schema.ins_tables group by stable_name;
sql explain verbose true select last(*) from st1;
sql explain verbose true select last_row(*) from st1;
sql explain verbose true select interp(f1) from tb1 where ts > now - 3m and ts < now range(now-3m,now) every(1m) fill(prev);
sql explain verbose true select _wstart, _wend, count(*) from tb1 EVENT_WINDOW start with f1 > 0 end with f1 < 10;
print ======== step4
sql explain analyze select ts from st1 where -2;
@ -96,6 +105,13 @@ sql explain analyze verbose true select * from information_schema.ins_stables wh
sql explain analyze verbose true select * from (select min(f1),count(*) a from st1 where f1 > 0) where a < 0;
sql explain analyze verbose true select count(f1) from st1 group by tbname;
sql explain analyze verbose true select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0;
sql explain analyze verbose true select diff(f1) as f11 from tb1 order by f11;
sql explain analyze verbose true select count(*) from st1 where ts > now - 3m and ts < now interval(10s) fill(linear);
sql explain analyze verbose true select count(*) from information_schema.ins_tables group by stable_name;
sql explain analyze verbose true select last(*) from st1;
sql explain analyze verbose true select last_row(*) from st1;
sql explain analyze verbose true select interp(f1) from tb1 where ts > now - 3m and ts < now range(now-3m,now) every(1m) fill(prev);
sql explain analyze verbose true select _wstart, _wend, count(*) from tb1 EVENT_WINDOW start with f1 > 0 end with f1 < 10;
#not pass case
#sql explain verbose true select count(*),sum(f1) as aa from tb1 where (f1 > 0 or f1 < -1) and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00' order by aa;

107
tests/script/tsim/query/tableCount.sim Normal file
View File

@ -0,0 +1,107 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
sql drop database if exists db1;
sql create database db1 vgroups 3;
sql create database db1;
sql use db1;
sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create table tba1 using sta tags(1, 1, 1);
sql create table tba2 using sta tags(2, 2, 2);
sql create table tba3 using sta tags(3, 3, 3);
sql create table tba4 using sta tags(3, 3, 3);
sql create table tba5 using sta tags(3, 3, 3);
sql create table tba6 using sta tags(3, 3, 3);
sql create table tba7 using sta tags(3, 3, 3);
sql create table tba8 using sta tags(3, 3, 3);
sql create table tbb1 using stb tags(4, 4, 4);
sql create table tbb2 using stb tags(5, 5, 5);
sql create table tbb3 using stb tags(6, 6, 6);
sql create table tbb4 using stb tags(4, 4, 4);
sql create table tbb5 using stb tags(5, 5, 5);
sql create table tbb6 using stb tags(6, 6, 6);
sql create table tbb7 using stb tags(7, 7, 7);
sql create table tbb8 using stb tags(8, 8, 8);
sql create table tbn1 (ts timestamp, f1 int);
sql create database db2 vgroups 3;
sql create database db2;
sql use db2;
sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int);
sql create table tba1 using sta tags(1, 1, 1);
sql create table tba2 using sta tags(2, 2, 2);
sql create table tbb1 using stb tags(4, 4, 4);
sql create table tbb2 using stb tags(5, 5, 5);
sql create table tbb3 using stb tags(6, 6, 6);
sql select count(table_name) from information_schema.ins_tables group by stable_name;
if $rows != 3 then
return -1
endi
sql select count(table_name) from information_schema.ins_tables group by db_name;
if $rows != 4 then
return -1
endi
sql select count(table_name) from information_schema.ins_tables group by db_name, stable_name;
if $rows != 7 then
return -1
endi
sql select stable_name,count(table_name) from information_schema.ins_tables group by stable_name order by stable_name;
if $rows != 3 then
return -1
endi
if $data01 != 30 then
return -1
endi
if $data11 != 10 then
return -1
endi
if $data21 != 11 then
return -1
endi
sql select db_name,count(table_name) from information_schema.ins_tables group by db_name order by db_name;
if $rows != 4 then
return -1
endi
if $data01 != 17 then
return -1
endi
if $data11 != 5 then
return -1
endi
if $data21 != 24 then
return -1
endi
if $data31 != 5 then
return -1
endi
sql select db_name,stable_name,count(table_name) from information_schema.ins_tables group by db_name, stable_name order by db_name, stable_name;
if $rows != 7 then
return -1
endi
if $data02 != 1 then
return -1
endi
if $data12 != 8 then
return -1
endi
if $data22 != 8 then
return -1
endi
if $data32 != 2 then
return -1
endi
if $data42 != 3 then
return -1
endi
if $data52 != 24 then
return -1
endi
if $data62 != 5 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT

10
tests/script/tsim/show/basic.sim
View File

@ -244,5 +244,15 @@ if $rows <= 0 then
return -1
endi
sql show cluster alive;
if $rows <= 0 then
return -1
endi
sql show db.alive;
if $rows <= 0 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT

9
tests/script/tsim/stream/deleteState.sim
View File

@ -51,13 +51,8 @@ if $loop_count == 10 then
return -1
endi
if $data01 != 1 then
print =====data01=$data01
goto loop1
endi
if $data02 != NULL then
print =====data02=$data02
if $rows != 0 then
print =====rows=$rows
goto loop1
endi

9
tests/script/tsim/stream/partitionbyColumnState.sim
View File

@ -27,13 +27,8 @@ if $loop_count == 10 then
return -1
endi
if $data01 != 1 then
print =====data01=$data01
goto loop0
endi
if $data02 != NULL then
print =====data02=$data02
if $rows != 0 then
print =====rows=$rows
goto loop0
endi

101
tests/script/tsim/stream/state1.sim Normal file
View File

@ -0,0 +1,101 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print =============== create database
sql create database test vgroups 4;
sql select * from information_schema.ins_databases;
if $rows != 3 then
return -1
endi
print $data00 $data01 $data02
sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double, id int);
print create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart, count(*) c1 from t1 state_window(a);
sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart, count(*) c1 from t1 state_window(a);
sql insert into t1(ts) values(1648791213000);
$loop_count = 0
loop0:
sleep 300
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
sql select * from streamt1;
print data00 data01
print data10 data11
if $rows != 0 then
print =====rows=$rows
goto loop0
endi
sql insert into t1 values(1648791214000,1,2,3,1.0,3);
$loop_count = 0
loop1:
sleep 300
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
sql select * from streamt1;
print data00 data01
print data10 data11
if $rows != 1 then
print =====rows=$rows
goto loop1
endi
sql insert into t1 values(1648791215000,2,2,3,1.0,4);
$loop_count = 0
loop2:
sleep 300
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
sql select * from streamt1;
if $rows != 2 then
print =====rows=$rows
goto loop2
endi
sql insert into t1(ts) values(1648791216000);
$loop_count = 0
loop3:
sleep 300
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
sql select * from streamt1;
if $rows != 2 then
print =====rows=$rows
goto loop2
endi
print state1 end
system sh/stop_dnodes.sh

12
tests/script/tsim/valgrind/checkError7.sim
View File

@ -8,8 +8,9 @@ sql create database d1
sql use d1
$x = 0
while $x < 128
while $x < 5
$tb = d1.s . $x
print create table $tb (ts timestamp, i int) tags (j int)
sql create table $tb (ts timestamp, i int) tags (j int)
$x = $x + 1
endw
@ -17,7 +18,7 @@ endw
print ======================== describe stables
# TODO : create stable error
$m = 0
while $m < 128
while $m < 5
$tb = s . $m
$filter = ' . $tb
$filter = $filter . '
@ -36,15 +37,16 @@ print ======================== show stables
sql show d1.stables
print num of stables is $rows
if $rows != 128 then
if $rows != 5 then
return -1
endi
print ======================== create table
$x = 0
while $x < 424
while $x < 42
$tb = d1.t . $x
print create table $tb using d1.s0 tags( $x )
sql create table $tb using d1.s0 tags( $x )
$x = $x + 1
endw
@ -54,7 +56,7 @@ print ======================== show stables
sql show d1.tables
print num of tables is $rows
if $rows != 424 then
if $rows != 42 then
return -1
endi

10
tests/system-test/0-others/information_schema.py
View File

@ -101,10 +101,18 @@ class TDTestCase:
tdSql.checkEqual(i[1],len(self.perf_list))
elif i[0].lower() == self.dbname:
tdSql.checkEqual(i[1],self.tbnum+1)
def ins_columns_check(self):
tdSql.execute('create database db2 vgroups 2 replica 1')
tdSql.execute('create table db2.stb2 (ts timestamp,c0 int,c1 int, c2 double, c3 float, c4 binary(1000), c5 nchar(100),c7 bigint, c8 bool, c9 smallint) tags(t0 int)')
for i in range(2000):
tdSql.execute("create table db2.ctb%d using db2.stb2 tags(%d)" %(i,i))
tdSql.query(f'select * from information_schema.ins_columns where db_name="db2" and table_type="CHILD_TABLE"')
tdSql.checkEqual(20000,len(tdSql.queryResult))
print("number of ins_columns of child table in db2 is %s" % len(tdSql.queryResult))
def run(self):
self.prepare_data()
self.count_check()
self.ins_columns_check()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)

87
tests/system-test/1-insert/insert_wide_column.py Normal file
View File

@ -0,0 +1,87 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import threading
import random
import string
import time
from util.log import *
from util.cases import *
from util.sql import *
from util.autogen import *
#
# Test Main class
#
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
self.autoGen = AutoGen()
def query_test(self, stbname):
sql = f"select count(*) from {stbname}"
tdSql.execute(sql)
sql = f"select * from {stbname} order by ts desc;"
tdSql.execute(sql)
sql = f"select * from (select * from {stbname} where c1=c2 or c3=c4 or c5=c6) order by ts desc;"
tdSql.execute(sql)
tdLog.info(" test query ok!")
def test_db(self, dbname, stbname, childname, tag_cnt, column_cnt, child_cnt, insert_rows, binary_len, nchar_len):
self.autoGen.create_db(dbname)
self.autoGen.create_stable(stbname, tag_cnt, column_cnt, binary_len, nchar_len)
self.autoGen.create_child(stbname, childname, child_cnt)
self.autoGen.insert_data(insert_rows)
self.autoGen.insert_samets(insert_rows)
self.query_test(stbname)
def run(self):
dbname = "test"
stbname = "st"
childname = "d"
child_cnt = 2
insert_rows = 10
tag_cnt = 15
column_cnt = 20
binary_len = 10240
nchar_len = 1025
self.autoGen.set_batch_size(1)
# normal
self.test_db(dbname, stbname, childname, tag_cnt, column_cnt, child_cnt, insert_rows, binary_len, nchar_len)
# max
dbname = "test_max_col"
child_cnt = 3
insert_rows = 50
tag_cnt = 128
binary_len = 3
nchar_len = 4
column_cnt = 4096 - tag_cnt
self.autoGen.set_batch_size(1)
self.test_db(dbname, stbname, childname, tag_cnt, column_cnt, child_cnt, insert_rows, binary_len, nchar_len)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

15
tests/system-test/2-query/Today.py
View File

@ -16,6 +16,7 @@ class TDTestCase:
tdSql.init(conn.cursor())
self.today_date = datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d")
self.today_ts = datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d").timestamp()
self.today_ts_ns = 0
self.time_unit = ['b','u','a','s','m','h','d','w']
self.error_param = ['1.5','abc','!@#','"abc"','today()']
self.arithmetic_operators = ['+','-','*','/']
@ -74,8 +75,10 @@ class TDTestCase:
num_same = 0
if v.lower() == 'timestamp':
tdSql.query(f'select {k} from {tbname}')
for i in tdSql.queryResult:
if precision == 'ms':
self.today_ts_trans = int(self.today_ts)*1000
if int(i[0].timestamp())*1000 > int(self.today_ts)*1000:
num_up += 1
elif int(i[0].timestamp())*1000 == int(self.today_ts)*1000:
@ -83,6 +86,7 @@ class TDTestCase:
elif int(i[0].timestamp())*1000 < int(self.today_ts)*1000:
num_down += 1
elif precision == 'us':
self.today_ts_trans = int(self.today_ts)*1000000
if int(i[0].timestamp())*1000000 > int(self.today_ts)*1000000:
num_up += 1
elif int(i[0].timestamp())*1000000 == int(self.today_ts)*1000000:
@ -90,6 +94,7 @@ class TDTestCase:
elif int(i[0].timestamp())*1000000 < int(self.today_ts)*1000000:
num_down += 1
elif precision == 'ns':
self.today_ts_trans = int(self.today_ts)*1000000000
if i[0] > int(self.today_ts)*1000000000:
num_up += 1
elif i[0] == int(self.today_ts)*1000000000:
@ -97,8 +102,9 @@ class TDTestCase:
elif i[0] < int(self.today_ts)*1000000000:
num_down += 1
tdSql.query(f"select today() from {tbname}")
tdSql.checkRows(len(values_list)*tb_num)
tdSql.checkData(0, 0, str(self.today_date))
tdSql.checkRows(len(values_list)*tb_num)
print(self.today_ts_trans,self.today_ts,precision,num_up,num_down,i[0])
tdSql.checkData(0, 0, self.today_ts_trans)
tdSql.query(f"select * from {tbname} where {k}=today()")
if tb == 'tb':
tdSql.checkRows(num_same*tb_num)
@ -149,11 +155,12 @@ class TDTestCase:
if tb == 'tb':
tdSql.checkRows(num_same*tb_num)
for i in range(num_same*tb_num):
tdSql.checkData(i, 0, str(self.today_date))
print(self.today_ts_trans,precision,num_up,num_down)
tdSql.checkData(i, 0, self.today_ts_trans)
elif tb == 'stb':
tdSql.checkRows(num_same)
for i in range(num_same):
tdSql.checkData(i, 0, str(self.today_date))
tdSql.checkData(i, 0, self.today_ts_trans)
def today_check_ntb(self):
for time_unit in self.db_percision:

25
tests/system-test/2-query/cast.py
View File

@ -65,7 +65,6 @@ class TDTestCase:
data_ct4_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
tdSql.query(f"select c1 from {self.dbname}.t1")
data_t1_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
tdLog.printNoPrefix("==========step2: cast int to bigint, expect no changes")
tdSql.query(f"select cast(c1 as bigint) as b from {self.dbname}.ct4")
@ -108,12 +107,12 @@ class TDTestCase:
tdSql.query(f"select cast(c1 as timestamp) as b from {self.dbname}.t1")
for i in range(len(data_t1_c1)):
if data_ct4_c1[i] is None:
if data_t1_c1[i] is None:
tdSql.checkData( i, 0 , None )
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c1[i]/1000)
date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c1[i]/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
@ -225,12 +224,12 @@ class TDTestCase:
tdSql.query(f"select cast(c3 as timestamp) as b from {self.dbname}.t1")
for i in range(len(data_t1_c3)):
if data_ct4_c3[i] is None:
if data_t1_c3[i] is None:
tdSql.checkData( i, 0 , None )
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c3[i]/1000)
date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c3[i]/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
@ -282,12 +281,12 @@ class TDTestCase:
tdSql.query(f"select cast(c4 as timestamp) as b from {self.dbname}.t1")
for i in range(len(data_t1_c4)):
if data_ct4_c4[i] is None:
if data_t1_c4[i] is None:
tdSql.checkData( i, 0 , None )
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c4[i]/1000)
date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c4[i]/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
@ -329,7 +328,7 @@ class TDTestCase:
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c5[i]/1000))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c5[i])/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
tdSql.query(f"select cast(c5 as timestamp) as b from {self.dbname}.t1")
@ -339,7 +338,7 @@ class TDTestCase:
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c5[i]/1000))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c5[i])/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
@ -385,7 +384,7 @@ class TDTestCase:
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c6[i]/1000))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c6[i])/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
@ -398,7 +397,7 @@ class TDTestCase:
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c6[i]/1000))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c6[i])/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
@ -439,7 +438,7 @@ class TDTestCase:
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c7[i]/1000))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c7[i])/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)
tdSql.query(f"select cast(c7 as timestamp) as b from {self.dbname}.t1")
@ -449,7 +448,7 @@ class TDTestCase:
else:
utc_zone = datetime.timezone.utc
utc_8 = datetime.timezone(datetime.timedelta(hours=8))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c7[i]/1000))
date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c7[i])/1000)
date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
tdSql.checkData( i, 0, date_data)

2
tests/system-test/2-query/json_tag.py
View File

@ -301,7 +301,7 @@ class TDTestCase:
# where json value is not exist
tdSql.query(f"select * from {dbname}.jsons1 where jtag->'tag1' is null")
tdSql.checkData(0, 0, 'jsons1_9')
# tdSql.checkData(0, 0, 'jsons1_9')
tdSql.checkRows(2)
tdSql.query(f"select * from {dbname}.jsons1 where jtag->'tag4' is null")
tdSql.checkRows(9)

44
tests/system-test/2-query/limit.py
View File

@ -140,7 +140,7 @@ class TDTestCase:
val2 = paraDict["ctbNum"] - 1
# select count(*), t1, t2, t3, t4, t5, t6 from $stb where t1 > $val1 and t1 < $val2 group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0
sqlStr = f"select count(*), t1, t2, t3, t4, t5, t6 from %s where t1 > %d and t1 < %d group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0"%(paraDict["stbName"], val1, val2)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(1)
tdSql.checkData(0, 0, paraDict["rowsPerTbl"])
@ -152,7 +152,7 @@ class TDTestCase:
# select count(*), t3, t4 from $stb where t2 like '%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 2 offset 0
sqlStr = f"select count(*), t3, t4 from %s where t2 like '%%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 2 offset 0"%(paraDict["stbName"])
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(2)
tdSql.checkData(0, 0, paraDict["rowsPerTbl"])
@ -164,7 +164,7 @@ class TDTestCase:
# select count(*) from $stb where t2 like '%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 1 offset 1
sqlStr = f"select count(*) from %s where t2 like '%%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 1 offset 1"%(paraDict["stbName"])
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(1)
@ -179,18 +179,18 @@ class TDTestCase:
tb = paraDict["ctbPrefix"] + '0'
# select _wstart, max(c1) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1) limit 10 offset 1
sqlStr = f"select _wstart, max(c1) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1) limit 10 offset 1"%(tb, ts0, tsu)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(10)
tdSql.checkData(0, 0, "18-09-17 09:05:00.000")
tdSql.checkData(0, 0, "2018-09-17 09:05:00.000")
tdSql.checkData(0, 1, -1)
tdSql.checkData(1, 1, 1)
tdSql.checkData(9, 0, "18-09-17 09:50:00.000")
tdSql.checkData(9, 0, "2018-09-17 09:50:00.000")
tdSql.checkData(9, 1, 5)
tb5 = paraDict["ctbPrefix"] + '5'
sqlStr = f"select max(c1), min(c2) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1, -2) limit 10 offset 1"%(tb5, ts0, tsu)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(10)
tdSql.checkData(0, 0, -1)
@ -206,22 +206,22 @@ class TDTestCase:
limit = paraDict["rowsPerTbl"]
offset = limit / 2
sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb, ts0, tsu, limit, offset)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(limit)
tdSql.checkData(0, 1, 0)
sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 8200"
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(8200)
sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 100000;"
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10 offset 8190;"
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(10)
tdSql.checkData(0, 0, 5)
@ -231,7 +231,7 @@ class TDTestCase:
sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10 offset 10001;"
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(10)
tdSql.checkData(0, 0, -1000)
@ -240,13 +240,13 @@ class TDTestCase:
tdSql.checkData(3, 0, 2)
sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10000 offset 10001;"
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(9998)
sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 100 offset 20001;"
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(0)
@ -254,7 +254,7 @@ class TDTestCase:
limit = paraDict["rowsPerTbl"]
offset = limit / 2
sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(linear) limit %d offset %d"%(tb,ts0,tsu,limit, offset)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(limit)
tdSql.checkData(0, 1, 0)
@ -272,7 +272,7 @@ class TDTestCase:
limit = paraDict["rowsPerTbl"]
offset = limit / 2
sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(prev) limit %d offset %d"%(tb,ts0,tsu,limit, offset)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(limit)
@ -280,7 +280,7 @@ class TDTestCase:
limit = paraDict["rowsPerTbl"]
offset = limit / 2 + 10
sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d and c1 = 5 interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb,ts0,tsu,limit, offset)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(limit)
tdSql.checkData(0, 1, 5)
@ -298,7 +298,7 @@ class TDTestCase:
limit = paraDict["rowsPerTbl"]
offset = limit * 2 - 11
sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d and c1 = 5 interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb,ts0,tsu,limit, offset)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(10)
tdSql.checkData(0, 1, -1)
@ -314,7 +314,7 @@ class TDTestCase:
### [TBASE-350]
## stb + interval + fill + group by + limit offset
sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') limit 2 offset 10"
print("====sql:%s"%(sqlStr))
tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(2)
@ -322,7 +322,7 @@ class TDTestCase:
offset = paraDict["rowsPerTbl"] * 2
offset = offset - 2
sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') order by t1 limit %d offset %d"%(limit, offset)
print("====sql:%s"%(sqlStr))
# tdLog.info("====sql:%s"%(sqlStr))
tdSql.query(sqlStr)
tdSql.checkRows(1)
tdSql.checkData(0, 0, 9)
@ -339,7 +339,7 @@ class TDTestCase:
tdLog.printNoPrefix("======== test case 1 end ...... ")
def run(self):
tdSql.prepare()
# tdSql.prepare()
self.prepareTestEnv()
self.tmqCase1()

69
tests/system-test/2-query/timeerror.py Normal file
View File

@ -0,0 +1,69 @@
import taos
import sys
from time import sleep
from util.log import *
from util.sql import *
from util.cases import *
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
def run(self):
rows = 10
tdSql.execute("create database dbus PRECISION 'us' ")
tdSql.execute("create database dbns PRECISION 'ns' ")
tdSql.execute("create database dbms PRECISION 'ms' ")
dball = ["dbns","dbus","dbms"]
for i in range(len(dball)):
dbname = dball[i]
stb = f"{dbname}.stb1"
if i == 0 :
qts = 1678883666951061471
qts1 = 1678883666951061471
qtime = "2023-03-15 20:34:26.951061471"
elif i == 1 :
qts = 1678883666951061
qts1 = 1678883666951061
qtime = "2023-03-15 20:34:26.951061"
else:
qts = 1678883666951
qts1 = 1678883666953
qtime = "2023-03-15 20:34:26.951"
tdSql.execute(f"use {dbname}")
tdLog.printNoPrefix("==========step1:create table")
tdSql.execute(
f'''create table if not exists {stb}
(ts timestamp, c1 int, c2 float, c3 bigint, c4 double, c5 smallint, c6 tinyint)
tags(location binary(64), type int, isused bool , family nchar(64))'''
)
tdSql.execute(f"create table {dbname}.t1 using {stb} tags('beijing', 1, 1, 'nchar1')")
tdSql.execute(f"create table {dbname}.t2 using {stb} tags('shanghai', 2, 0, 'nchar2')")
tdSql.execute(f"create table {dbname}.t3 using {stb} tags('shanghai', 3, 0, 'nchar3')")
tdLog.printNoPrefix("==========step2:insert data")
tdSql.execute(
f"insert into {dbname}.t3 values ({qts}, null , {-3.4*10**38}, null , {-1.7*10**308}, null , null)"
)
tdLog.printNoPrefix("==========step3:query timestamp type")
tdSql.query(f"select ts from {dbname}.t3 limit 1")
tdSql.checkData(0,0,qtime)
tdSql.checkData(0,0,qts)
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

2
tests/system-test/6-cluster/5dnode1mnode.py
View File

@ -29,7 +29,7 @@ class TDTestCase:
self.master_dnode = self.TDDnodes.dnodes[0]
self.host=self.master_dnode.cfgDict["fqdn"]
conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir)
tdSql.init(conn1.cursor())
tdSql.init(conn1.cursor(), True)
def getBuildPath(self):

5
tests/system-test/7-tmq/subscribeDb1.py
View File

@ -123,8 +123,9 @@ class TDTestCase:
pre_insert = "insert into "
sql = pre_insert
t = time.time()
startTs = int(round(t * 1000))
# t = 1678609778776 #time.time()
t = 1600000000000
startTs = t #int(round(t * 1000))
#tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
for i in range(ctbNum):
sql += " %s_%d values "%(stbName,i)

24
utils/test/c/tmqSim.c
View File

@ -386,7 +386,7 @@ void addRowsToVgroupId(SThreadInfo* pInfo, int32_t vgroupId, int32_t rows) {
pInfo->rowsOfPerVgroups[pInfo->numOfVgroups][1] += rows;
pInfo->numOfVgroups++;
taosFprintfFile(g_fp, "consume id %d, add one new vogroup id: %d\n", pInfo->consumerId, vgroupId);
taosFprintfFile(g_fp, "consume id %d, add new vgroupId:%d\n", pInfo->consumerId, vgroupId);
if (pInfo->numOfVgroups > MAX_VGROUP_CNT) {
taosFprintfFile(g_fp, "====consume id %d, vgroup num %d over than 32. new vgroupId: %d\n", pInfo->consumerId,
pInfo->numOfVgroups, vgroupId);
@ -578,18 +578,25 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn
char buf[1024];
int32_t totalRows = 0;
// printf("topic: %s\n", tmq_get_topic_name(msg));
int32_t vgroupId = tmq_get_vgroup_id(msg);
const char* dbName = tmq_get_db_name(msg);
taosFprintfFile(g_fp, "consumerId: %d, msg index:%d\n", pInfo->consumerId, msgIndex);
taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table",
tmq_get_topic_name(msg), vgroupId);
int32_t index = 0;
for (index = 0; index < pInfo->numOfVgroups; index++) {
if (vgroupId == pInfo->rowsOfPerVgroups[index][0]) {
break;
}
}
taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId:%d, currentRows:%d\n", dbName != NULL ? dbName : "invalid table",
tmq_get_topic_name(msg), vgroupId, pInfo->rowsOfPerVgroups[index][1]);
while (1) {
TAOS_ROW row = taos_fetch_row(msg);
if (row == NULL) break;
if (row == NULL) {
break;
}
TAOS_FIELD* fields = taos_fetch_fields(msg);
int32_t numOfFields = taos_field_count(msg);
@ -607,7 +614,6 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn
#endif
dumpToFileForCheck(pInfo->pConsumeRowsFile, row, fields, length, numOfFields, precision);
taos_print_row(buf, row, fields, numOfFields);
if (0 != g_stConfInfo.showRowFlag) {
@ -621,7 +627,6 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn
}
addRowsToVgroupId(pInfo, vgroupId, totalRows);
return totalRows;
}
@ -730,9 +735,7 @@ void build_consumer(SThreadInfo* pInfo) {
}
pInfo->tmq = tmq_consumer_new(conf, NULL, 0);
tmq_conf_destroy(conf);
return;
}
@ -817,7 +820,6 @@ void loop_consume(SThreadInfo* pInfo) {
}
taos_free_result(tmqMsg);
totalMsgs++;
int64_t currentPrintTime = taosGetTimestampMs();