Merge remote-tracking branch 'origin/3.0' into feature/check
This commit is contained in:
Shengliang Guan
commit
bd5d21b2fd
|
|
@ -167,7 +167,7 @@ tmq_t* build_consumer() {
|
|||
tmq_conf_set(conf, "td.connect.pass", "taosdata");
|
||||
/*tmq_conf_set(conf, "td.connect.db", "abc1");*/
|
||||
tmq_conf_set(conf, "msg.with.table.name", "true");
|
||||
tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL);
|
||||
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
|
||||
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
|
||||
assert(tmq);
|
||||
return tmq;
|
||||
|
|
@ -176,6 +176,7 @@ tmq_t* build_consumer() {
|
|||
tmq_list_t* build_topic_list() {
|
||||
tmq_list_t* topic_list = tmq_list_new();
|
||||
tmq_list_append(topic_list, "topic_ctb_column");
|
||||
/*tmq_list_append(topic_list, "tmq_test_db_multi_insert_topic");*/
|
||||
return topic_list;
|
||||
}
|
||||
|
||||
|
|
@ -190,7 +191,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
|
|||
int32_t cnt = 0;
|
||||
/*clock_t startTime = clock();*/
|
||||
while (running) {
|
||||
TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 500);
|
||||
TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 0);
|
||||
if (tmqmessage) {
|
||||
cnt++;
|
||||
/*printf("get data\n");*/
|
||||
|
|
|
|||
|
|
@ -233,6 +233,8 @@ DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
|
|||
DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t wait_time);
|
||||
DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
|
||||
DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async);
|
||||
DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param);
|
||||
DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets);
|
||||
#if 0
|
||||
DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async);
|
||||
DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset);
|
||||
|
|
@ -251,7 +253,7 @@ typedef enum tmq_conf_res_t tmq_conf_res_t;
|
|||
DLL_EXPORT tmq_conf_t *tmq_conf_new();
|
||||
DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
|
||||
DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf);
|
||||
DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
|
||||
DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
|
||||
|
||||
/* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
|
||||
|
||||
|
|
|
|||
|
|
@ -403,6 +403,19 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWra
|
|||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaWrapper* pSW) {
|
||||
if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1;
|
||||
if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1;
|
||||
|
||||
pSW->pSchema = (SSchema*)tDecoderMalloc(pDecoder, pSW->nCols * sizeof(SSchema));
|
||||
if (pSW->pSchema == NULL) return -1;
|
||||
for (int32_t i = 0; i < pSW->nCols; i++) {
|
||||
if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols);
|
||||
|
||||
typedef struct {
|
||||
|
|
|
|||
|
|
@ -668,6 +668,8 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_UDF_INVALID_STATE TAOS_DEF_ERROR_CODE(0, 0x2906)
|
||||
#define TSDB_CODE_UDF_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2907)
|
||||
#define TSDB_CODE_UDF_NO_FUNC_HANDLE TAOS_DEF_ERROR_CODE(0, 0x2908)
|
||||
#define TSDB_CODE_UDF_INVALID_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x2909)
|
||||
#define TSDB_CODE_UDF_INVALID_OUTPUT_TYPE TAOS_DEF_ERROR_CODE(0, 0x290A)
|
||||
|
||||
#define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000)
|
||||
#define TSDB_CODE_SML_INVALID_PRECISION_TYPE TAOS_DEF_ERROR_CODE(0, 0x3001)
|
||||
|
|
|
|||
|
|
@ -182,10 +182,14 @@ typedef struct {
|
|||
|
||||
typedef struct {
|
||||
tmq_t* tmq;
|
||||
int32_t async;
|
||||
int8_t async;
|
||||
int8_t automatic;
|
||||
int8_t freeOffsets;
|
||||
tmq_commit_cb* userCb;
|
||||
tsem_t rspSem;
|
||||
tmq_resp_err_t rspErr;
|
||||
SArray* offsets;
|
||||
void* userParam;
|
||||
} SMqCommitCbParam;
|
||||
|
||||
tmq_conf_t* tmq_conf_new() {
|
||||
|
|
@ -314,6 +318,142 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
|
|||
return sprintf(dst, "%s:%d", topicName, vg);
|
||||
}
|
||||
|
||||
int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
|
||||
pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL;
|
||||
if (pParam->async) {
|
||||
if (pParam->automatic && pParam->tmq->commitCb) {
|
||||
pParam->tmq->commitCb(pParam->tmq, pParam->rspErr, (tmq_topic_vgroup_list_t*)pParam->offsets,
|
||||
pParam->tmq->commitCbUserParam);
|
||||
} else if (!pParam->automatic && pParam->userCb) {
|
||||
pParam->userCb(pParam->tmq, pParam->rspErr, (tmq_topic_vgroup_list_t*)pParam->offsets, pParam->userParam);
|
||||
}
|
||||
|
||||
if (pParam->freeOffsets) {
|
||||
taosArrayDestroy(pParam->offsets);
|
||||
}
|
||||
|
||||
taosMemoryFree(pParam);
|
||||
} else {
|
||||
tsem_post(&pParam->rspSem);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tmqCommitInner(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_t automatic, int8_t async,
|
||||
tmq_commit_cb* userCb, void* userParam) {
|
||||
SMqCMCommitOffsetReq req;
|
||||
SArray* pOffsets = NULL;
|
||||
void* buf = NULL;
|
||||
SMqCommitCbParam* pParam = NULL;
|
||||
SMsgSendInfo* sendInfo = NULL;
|
||||
int8_t freeOffsets;
|
||||
int32_t code = -1;
|
||||
|
||||
if (offsets == NULL) {
|
||||
freeOffsets = 1;
|
||||
pOffsets = taosArrayInit(0, sizeof(SMqOffset));
|
||||
for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
|
||||
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
|
||||
for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
|
||||
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
|
||||
SMqOffset offset;
|
||||
tstrncpy(offset.topicName, pTopic->topicName, TSDB_TOPIC_FNAME_LEN);
|
||||
tstrncpy(offset.cgroup, tmq->groupId, TSDB_CGROUP_LEN);
|
||||
offset.vgId = pVg->vgId;
|
||||
offset.offset = pVg->currentOffset;
|
||||
taosArrayPush(pOffsets, &offset);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
freeOffsets = 0;
|
||||
pOffsets = (SArray*)&offsets->container;
|
||||
}
|
||||
|
||||
req.num = (int32_t)pOffsets->size;
|
||||
req.offsets = pOffsets->pData;
|
||||
|
||||
SEncoder encoder;
|
||||
|
||||
tEncoderInit(&encoder, NULL, 0);
|
||||
code = tEncodeSMqCMCommitOffsetReq(&encoder, &req);
|
||||
if (code < 0) {
|
||||
goto END;
|
||||
}
|
||||
int32_t tlen = encoder.pos;
|
||||
buf = taosMemoryMalloc(tlen);
|
||||
if (buf == NULL) {
|
||||
tEncoderClear(&encoder);
|
||||
goto END;
|
||||
}
|
||||
tEncoderClear(&encoder);
|
||||
|
||||
tEncoderInit(&encoder, buf, tlen);
|
||||
tEncodeSMqCMCommitOffsetReq(&encoder, &req);
|
||||
tEncoderClear(&encoder);
|
||||
|
||||
pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam));
|
||||
if (pParam == NULL) {
|
||||
goto END;
|
||||
}
|
||||
pParam->tmq = tmq;
|
||||
pParam->automatic = automatic;
|
||||
pParam->async = async;
|
||||
pParam->offsets = pOffsets;
|
||||
pParam->freeOffsets = freeOffsets;
|
||||
pParam->userCb = userCb;
|
||||
pParam->userParam = userParam;
|
||||
if (!async) tsem_init(&pParam->rspSem, 0, 0);
|
||||
|
||||
sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo));
|
||||
if (sendInfo == NULL) goto END;
|
||||
sendInfo->msgInfo = (SDataBuf){
|
||||
.pData = buf,
|
||||
.len = tlen,
|
||||
.handle = NULL,
|
||||
};
|
||||
|
||||
sendInfo->requestId = generateRequestId();
|
||||
sendInfo->requestObjRefId = 0;
|
||||
sendInfo->param = pParam;
|
||||
sendInfo->fp = tmqCommitCb;
|
||||
sendInfo->msgType = TDMT_MND_MQ_COMMIT_OFFSET;
|
||||
|
||||
SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
|
||||
|
||||
int64_t transporterId = 0;
|
||||
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
|
||||
|
||||
if (!async) {
|
||||
tsem_wait(&pParam->rspSem);
|
||||
code = pParam->rspErr;
|
||||
tsem_destroy(&pParam->rspSem);
|
||||
taosMemoryFree(pParam);
|
||||
}
|
||||
|
||||
// avoid double free if msg is sent
|
||||
buf = NULL;
|
||||
|
||||
code = 0;
|
||||
END:
|
||||
if (buf) taosMemoryFree(buf);
|
||||
/*if (pParam) taosMemoryFree(pParam);*/
|
||||
/*if (sendInfo) taosMemoryFree(sendInfo);*/
|
||||
|
||||
if (code != 0 && async) {
|
||||
if (automatic) {
|
||||
tmq->commitCb(tmq, TMQ_RESP_ERR__FAIL, (tmq_topic_vgroup_list_t*)pOffsets, tmq->commitCbUserParam);
|
||||
} else {
|
||||
userCb(tmq, TMQ_RESP_ERR__FAIL, (tmq_topic_vgroup_list_t*)pOffsets, userParam);
|
||||
}
|
||||
}
|
||||
|
||||
if (!async && freeOffsets) {
|
||||
taosArrayDestroy(pOffsets);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
void tmqAssignDelayedHbTask(void* param, void* tmrId) {
|
||||
tmq_t* tmq = (tmq_t*)param;
|
||||
int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM);
|
||||
|
|
@ -350,7 +490,8 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) {
|
|||
tmqAskEp(tmq, true);
|
||||
taosTmrReset(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer, &tmq->hbTimer);
|
||||
} else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
|
||||
tmq_commit(tmq, NULL, true);
|
||||
/*tmq_commit(tmq, NULL, true);*/
|
||||
tmqCommitInner(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam);
|
||||
taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer, &tmq->commitTimer);
|
||||
} else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) {
|
||||
} else {
|
||||
|
|
@ -385,32 +526,11 @@ void tmqClearUnhandleMsg(tmq_t* tmq) {
|
|||
int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param;
|
||||
pParam->rspErr = code;
|
||||
tmq_t* tmq = pParam->tmq;
|
||||
/*tmq_t* tmq = pParam->tmq;*/
|
||||
tsem_post(&pParam->rspSem);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
|
||||
SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
|
||||
pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL;
|
||||
if (pParam->tmq->commitCb) {
|
||||
pParam->tmq->commitCb(pParam->tmq, pParam->rspErr, NULL, pParam->tmq->commitCbUserParam);
|
||||
}
|
||||
if (!pParam->async)
|
||||
tsem_post(&pParam->rspSem);
|
||||
else {
|
||||
if (pParam->offsets) {
|
||||
taosArrayDestroy(pParam->offsets);
|
||||
}
|
||||
tsem_destroy(&pParam->rspSem);
|
||||
/*if (pParam->pArray) {*/
|
||||
/*taosArrayDestroy(pParam->pArray);*/
|
||||
/*}*/
|
||||
taosMemoryFree(pParam);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) {
|
||||
if (*topics == NULL) {
|
||||
*topics = tmq_list_new();
|
||||
|
|
@ -541,6 +661,8 @@ FAIL:
|
|||
}
|
||||
|
||||
tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) {
|
||||
return tmqCommitInner(tmq, offsets, 0, async, tmq->commitCb, tmq->commitCbUserParam);
|
||||
#if 0
|
||||
// TODO: add read write lock
|
||||
SRequestObj* pRequest = NULL;
|
||||
tmq_resp_err_t resp = TMQ_RESP_ERR__SUCCESS;
|
||||
|
|
@ -627,6 +749,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in
|
|||
}
|
||||
|
||||
return resp;
|
||||
#endif
|
||||
}
|
||||
|
||||
tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
|
||||
|
|
@ -723,7 +846,7 @@ FAIL:
|
|||
return code;
|
||||
}
|
||||
|
||||
void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* param) {
|
||||
void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* param) {
|
||||
//
|
||||
conf->commitCb = cb;
|
||||
conf->commitCbUserParam = param;
|
||||
|
|
@ -1384,3 +1507,10 @@ const char* tmq_get_table_name(TAOS_RES* res) {
|
|||
}
|
||||
return NULL;
|
||||
}
|
||||
DLL_EXPORT void tmq_commit_async(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, tmq_commit_cb* cb, void* param) {
|
||||
tmqCommitInner(tmq, offsets, 0, 1, cb, param);
|
||||
}
|
||||
|
||||
DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) {
|
||||
return tmqCommitInner(tmq, offsets, 0, 0, NULL, NULL);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -196,6 +196,8 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) {
|
|||
}
|
||||
}
|
||||
|
||||
tDecoderClear(&decoder);
|
||||
|
||||
if (mndTransPrepare(pMnode, pTrans) != 0) {
|
||||
mError("mq-commit-offset-trans:%d, failed to prepare since %s", pTrans->id, terrstr());
|
||||
mndTransDrop(pTrans);
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ struct SSmaStatItem {
|
|||
* N.B. only applicable to tsma
|
||||
*/
|
||||
int8_t state; // ETsdbSmaStat
|
||||
SHashObj *expiredWindows; // key: skey of time window, value: N/A
|
||||
SHashObj *expiredWindows; // key: skey of time window, value: version
|
||||
STSma *pTSma; // cache schema
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ typedef struct {
|
|||
TSKEY minKey;
|
||||
} SRtn;
|
||||
|
||||
#define TSDB_DATA_DIR_LEN 6
|
||||
#define TSDB_DATA_DIR_LEN 6 // adapt accordingly
|
||||
struct STsdb {
|
||||
char *path;
|
||||
SVnode *pVnode;
|
||||
|
|
|
|||
|
|
@ -56,8 +56,8 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
|
|||
if (tDecodeCStr(pCoder, &pME->name) < 0) return -1;
|
||||
|
||||
if (pME->type == TSDB_SUPER_TABLE) {
|
||||
if (tDecodeSSchemaWrapper(pCoder, &pME->stbEntry.schema) < 0) return -1;
|
||||
if (tDecodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
|
||||
if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schema) < 0) return -1;
|
||||
if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
|
||||
} else if (pME->type == TSDB_CHILD_TABLE) {
|
||||
if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1;
|
||||
if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1;
|
||||
|
|
@ -67,9 +67,9 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
|
|||
if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1;
|
||||
if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
|
||||
if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1;
|
||||
if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1;
|
||||
if (tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schema) < 0) return -1;
|
||||
} else if (pME->type == TSDB_TSMA_TABLE) {
|
||||
pME->smaEntry.tsma = taosMemoryCalloc(1, sizeof(STSma));
|
||||
pME->smaEntry.tsma = tDecoderMalloc(pCoder, sizeof(STSma));
|
||||
if(!pME->smaEntry.tsma) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
|
|
|
|||
|
|
@ -394,11 +394,6 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) {
|
|||
goto _err;
|
||||
}
|
||||
|
||||
SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
|
||||
if (pCur == NULL) {
|
||||
goto _err;
|
||||
}
|
||||
|
||||
SMetaReader mr = {0};
|
||||
metaReaderInit(&mr, pMeta, 0);
|
||||
int64_t smaId;
|
||||
|
|
@ -442,12 +437,10 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) {
|
|||
|
||||
metaReaderClear(&mr);
|
||||
taosArrayDestroy(pSmaIds);
|
||||
metaCloseSmaCursor(pCur);
|
||||
return pSW;
|
||||
_err:
|
||||
metaReaderClear(&mr);
|
||||
taosArrayDestroy(pSmaIds);
|
||||
metaCloseSmaCursor(pCur);
|
||||
tdFreeTSmaWrapper(pSW, deepCopy);
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -55,12 +55,13 @@ static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int
|
|||
}
|
||||
|
||||
static int32_t smaOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) {
|
||||
int ret;
|
||||
tdb_cmpr_fn_t compFunc;
|
||||
|
||||
// Create a database
|
||||
compFunc = tdSmaKeyCmpr;
|
||||
ret = tdbDbOpen(pFName, -1, -1, compFunc, pEnv, ppDB);
|
||||
if(tdbDbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -76,7 +77,7 @@ int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF) {
|
|||
|
||||
// Open DBF
|
||||
if (smaOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) {
|
||||
terrno = TSDB_CODE_TDB_INIT_FAILED;
|
||||
smaError("failed to open DBF: %s", pDBF->path);
|
||||
smaCloseDBDb(pDBF->pDB);
|
||||
return -1;
|
||||
}
|
||||
|
|
@ -97,9 +98,10 @@ int32_t smaCloseDBF(SDBFile *pDBF) {
|
|||
int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) {
|
||||
int32_t ret;
|
||||
|
||||
ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
|
||||
printf("save tsma data into %s, keyLen:%d valLen:%d txn:%p\n", pDBF->path, keyLen, valLen, txn);
|
||||
ret = tdbDbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
|
||||
if (ret < 0) {
|
||||
smaError("failed to create insert sma data into db, ret = %d", ret);
|
||||
smaError("failed to upsert tsma data into db, ret = %d", ret);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -113,7 +115,7 @@ void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_
|
|||
ret = tdbDbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen);
|
||||
|
||||
if (ret < 0) {
|
||||
smaError("failed to get sma data from db, ret = %d", ret);
|
||||
smaError("failed to get tsma data from db, ret = %d", ret);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,21 +16,22 @@
|
|||
#include "sma.h"
|
||||
#include "tsdb.h"
|
||||
|
||||
typedef STsdbCfg STSmaKeepCfg;
|
||||
|
||||
#undef _TEST_SMA_PRINT_DEBUG_LOG_
|
||||
#define SMA_STORAGE_TSDB_DAYS 30
|
||||
#define SMA_STORAGE_TSDB_TIMES 10
|
||||
#define SMA_STORAGE_SPLIT_HOURS 24
|
||||
#define SMA_KEY_LEN 16 // TSKEY+groupId 8+8
|
||||
#define SMA_DROP_EXPIRED_TIME 10 // default is 10 seconds
|
||||
#define SMA_STORAGE_TSDB_MINUTES 86400
|
||||
#define SMA_STORAGE_TSDB_TIMES 10
|
||||
#define SMA_STORAGE_SPLIT_FACTOR 144 // least records in tsma file
|
||||
#define SMA_KEY_LEN 16 // TSKEY+groupId 8+8
|
||||
#define SMA_DROP_EXPIRED_TIME 10 // default is 10 seconds
|
||||
|
||||
#define SMA_STATE_ITEM_HASH_SLOT 32
|
||||
|
||||
|
||||
typedef struct {
|
||||
SSma *pSma;
|
||||
SDBFile dFile;
|
||||
const SArray *pDataBlocks; // sma data
|
||||
int32_t interval; // interval with the precision of DB
|
||||
int64_t interval; // interval with the precision of DB
|
||||
} STSmaWriteH;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -42,10 +43,10 @@ typedef struct {
|
|||
STsdb *pTsdb;
|
||||
SSma *pSma;
|
||||
SDBFile dFile;
|
||||
int32_t interval; // interval with the precision of DB
|
||||
int64_t interval; // interval with the precision of DB
|
||||
int32_t blockSize; // size of SMA block item
|
||||
int32_t days;
|
||||
int8_t storageLevel;
|
||||
int8_t days;
|
||||
SmaFsIter smaFsIter;
|
||||
} STSmaReadH;
|
||||
|
||||
|
|
@ -58,9 +59,9 @@ typedef enum {
|
|||
// static func
|
||||
|
||||
static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted);
|
||||
static int32_t tdGetSmaStorageLevel(int64_t interval, int8_t intervalUnit);
|
||||
static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval);
|
||||
static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval,
|
||||
int8_t intervalUnit);
|
||||
int8_t intervalUnit);
|
||||
static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit);
|
||||
static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH);
|
||||
static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel);
|
||||
|
|
@ -92,9 +93,10 @@ static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid);
|
|||
* @return int32_t
|
||||
*/
|
||||
static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit) {
|
||||
STSmaKeepCfg *pCfg = SMA_TSDB_CFG(pSma);
|
||||
pSmaH->pSma = pSma;
|
||||
pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true);
|
||||
pSmaH->storageLevel = tdGetSmaStorageLevel(interval, intervalUnit);
|
||||
pSmaH->storageLevel = tdGetSmaStorageLevel(pCfg, interval);
|
||||
pSmaH->days = tdGetTSmaDays(pSma, pSmaH->interval, pSmaH->storageLevel);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
|
@ -275,11 +277,13 @@ static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t f
|
|||
*/
|
||||
static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel) {
|
||||
STsdbCfg *pCfg = SMA_TSDB_CFG(pSma);
|
||||
int32_t daysPerFile = pCfg->days;
|
||||
int32_t daysPerFile = pCfg->days; // unit is minute
|
||||
|
||||
if (storageLevel == SMA_STORAGE_LEVEL_TSDB) {
|
||||
int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]);
|
||||
daysPerFile = days > SMA_STORAGE_TSDB_DAYS ? days : SMA_STORAGE_TSDB_DAYS;
|
||||
int32_t minutes = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]);
|
||||
if (minutes > SMA_STORAGE_TSDB_MINUTES) {
|
||||
daysPerFile = SMA_STORAGE_TSDB_MINUTES;
|
||||
}
|
||||
}
|
||||
|
||||
return daysPerFile;
|
||||
|
|
@ -288,45 +292,14 @@ static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel)
|
|||
/**
|
||||
* @brief Judge the tSma storage level
|
||||
*
|
||||
* @param pCfg
|
||||
* @param interval
|
||||
* @param intervalUnit
|
||||
* @return int32_t
|
||||
*/
|
||||
static int32_t tdGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
|
||||
// TODO: configurable for SMA_STORAGE_SPLIT_HOURS?
|
||||
switch (intervalUnit) {
|
||||
case TIME_UNIT_HOUR:
|
||||
if (interval < SMA_STORAGE_SPLIT_HOURS) {
|
||||
return SMA_STORAGE_LEVEL_DFILESET;
|
||||
}
|
||||
break;
|
||||
case TIME_UNIT_MINUTE:
|
||||
if (interval < 60 * SMA_STORAGE_SPLIT_HOURS) {
|
||||
return SMA_STORAGE_LEVEL_DFILESET;
|
||||
}
|
||||
break;
|
||||
case TIME_UNIT_SECOND:
|
||||
if (interval < 3600 * SMA_STORAGE_SPLIT_HOURS) {
|
||||
return SMA_STORAGE_LEVEL_DFILESET;
|
||||
}
|
||||
break;
|
||||
case TIME_UNIT_MILLISECOND:
|
||||
if (interval < 3600 * 1e3 * SMA_STORAGE_SPLIT_HOURS) {
|
||||
return SMA_STORAGE_LEVEL_DFILESET;
|
||||
}
|
||||
break;
|
||||
case TIME_UNIT_MICROSECOND:
|
||||
if (interval < 3600 * 1e6 * SMA_STORAGE_SPLIT_HOURS) {
|
||||
return SMA_STORAGE_LEVEL_DFILESET;
|
||||
}
|
||||
break;
|
||||
case TIME_UNIT_NANOSECOND:
|
||||
if (interval < 3600 * 1e9 * SMA_STORAGE_SPLIT_HOURS) {
|
||||
return SMA_STORAGE_LEVEL_DFILESET;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval) {
|
||||
int64_t mInterval = convertTimeFromPrecisionToUnit(interval, pCfg->precision, TIME_UNIT_MINUTE);
|
||||
if (pCfg->days / mInterval >= SMA_STORAGE_SPLIT_FACTOR) {
|
||||
return SMA_STORAGE_LEVEL_DFILESET;
|
||||
}
|
||||
return SMA_STORAGE_LEVEL_TSDB;
|
||||
}
|
||||
|
|
@ -346,6 +319,7 @@ static int32_t tdGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
|
|||
int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
|
||||
STsdbCfg *pCfg = SMA_TSDB_CFG(pSma);
|
||||
const SArray *pDataBlocks = (const SArray *)msg;
|
||||
int64_t testSkey = TSKEY_INITIAL_VAL;
|
||||
|
||||
// TODO: destroy SSDataBlocks(msg)
|
||||
|
||||
|
|
@ -403,8 +377,8 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
|
|||
}
|
||||
|
||||
// Step 1: Judge the storage level and days
|
||||
int32_t storageLevel = tdGetSmaStorageLevel(pTSma->interval, pTSma->intervalUnit);
|
||||
int32_t daysPerFile = tdGetTSmaDays(pSma, tSmaH.interval, storageLevel);
|
||||
int32_t storageLevel = tdGetSmaStorageLevel(pCfg, tSmaH.interval);
|
||||
int32_t minutePerFile = tdGetTSmaDays(pSma, tSmaH.interval, storageLevel);
|
||||
|
||||
char smaKey[SMA_KEY_LEN] = {0}; // key: skey + groupId
|
||||
char dataBuf[512] = {0}; // val: aggr data // TODO: handle 512 buffer?
|
||||
|
|
@ -432,6 +406,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
|
|||
if (!isStartKey) {
|
||||
isStartKey = true;
|
||||
skey = *(TSKEY *)var;
|
||||
testSkey = skey;
|
||||
printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId);
|
||||
tdEncodeTSmaKey(groupId, skey, &pSmaKey);
|
||||
} else {
|
||||
|
|
@ -503,9 +478,10 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
|
|||
break;
|
||||
}
|
||||
}
|
||||
printf("\n");
|
||||
// if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) {
|
||||
if (tlen > 0) {
|
||||
int32_t fid = (int32_t)(TSDB_KEY_FID(skey, daysPerFile, pCfg->precision));
|
||||
int32_t fid = (int32_t)(TSDB_KEY_FID(skey, minutePerFile, pCfg->precision));
|
||||
|
||||
// Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index
|
||||
// file
|
||||
|
|
@ -517,6 +493,8 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
|
|||
smaCloseDBF(&tSmaH.dFile);
|
||||
}
|
||||
tdSetTSmaDataFile(&tSmaH, indexUid, fid);
|
||||
smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 " queryKey:%" PRIi64,
|
||||
SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey);
|
||||
if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) {
|
||||
smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma),
|
||||
tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno));
|
||||
|
|
@ -528,16 +506,17 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
|
|||
}
|
||||
|
||||
if (tdInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) {
|
||||
smaWarn("vgId:%d insert tSma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64
|
||||
" since %s",
|
||||
SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno));
|
||||
smaWarn("vgId:%d insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64
|
||||
" since %s",
|
||||
SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno));
|
||||
tdSmaEndCommit(pEnv);
|
||||
tdDestroyTSmaWriteH(&tSmaH);
|
||||
tdUnRefSmaStat(pSma, pStat);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
smaDebug("vgId:%d insert tSma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64,
|
||||
SMA_VID(pSma), indexUid, skey, groupId);
|
||||
|
||||
smaDebug("vgId:%d insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64,
|
||||
SMA_VID(pSma), indexUid, skey, groupId);
|
||||
// TODO:tsdbEndTSmaCommit();
|
||||
|
||||
// Step 3: reset the SSmaStat
|
||||
|
|
@ -547,7 +526,6 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
|
|||
SMA_VID(pSma), skey, tlen, indexUid);
|
||||
}
|
||||
|
||||
printf("\n");
|
||||
}
|
||||
}
|
||||
tdSmaEndCommit(pEnv); // TODO: not commit for every insert
|
||||
|
|
@ -579,14 +557,14 @@ static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyL
|
|||
TXN *txn) {
|
||||
SDBFile *pDBFile = &pSmaH->dFile;
|
||||
|
||||
// TODO: insert sma data blocks into B+Tree(TDB)
|
||||
// TODO: insert tsma data blocks into B+Tree(TDB)
|
||||
if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) {
|
||||
smaWarn("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
|
||||
SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
|
||||
smaWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
|
||||
SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
smaDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed",
|
||||
SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
|
||||
smaDebug("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed",
|
||||
SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
|
||||
|
||||
#ifdef _TEST_SMA_PRINT_DEBUG_LOG_
|
||||
uint32_t valueSize = 0;
|
||||
|
|
@ -776,6 +754,8 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query
|
|||
tdUnRefSmaStat(pSma, pStat);
|
||||
|
||||
tdInitTSmaFile(&tReadH, indexUid, querySKey);
|
||||
smaDebug("### vgId:%d read from DBF %s days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi8 " queryKey:%" PRIi64,
|
||||
SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey);
|
||||
if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) {
|
||||
smaWarn("vgId:%d open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno));
|
||||
return TSDB_CODE_FAILED;
|
||||
|
|
@ -783,7 +763,7 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query
|
|||
|
||||
char smaKey[SMA_KEY_LEN] = {0};
|
||||
void *pSmaKey = &smaKey;
|
||||
int64_t queryGroupId = 1;
|
||||
int64_t queryGroupId = 0;
|
||||
tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey);
|
||||
|
||||
smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma),
|
||||
|
|
@ -915,8 +895,8 @@ static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t inde
|
|||
terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META;
|
||||
taosHashCleanup(pItem->expiredWindows);
|
||||
taosMemoryFree(pItem);
|
||||
smaWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid,
|
||||
tstrerror(terrno));
|
||||
smaWarn("vgId:%d set expire window, get tsma meta failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma),
|
||||
indexUid, tstrerror(terrno));
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
pItem->pTSma = pTSma;
|
||||
|
|
@ -1021,7 +1001,7 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version)
|
|||
pSW = tdFreeTSmaWrapper(pSW, false);
|
||||
break;
|
||||
}
|
||||
if (!pSW || (pTSma->tableUid != msgIter.suid)) {
|
||||
if (!pSW || (pTSma && (pTSma->tableUid != msgIter.suid))) {
|
||||
if (pSW) {
|
||||
pSW = tdFreeTSmaWrapper(pSW, false);
|
||||
}
|
||||
|
|
@ -1043,6 +1023,7 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version)
|
|||
interval.slidingUnit = pTSma->slidingUnit;
|
||||
}
|
||||
|
||||
// TODO: process multiple tsma for one table uid
|
||||
TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision);
|
||||
|
||||
if (lastWinSKey != winSKey) {
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
|
|||
int slen = 0;
|
||||
|
||||
*ppTsdb = NULL;
|
||||
slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + TSDB_DATA_DIR_LEN + 3;
|
||||
slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + 3;
|
||||
|
||||
// create handle
|
||||
pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen);
|
||||
|
|
@ -73,7 +73,8 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
|
|||
goto _err;
|
||||
}
|
||||
|
||||
tsdbDebug("vgId:%d tsdb is opened for %s", TD_VID(pVnode), pTsdb->path);
|
||||
tsdbDebug("vgId:%d tsdb is opened for %s, days:%d, keep:%d,%d,%d", TD_VID(pVnode), pTsdb->path, pTsdb->keepCfg.days,
|
||||
pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2);
|
||||
|
||||
*ppTsdb = pTsdb;
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -372,13 +372,13 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle,
|
|||
}
|
||||
|
||||
if (level == TSDB_RETENTION_L0) {
|
||||
tsdbDebug("%p rsma level %d is selected to query", pReadHandle, TSDB_RETENTION_L0);
|
||||
tsdbDebug("vgId:%d read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L0);
|
||||
return VND_RSMA0(pVnode);
|
||||
} else if (level == TSDB_RETENTION_L1) {
|
||||
tsdbDebug("%p rsma level %d is selected to query", pReadHandle, TSDB_RETENTION_L1);
|
||||
tsdbDebug("vgId:%d read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L1);
|
||||
return VND_RSMA1(pVnode);
|
||||
} else {
|
||||
tsdbDebug("%p rsma level %d is selected to query", pReadHandle, TSDB_RETENTION_L2);
|
||||
tsdbDebug("vgId:%d read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L2);
|
||||
return VND_RSMA2(pVnode);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -876,13 +876,13 @@ static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t ke
|
|||
TXN *txn) {
|
||||
SDBFile *pDBFile = &pSmaH->dFile;
|
||||
|
||||
// TODO: insert sma data blocks into B+Tree(TDB)
|
||||
// TODO: insert tsma data blocks into B+Tree(TDB)
|
||||
if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) {
|
||||
tsdbWarn("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
|
||||
tsdbWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
|
||||
REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
tsdbDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed",
|
||||
tsdbDebug("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed",
|
||||
REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
|
||||
|
||||
#ifdef _TEST_SMA_PRINT_DEBUG_LOG_
|
||||
|
|
@ -1245,7 +1245,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
|
|||
}
|
||||
|
||||
if (tsdbInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) {
|
||||
tsdbWarn("vgId:%d insert tSma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64
|
||||
tsdbWarn("vgId:%d insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64
|
||||
" since %s",
|
||||
REPO_ID(pTsdb), indexUid, skey, groupId, tstrerror(terrno));
|
||||
tsdbSmaEndCommit(pEnv);
|
||||
|
|
@ -1253,7 +1253,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
|
|||
tsdbUnRefSmaStat(pTsdb, pStat);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
tsdbDebug("vgId:%d insert tSma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64,
|
||||
tsdbDebug("vgId:%d insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64,
|
||||
REPO_ID(pTsdb), indexUid, skey, groupId);
|
||||
// TODO:tsdbEndTSmaCommit();
|
||||
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
|
|||
|
||||
// open sma
|
||||
if (smaOpen(pVnode)) {
|
||||
vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
|
||||
vError("vgId:%d failed to open vnode sma since %s", TD_VID(pVnode), tstrerror(terrno));
|
||||
goto _err;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -85,6 +85,11 @@ bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo)
|
|||
int32_t spreadFunction(SqlFunctionCtx* pCtx);
|
||||
int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
|
||||
|
||||
bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
|
||||
bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
|
||||
int32_t elapsedFunction(SqlFunctionCtx* pCtx);
|
||||
int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
|
||||
|
||||
bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
|
||||
bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
|
||||
int32_t histogramFunction(SqlFunctionCtx* pCtx);
|
||||
|
|
|
|||
|
|
@ -226,6 +226,27 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
|
||||
int32_t paraNum = LIST_LENGTH(pFunc->pParameterList);
|
||||
if (1 != paraNum && 2 != paraNum) {
|
||||
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
|
||||
if (QUERY_NODE_COLUMN != nodeType(pPara)) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"The input parameter of ELAPSED function can only be column");
|
||||
}
|
||||
|
||||
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
|
||||
if (TSDB_DATA_TYPE_TIMESTAMP != paraType) {
|
||||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE};
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t translateLeastSQR(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
|
||||
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
|
||||
if (3 != numOfParams) {
|
||||
|
|
@ -794,6 +815,17 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
.processFunc = spreadFunction,
|
||||
.finalizeFunc = spreadFinalize
|
||||
},
|
||||
{
|
||||
.name = "elapsed",
|
||||
.type = FUNCTION_TYPE_ELAPSED,
|
||||
.classification = FUNC_MGT_AGG_FUNC,
|
||||
.dataRequiredFunc = statisDataRequired,
|
||||
.translateFunc = translateElapsed,
|
||||
.getEnvFunc = getElapsedFuncEnv,
|
||||
.initFunc = elapsedFunctionSetup,
|
||||
.processFunc = elapsedFunction,
|
||||
.finalizeFunc = elapsedFinalize
|
||||
},
|
||||
{
|
||||
.name = "last_row",
|
||||
.type = FUNCTION_TYPE_LAST_ROW,
|
||||
|
|
|
|||
|
|
@ -112,6 +112,13 @@ typedef struct SSpreadInfo {
|
|||
double max;
|
||||
} SSpreadInfo;
|
||||
|
||||
typedef struct SElapsedInfo {
|
||||
double result;
|
||||
TSKEY min;
|
||||
TSKEY max;
|
||||
int64_t timeUnit;
|
||||
} SElapsedInfo;
|
||||
|
||||
typedef struct SHistoFuncBin {
|
||||
double lower;
|
||||
double upper;
|
||||
|
|
@ -2494,6 +2501,116 @@ int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
|||
return functionFinalize(pCtx, pBlock);
|
||||
}
|
||||
|
||||
bool getElapsedFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
|
||||
pEnv->calcMemSize = sizeof(SElapsedInfo);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
|
||||
if (!functionSetup(pCtx, pResultInfo)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
|
||||
pInfo->result = 0;
|
||||
pInfo->min = MAX_TS_KEY;
|
||||
pInfo->max = 0;
|
||||
|
||||
if (pCtx->numOfParams == 3) {
|
||||
pInfo->timeUnit = pCtx->param[1].param.i;
|
||||
} else {
|
||||
pInfo->timeUnit = 1;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
int32_t elapsedFunction(SqlFunctionCtx *pCtx) {
|
||||
int32_t numOfElems = 0;
|
||||
|
||||
// Only the pre-computing information loaded and actual data does not loaded
|
||||
SInputColumnInfoData* pInput = &pCtx->input;
|
||||
SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0];
|
||||
|
||||
SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
|
||||
|
||||
numOfElems = pInput->numOfRows; //since this is the primary timestamp, no need to exclude NULL values
|
||||
if (numOfElems == 0) {
|
||||
goto _elapsed_over;
|
||||
}
|
||||
|
||||
if (pInput->colDataAggIsSet) {
|
||||
|
||||
if (pInfo->min == MAX_TS_KEY) {
|
||||
pInfo->min = GET_INT64_VAL(&pAgg->min);
|
||||
pInfo->max = GET_INT64_VAL(&pAgg->max);
|
||||
} else {
|
||||
if (pCtx->order == TSDB_ORDER_ASC) {
|
||||
pInfo->max = GET_INT64_VAL(&pAgg->max);
|
||||
} else {
|
||||
pInfo->min = GET_INT64_VAL(&pAgg->min);
|
||||
}
|
||||
}
|
||||
} else { // computing based on the true data block
|
||||
if (0 == pCtx->size) {
|
||||
if (pCtx->order == TSDB_ORDER_DESC) {
|
||||
if (pCtx->end.key != INT64_MIN) {
|
||||
pInfo->min = pCtx->end.key;
|
||||
}
|
||||
} else {
|
||||
if (pCtx->end.key != INT64_MIN) {
|
||||
pInfo->max = pCtx->end.key + 1;
|
||||
}
|
||||
}
|
||||
goto _elapsed_over;
|
||||
}
|
||||
|
||||
SColumnInfoData* pCol = pInput->pData[0];
|
||||
|
||||
int32_t start = pInput->startRowIndex;
|
||||
TSKEY* ptsList = (int64_t*)colDataGetData(pCol, start);
|
||||
if (pCtx->order == TSDB_ORDER_DESC) {
|
||||
if (pCtx->start.key == INT64_MIN) {
|
||||
pInfo->max = (pInfo->max < ptsList[pCtx->size - 1]) ? ptsList[pCtx->size - 1] : pInfo->max;
|
||||
} else {
|
||||
pInfo->max = pCtx->start.key + 1;
|
||||
}
|
||||
|
||||
if (pCtx->end.key != INT64_MIN) {
|
||||
pInfo->min = pCtx->end.key;
|
||||
} else {
|
||||
pInfo->min = ptsList[0];
|
||||
}
|
||||
} else {
|
||||
if (pCtx->start.key == INT64_MIN) {
|
||||
pInfo->min = (pInfo->min > ptsList[0]) ? ptsList[0] : pInfo->min;
|
||||
} else {
|
||||
pInfo->min = pCtx->start.key;
|
||||
}
|
||||
|
||||
if (pCtx->end.key != INT64_MIN) {
|
||||
pInfo->max = pCtx->end.key + 1;
|
||||
} else {
|
||||
pInfo->max = ptsList[pCtx->size - 1];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_elapsed_over:
|
||||
// data in the check operation are all null, not output
|
||||
SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
||||
SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
|
||||
double result = (double)pInfo->max - (double)pInfo->min;
|
||||
result = (result >= 0) ? result : -result;
|
||||
pInfo->result = result / pInfo->timeUnit;
|
||||
return functionFinalize(pCtx, pBlock);
|
||||
}
|
||||
|
||||
bool getHistogramFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
|
||||
pEnv->calcMemSize = sizeof(SHistoFuncInfo) + HISTOGRAM_MAX_BINS_NUM * sizeof(SHistoFuncBin);
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -1605,7 +1605,12 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult
|
|||
return false;
|
||||
}
|
||||
udfRes->interResNum = buf.numOfResult;
|
||||
memcpy(udfRes->interResBuf, buf.buf, buf.bufLen);
|
||||
if (buf.bufLen <= session->bufSize) {
|
||||
memcpy(udfRes->interResBuf, buf.buf, buf.bufLen);
|
||||
} else {
|
||||
fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize);
|
||||
return false;
|
||||
}
|
||||
freeUdfInterBuf(&buf);
|
||||
return true;
|
||||
}
|
||||
|
|
@ -1655,7 +1660,12 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
|
|||
newState.numOfResult = 0;
|
||||
} else {
|
||||
udfRes->interResNum = newState.numOfResult;
|
||||
memcpy(udfRes->interResBuf, newState.buf, newState.bufLen);
|
||||
if (newState.bufLen <= session->bufSize) {
|
||||
memcpy(udfRes->interResBuf, newState.buf, newState.bufLen);
|
||||
} else {
|
||||
fnError("udfc inter buf size %d is greater than function bufSize %d", newState.bufLen, session->bufSize);
|
||||
udfCode = TSDB_CODE_UDF_INVALID_BUFSIZE;
|
||||
}
|
||||
}
|
||||
if (newState.numOfResult == 1 || state.numOfResult == 1) {
|
||||
GET_RES_INFO(pCtx)->numOfRes = 1;
|
||||
|
|
@ -1688,9 +1698,15 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
|
|||
fnError("udfAggFinalize error. doCallUdfAggFinalize step. udf code:%d", udfCallCode);
|
||||
GET_RES_INFO(pCtx)->numOfRes = 0;
|
||||
} else {
|
||||
memcpy(udfRes->finalResBuf, resultBuf.buf, session->outputLen);
|
||||
udfRes->finalResNum = resultBuf.numOfResult;
|
||||
GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum;
|
||||
if (resultBuf.bufLen <= session->outputLen) {
|
||||
memcpy(udfRes->finalResBuf, resultBuf.buf, session->outputLen);
|
||||
udfRes->finalResNum = resultBuf.numOfResult;
|
||||
GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum;
|
||||
} else {
|
||||
fnError("udfc inter buf size %d is greater than function output size %d", resultBuf.bufLen, session->outputLen);
|
||||
GET_RES_INFO(pCtx)->numOfRes = 0;
|
||||
udfCallCode = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE;
|
||||
}
|
||||
}
|
||||
|
||||
freeUdfInterBuf(&resultBuf);
|
||||
|
|
|
|||
|
|
@ -616,9 +616,8 @@ void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
|
|||
}
|
||||
|
||||
void udfdOnNewConnection(uv_stream_t *server, int status) {
|
||||
fnDebug("new connection");
|
||||
if (status < 0) {
|
||||
// TODO
|
||||
fnError("udfd new connection error. code: %s", uv_strerror(status));
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,12 +20,13 @@
|
|||
#include "tcompare.h"
|
||||
#include "tdataformat.h"
|
||||
#include "ttypes.h"
|
||||
#include "tvariant.h"
|
||||
|
||||
char JSON_COLUMN[] = "JSON";
|
||||
char JSON_VALUE_DELIM = '&';
|
||||
|
||||
char* indexInt2str(int64_t val, char* dst, int radix) {
|
||||
char buffer[65];
|
||||
char buffer[65] = {0};
|
||||
char* p;
|
||||
int64_t new_val;
|
||||
uint64_t uval = (uint64_t)val;
|
||||
|
|
@ -74,28 +75,70 @@ static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
|
|||
return tCompare(func, QUERY_GREATER_EQUAL, a, b, type);
|
||||
}
|
||||
TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) {
|
||||
if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR) {
|
||||
if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY) {
|
||||
return tDoCompare(func, cmptype, a, b);
|
||||
}
|
||||
#if 1
|
||||
int8_t bytes = tDataTypes[dtype].bytes;
|
||||
if (bytes == 1) {
|
||||
int8_t va = taosStr2int64(a);
|
||||
int8_t vb = taosStr2int64(b);
|
||||
return tDoCompare(func, cmptype, &va, &vb);
|
||||
} else if (bytes == 2) {
|
||||
int16_t va = taosStr2int64(a);
|
||||
int16_t vb = taosStr2int64(b);
|
||||
return tDoCompare(func, cmptype, &va, &vb);
|
||||
} else if (bytes == 4) {
|
||||
int32_t va = taosStr2int64(a);
|
||||
int32_t vb = taosStr2int64(b);
|
||||
return tDoCompare(func, cmptype, &va, &vb);
|
||||
} else {
|
||||
if (dtype == TSDB_DATA_TYPE_TIMESTAMP) {
|
||||
int64_t va = taosStr2int64(a);
|
||||
int64_t vb = taosStr2int64(b);
|
||||
return tDoCompare(func, cmptype, &va, &vb);
|
||||
} else if (dtype == TSDB_DATA_TYPE_BOOL || dtype == TSDB_DATA_TYPE_UTINYINT) {
|
||||
uint8_t va = taosStr2int64(a);
|
||||
uint8_t vb = taosStr2int64(b);
|
||||
return tDoCompare(func, cmptype, &va, &vb);
|
||||
} else if (dtype == TSDB_DATA_TYPE_TINYINT) {
|
||||
int8_t va = taosStr2int64(a);
|
||||
int8_t vb = taosStr2int64(b);
|
||||
return tDoCompare(func, cmptype, &va, &vb);
|
||||
} else if (dtype == TSDB_DATA_TYPE_SMALLINT) {
|
||||
int16_t va = taosStr2int64(a);
|
||||
int16_t vb = taosStr2int64(b);
|
||||
return tDoCompare(func, cmptype, &va, &vb);
|
||||
} else if (dtype == TSDB_DATA_TYPE_USMALLINT) {
|
||||
uint16_t va = taosStr2int64(a);
|
||||
uint16_t vb = taosStr2int64(b);
|
||||
return tDoCompare(func, cmptype, &va, &vb);
|
||||
} else if (dtype == TSDB_DATA_TYPE_INT) {
|
||||
int32_t va = taosStr2int64(a);
|
||||
int32_t vb = taosStr2int64(b);
|
||||
return tDoCompare(func, cmptype, &va, &vb);
|
||||
} else if (dtype == TSDB_DATA_TYPE_UINT) {
|
||||
uint32_t va = taosStr2int64(a);
|
||||
uint32_t vb = taosStr2int64(b);
|
||||
return tDoCompare(func, cmptype, &va, &vb);
|
||||
} else if (dtype == TSDB_DATA_TYPE_BIGINT) {
|
||||
int64_t va = taosStr2int64(a);
|
||||
int64_t vb = taosStr2int64(b);
|
||||
return tDoCompare(func, cmptype, &va, &vb);
|
||||
} else if (dtype == TSDB_DATA_TYPE_UBIGINT) {
|
||||
uint64_t va, vb;
|
||||
if (0 != toUInteger(a, strlen(a), 10, &va) || 0 != toUInteger(b, strlen(b), 10, &vb)) {
|
||||
return CONTINUE;
|
||||
}
|
||||
return tDoCompare(func, cmptype, &va, &vb);
|
||||
} else if (dtype == TSDB_DATA_TYPE_FLOAT) {
|
||||
float va = strtod(a, NULL);
|
||||
if (errno == ERANGE && va == -1) {
|
||||
return CONTINUE;
|
||||
}
|
||||
float vb = strtod(b, NULL);
|
||||
if (errno == ERANGE && va == -1) {
|
||||
return CONTINUE;
|
||||
}
|
||||
return tDoCompare(func, cmptype, &va, &vb);
|
||||
} else if (dtype == TSDB_DATA_TYPE_DOUBLE) {
|
||||
double va = strtod(a, NULL);
|
||||
if (errno == ERANGE && va == -1) {
|
||||
return CONTINUE;
|
||||
}
|
||||
double vb = strtod(b, NULL);
|
||||
if (errno == ERANGE && va == -1) {
|
||||
return CONTINUE;
|
||||
}
|
||||
return tDoCompare(func, cmptype, &va, &vb);
|
||||
}
|
||||
assert(0);
|
||||
#endif
|
||||
}
|
||||
TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) {
|
||||
|
|
@ -248,20 +291,16 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
|
|||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY
|
||||
#if 1
|
||||
tlen = taosEncodeBinary(NULL, src, strlen(src));
|
||||
*dst = taosMemoryCalloc(1, tlen + 1);
|
||||
tlen = taosEncodeBinary(dst, src, strlen(src));
|
||||
break;
|
||||
#endif
|
||||
}
|
||||
case TSDB_DATA_TYPE_VARBINARY:
|
||||
#if 1
|
||||
tlen = taosEncodeBinary(NULL, src, strlen(src));
|
||||
*dst = taosMemoryCalloc(1, tlen + 1);
|
||||
tlen = taosEncodeBinary(dst, src, strlen(src));
|
||||
break;
|
||||
#endif
|
||||
default:
|
||||
TASSERT(0);
|
||||
break;
|
||||
|
|
@ -271,87 +310,73 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
|
|||
return tlen;
|
||||
}
|
||||
int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
|
||||
int tlen = tDataTypes[type].bytes;
|
||||
|
||||
int tlen = tDataTypes[type].bytes;
|
||||
int32_t bufSize = 64;
|
||||
switch (type) {
|
||||
case TSDB_DATA_TYPE_TIMESTAMP:
|
||||
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
|
||||
*dst = taosMemoryCalloc(1, bufSize + 1);
|
||||
indexInt2str(*(int64_t*)src, *dst, -1);
|
||||
break;
|
||||
case TSDB_DATA_TYPE_BOOL:
|
||||
case TSDB_DATA_TYPE_UTINYINT:
|
||||
// tlen = taosEncodeFixedU8(NULL, *(uint8_t*)src);
|
||||
//*dst = taosMemoryCalloc(1, tlen + 1);
|
||||
// tlen = taosEncodeFixedU8(dst, *(uint8_t*)src);
|
||||
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
|
||||
*dst = taosMemoryCalloc(1, bufSize + 1);
|
||||
indexInt2str(*(uint8_t*)src, *dst, 1);
|
||||
break;
|
||||
case TSDB_DATA_TYPE_TINYINT:
|
||||
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
|
||||
*dst = taosMemoryCalloc(1, bufSize + 1);
|
||||
indexInt2str(*(int8_t*)src, *dst, 1);
|
||||
break;
|
||||
case TSDB_DATA_TYPE_SMALLINT:
|
||||
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
|
||||
*dst = taosMemoryCalloc(1, bufSize + 1);
|
||||
indexInt2str(*(int16_t*)src, *dst, -1);
|
||||
break;
|
||||
case TSDB_DATA_TYPE_USMALLINT:
|
||||
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
|
||||
*dst = taosMemoryCalloc(1, bufSize + 1);
|
||||
indexInt2str(*(uint16_t*)src, *dst, -1);
|
||||
break;
|
||||
case TSDB_DATA_TYPE_INT:
|
||||
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
|
||||
*dst = taosMemoryCalloc(1, bufSize + 1);
|
||||
indexInt2str(*(int32_t*)src, *dst, -1);
|
||||
break;
|
||||
case TSDB_DATA_TYPE_FLOAT:
|
||||
tlen = taosEncodeBinary(NULL, src, sizeof(float));
|
||||
*dst = taosMemoryCalloc(1, tlen + 1);
|
||||
tlen = taosEncodeBinary(dst, src, sizeof(float));
|
||||
*dst = (char*) * dst - tlen;
|
||||
break;
|
||||
case TSDB_DATA_TYPE_UINT:
|
||||
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
|
||||
*dst = taosMemoryCalloc(1, bufSize + 1);
|
||||
indexInt2str(*(uint32_t*)src, *dst, 1);
|
||||
break;
|
||||
case TSDB_DATA_TYPE_BIGINT:
|
||||
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
|
||||
indexInt2str(*(int64_t*)src, *dst, 1);
|
||||
break;
|
||||
case TSDB_DATA_TYPE_DOUBLE:
|
||||
tlen = taosEncodeBinary(NULL, src, sizeof(double));
|
||||
*dst = taosMemoryCalloc(1, tlen + 1);
|
||||
tlen = taosEncodeBinary(dst, src, sizeof(double));
|
||||
*dst = (char*) * dst - tlen;
|
||||
*dst = taosMemoryCalloc(1, bufSize + 1);
|
||||
sprintf(*dst, "%" PRIu64, *(uint64_t*)src);
|
||||
break;
|
||||
case TSDB_DATA_TYPE_UBIGINT:
|
||||
assert(0);
|
||||
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
|
||||
*dst = taosMemoryCalloc(1, bufSize + 1);
|
||||
indexInt2str(*(uint64_t*)src, *dst, 1);
|
||||
case TSDB_DATA_TYPE_FLOAT:
|
||||
*dst = taosMemoryCalloc(1, bufSize + 1);
|
||||
sprintf(*dst, "%.9lf", *(float*)src);
|
||||
break;
|
||||
case TSDB_DATA_TYPE_DOUBLE:
|
||||
*dst = taosMemoryCalloc(1, bufSize + 1);
|
||||
sprintf(*dst, "%.9lf", *(double*)src);
|
||||
break;
|
||||
case TSDB_DATA_TYPE_NCHAR: {
|
||||
tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
|
||||
*dst = taosMemoryCalloc(1, tlen + 1);
|
||||
tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
|
||||
*dst = (char*) * dst - tlen;
|
||||
|
||||
*dst = *dst - tlen;
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY
|
||||
#if 1
|
||||
tlen = taosEncodeBinary(NULL, src, strlen(src));
|
||||
*dst = taosMemoryCalloc(1, tlen + 1);
|
||||
tlen = taosEncodeBinary(dst, src, strlen(src));
|
||||
*dst = (char*) * dst - tlen;
|
||||
*dst = (char*)*dst - tlen;
|
||||
break;
|
||||
#endif
|
||||
}
|
||||
case TSDB_DATA_TYPE_VARBINARY:
|
||||
#if 1
|
||||
tlen = taosEncodeBinary(NULL, src, strlen(src));
|
||||
*dst = taosMemoryCalloc(1, tlen + 1);
|
||||
tlen = taosEncodeBinary(dst, src, strlen(src));
|
||||
*dst = (char*) * dst - tlen;
|
||||
*dst = (char*)*dst - tlen;
|
||||
break;
|
||||
#endif
|
||||
default:
|
||||
TASSERT(0);
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ p *
|
|||
#include "indexFstCountingWriter.h"
|
||||
#include "indexUtil.h"
|
||||
#include "taosdef.h"
|
||||
#include "taoserror.h"
|
||||
#include "tcoding.h"
|
||||
#include "tcompare.h"
|
||||
|
||||
|
|
@ -472,16 +473,16 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR
|
|||
|
||||
int32_t sz = 0;
|
||||
char* ch = (char*)fstSliceData(s, &sz);
|
||||
char* tmp = taosMemoryCalloc(1, sz + 1);
|
||||
memcpy(tmp, ch, sz);
|
||||
// char* tmp = taosMemoryCalloc(1, sz + 1);
|
||||
// memcpy(tmp, ch, sz);
|
||||
|
||||
if (0 != strncmp(tmp, p, skip)) {
|
||||
if (0 != strncmp(ch, p, skip)) {
|
||||
swsResultDestroy(rt);
|
||||
taosMemoryFree(tmp);
|
||||
// taosMemoryFree(tmp);
|
||||
break;
|
||||
}
|
||||
|
||||
TExeCond cond = cmpFn(tmp + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType));
|
||||
TExeCond cond = cmpFn(ch + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType));
|
||||
|
||||
if (MATCH == cond) {
|
||||
tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total);
|
||||
|
|
@ -490,7 +491,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR
|
|||
swsResultDestroy(rt);
|
||||
break;
|
||||
}
|
||||
taosMemoryFree(tmp);
|
||||
// taosMemoryFree(tmp);
|
||||
swsResultDestroy(rt);
|
||||
}
|
||||
streamWithStateDestroy(st);
|
||||
|
|
@ -533,10 +534,12 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c
|
|||
tfileGenFileFullName(fullname, path, suid, colName, version);
|
||||
|
||||
WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024);
|
||||
indexInfo("open read file name:%s, file size: %d", wc->file.buf, wc->file.size);
|
||||
if (wc == NULL) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
indexError("failed to open readonly file: %s, reason: %s", fullname, terrstr());
|
||||
return NULL;
|
||||
}
|
||||
indexInfo("open read file name:%s, file size: %d", wc->file.buf, wc->file.size);
|
||||
|
||||
TFileReader* reader = tfileReaderCreate(wc);
|
||||
return reader;
|
||||
|
|
@ -613,9 +616,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
|
|||
if (tfileWriteData(tw, v) != 0) {
|
||||
indexError("failed to write data: %s, offset: %d len: %d", v->colVal, v->offset,
|
||||
(int)taosArrayGetSize(v->tableId));
|
||||
// printf("write faile\n");
|
||||
} else {
|
||||
// printf("write sucee\n");
|
||||
// indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset,
|
||||
// (int)taosArrayGetSize(v->tableId));
|
||||
|
||||
|
|
|
|||
|
|
@ -553,7 +553,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) {
|
|||
float val = 2.0;
|
||||
std::string colName("test1");
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i);
|
||||
WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i + 1000);
|
||||
}
|
||||
}
|
||||
{
|
||||
|
|
|
|||
|
|
@ -15,10 +15,10 @@
|
|||
#include "tdbInt.h"
|
||||
|
||||
struct SPCache {
|
||||
int pageSize;
|
||||
int cacheSize;
|
||||
int szPage;
|
||||
int nPages;
|
||||
SPage **aPage;
|
||||
tdb_mutex_t mutex;
|
||||
SPage *pList;
|
||||
int nFree;
|
||||
SPage *pFree;
|
||||
int nPage;
|
||||
|
|
@ -52,13 +52,14 @@ int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
|
|||
void *pPtr;
|
||||
SPage *pPgHdr;
|
||||
|
||||
pCache = (SPCache *)tdbOsCalloc(1, sizeof(*pCache));
|
||||
pCache = (SPCache *)tdbOsCalloc(1, sizeof(*pCache) + sizeof(SPage *) * cacheSize);
|
||||
if (pCache == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
pCache->pageSize = pageSize;
|
||||
pCache->cacheSize = cacheSize;
|
||||
pCache->szPage = pageSize;
|
||||
pCache->nPages = cacheSize;
|
||||
pCache->aPage = (SPage **)&pCache[1];
|
||||
|
||||
if (tdbPCacheOpenImpl(pCache) < 0) {
|
||||
tdbOsFree(pCache);
|
||||
|
|
@ -84,7 +85,7 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
|
|||
|
||||
pPage = tdbPCacheFetchImpl(pCache, pPgid, pTxn);
|
||||
if (pPage) {
|
||||
TDB_REF_PAGE(pPage);
|
||||
tdbRefPage(pPage);
|
||||
}
|
||||
|
||||
tdbPCacheUnlock(pCache);
|
||||
|
|
@ -97,7 +98,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
|
|||
|
||||
ASSERT(pTxn);
|
||||
|
||||
nRef = TDB_UNREF_PAGE(pPage);
|
||||
nRef = tdbUnrefPage(pPage);
|
||||
ASSERT(nRef >= 0);
|
||||
|
||||
if (nRef == 0) {
|
||||
|
|
@ -105,7 +106,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
|
|||
|
||||
// test the nRef again to make sure
|
||||
// it is safe th handle the page
|
||||
nRef = TDB_GET_PAGE_REF(pPage);
|
||||
nRef = tdbGetPageRef(pPage);
|
||||
if (nRef == 0) {
|
||||
if (pPage->isLocal) {
|
||||
tdbPCacheUnpinPage(pCache, pPage);
|
||||
|
|
@ -123,7 +124,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
|
|||
}
|
||||
}
|
||||
|
||||
int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; }
|
||||
int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->szPage; }
|
||||
|
||||
static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
|
||||
int ret = 0;
|
||||
|
|
@ -168,7 +169,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
|
|||
|
||||
// 4. Try a create new page
|
||||
if (!pPage) {
|
||||
ret = tdbPageCreate(pCache->pageSize, &pPage, pTxn->xMalloc, pTxn->xArg);
|
||||
ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg);
|
||||
if (ret < 0) {
|
||||
// TODO
|
||||
ASSERT(0);
|
||||
|
|
@ -178,7 +179,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
|
|||
// init the page fields
|
||||
pPage->isAnchor = 0;
|
||||
pPage->isLocal = 0;
|
||||
TDB_INIT_PAGE_REF(pPage);
|
||||
pPage->nRef = 0;
|
||||
pPage->id = -1;
|
||||
}
|
||||
|
||||
// 5. Page here are just created from a free list
|
||||
|
|
@ -212,20 +214,25 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
|
|||
}
|
||||
|
||||
static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) {
|
||||
if (!PAGE_IS_PINNED(pPage)) {
|
||||
if (pPage->pLruNext != NULL) {
|
||||
ASSERT(tdbGetPageRef(pPage) == 0);
|
||||
|
||||
pPage->pLruPrev->pLruNext = pPage->pLruNext;
|
||||
pPage->pLruNext->pLruPrev = pPage->pLruPrev;
|
||||
pPage->pLruNext = NULL;
|
||||
|
||||
pCache->nRecyclable--;
|
||||
|
||||
tdbTrace("pin page %d", pPage->id);
|
||||
}
|
||||
}
|
||||
|
||||
static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) {
|
||||
i32 nRef;
|
||||
|
||||
ASSERT(pPage->isLocal);
|
||||
ASSERT(!pPage->isDirty);
|
||||
ASSERT(TDB_GET_PAGE_REF(pPage) == 0);
|
||||
ASSERT(tdbGetPageRef(pPage) == 0);
|
||||
|
||||
ASSERT(pPage->pLruNext == NULL);
|
||||
|
||||
|
|
@ -235,6 +242,8 @@ static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) {
|
|||
pCache->lru.pLruNext = pPage;
|
||||
|
||||
pCache->nRecyclable++;
|
||||
|
||||
tdbTrace("unpin page %d", pPage->id);
|
||||
}
|
||||
|
||||
static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) {
|
||||
|
|
@ -248,6 +257,8 @@ static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) {
|
|||
*ppPage = pPage->pHashNext;
|
||||
|
||||
pCache->nPage--;
|
||||
|
||||
tdbTrace("remove page %d to hash", pPage->id);
|
||||
}
|
||||
|
||||
static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) {
|
||||
|
|
@ -259,6 +270,8 @@ static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) {
|
|||
pCache->pgHash[h] = pPage;
|
||||
|
||||
pCache->nPage++;
|
||||
|
||||
tdbTrace("add page %d to hash", pPage->id);
|
||||
}
|
||||
|
||||
static int tdbPCacheOpenImpl(SPCache *pCache) {
|
||||
|
|
@ -272,8 +285,8 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
|
|||
// Open the free list
|
||||
pCache->nFree = 0;
|
||||
pCache->pFree = NULL;
|
||||
for (int i = 0; i < pCache->cacheSize; i++) {
|
||||
ret = tdbPageCreate(pCache->pageSize, &pPage, tdbDefaultMalloc, NULL);
|
||||
for (int i = 0; i < pCache->nPages; i++) {
|
||||
ret = tdbPageCreate(pCache->szPage, &pPage, tdbDefaultMalloc, NULL);
|
||||
if (ret < 0) {
|
||||
// TODO: handle error
|
||||
return -1;
|
||||
|
|
@ -282,7 +295,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
|
|||
// pPage->pgid = 0;
|
||||
pPage->isAnchor = 0;
|
||||
pPage->isLocal = 1;
|
||||
TDB_INIT_PAGE_REF(pPage);
|
||||
pPage->nRef = 0;
|
||||
pPage->pHashNext = NULL;
|
||||
pPage->pLruNext = NULL;
|
||||
pPage->pLruPrev = NULL;
|
||||
|
|
@ -294,13 +307,13 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
|
|||
pCache->nFree++;
|
||||
|
||||
// add to local list
|
||||
pPage->pCacheNext = pCache->pList;
|
||||
pCache->pList = pPage;
|
||||
pPage->id = i;
|
||||
pCache->aPage[i] = pPage;
|
||||
}
|
||||
|
||||
// Open the hash table
|
||||
pCache->nPage = 0;
|
||||
pCache->nHash = pCache->cacheSize < 8 ? 8 : pCache->cacheSize;
|
||||
pCache->nHash = pCache->nPages < 8 ? 8 : pCache->nPages;
|
||||
pCache->pgHash = (SPage **)tdbOsCalloc(pCache->nHash, sizeof(SPage *));
|
||||
if (pCache->pgHash == NULL) {
|
||||
// TODO
|
||||
|
|
@ -317,11 +330,11 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
|
|||
}
|
||||
|
||||
static int tdbPCacheCloseImpl(SPCache *pCache) {
|
||||
SPage *pPage;
|
||||
|
||||
for (pPage = pCache->pList; pPage; pPage = pCache->pList) {
|
||||
pCache->pList = pPage->pCacheNext;
|
||||
tdbPageDestroy(pPage, tdbDefaultFree, NULL);
|
||||
for (i32 iPage = 0; iPage < pCache->nPages; iPage++) {
|
||||
if (pCache->aPage[iPage]) {
|
||||
tdbPageDestroy(pCache->aPage[iPage], tdbDefaultFree, NULL);
|
||||
pCache->aPage[iPage] = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
tdbOsFree(pCache->pgHash);
|
||||
|
|
|
|||
|
|
@ -149,7 +149,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
|
|||
if (pPage->isDirty) return 0;
|
||||
|
||||
// ref page one more time so the page will not be release
|
||||
TDB_REF_PAGE(pPage);
|
||||
tdbRefPage(pPage);
|
||||
|
||||
// Set page as dirty
|
||||
pPage->isDirty = 1;
|
||||
|
|
|
|||
|
|
@ -18,10 +18,23 @@
|
|||
|
||||
#include "tdb.h"
|
||||
|
||||
#include "tlog.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// clang-format off
|
||||
extern int32_t tdbDebugFlag;
|
||||
|
||||
#define tdbFatal(...) do { if (tdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0)
|
||||
#define tdbError(...) do { if (tdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0)
|
||||
#define tdbWarn(...) do { if (tdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0)
|
||||
#define tdbInfo(...) do { if (tdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0)
|
||||
#define tdbDebug(...) do { if (tdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", DEBUG_DEBUG, tdbDebugFlag, __VA_ARGS__); }} while(0)
|
||||
#define tdbTrace(...) do { if (tdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", DEBUG_TRACE, tdbDebugFlag, __VA_ARGS__); }} while(0)
|
||||
// clang-format on
|
||||
|
||||
typedef int8_t i8;
|
||||
typedef int16_t i16;
|
||||
typedef int32_t i32;
|
||||
|
|
@ -161,25 +174,21 @@ void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn);
|
|||
int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
|
||||
|
||||
// tdbPCache.c ====================================
|
||||
#define TDB_PCACHE_PAGE \
|
||||
u8 isAnchor; \
|
||||
u8 isLocal; \
|
||||
u8 isDirty; \
|
||||
i32 nRef; \
|
||||
SPage *pCacheNext; \
|
||||
SPage *pFreeNext; \
|
||||
SPage *pHashNext; \
|
||||
SPage *pLruNext; \
|
||||
SPage *pLruPrev; \
|
||||
SPage *pDirtyNext; \
|
||||
SPager *pPager; \
|
||||
SPgid pgid;
|
||||
#define TDB_PCACHE_PAGE \
|
||||
u8 isAnchor; \
|
||||
u8 isLocal; \
|
||||
u8 isDirty; \
|
||||
volatile i32 nRef; \
|
||||
i32 id; \
|
||||
SPage *pFreeNext; \
|
||||
SPage *pHashNext; \
|
||||
SPage *pLruNext; \
|
||||
SPage *pLruPrev; \
|
||||
SPage *pDirtyNext; \
|
||||
SPager *pPager; \
|
||||
SPgid pgid;
|
||||
|
||||
// For page ref
|
||||
#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0)
|
||||
#define TDB_REF_PAGE(pPage) atomic_add_fetch_32(&((pPage)->nRef), 1)
|
||||
#define TDB_UNREF_PAGE(pPage) atomic_sub_fetch_32(&((pPage)->nRef), 1)
|
||||
#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef))
|
||||
|
||||
int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache);
|
||||
int tdbPCacheClose(SPCache *pCache);
|
||||
|
|
@ -246,6 +255,20 @@ struct SPage {
|
|||
TDB_PCACHE_PAGE
|
||||
};
|
||||
|
||||
static inline i32 tdbRefPage(SPage *pPage) {
|
||||
i32 nRef = atomic_add_fetch_32(&((pPage)->nRef), 1);
|
||||
tdbTrace("ref page %d, nRef %d", pPage->id, nRef);
|
||||
return nRef;
|
||||
}
|
||||
|
||||
static inline i32 tdbUnrefPage(SPage *pPage) {
|
||||
i32 nRef = atomic_sub_fetch_32(&((pPage)->nRef), 1);
|
||||
tdbTrace("unref page %d, nRef %d", pPage->id, nRef);
|
||||
return nRef;
|
||||
}
|
||||
|
||||
#define tdbGetPageRef(pPage) atomic_load_32(&((pPage)->nRef))
|
||||
|
||||
// For page lock
|
||||
#define P_LOCK_SUCC 0
|
||||
#define P_LOCK_BUSY 1
|
||||
|
|
|
|||
|
|
@ -470,6 +470,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE, "udf load failure")
|
|||
TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE, "udf invalid state")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT, "udf invalid function input")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_UDF_NO_FUNC_HANDLE, "udf no function handle")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_BUFSIZE, "udf invalid bufsize")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_OUTPUT_TYPE, "udf invalid output type")
|
||||
|
||||
//schemaless
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp precision type")
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@
|
|||
*/
|
||||
|
||||
#define _DEFAULT_SOURCE
|
||||
#include "os.h"
|
||||
#include "tlog.h"
|
||||
#include "os.h"
|
||||
#include "tutil.h"
|
||||
|
||||
#define LOG_MAX_LINE_SIZE (1024)
|
||||
|
|
@ -90,6 +90,7 @@ int32_t qDebugFlag = 131;
|
|||
int32_t wDebugFlag = 135;
|
||||
int32_t sDebugFlag = 135;
|
||||
int32_t tsdbDebugFlag = 131;
|
||||
int32_t tdbDebugFlag = 131;
|
||||
int32_t tqDebugFlag = 135;
|
||||
int32_t fsDebugFlag = 135;
|
||||
int32_t metaDebugFlag = 135;
|
||||
|
|
|
|||
|
|
@ -35,53 +35,26 @@ class TDTestCase:
|
|||
for char_col in CHAR_COL:
|
||||
query_condition.extend(
|
||||
(
|
||||
f"{tbname}.{char_col}",
|
||||
f"upper( {tbname}.{char_col} )",
|
||||
f"char_length( {tbname}.{char_col} )",
|
||||
f"concat( {tbname}.{char_col}, {tbname}.{char_col} )",
|
||||
f"concat_ws( '_', {tbname}.{char_col}, {tbname}.{char_col} )",
|
||||
f"length( {tbname}.{char_col} )",
|
||||
f"lower( {tbname}.{char_col} )",
|
||||
f"ltrim( {tbname}.{char_col} )",
|
||||
f"rtrim( {tbname}.{char_col} )",
|
||||
f"substr( {tbname}.{char_col}, 1 )",
|
||||
f"count( {tbname}.{char_col} )",
|
||||
f"cast( {tbname}.{char_col} as nchar(3) )",
|
||||
f"cast( {tbname}.{char_col} as nchar(8) )",
|
||||
)
|
||||
)
|
||||
query_condition.extend( f"cast( {tbname}.{un_char_col} as binary(16) ) " for un_char_col in NUM_COL)
|
||||
query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL )
|
||||
query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{un_char_col} as binary(32) ) " for un_char_col in NUM_COL )
|
||||
|
||||
for num_col in NUM_COL:
|
||||
query_condition.extend(
|
||||
(
|
||||
f"{tbname}.{num_col}",
|
||||
f"ceil( {tbname}.{num_col} )",
|
||||
f"abs( {tbname}.{num_col} )",
|
||||
f"acos( {tbname}.{num_col} )",
|
||||
f"asin( {tbname}.{num_col} )",
|
||||
f"atan( {tbname}.{num_col} )",
|
||||
f"cos( {tbname}.{num_col} )",
|
||||
f"floor( {tbname}.{num_col} )",
|
||||
f"log( {tbname}.{num_col}, {tbname}.{num_col})",
|
||||
f"sin( {tbname}.{num_col} )",
|
||||
f"sqrt( {tbname}.{num_col} )",
|
||||
f"tan( {tbname}.{num_col} )",
|
||||
f"round( {tbname}.{num_col} )",
|
||||
f"max( {tbname}.{num_col} )",
|
||||
f"sum( {tbname}.{num_col} )",
|
||||
f"count( {tbname}.{num_col} )",
|
||||
f"min( {tbname}.{num_col} )",
|
||||
)
|
||||
)
|
||||
query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_2}" for num_col_2 in NUM_COL )
|
||||
query_condition.extend( f"{tbname}.{num_col} + {tbname}.{char_col} " for char_col in CHAR_COL )
|
||||
|
||||
query_condition.extend(
|
||||
(
|
||||
''' "test1234!@#$%^&*():'><?/.,][}{" ''',
|
||||
''' "test12" ''',
|
||||
# 1010,
|
||||
)
|
||||
|
|
@ -208,8 +181,8 @@ class TDTestCase:
|
|||
)
|
||||
)
|
||||
|
||||
return filter(None, sqls)
|
||||
# return list(filter(None, sqls))
|
||||
# return filter(None, sqls)
|
||||
return list(filter(None, sqls))
|
||||
|
||||
def __get_type(self, col):
|
||||
if tdSql.cursor.istype(col, "BOOL"):
|
||||
|
|
@ -245,34 +218,55 @@ class TDTestCase:
|
|||
|
||||
def union_check(self):
|
||||
sqls = self.sql_list()
|
||||
for sql1 in sqls:
|
||||
tdSql.query(sql1)
|
||||
for i in range(len(sqls)):
|
||||
tdSql.query(sqls[i])
|
||||
res1_type = self.__get_type(0)
|
||||
for sql2 in sqls:
|
||||
tdSql.query(sql2)
|
||||
union_type = False
|
||||
for j in range(len(sqls[i:])):
|
||||
tdSql.query(sqls[j+i])
|
||||
order_union_type = False
|
||||
rev_order_type = False
|
||||
all_union_type = False
|
||||
res2_type = self.__get_type(0)
|
||||
|
||||
if res1_type in ( "BIGINT" , "NCHAR" ):
|
||||
union_type = True
|
||||
elif res2_type == res1_type:
|
||||
union_type = True
|
||||
if res2_type == res1_type:
|
||||
all_union_type = True
|
||||
elif res1_type in ( "BIGINT" , "NCHAR" ) and res2_type in ("BIGINT" , "NCHAR"):
|
||||
all_union_type = True
|
||||
elif res1_type in ("BIGINT", "NCHAR"):
|
||||
order_union_type = True
|
||||
elif res2_type in ("BIGINT", "NCHAR"):
|
||||
rev_order_type = True
|
||||
elif res1_type == "TIMESAMP" and res2_type not in ("BINARY", "NCHAR"):
|
||||
union_type = True
|
||||
order_union_type = True
|
||||
elif res2_type == "TIMESAMP" and res1_type not in ("BINARY", "NCHAR"):
|
||||
rev_order_type = True
|
||||
elif res1_type == "BINARY" and res2_type != "NCHAR":
|
||||
union_type = True
|
||||
order_union_type = True
|
||||
elif res2_type == "BINARY" and res1_type != "NCHAR":
|
||||
rev_order_type = True
|
||||
|
||||
if union_type:
|
||||
tdSql.query(f"{sql1} union {sql2}")
|
||||
if all_union_type:
|
||||
tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
|
||||
tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
tdSql.query(f"{sql1} union all {sql2}")
|
||||
tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
|
||||
tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
elif order_union_type:
|
||||
tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
|
||||
tdSql.checkCols(1)
|
||||
tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
|
||||
tdSql.checkCols(1)
|
||||
elif rev_order_type:
|
||||
tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
else:
|
||||
tdSql.error(f"{sql1} union {sql2}")
|
||||
tdSql.error(f"{sqls[i]} union {sqls[j+i]}")
|
||||
|
||||
def __test_error(self):
|
||||
|
||||
|
||||
tdSql.error( "show tables union show tables" )
|
||||
tdSql.error( "create table errtb1 union all create table errtb2" )
|
||||
tdSql.error( "drop table ct1 union all drop table ct3" )
|
||||
|
|
|
|||
|
|
@ -0,0 +1,405 @@
|
|||
import datetime
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
|
||||
PRIMARY_COL = "ts"
|
||||
|
||||
INT_COL = "c1"
|
||||
BINT_COL = "c2"
|
||||
SINT_COL = "c3"
|
||||
TINT_COL = "c4"
|
||||
FLOAT_COL = "c5"
|
||||
DOUBLE_COL = "c6"
|
||||
BOOL_COL = "c7"
|
||||
|
||||
BINARY_COL = "c8"
|
||||
NCHAR_COL = "c9"
|
||||
TS_COL = "c10"
|
||||
|
||||
NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
|
||||
CHAR_COL = [ BINARY_COL, NCHAR_COL, ]
|
||||
BOOLEAN_COL = [ BOOL_COL, ]
|
||||
TS_TYPE_COL = [ TS_COL, ]
|
||||
|
||||
class TDTestCase:
|
||||
|
||||
def init(self, conn, logSql):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor())
|
||||
|
||||
def __query_condition(self,tbname):
|
||||
query_condition = []
|
||||
for char_col in CHAR_COL:
|
||||
query_condition.extend(
|
||||
(
|
||||
f"{tbname}.{char_col}",
|
||||
f"upper( {tbname}.{char_col} )",
|
||||
f"char_length( {tbname}.{char_col} )",
|
||||
f"concat( {tbname}.{char_col}, {tbname}.{char_col} )",
|
||||
)
|
||||
)
|
||||
|
||||
for num_col in NUM_COL:
|
||||
query_condition.extend(
|
||||
(
|
||||
f"{tbname}.{num_col}",
|
||||
f"ceil( {tbname}.{num_col} )",
|
||||
f"abs( {tbname}.{num_col} )",
|
||||
f"acos( {tbname}.{num_col} )",
|
||||
f"max( {tbname}.{num_col} )",
|
||||
)
|
||||
)
|
||||
|
||||
query_condition.extend(
|
||||
(
|
||||
# 1010,
|
||||
)
|
||||
)
|
||||
|
||||
return query_condition
|
||||
|
||||
def __join_condition(self, tb_list, filter=PRIMARY_COL, INNER=False):
|
||||
table_reference = tb_list[0]
|
||||
join_condition = table_reference
|
||||
join = "inner join" if INNER else "join"
|
||||
for i in range(len(tb_list[1:])):
|
||||
join_condition += f" {join} {tb_list[i+1]} on {table_reference}.{filter}={tb_list[i+1]}.{filter}"
|
||||
|
||||
return join_condition
|
||||
|
||||
def __where_condition(self, col=None, tbname=None, query_conditon=None):
|
||||
if query_conditon and isinstance(query_conditon, str):
|
||||
if query_conditon.startswith("count"):
|
||||
query_conditon = query_conditon[6:-1]
|
||||
elif query_conditon.startswith("max"):
|
||||
query_conditon = query_conditon[4:-1]
|
||||
elif query_conditon.startswith("sum"):
|
||||
query_conditon = query_conditon[4:-1]
|
||||
elif query_conditon.startswith("min"):
|
||||
query_conditon = query_conditon[4:-1]
|
||||
|
||||
|
||||
if query_conditon:
|
||||
return f" where {query_conditon} is not null"
|
||||
if col in NUM_COL:
|
||||
return f" where abs( {tbname}.{col} ) >= 0"
|
||||
if col in CHAR_COL:
|
||||
return f" where lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' "
|
||||
if col in BOOLEAN_COL:
|
||||
return f" where {tbname}.{col} in (false, true) "
|
||||
if col in TS_TYPE_COL or col in PRIMARY_COL:
|
||||
return f" where cast( {tbname}.{col} as binary(16) ) is not null "
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def __group_condition(self, col, having = None):
|
||||
if isinstance(col, str):
|
||||
if col.startswith("count"):
|
||||
col = col[6:-1]
|
||||
elif col.startswith("max"):
|
||||
col = col[4:-1]
|
||||
elif col.startswith("sum"):
|
||||
col = col[4:-1]
|
||||
elif col.startswith("min"):
|
||||
col = col[4:-1]
|
||||
return f" group by {col} having {having}" if having else f" group by {col} "
|
||||
|
||||
def __single_sql(self, select_clause, from_clause, where_condition="", group_condition=""):
|
||||
if isinstance(select_clause, str) and "on" not in from_clause and select_clause.split(".")[0] != from_clause.split(".")[0]:
|
||||
return
|
||||
return f"select {select_clause} from {from_clause} {where_condition} {group_condition}"
|
||||
|
||||
|
||||
@property
|
||||
def __join_tblist(self):
|
||||
return [
|
||||
["ct1", "ct2"],
|
||||
["ct1", "ct4"],
|
||||
["ct1", "t1"],
|
||||
["ct2", "ct4"],
|
||||
["ct2", "t1"],
|
||||
["ct4", "t1"],
|
||||
# ["ct1", "ct2", "ct4"],
|
||||
# ["ct1", "ct2", "t1"],
|
||||
# ["ct1", "ct4", "t1"],
|
||||
# ["ct2", "ct4", "t1"],
|
||||
# ["ct1", "ct2", "ct4", "t1"],
|
||||
]
|
||||
|
||||
@property
|
||||
def __tb_liast(self):
|
||||
return [
|
||||
"ct1",
|
||||
"ct2",
|
||||
"ct4",
|
||||
"t1",
|
||||
]
|
||||
|
||||
def sql_list(self):
|
||||
sqls = []
|
||||
__join_tblist = self.__join_tblist
|
||||
for join_tblist in __join_tblist:
|
||||
for join_tb in join_tblist:
|
||||
select_claus_list = self.__query_condition(join_tb)
|
||||
for select_claus in select_claus_list:
|
||||
group_claus = self.__group_condition( col=select_claus)
|
||||
where_claus = self.__where_condition(query_conditon=select_claus)
|
||||
having_claus = self.__group_condition( col=select_claus, having=f"{select_claus} is not null")
|
||||
sqls.extend(
|
||||
(
|
||||
self.__single_sql(select_claus, join_tb, where_claus, group_claus),
|
||||
self.__single_sql(select_claus, join_tb, where_claus, having_claus),
|
||||
self.__single_sql(select_claus, self.__join_condition(join_tblist), where_claus, having_claus),
|
||||
self.__single_sql(select_claus, self.__join_condition(join_tblist, INNER=True), where_claus, having_claus),
|
||||
self.__single_sql(select_claus, join_tb, where_claus),
|
||||
self.__single_sql(select_claus, join_tb, having_claus),
|
||||
self.__single_sql(select_claus, join_tb, group_claus),
|
||||
self.__single_sql(select_claus, join_tb),
|
||||
|
||||
)
|
||||
)
|
||||
__no_join_tblist = self.__tb_liast
|
||||
for tb in __no_join_tblist:
|
||||
select_claus_list = self.__query_condition(tb)
|
||||
for select_claus in select_claus_list:
|
||||
group_claus = self.__group_condition(col=select_claus)
|
||||
where_claus = self.__where_condition(query_conditon=select_claus)
|
||||
having_claus = self.__group_condition(col=select_claus, having=f"{select_claus} is not null")
|
||||
sqls.extend(
|
||||
(
|
||||
self.__single_sql(select_claus, join_tb, where_claus, group_claus),
|
||||
self.__single_sql(select_claus, join_tb, where_claus, having_claus),
|
||||
self.__single_sql(select_claus, join_tb, where_claus),
|
||||
self.__single_sql(select_claus, join_tb, group_claus),
|
||||
self.__single_sql(select_claus, join_tb, having_claus),
|
||||
self.__single_sql(select_claus, join_tb),
|
||||
)
|
||||
)
|
||||
|
||||
# return filter(None, sqls)
|
||||
return list(filter(None, sqls))
|
||||
|
||||
def __get_type(self, col):
|
||||
if tdSql.cursor.istype(col, "BOOL"):
|
||||
return "BOOL"
|
||||
if tdSql.cursor.istype(col, "INT"):
|
||||
return "INT"
|
||||
if tdSql.cursor.istype(col, "BIGINT"):
|
||||
return "BIGINT"
|
||||
if tdSql.cursor.istype(col, "TINYINT"):
|
||||
return "TINYINT"
|
||||
if tdSql.cursor.istype(col, "SMALLINT"):
|
||||
return "SMALLINT"
|
||||
if tdSql.cursor.istype(col, "FLOAT"):
|
||||
return "FLOAT"
|
||||
if tdSql.cursor.istype(col, "DOUBLE"):
|
||||
return "DOUBLE"
|
||||
if tdSql.cursor.istype(col, "BINARY"):
|
||||
return "BINARY"
|
||||
if tdSql.cursor.istype(col, "NCHAR"):
|
||||
return "NCHAR"
|
||||
if tdSql.cursor.istype(col, "TIMESTAMP"):
|
||||
return "TIMESTAMP"
|
||||
if tdSql.cursor.istype(col, "JSON"):
|
||||
return "JSON"
|
||||
if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
|
||||
return "TINYINT UNSIGNED"
|
||||
if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
|
||||
return "SMALLINT UNSIGNED"
|
||||
if tdSql.cursor.istype(col, "INT UNSIGNED"):
|
||||
return "INT UNSIGNED"
|
||||
if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
|
||||
return "BIGINT UNSIGNED"
|
||||
|
||||
def union_check(self):
|
||||
sqls = self.sql_list()
|
||||
for i in range(len(sqls)):
|
||||
tdSql.query(sqls[i])
|
||||
res1_type = self.__get_type(0)
|
||||
for j in range(len(sqls[i:])):
|
||||
tdSql.query(sqls[j+i])
|
||||
order_union_type = False
|
||||
rev_order_type = False
|
||||
all_union_type = False
|
||||
res2_type = self.__get_type(0)
|
||||
|
||||
if res2_type == res1_type:
|
||||
all_union_type = True
|
||||
elif res1_type in ( "BIGINT" , "NCHAR" ) and res2_type in ("BIGINT" , "NCHAR"):
|
||||
all_union_type = True
|
||||
elif res1_type in ("BIGINT", "NCHAR"):
|
||||
order_union_type = True
|
||||
elif res2_type in ("BIGINT", "NCHAR"):
|
||||
rev_order_type = True
|
||||
elif res1_type == "TIMESAMP" and res2_type not in ("BINARY", "NCHAR"):
|
||||
order_union_type = True
|
||||
elif res2_type == "TIMESAMP" and res1_type not in ("BINARY", "NCHAR"):
|
||||
rev_order_type = True
|
||||
elif res1_type == "BINARY" and res2_type != "NCHAR":
|
||||
order_union_type = True
|
||||
elif res2_type == "BINARY" and res1_type != "NCHAR":
|
||||
rev_order_type = True
|
||||
|
||||
if all_union_type:
|
||||
tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
|
||||
tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
|
||||
tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
elif order_union_type:
|
||||
tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
|
||||
tdSql.checkCols(1)
|
||||
tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
|
||||
tdSql.checkCols(1)
|
||||
elif rev_order_type:
|
||||
tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
else:
|
||||
tdSql.error(f"{sqls[i]} union {sqls[j+i]}")
|
||||
|
||||
def __test_error(self):
|
||||
|
||||
tdSql.error( "show tables union show tables" )
|
||||
tdSql.error( "create table errtb1 union all create table errtb2" )
|
||||
tdSql.error( "drop table ct1 union all drop table ct3" )
|
||||
tdSql.error( "select c1 from ct1 union all drop table ct3" )
|
||||
tdSql.error( "select c1 from ct1 union all '' " )
|
||||
tdSql.error( " '' union all select c1 from ct1 " )
|
||||
tdSql.error( "select c1 from ct1 union select c1 from ct2 union select c1 from ct4 ")
|
||||
|
||||
def all_test(self):
|
||||
self.__test_error()
|
||||
self.union_check()
|
||||
|
||||
|
||||
def __create_tb(self):
|
||||
|
||||
tdLog.printNoPrefix("==========step1:create table")
|
||||
create_stb_sql = f'''create table stb1(
|
||||
ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
|
||||
{FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
|
||||
{BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
|
||||
) tags (t1 int)
|
||||
'''
|
||||
create_ntb_sql = f'''create table t1(
|
||||
ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
|
||||
{FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
|
||||
{BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
|
||||
)
|
||||
'''
|
||||
tdSql.execute(create_stb_sql)
|
||||
tdSql.execute(create_ntb_sql)
|
||||
|
||||
for i in range(4):
|
||||
tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
|
||||
{ i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}
|
||||
|
||||
def __insert_data(self, rows):
|
||||
now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
|
||||
for i in range(rows):
|
||||
tdSql.execute(
|
||||
f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
|
||||
)
|
||||
tdSql.execute(
|
||||
f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
|
||||
)
|
||||
tdSql.execute(
|
||||
f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
|
||||
)
|
||||
tdSql.execute(
|
||||
f'''insert into ct1 values
|
||||
( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } )
|
||||
( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } )
|
||||
'''
|
||||
)
|
||||
|
||||
tdSql.execute(
|
||||
f'''insert into ct4 values
|
||||
( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
(
|
||||
{ now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127,
|
||||
{ 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000}
|
||||
)
|
||||
(
|
||||
{ now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126,
|
||||
{ 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000}
|
||||
)
|
||||
'''
|
||||
)
|
||||
|
||||
tdSql.execute(
|
||||
f'''insert into ct2 values
|
||||
( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
(
|
||||
{ now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126,
|
||||
{ -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
|
||||
)
|
||||
(
|
||||
{ now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127,
|
||||
{ - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
|
||||
)
|
||||
'''
|
||||
)
|
||||
|
||||
for i in range(rows):
|
||||
insert_data = f'''insert into t1 values
|
||||
( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2},
|
||||
"binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } )
|
||||
'''
|
||||
tdSql.execute(insert_data)
|
||||
tdSql.execute(
|
||||
f'''insert into t1 values
|
||||
( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127,
|
||||
{ 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 },
|
||||
"binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
|
||||
)
|
||||
(
|
||||
{ now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126,
|
||||
{ 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 },
|
||||
"binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
|
||||
)
|
||||
'''
|
||||
)
|
||||
|
||||
|
||||
def run(self):
|
||||
tdSql.prepare()
|
||||
|
||||
tdLog.printNoPrefix("==========step1:create table")
|
||||
self.__create_tb()
|
||||
|
||||
tdLog.printNoPrefix("==========step2:insert data")
|
||||
self.rows = 10
|
||||
self.__insert_data(self.rows)
|
||||
|
||||
tdLog.printNoPrefix("==========step3:all check")
|
||||
self.all_test()
|
||||
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
|
||||
tdSql.execute("use db")
|
||||
|
||||
tdLog.printNoPrefix("==========step4:after wal, all check again ")
|
||||
self.all_test()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -0,0 +1,406 @@
|
|||
import datetime
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
|
||||
PRIMARY_COL = "ts"
|
||||
|
||||
INT_COL = "c1"
|
||||
BINT_COL = "c2"
|
||||
SINT_COL = "c3"
|
||||
TINT_COL = "c4"
|
||||
FLOAT_COL = "c5"
|
||||
DOUBLE_COL = "c6"
|
||||
BOOL_COL = "c7"
|
||||
|
||||
BINARY_COL = "c8"
|
||||
NCHAR_COL = "c9"
|
||||
TS_COL = "c10"
|
||||
|
||||
NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
|
||||
CHAR_COL = [ BINARY_COL, NCHAR_COL, ]
|
||||
BOOLEAN_COL = [ BOOL_COL, ]
|
||||
TS_TYPE_COL = [ TS_COL, ]
|
||||
|
||||
class TDTestCase:
|
||||
|
||||
def init(self, conn, logSql):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor())
|
||||
|
||||
def __query_condition(self,tbname):
|
||||
query_condition = []
|
||||
for char_col in CHAR_COL:
|
||||
query_condition.extend(
|
||||
(
|
||||
f"concat_ws( '_', {tbname}.{char_col}, {tbname}.{char_col} )",
|
||||
f"length( {tbname}.{char_col} )",
|
||||
f"lower( {tbname}.{char_col} )",
|
||||
f"ltrim( {tbname}.{char_col} )",
|
||||
)
|
||||
)
|
||||
|
||||
for num_col in NUM_COL:
|
||||
query_condition.extend(
|
||||
(
|
||||
f"asin( {tbname}.{num_col} )",
|
||||
f"atan( {tbname}.{num_col} )",
|
||||
f"cos( {tbname}.{num_col} )",
|
||||
f"sum( {tbname}.{num_col} )",
|
||||
)
|
||||
)
|
||||
query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_2}" for num_col_2 in NUM_COL )
|
||||
|
||||
query_condition.extend(
|
||||
(
|
||||
''' "test1234!@#$%^&*():'><?/.,][}{" ''',
|
||||
# 1010,
|
||||
)
|
||||
)
|
||||
|
||||
return query_condition
|
||||
|
||||
def __join_condition(self, tb_list, filter=PRIMARY_COL, INNER=False):
|
||||
table_reference = tb_list[0]
|
||||
join_condition = table_reference
|
||||
join = "inner join" if INNER else "join"
|
||||
for i in range(len(tb_list[1:])):
|
||||
join_condition += f" {join} {tb_list[i+1]} on {table_reference}.{filter}={tb_list[i+1]}.{filter}"
|
||||
|
||||
return join_condition
|
||||
|
||||
def __where_condition(self, col=None, tbname=None, query_conditon=None):
|
||||
if query_conditon and isinstance(query_conditon, str):
|
||||
if query_conditon.startswith("count"):
|
||||
query_conditon = query_conditon[6:-1]
|
||||
elif query_conditon.startswith("max"):
|
||||
query_conditon = query_conditon[4:-1]
|
||||
elif query_conditon.startswith("sum"):
|
||||
query_conditon = query_conditon[4:-1]
|
||||
elif query_conditon.startswith("min"):
|
||||
query_conditon = query_conditon[4:-1]
|
||||
|
||||
|
||||
if query_conditon:
|
||||
return f" where {query_conditon} is not null"
|
||||
if col in NUM_COL:
|
||||
return f" where abs( {tbname}.{col} ) >= 0"
|
||||
if col in CHAR_COL:
|
||||
return f" where lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' "
|
||||
if col in BOOLEAN_COL:
|
||||
return f" where {tbname}.{col} in (false, true) "
|
||||
if col in TS_TYPE_COL or col in PRIMARY_COL:
|
||||
return f" where cast( {tbname}.{col} as binary(16) ) is not null "
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def __group_condition(self, col, having = None):
|
||||
if isinstance(col, str):
|
||||
if col.startswith("count"):
|
||||
col = col[6:-1]
|
||||
elif col.startswith("max"):
|
||||
col = col[4:-1]
|
||||
elif col.startswith("sum"):
|
||||
col = col[4:-1]
|
||||
elif col.startswith("min"):
|
||||
col = col[4:-1]
|
||||
return f" group by {col} having {having}" if having else f" group by {col} "
|
||||
|
||||
def __single_sql(self, select_clause, from_clause, where_condition="", group_condition=""):
|
||||
if isinstance(select_clause, str) and "on" not in from_clause and select_clause.split(".")[0] != from_clause.split(".")[0]:
|
||||
return
|
||||
return f"select {select_clause} from {from_clause} {where_condition} {group_condition}"
|
||||
|
||||
|
||||
@property
|
||||
def __join_tblist(self):
|
||||
return [
|
||||
["ct1", "ct2"],
|
||||
["ct1", "ct4"],
|
||||
["ct1", "t1"],
|
||||
["ct2", "ct4"],
|
||||
["ct2", "t1"],
|
||||
["ct4", "t1"],
|
||||
# ["ct1", "ct2", "ct4"],
|
||||
# ["ct1", "ct2", "t1"],
|
||||
# ["ct1", "ct4", "t1"],
|
||||
# ["ct2", "ct4", "t1"],
|
||||
# ["ct1", "ct2", "ct4", "t1"],
|
||||
]
|
||||
|
||||
@property
|
||||
def __tb_liast(self):
|
||||
return [
|
||||
"ct1",
|
||||
"ct2",
|
||||
"ct4",
|
||||
"t1",
|
||||
]
|
||||
|
||||
def sql_list(self):
|
||||
sqls = []
|
||||
__join_tblist = self.__join_tblist
|
||||
for join_tblist in __join_tblist:
|
||||
for join_tb in join_tblist:
|
||||
select_claus_list = self.__query_condition(join_tb)
|
||||
for select_claus in select_claus_list:
|
||||
group_claus = self.__group_condition( col=select_claus)
|
||||
where_claus = self.__where_condition(query_conditon=select_claus)
|
||||
having_claus = self.__group_condition( col=select_claus, having=f"{select_claus} is not null")
|
||||
sqls.extend(
|
||||
(
|
||||
self.__single_sql(select_claus, join_tb, where_claus, group_claus),
|
||||
self.__single_sql(select_claus, join_tb, where_claus, having_claus),
|
||||
self.__single_sql(select_claus, self.__join_condition(join_tblist), where_claus, having_claus),
|
||||
self.__single_sql(select_claus, self.__join_condition(join_tblist, INNER=True), where_claus, having_claus),
|
||||
self.__single_sql(select_claus, join_tb, where_claus),
|
||||
self.__single_sql(select_claus, join_tb, having_claus),
|
||||
self.__single_sql(select_claus, join_tb, group_claus),
|
||||
self.__single_sql(select_claus, join_tb),
|
||||
|
||||
)
|
||||
)
|
||||
__no_join_tblist = self.__tb_liast
|
||||
for tb in __no_join_tblist:
|
||||
select_claus_list = self.__query_condition(tb)
|
||||
for select_claus in select_claus_list:
|
||||
group_claus = self.__group_condition(col=select_claus)
|
||||
where_claus = self.__where_condition(query_conditon=select_claus)
|
||||
having_claus = self.__group_condition(col=select_claus, having=f"{select_claus} is not null")
|
||||
sqls.extend(
|
||||
(
|
||||
self.__single_sql(select_claus, join_tb, where_claus, group_claus),
|
||||
self.__single_sql(select_claus, join_tb, where_claus, having_claus),
|
||||
self.__single_sql(select_claus, join_tb, where_claus),
|
||||
self.__single_sql(select_claus, join_tb, group_claus),
|
||||
self.__single_sql(select_claus, join_tb, having_claus),
|
||||
self.__single_sql(select_claus, join_tb),
|
||||
)
|
||||
)
|
||||
|
||||
# return filter(None, sqls)
|
||||
return list(filter(None, sqls))
|
||||
|
||||
def __get_type(self, col):
|
||||
if tdSql.cursor.istype(col, "BOOL"):
|
||||
return "BOOL"
|
||||
if tdSql.cursor.istype(col, "INT"):
|
||||
return "INT"
|
||||
if tdSql.cursor.istype(col, "BIGINT"):
|
||||
return "BIGINT"
|
||||
if tdSql.cursor.istype(col, "TINYINT"):
|
||||
return "TINYINT"
|
||||
if tdSql.cursor.istype(col, "SMALLINT"):
|
||||
return "SMALLINT"
|
||||
if tdSql.cursor.istype(col, "FLOAT"):
|
||||
return "FLOAT"
|
||||
if tdSql.cursor.istype(col, "DOUBLE"):
|
||||
return "DOUBLE"
|
||||
if tdSql.cursor.istype(col, "BINARY"):
|
||||
return "BINARY"
|
||||
if tdSql.cursor.istype(col, "NCHAR"):
|
||||
return "NCHAR"
|
||||
if tdSql.cursor.istype(col, "TIMESTAMP"):
|
||||
return "TIMESTAMP"
|
||||
if tdSql.cursor.istype(col, "JSON"):
|
||||
return "JSON"
|
||||
if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
|
||||
return "TINYINT UNSIGNED"
|
||||
if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
|
||||
return "SMALLINT UNSIGNED"
|
||||
if tdSql.cursor.istype(col, "INT UNSIGNED"):
|
||||
return "INT UNSIGNED"
|
||||
if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
|
||||
return "BIGINT UNSIGNED"
|
||||
|
||||
def union_check(self):
|
||||
sqls = self.sql_list()
|
||||
for i in range(len(sqls)):
|
||||
tdSql.query(sqls[i])
|
||||
res1_type = self.__get_type(0)
|
||||
for j in range(len(sqls[i:])):
|
||||
tdSql.query(sqls[j+i])
|
||||
order_union_type = False
|
||||
rev_order_type = False
|
||||
all_union_type = False
|
||||
res2_type = self.__get_type(0)
|
||||
|
||||
if res2_type == res1_type:
|
||||
all_union_type = True
|
||||
elif res1_type in ( "BIGINT" , "NCHAR" ) and res2_type in ("BIGINT" , "NCHAR"):
|
||||
all_union_type = True
|
||||
elif res1_type in ("BIGINT", "NCHAR"):
|
||||
order_union_type = True
|
||||
elif res2_type in ("BIGINT", "NCHAR"):
|
||||
rev_order_type = True
|
||||
elif res1_type == "TIMESAMP" and res2_type not in ("BINARY", "NCHAR"):
|
||||
order_union_type = True
|
||||
elif res2_type == "TIMESAMP" and res1_type not in ("BINARY", "NCHAR"):
|
||||
rev_order_type = True
|
||||
elif res1_type == "BINARY" and res2_type != "NCHAR":
|
||||
order_union_type = True
|
||||
elif res2_type == "BINARY" and res1_type != "NCHAR":
|
||||
rev_order_type = True
|
||||
|
||||
if all_union_type:
|
||||
tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
|
||||
tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
|
||||
tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
elif order_union_type:
|
||||
tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
|
||||
tdSql.checkCols(1)
|
||||
tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
|
||||
tdSql.checkCols(1)
|
||||
elif rev_order_type:
|
||||
tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
else:
|
||||
tdSql.error(f"{sqls[i]} union {sqls[j+i]}")
|
||||
|
||||
def __test_error(self):
|
||||
|
||||
tdSql.error( "show tables union show tables" )
|
||||
tdSql.error( "create table errtb1 union all create table errtb2" )
|
||||
tdSql.error( "drop table ct1 union all drop table ct3" )
|
||||
tdSql.error( "select c1 from ct1 union all drop table ct3" )
|
||||
tdSql.error( "select c1 from ct1 union all '' " )
|
||||
tdSql.error( " '' union all select c1 from ct1 " )
|
||||
tdSql.error( "select c1 from ct1 union select c1 from ct2 union select c1 from ct4 ")
|
||||
|
||||
def all_test(self):
|
||||
self.__test_error()
|
||||
self.union_check()
|
||||
|
||||
|
||||
def __create_tb(self):
|
||||
|
||||
tdLog.printNoPrefix("==========step1:create table")
|
||||
create_stb_sql = f'''create table stb1(
|
||||
ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
|
||||
{FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
|
||||
{BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
|
||||
) tags (t1 int)
|
||||
'''
|
||||
create_ntb_sql = f'''create table t1(
|
||||
ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
|
||||
{FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
|
||||
{BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
|
||||
)
|
||||
'''
|
||||
tdSql.execute(create_stb_sql)
|
||||
tdSql.execute(create_ntb_sql)
|
||||
|
||||
for i in range(4):
|
||||
tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
|
||||
{ i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}
|
||||
|
||||
def __insert_data(self, rows):
|
||||
now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
|
||||
for i in range(rows):
|
||||
tdSql.execute(
|
||||
f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
|
||||
)
|
||||
tdSql.execute(
|
||||
f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
|
||||
)
|
||||
tdSql.execute(
|
||||
f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
|
||||
)
|
||||
tdSql.execute(
|
||||
f'''insert into ct1 values
|
||||
( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } )
|
||||
( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } )
|
||||
'''
|
||||
)
|
||||
|
||||
tdSql.execute(
|
||||
f'''insert into ct4 values
|
||||
( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
(
|
||||
{ now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127,
|
||||
{ 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000}
|
||||
)
|
||||
(
|
||||
{ now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126,
|
||||
{ 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000}
|
||||
)
|
||||
'''
|
||||
)
|
||||
|
||||
tdSql.execute(
|
||||
f'''insert into ct2 values
|
||||
( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
(
|
||||
{ now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126,
|
||||
{ -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
|
||||
)
|
||||
(
|
||||
{ now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127,
|
||||
{ - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
|
||||
)
|
||||
'''
|
||||
)
|
||||
|
||||
for i in range(rows):
|
||||
insert_data = f'''insert into t1 values
|
||||
( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2},
|
||||
"binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } )
|
||||
'''
|
||||
tdSql.execute(insert_data)
|
||||
tdSql.execute(
|
||||
f'''insert into t1 values
|
||||
( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127,
|
||||
{ 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 },
|
||||
"binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
|
||||
)
|
||||
(
|
||||
{ now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126,
|
||||
{ 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 },
|
||||
"binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
|
||||
)
|
||||
'''
|
||||
)
|
||||
|
||||
|
||||
def run(self):
|
||||
tdSql.prepare()
|
||||
|
||||
tdLog.printNoPrefix("==========step1:create table")
|
||||
self.__create_tb()
|
||||
|
||||
tdLog.printNoPrefix("==========step2:insert data")
|
||||
self.rows = 10
|
||||
self.__insert_data(self.rows)
|
||||
|
||||
tdLog.printNoPrefix("==========step3:all check")
|
||||
self.all_test()
|
||||
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
|
||||
tdSql.execute("use db")
|
||||
|
||||
tdLog.printNoPrefix("==========step4:after wal, all check again ")
|
||||
self.all_test()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -0,0 +1,406 @@
|
|||
import datetime
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
|
||||
PRIMARY_COL = "ts"
|
||||
|
||||
INT_COL = "c1"
|
||||
BINT_COL = "c2"
|
||||
SINT_COL = "c3"
|
||||
TINT_COL = "c4"
|
||||
FLOAT_COL = "c5"
|
||||
DOUBLE_COL = "c6"
|
||||
BOOL_COL = "c7"
|
||||
|
||||
BINARY_COL = "c8"
|
||||
NCHAR_COL = "c9"
|
||||
TS_COL = "c10"
|
||||
|
||||
NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
|
||||
CHAR_COL = [ BINARY_COL, NCHAR_COL, ]
|
||||
BOOLEAN_COL = [ BOOL_COL, ]
|
||||
TS_TYPE_COL = [ TS_COL, ]
|
||||
|
||||
class TDTestCase:
|
||||
|
||||
def init(self, conn, logSql):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor())
|
||||
|
||||
def __query_condition(self,tbname):
|
||||
query_condition = []
|
||||
for char_col in CHAR_COL:
|
||||
query_condition.extend(
|
||||
(
|
||||
f"cast( {tbname}.{char_col} as nchar(8) )",
|
||||
)
|
||||
)
|
||||
query_condition.extend( f"cast( {tbname}.{un_char_col} as binary(16) ) " for un_char_col in NUM_COL)
|
||||
query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL )
|
||||
query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{un_char_col} as binary(32) ) " for un_char_col in NUM_COL )
|
||||
|
||||
for num_col in NUM_COL:
|
||||
query_condition.extend(
|
||||
(
|
||||
f"tan( {tbname}.{num_col} )",
|
||||
f"round( {tbname}.{num_col} )",
|
||||
f"count( {tbname}.{num_col} )",
|
||||
f"min( {tbname}.{num_col} )",
|
||||
)
|
||||
)
|
||||
query_condition.extend( f"{tbname}.{num_col} + {tbname}.{char_col} " for char_col in CHAR_COL )
|
||||
|
||||
query_condition.extend(
|
||||
(
|
||||
''' "test12" ''',
|
||||
# 1010,
|
||||
)
|
||||
)
|
||||
|
||||
return query_condition
|
||||
|
||||
def __join_condition(self, tb_list, filter=PRIMARY_COL, INNER=False):
|
||||
table_reference = tb_list[0]
|
||||
join_condition = table_reference
|
||||
join = "inner join" if INNER else "join"
|
||||
for i in range(len(tb_list[1:])):
|
||||
join_condition += f" {join} {tb_list[i+1]} on {table_reference}.{filter}={tb_list[i+1]}.{filter}"
|
||||
|
||||
return join_condition
|
||||
|
||||
def __where_condition(self, col=None, tbname=None, query_conditon=None):
|
||||
if query_conditon and isinstance(query_conditon, str):
|
||||
if query_conditon.startswith("count"):
|
||||
query_conditon = query_conditon[6:-1]
|
||||
elif query_conditon.startswith("max"):
|
||||
query_conditon = query_conditon[4:-1]
|
||||
elif query_conditon.startswith("sum"):
|
||||
query_conditon = query_conditon[4:-1]
|
||||
elif query_conditon.startswith("min"):
|
||||
query_conditon = query_conditon[4:-1]
|
||||
|
||||
|
||||
if query_conditon:
|
||||
return f" where {query_conditon} is not null"
|
||||
if col in NUM_COL:
|
||||
return f" where abs( {tbname}.{col} ) >= 0"
|
||||
if col in CHAR_COL:
|
||||
return f" where lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' "
|
||||
if col in BOOLEAN_COL:
|
||||
return f" where {tbname}.{col} in (false, true) "
|
||||
if col in TS_TYPE_COL or col in PRIMARY_COL:
|
||||
return f" where cast( {tbname}.{col} as binary(16) ) is not null "
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def __group_condition(self, col, having = None):
|
||||
if isinstance(col, str):
|
||||
if col.startswith("count"):
|
||||
col = col[6:-1]
|
||||
elif col.startswith("max"):
|
||||
col = col[4:-1]
|
||||
elif col.startswith("sum"):
|
||||
col = col[4:-1]
|
||||
elif col.startswith("min"):
|
||||
col = col[4:-1]
|
||||
return f" group by {col} having {having}" if having else f" group by {col} "
|
||||
|
||||
def __single_sql(self, select_clause, from_clause, where_condition="", group_condition=""):
|
||||
if isinstance(select_clause, str) and "on" not in from_clause and select_clause.split(".")[0] != from_clause.split(".")[0]:
|
||||
return
|
||||
return f"select {select_clause} from {from_clause} {where_condition} {group_condition}"
|
||||
|
||||
|
||||
@property
|
||||
def __join_tblist(self):
|
||||
return [
|
||||
["ct1", "ct2"],
|
||||
["ct1", "ct4"],
|
||||
["ct1", "t1"],
|
||||
["ct2", "ct4"],
|
||||
["ct2", "t1"],
|
||||
["ct4", "t1"],
|
||||
# ["ct1", "ct2", "ct4"],
|
||||
# ["ct1", "ct2", "t1"],
|
||||
# ["ct1", "ct4", "t1"],
|
||||
# ["ct2", "ct4", "t1"],
|
||||
# ["ct1", "ct2", "ct4", "t1"],
|
||||
]
|
||||
|
||||
@property
|
||||
def __tb_liast(self):
|
||||
return [
|
||||
"ct1",
|
||||
"ct2",
|
||||
"ct4",
|
||||
"t1",
|
||||
]
|
||||
|
||||
def sql_list(self):
|
||||
sqls = []
|
||||
__join_tblist = self.__join_tblist
|
||||
for join_tblist in __join_tblist:
|
||||
for join_tb in join_tblist:
|
||||
select_claus_list = self.__query_condition(join_tb)
|
||||
for select_claus in select_claus_list:
|
||||
group_claus = self.__group_condition( col=select_claus)
|
||||
where_claus = self.__where_condition(query_conditon=select_claus)
|
||||
having_claus = self.__group_condition( col=select_claus, having=f"{select_claus} is not null")
|
||||
sqls.extend(
|
||||
(
|
||||
self.__single_sql(select_claus, join_tb, where_claus, group_claus),
|
||||
self.__single_sql(select_claus, join_tb, where_claus, having_claus),
|
||||
self.__single_sql(select_claus, self.__join_condition(join_tblist), where_claus, having_claus),
|
||||
self.__single_sql(select_claus, self.__join_condition(join_tblist, INNER=True), where_claus, having_claus),
|
||||
self.__single_sql(select_claus, join_tb, where_claus),
|
||||
self.__single_sql(select_claus, join_tb, having_claus),
|
||||
self.__single_sql(select_claus, join_tb, group_claus),
|
||||
self.__single_sql(select_claus, join_tb),
|
||||
|
||||
)
|
||||
)
|
||||
__no_join_tblist = self.__tb_liast
|
||||
for tb in __no_join_tblist:
|
||||
select_claus_list = self.__query_condition(tb)
|
||||
for select_claus in select_claus_list:
|
||||
group_claus = self.__group_condition(col=select_claus)
|
||||
where_claus = self.__where_condition(query_conditon=select_claus)
|
||||
having_claus = self.__group_condition(col=select_claus, having=f"{select_claus} is not null")
|
||||
sqls.extend(
|
||||
(
|
||||
self.__single_sql(select_claus, join_tb, where_claus, group_claus),
|
||||
self.__single_sql(select_claus, join_tb, where_claus, having_claus),
|
||||
self.__single_sql(select_claus, join_tb, where_claus),
|
||||
self.__single_sql(select_claus, join_tb, group_claus),
|
||||
self.__single_sql(select_claus, join_tb, having_claus),
|
||||
self.__single_sql(select_claus, join_tb),
|
||||
)
|
||||
)
|
||||
|
||||
# return filter(None, sqls)
|
||||
return list(filter(None, sqls))
|
||||
|
||||
def __get_type(self, col):
|
||||
if tdSql.cursor.istype(col, "BOOL"):
|
||||
return "BOOL"
|
||||
if tdSql.cursor.istype(col, "INT"):
|
||||
return "INT"
|
||||
if tdSql.cursor.istype(col, "BIGINT"):
|
||||
return "BIGINT"
|
||||
if tdSql.cursor.istype(col, "TINYINT"):
|
||||
return "TINYINT"
|
||||
if tdSql.cursor.istype(col, "SMALLINT"):
|
||||
return "SMALLINT"
|
||||
if tdSql.cursor.istype(col, "FLOAT"):
|
||||
return "FLOAT"
|
||||
if tdSql.cursor.istype(col, "DOUBLE"):
|
||||
return "DOUBLE"
|
||||
if tdSql.cursor.istype(col, "BINARY"):
|
||||
return "BINARY"
|
||||
if tdSql.cursor.istype(col, "NCHAR"):
|
||||
return "NCHAR"
|
||||
if tdSql.cursor.istype(col, "TIMESTAMP"):
|
||||
return "TIMESTAMP"
|
||||
if tdSql.cursor.istype(col, "JSON"):
|
||||
return "JSON"
|
||||
if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
|
||||
return "TINYINT UNSIGNED"
|
||||
if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
|
||||
return "SMALLINT UNSIGNED"
|
||||
if tdSql.cursor.istype(col, "INT UNSIGNED"):
|
||||
return "INT UNSIGNED"
|
||||
if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
|
||||
return "BIGINT UNSIGNED"
|
||||
|
||||
def union_check(self):
|
||||
sqls = self.sql_list()
|
||||
for i in range(len(sqls)):
|
||||
tdSql.query(sqls[i])
|
||||
res1_type = self.__get_type(0)
|
||||
for j in range(len(sqls[i:])):
|
||||
tdSql.query(sqls[j+i])
|
||||
order_union_type = False
|
||||
rev_order_type = False
|
||||
all_union_type = False
|
||||
res2_type = self.__get_type(0)
|
||||
|
||||
if res2_type == res1_type:
|
||||
all_union_type = True
|
||||
elif res1_type in ( "BIGINT" , "NCHAR" ) and res2_type in ("BIGINT" , "NCHAR"):
|
||||
all_union_type = True
|
||||
elif res1_type in ("BIGINT", "NCHAR"):
|
||||
order_union_type = True
|
||||
elif res2_type in ("BIGINT", "NCHAR"):
|
||||
rev_order_type = True
|
||||
elif res1_type == "TIMESAMP" and res2_type not in ("BINARY", "NCHAR"):
|
||||
order_union_type = True
|
||||
elif res2_type == "TIMESAMP" and res1_type not in ("BINARY", "NCHAR"):
|
||||
rev_order_type = True
|
||||
elif res1_type == "BINARY" and res2_type != "NCHAR":
|
||||
order_union_type = True
|
||||
elif res2_type == "BINARY" and res1_type != "NCHAR":
|
||||
rev_order_type = True
|
||||
|
||||
if all_union_type:
|
||||
tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
|
||||
tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
|
||||
tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
elif order_union_type:
|
||||
tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
|
||||
tdSql.checkCols(1)
|
||||
tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
|
||||
tdSql.checkCols(1)
|
||||
elif rev_order_type:
|
||||
tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
|
||||
tdSql.checkCols(1)
|
||||
else:
|
||||
tdSql.error(f"{sqls[i]} union {sqls[j+i]}")
|
||||
|
||||
def __test_error(self):
|
||||
|
||||
tdSql.error( "show tables union show tables" )
|
||||
tdSql.error( "create table errtb1 union all create table errtb2" )
|
||||
tdSql.error( "drop table ct1 union all drop table ct3" )
|
||||
tdSql.error( "select c1 from ct1 union all drop table ct3" )
|
||||
tdSql.error( "select c1 from ct1 union all '' " )
|
||||
tdSql.error( " '' union all select c1 from ct1 " )
|
||||
tdSql.error( "select c1 from ct1 union select c1 from ct2 union select c1 from ct4 ")
|
||||
|
||||
def all_test(self):
|
||||
self.__test_error()
|
||||
self.union_check()
|
||||
|
||||
|
||||
def __create_tb(self):
|
||||
|
||||
tdLog.printNoPrefix("==========step1:create table")
|
||||
create_stb_sql = f'''create table stb1(
|
||||
ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
|
||||
{FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
|
||||
{BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
|
||||
) tags (t1 int)
|
||||
'''
|
||||
create_ntb_sql = f'''create table t1(
|
||||
ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
|
||||
{FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
|
||||
{BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
|
||||
)
|
||||
'''
|
||||
tdSql.execute(create_stb_sql)
|
||||
tdSql.execute(create_ntb_sql)
|
||||
|
||||
for i in range(4):
|
||||
tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
|
||||
{ i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}
|
||||
|
||||
def __insert_data(self, rows):
|
||||
now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
|
||||
for i in range(rows):
|
||||
tdSql.execute(
|
||||
f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
|
||||
)
|
||||
tdSql.execute(
|
||||
f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
|
||||
)
|
||||
tdSql.execute(
|
||||
f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
|
||||
)
|
||||
tdSql.execute(
|
||||
f'''insert into ct1 values
|
||||
( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } )
|
||||
( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } )
|
||||
'''
|
||||
)
|
||||
|
||||
tdSql.execute(
|
||||
f'''insert into ct4 values
|
||||
( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
(
|
||||
{ now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127,
|
||||
{ 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000}
|
||||
)
|
||||
(
|
||||
{ now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126,
|
||||
{ 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000}
|
||||
)
|
||||
'''
|
||||
)
|
||||
|
||||
tdSql.execute(
|
||||
f'''insert into ct2 values
|
||||
( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
(
|
||||
{ now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126,
|
||||
{ -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
|
||||
)
|
||||
(
|
||||
{ now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127,
|
||||
{ - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
|
||||
)
|
||||
'''
|
||||
)
|
||||
|
||||
for i in range(rows):
|
||||
insert_data = f'''insert into t1 values
|
||||
( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2},
|
||||
"binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } )
|
||||
'''
|
||||
tdSql.execute(insert_data)
|
||||
tdSql.execute(
|
||||
f'''insert into t1 values
|
||||
( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
|
||||
( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127,
|
||||
{ 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 },
|
||||
"binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
|
||||
)
|
||||
(
|
||||
{ now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126,
|
||||
{ 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 },
|
||||
"binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
|
||||
)
|
||||
'''
|
||||
)
|
||||
|
||||
|
||||
def run(self):
|
||||
tdSql.prepare()
|
||||
|
||||
tdLog.printNoPrefix("==========step1:create table")
|
||||
self.__create_tb()
|
||||
|
||||
tdLog.printNoPrefix("==========step2:insert data")
|
||||
self.rows = 10
|
||||
self.__insert_data(self.rows)
|
||||
|
||||
tdLog.printNoPrefix("==========step3:all check")
|
||||
self.all_test()
|
||||
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
|
||||
tdSql.execute("use db")
|
||||
|
||||
tdLog.printNoPrefix("==========step4:after wal, all check again ")
|
||||
self.all_test()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -0,0 +1,487 @@
|
|||
|
||||
import taos
|
||||
import sys
|
||||
import time
|
||||
import socket
|
||||
import os
|
||||
import threading
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
|
||||
class TDTestCase:
|
||||
hostname = socket.gethostname()
|
||||
#rpcDebugFlagVal = '143'
|
||||
#clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
|
||||
#clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal
|
||||
#updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
|
||||
#updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal
|
||||
#print ("===================: ", updatecfgDict)
|
||||
|
||||
def init(self, conn, logSql):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
#tdSql.init(conn.cursor())
|
||||
tdSql.init(conn.cursor(), logSql) # output sql.txt file
|
||||
|
||||
def getBuildPath(self):
|
||||
selfPath = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
if ("community" in selfPath):
|
||||
projPath = selfPath[:selfPath.find("community")]
|
||||
else:
|
||||
projPath = selfPath[:selfPath.find("tests")]
|
||||
|
||||
for root, dirs, files in os.walk(projPath):
|
||||
if ("taosd" in files):
|
||||
rootRealPath = os.path.dirname(os.path.realpath(root))
|
||||
if ("packaging" not in rootRealPath):
|
||||
buildPath = root[:len(root) - len("/build/bin")]
|
||||
break
|
||||
return buildPath
|
||||
|
||||
def newcur(self,cfg,host,port):
|
||||
user = "root"
|
||||
password = "taosdata"
|
||||
con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port)
|
||||
cur=con.cursor()
|
||||
print(cur)
|
||||
return cur
|
||||
|
||||
def initConsumerTable(self,cdbName='cdb'):
|
||||
tdLog.info("create consume database, and consume info table, and consume result table")
|
||||
tdSql.query("create database if not exists %s vgroups 1"%(cdbName))
|
||||
tdSql.query("drop table if exists %s.consumeinfo "%(cdbName))
|
||||
tdSql.query("drop table if exists %s.consumeresult "%(cdbName))
|
||||
|
||||
tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName)
|
||||
tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
|
||||
|
||||
def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'):
|
||||
sql = "insert into %s.consumeinfo values "%cdbName
|
||||
sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit)
|
||||
tdLog.info("consume info sql: %s"%sql)
|
||||
tdSql.query(sql)
|
||||
|
||||
def selectConsumeResult(self,expectRows,cdbName='cdb'):
|
||||
resultList=[]
|
||||
while 1:
|
||||
tdSql.query("select * from %s.consumeresult"%cdbName)
|
||||
#tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
|
||||
if tdSql.getRows() == expectRows:
|
||||
break
|
||||
else:
|
||||
time.sleep(5)
|
||||
|
||||
for i in range(expectRows):
|
||||
tdLog.info ("ts: %s, consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 0), tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3)))
|
||||
resultList.append(tdSql.getData(i , 3))
|
||||
|
||||
return resultList
|
||||
|
||||
def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0):
|
||||
shellCmd = 'nohup '
|
||||
if valgrind == 1:
|
||||
logFile = cfgPath + '/../log/valgrind-tmq.log'
|
||||
shellCmd = 'nohup valgrind --log-file=' + logFile
|
||||
shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes '
|
||||
|
||||
shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath
|
||||
shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName)
|
||||
shellCmd += "> /dev/null 2>&1 &"
|
||||
tdLog.info(shellCmd)
|
||||
os.system(shellCmd)
|
||||
|
||||
def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl):
|
||||
tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups))
|
||||
tsql.execute("use %s" %dbName)
|
||||
tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName)
|
||||
pre_create = "create table"
|
||||
sql = pre_create
|
||||
#tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname))
|
||||
for i in range(ctbNum):
|
||||
sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1)
|
||||
if (i > 0) and (i%100 == 0):
|
||||
tsql.execute(sql)
|
||||
sql = pre_create
|
||||
if sql != pre_create:
|
||||
tsql.execute(sql)
|
||||
|
||||
event.set()
|
||||
tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum))
|
||||
return
|
||||
|
||||
def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs):
|
||||
tdLog.debug("start to insert data ............")
|
||||
tsql.execute("use %s" %dbName)
|
||||
pre_insert = "insert into "
|
||||
sql = pre_insert
|
||||
|
||||
t = time.time()
|
||||
startTs = int(round(t * 1000))
|
||||
#tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
|
||||
for i in range(ctbNum):
|
||||
sql += " %s_%d values "%(stbName,i)
|
||||
for j in range(rowsPerTbl):
|
||||
sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
|
||||
if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
|
||||
tsql.execute(sql)
|
||||
if j < rowsPerTbl - 1:
|
||||
sql = "insert into %s_%d values " %(stbName,i)
|
||||
else:
|
||||
sql = "insert into "
|
||||
#end sql
|
||||
if sql != pre_insert:
|
||||
#print("insert sql:%s"%sql)
|
||||
tsql.execute(sql)
|
||||
tdLog.debug("insert data ............ [OK]")
|
||||
return
|
||||
|
||||
def prepareEnv(self, **parameterDict):
|
||||
print ("input parameters:")
|
||||
print (parameterDict)
|
||||
# create new connector for my thread
|
||||
tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030)
|
||||
self.create_tables(tsql,\
|
||||
parameterDict["dbName"],\
|
||||
parameterDict["vgroups"],\
|
||||
parameterDict["stbName"],\
|
||||
parameterDict["ctbNum"],\
|
||||
parameterDict["rowsPerTbl"])
|
||||
|
||||
self.insert_data(tsql,\
|
||||
parameterDict["dbName"],\
|
||||
parameterDict["stbName"],\
|
||||
parameterDict["ctbNum"],\
|
||||
parameterDict["rowsPerTbl"],\
|
||||
parameterDict["batchNum"],\
|
||||
parameterDict["startTs"])
|
||||
return
|
||||
|
||||
def tmqCase8(self, cfgPath, buildPath):
|
||||
tdLog.printNoPrefix("======== test case 8: Produce while one consume to subscribe one db, inclue 1 stb")
|
||||
tdLog.info("step 1: create database, stb, ctb and insert data")
|
||||
# create and start thread
|
||||
parameterDict = {'cfg': '', \
|
||||
'dbName': 'db8', \
|
||||
'vgroups': 4, \
|
||||
'stbName': 'stb', \
|
||||
'ctbNum': 10, \
|
||||
'rowsPerTbl': 10000, \
|
||||
'batchNum': 100, \
|
||||
'startTs': 1640966400000} # 2022-01-01 00:00:00.000
|
||||
parameterDict['cfg'] = cfgPath
|
||||
|
||||
self.initConsumerTable()
|
||||
|
||||
tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
|
||||
|
||||
prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
|
||||
prepareEnvThread.start()
|
||||
|
||||
tdLog.info("create topics from db")
|
||||
topicName1 = 'topic_db1'
|
||||
|
||||
tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
|
||||
consumerId = 0
|
||||
expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] / 2
|
||||
topicList = topicName1
|
||||
ifcheckdata = 0
|
||||
ifManualCommit = 0
|
||||
keyList = 'group.id:cgrp1,\
|
||||
enable.auto.commit:false,\
|
||||
auto.commit.interval.ms:6000,\
|
||||
auto.offset.reset:earliest'
|
||||
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
event.wait()
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
pollDelay = 5
|
||||
showMsg = 1
|
||||
showRow = 1
|
||||
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
|
||||
|
||||
# wait for data ready
|
||||
prepareEnvThread.join()
|
||||
|
||||
tdLog.info("insert process end, and start to check consume result")
|
||||
expectRows = 1
|
||||
resultList = self.selectConsumeResult(expectRows)
|
||||
totalConsumeRows = 0
|
||||
for i in range(expectRows):
|
||||
totalConsumeRows += resultList[i]
|
||||
|
||||
if totalConsumeRows != expectrowcnt:
|
||||
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
|
||||
tdLog.exit("tmq consume rows error!")
|
||||
|
||||
|
||||
tdLog.info("again start consume processer")
|
||||
self.initConsumerTable()
|
||||
expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"]
|
||||
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
|
||||
expectRows = 1
|
||||
resultList = self.selectConsumeResult(expectRows)
|
||||
totalConsumeRows = 0
|
||||
for i in range(expectRows):
|
||||
totalConsumeRows += resultList[i]
|
||||
|
||||
if totalConsumeRows != expectrowcnt:
|
||||
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
|
||||
tdLog.exit("tmq consume rows error!")
|
||||
|
||||
tdSql.query("drop topic %s"%topicName1)
|
||||
|
||||
tdLog.printNoPrefix("======== test case 8 end ...... ")
|
||||
|
||||
def tmqCase9(self, cfgPath, buildPath):
|
||||
tdLog.printNoPrefix("======== test case 9: Produce while one consume to subscribe one db, inclue 1 stb")
|
||||
tdLog.info("step 1: create database, stb, ctb and insert data")
|
||||
# create and start thread
|
||||
parameterDict = {'cfg': '', \
|
||||
'dbName': 'db9', \
|
||||
'vgroups': 4, \
|
||||
'stbName': 'stb', \
|
||||
'ctbNum': 10, \
|
||||
'rowsPerTbl': 10000, \
|
||||
'batchNum': 100, \
|
||||
'startTs': 1640966400000} # 2022-01-01 00:00:00.000
|
||||
parameterDict['cfg'] = cfgPath
|
||||
|
||||
self.initConsumerTable()
|
||||
|
||||
tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
|
||||
|
||||
prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
|
||||
prepareEnvThread.start()
|
||||
|
||||
tdLog.info("create topics from db")
|
||||
topicName1 = 'topic_db1'
|
||||
|
||||
tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
|
||||
consumerId = 0
|
||||
expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] / 2
|
||||
topicList = topicName1
|
||||
ifcheckdata = 0
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1,\
|
||||
enable.auto.commit:false,\
|
||||
auto.commit.interval.ms:6000,\
|
||||
auto.offset.reset:earliest'
|
||||
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
event.wait()
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
pollDelay = 5
|
||||
showMsg = 1
|
||||
showRow = 1
|
||||
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
|
||||
|
||||
# wait for data ready
|
||||
prepareEnvThread.join()
|
||||
|
||||
tdLog.info("insert process end, and start to check consume result")
|
||||
expectRows = 1
|
||||
resultList = self.selectConsumeResult(expectRows)
|
||||
totalConsumeRows = 0
|
||||
for i in range(expectRows):
|
||||
totalConsumeRows += resultList[i]
|
||||
|
||||
if totalConsumeRows != expectrowcnt:
|
||||
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
|
||||
tdLog.exit("tmq consume rows error!")
|
||||
|
||||
|
||||
tdLog.info("again start consume processer")
|
||||
self.initConsumerTable()
|
||||
expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"]
|
||||
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
|
||||
expectRows = 1
|
||||
resultList = self.selectConsumeResult(expectRows)
|
||||
totalConsumeRows = 0
|
||||
for i in range(expectRows):
|
||||
totalConsumeRows += resultList[i]
|
||||
|
||||
if totalConsumeRows != expectrowcnt/2:
|
||||
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/2))
|
||||
tdLog.exit("tmq consume rows error!")
|
||||
|
||||
tdSql.query("drop topic %s"%topicName1)
|
||||
|
||||
tdLog.printNoPrefix("======== test case 9 end ...... ")
|
||||
|
||||
def tmqCase10(self, cfgPath, buildPath):
|
||||
tdLog.printNoPrefix("======== test case 10: Produce while one consume to subscribe one db, inclue 1 stb")
|
||||
tdLog.info("step 1: create database, stb, ctb and insert data")
|
||||
# create and start thread
|
||||
parameterDict = {'cfg': '', \
|
||||
'dbName': 'db10', \
|
||||
'vgroups': 4, \
|
||||
'stbName': 'stb', \
|
||||
'ctbNum': 10, \
|
||||
'rowsPerTbl': 10000, \
|
||||
'batchNum': 100, \
|
||||
'startTs': 1640966400000} # 2022-01-01 00:00:00.000
|
||||
parameterDict['cfg'] = cfgPath
|
||||
|
||||
self.initConsumerTable()
|
||||
|
||||
tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
|
||||
|
||||
prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
|
||||
prepareEnvThread.start()
|
||||
|
||||
tdLog.info("create topics from db")
|
||||
topicName1 = 'topic_db1'
|
||||
|
||||
tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
|
||||
consumerId = 0
|
||||
expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"]
|
||||
topicList = topicName1
|
||||
ifcheckdata = 0
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1,\
|
||||
enable.auto.commit:false,\
|
||||
auto.commit.interval.ms:6000,\
|
||||
auto.offset.reset:earliest'
|
||||
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
event.wait()
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
pollDelay = 5
|
||||
showMsg = 1
|
||||
showRow = 1
|
||||
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
|
||||
|
||||
time.sleep(2)
|
||||
tdLog.info("pkill consume processor")
|
||||
os.system('pkill tmq_sim')
|
||||
expectRows = 0
|
||||
resultList = self.selectConsumeResult(expectRows)
|
||||
|
||||
# wait for data ready
|
||||
prepareEnvThread.join()
|
||||
tdLog.info("insert process end, and start to check consume result")
|
||||
|
||||
tdLog.info("again start consume processer")
|
||||
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
|
||||
|
||||
expectRows = 1
|
||||
resultList = self.selectConsumeResult(expectRows)
|
||||
totalConsumeRows = 0
|
||||
for i in range(expectRows):
|
||||
totalConsumeRows += resultList[i]
|
||||
|
||||
if totalConsumeRows != expectrowcnt:
|
||||
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
|
||||
tdLog.exit("tmq consume rows error!")
|
||||
|
||||
tdSql.query("drop topic %s"%topicName1)
|
||||
|
||||
tdLog.printNoPrefix("======== test case 10 end ...... ")
|
||||
|
||||
def tmqCase11(self, cfgPath, buildPath):
|
||||
tdLog.printNoPrefix("======== test case 11: Produce while one consume to subscribe one db, inclue 1 stb")
|
||||
tdLog.info("step 1: create database, stb, ctb and insert data")
|
||||
# create and start thread
|
||||
parameterDict = {'cfg': '', \
|
||||
'dbName': 'db11', \
|
||||
'vgroups': 4, \
|
||||
'stbName': 'stb', \
|
||||
'ctbNum': 10, \
|
||||
'rowsPerTbl': 10000, \
|
||||
'batchNum': 100, \
|
||||
'startTs': 1640966400000} # 2022-01-01 00:00:00.000
|
||||
parameterDict['cfg'] = cfgPath
|
||||
|
||||
self.initConsumerTable()
|
||||
|
||||
tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups']))
|
||||
|
||||
prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
|
||||
prepareEnvThread.start()
|
||||
|
||||
tdLog.info("create topics from db")
|
||||
topicName1 = 'topic_db1'
|
||||
|
||||
tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
|
||||
consumerId = 0
|
||||
expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"]
|
||||
topicList = topicName1
|
||||
ifcheckdata = 0
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1,\
|
||||
enable.auto.commit:true,\
|
||||
auto.commit.interval.ms:1000,\
|
||||
auto.offset.reset:earliest'
|
||||
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
event.wait()
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
pollDelay = 5
|
||||
showMsg = 1
|
||||
showRow = 1
|
||||
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
|
||||
|
||||
time.sleep(5)
|
||||
tdLog.info("pkill consume processor")
|
||||
os.system('pkill tmq_sim')
|
||||
expectRows = 0
|
||||
resultList = self.selectConsumeResult(expectRows)
|
||||
|
||||
# wait for data ready
|
||||
prepareEnvThread.join()
|
||||
tdLog.info("insert process end, and start to check consume result")
|
||||
|
||||
tdLog.info("again start consume processer")
|
||||
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
|
||||
|
||||
expectRows = 1
|
||||
resultList = self.selectConsumeResult(expectRows)
|
||||
totalConsumeRows = 0
|
||||
for i in range(expectRows):
|
||||
totalConsumeRows += resultList[i]
|
||||
|
||||
if totalConsumeRows >= expectrowcnt or totalConsumeRows <= 0:
|
||||
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
|
||||
tdLog.exit("tmq consume rows error!")
|
||||
|
||||
tdSql.query("drop topic %s"%topicName1)
|
||||
|
||||
tdLog.printNoPrefix("======== test case 11 end ...... ")
|
||||
|
||||
def run(self):
|
||||
tdSql.prepare()
|
||||
|
||||
buildPath = self.getBuildPath()
|
||||
if (buildPath == ""):
|
||||
tdLog.exit("taosd not found!")
|
||||
else:
|
||||
tdLog.info("taosd found in %s" % buildPath)
|
||||
cfgPath = buildPath + "/../sim/psim/cfg"
|
||||
tdLog.info("cfgPath: %s" % cfgPath)
|
||||
|
||||
#self.tmqCase8(cfgPath, buildPath)
|
||||
#self.tmqCase9(cfgPath, buildPath)
|
||||
#self.tmqCase10(cfgPath, buildPath)
|
||||
self.tmqCase11(cfgPath, buildPath)
|
||||
# self.tmqCase12(cfgPath, buildPath)
|
||||
# self.tmqCase13(cfgPath, buildPath)
|
||||
# self.tmqCase14(cfgPath, buildPath)
|
||||
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
event = threading.Event()
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -22,8 +22,12 @@ python3 ./test.py -f 2-query/upper.py
|
|||
python3 ./test.py -f 2-query/lower.py
|
||||
python3 ./test.py -f 2-query/join.py
|
||||
python3 ./test.py -f 2-query/cast.py
|
||||
# python3 ./test.py -f 2-query/concat.py # after wal ,crash occured
|
||||
# python3 ./test.py -f 2-query/concat_ws.py
|
||||
python3 ./test.py -f 2-query/concat.py
|
||||
python3 ./test.py -f 2-query/concat_ws.py
|
||||
# python3 ./test.py -f 2-query/union.py
|
||||
# python3 ./test.py -f 2-query/union2.py
|
||||
# python3 ./test.py -f 2-query/union3.py
|
||||
# python3 ./test.py -f 2-query/union4.py
|
||||
|
||||
python3 ./test.py -f 2-query/timezone.py
|
||||
python3 ./test.py -f 2-query/Now.py
|
||||
|
|
@ -58,4 +62,4 @@ python3 ./test.py -f 2-query/nestedQuery.py
|
|||
|
||||
python3 ./test.py -f 7-tmq/basic5.py
|
||||
python3 ./test.py -f 7-tmq/subscribeDb.py
|
||||
|
||||
python3 ./test.py -f 7-tmq/subscribeDb1.py
|
||||
|
|
|
|||
|
|
@ -278,7 +278,7 @@ void build_consumer(SThreadInfo* pInfo) {
|
|||
|
||||
//tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName);
|
||||
|
||||
tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL);
|
||||
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
|
||||
|
||||
// tmq_conf_set(conf, "group.id", "cgrp1");
|
||||
for (int32_t i = 0; i < pInfo->numOfKey; i++) {
|
||||
|
|
@ -322,8 +322,11 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) {
|
|||
sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName,
|
||||
pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult);
|
||||
|
||||
taosFprintfFile(g_fp, "== save result sql: %s \n", sqlStr);
|
||||
|
||||
time_t tTime = taosGetTimestampSec();
|
||||
struct tm tm = *taosLocalTime(&tTime, NULL);
|
||||
taosFprintfFile(g_fp, "# save result: %d-%02d-%02d %02d:%02d:%02d, sql: %s\n", tm.tm_year + 1900, tm.tm_mon + 1,
|
||||
tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, sqlStr);
|
||||
|
||||
TAOS_RES* pRes = taos_query(pConn, sqlStr);
|
||||
if (taos_errno(pRes) != 0) {
|
||||
pError("error in save consumeinfo, reason:%s\n", taos_errstr(pRes));
|
||||
|
|
|
|||
Loading…
Reference in New Issue