xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix:error in getting snapshot for taosX

This commit is contained in:
wangmm0220 2022-08-25 16:55:12 +08:00
parent c7bf08d5af
commit 4cc2d361e1
8 changed files with 132 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
examples/c/tmq_taosx.c
View File

@ -226,19 +226,19 @@ int32_t init_env() {
} }
taos_free_result(pRes); taos_free_result(pRes);
// pRes = taos_query(pConn, "drop table ct3 ct1"); pRes = taos_query(pConn, "drop table ct3 ct1");
// if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
// printf("failed to drop child table ct3, reason:%s\n", taos_errstr(pRes)); printf("failed to drop child table ct3, reason:%s\n", taos_errstr(pRes));
// return -1; return -1;
// } }
// taos_free_result(pRes); taos_free_result(pRes);
//
// pRes = taos_query(pConn, "drop table st1"); pRes = taos_query(pConn, "drop table st1");
// if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
// printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes));
// return -1; return -1;
// } }
// taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create table if not exists n1(ts timestamp, c1 int, c2 nchar(4))"); pRes = taos_query(pConn, "create table if not exists n1(ts timestamp, c1 int, c2 nchar(4))");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
@ -289,12 +289,12 @@ int32_t init_env() {
} }
taos_free_result(pRes); taos_free_result(pRes);
// pRes = taos_query(pConn, "drop table n1"); pRes = taos_query(pConn, "drop table n1");
// if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
// printf("failed to drop normal table n1, reason:%s\n", taos_errstr(pRes)); printf("failed to drop normal table n1, reason:%s\n", taos_errstr(pRes));
// return -1; return -1;
// } }
// taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create table jt(ts timestamp, i int) tags(t json)"); pRes = taos_query(pConn, "create table jt(ts timestamp, i int) tags(t json)");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
@ -326,12 +326,12 @@ int32_t init_env() {
} }
taos_free_result(pRes); taos_free_result(pRes);
// pRes = taos_query(pConn, "drop table st1"); pRes = taos_query(pConn, "drop table st1");
// if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
// printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes));
// return -1; return -1;
// } }
// taos_free_result(pRes); taos_free_result(pRes);
taos_close(pConn); taos_close(pConn);
return 0; return 0;
@ -386,7 +386,7 @@ tmq_t* build_consumer() {
tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set(conf, "msg.with.table.name", "true");
tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "enable.auto.commit", "true");
tmq_conf_set(conf, "experimental.snapshot.enable", "false"); tmq_conf_set(conf, "experimental.snapshot.enable", "true");
/*tmq_conf_set(conf, "experimental.snapshot.enable", "true");*/ /*tmq_conf_set(conf, "experimental.snapshot.enable", "true");*/

13
source/client/src/taosx.c
View File

@ -1405,6 +1405,11 @@ static int32_t tmqWriteRaw(TAOS* taos, void* data, int32_t dataLen) {
} }
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){
uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName);
code = TSDB_CODE_SUCCESS;
continue;
}
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbName); uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbName);
goto end; goto end;
@ -1472,7 +1477,7 @@ static int32_t tmqWriteRaw(TAOS* taos, void* data, int32_t dataLen) {
SRowBuilder rb = {0}; SRowBuilder rb = {0};
tdSRowInit(&rb, sver); tdSRowInit(&rb, sver);
tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen); tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen);
int32_t dataLen = 0; int32_t totalLen = 0;
SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
for (int i = 0; i < pSW->nCols; i++) { for (int i = 0; i < pSW->nCols; i++) {
@ -1509,7 +1514,7 @@ static int32_t tmqWriteRaw(TAOS* taos, void* data, int32_t dataLen) {
tdSRowEnd(&rb); tdSRowEnd(&rb);
int32_t rowLen = TD_ROW_LEN(rowData); int32_t rowLen = TD_ROW_LEN(rowData);
rowData = POINTER_SHIFT(rowData, rowLen); rowData = POINTER_SHIFT(rowData, rowLen);
dataLen += rowLen; totalLen += rowLen;
} }
taosHashCleanup(schemaHash); taosHashCleanup(schemaHash);
@ -1518,8 +1523,8 @@ static int32_t tmqWriteRaw(TAOS* taos, void* data, int32_t dataLen) {
blk->sversion = htonl(sver); blk->sversion = htonl(sver);
blk->schemaLen = htonl(schemaLen); blk->schemaLen = htonl(schemaLen);
blk->numOfRows = htonl(rows); blk->numOfRows = htonl(rows);
blk->dataLen = htonl(dataLen); blk->dataLen = htonl(totalLen);
subReq->length += sizeof(SSubmitBlk) + schemaLen + dataLen; subReq->length += sizeof(SSubmitBlk) + schemaLen + totalLen;
subReq->numOfBlocks++; subReq->numOfBlocks++;
taosMemoryFreeClear(pTableMeta); taosMemoryFreeClear(pTableMeta);
} }

34
source/client/test/smlTest.cpp
View File

File diff suppressed because one or more lines are too long

32
source/dnode/vnode/src/meta/metaSnapshot.c
View File

@ -283,6 +283,15 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
STbDbKey *tmp = (STbDbKey*)pKey; STbDbKey *tmp = (STbDbKey*)pKey;
if (tmp->version > ctx->snapVersion) break; if (tmp->version > ctx->snapVersion) break;
SIdInfo* idData = (SIdInfo*)taosHashGet(ctx->idVersion, &tmp->uid, sizeof(tb_uid_t));
if(idData) {
continue;
}
if (tdbTbGet(pMeta->pUidIdx, &tmp->uid, sizeof(tb_uid_t), NULL, NULL) < 0) { // check if table exist for now, need optimize later
continue;
}
SDecoder dc = {0}; SDecoder dc = {0};
SMetaEntry me = {0}; SMetaEntry me = {0};
tDecoderInit(&dc, pVal, vLen); tDecoderInit(&dc, pVal, vLen);
@ -290,16 +299,17 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
if(ctx->subType == TOPIC_SUB_TYPE__TABLE){ if(ctx->subType == TOPIC_SUB_TYPE__TABLE){
if ((me.uid != ctx->suid && me.type == TSDB_SUPER_TABLE) || if ((me.uid != ctx->suid && me.type == TSDB_SUPER_TABLE) ||
(me.ctbEntry.suid != ctx->suid && me.type == TSDB_CHILD_TABLE)){ (me.ctbEntry.suid != ctx->suid && me.type == TSDB_CHILD_TABLE)){
tDecoderClear(&dc);
continue; continue;
} }
} }
SIdInfo* idData = (SIdInfo*)taosHashGet(ctx->idVersion, &tmp->uid, sizeof(tb_uid_t));
if(!idData){ taosArrayPush(ctx->idList, &tmp->uid);
taosArrayPush(ctx->idList, &tmp->uid); metaDebug("tmqsnap init idlist name:%s, uid:%" PRIi64, me.name, tmp->uid);
metaDebug("tmqsnap init idlist name:%s, uid:%" PRIi64, me.name, tmp->uid); SIdInfo info = {0};
SIdInfo info = {0}; taosHashPut(ctx->idVersion, &tmp->uid, sizeof(tb_uid_t), &info, sizeof(SIdInfo));
taosHashPut(ctx->idVersion, &tmp->uid, sizeof(tb_uid_t), &info, sizeof(SIdInfo));
} tDecoderClear(&dc);
} }
taosHashClear(ctx->idVersion); taosHashClear(ctx->idVersion);
@ -310,10 +320,11 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
STbDbKey *tmp = (STbDbKey*)pKey; STbDbKey *tmp = (STbDbKey*)pKey;
SIdInfo* idData = (SIdInfo*)taosHashGet(ctx->idVersion, &tmp->uid, sizeof(tb_uid_t)); SIdInfo* idData = (SIdInfo*)taosHashGet(ctx->idVersion, &tmp->uid, sizeof(tb_uid_t));
if(!idData){ if(idData){
SIdInfo info = {.version = tmp->version, .index = 0}; continue;
taosHashPut(ctx->idVersion, &tmp->uid, sizeof(tb_uid_t), &info, sizeof(SIdInfo));
} }
SIdInfo info = {.version = tmp->version, .index = 0};
taosHashPut(ctx->idVersion, &tmp->uid, sizeof(tb_uid_t), &info, sizeof(SIdInfo));
SDecoder dc = {0}; SDecoder dc = {0};
SMetaEntry me = {0}; SMetaEntry me = {0};
@ -322,6 +333,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
if(ctx->subType == TOPIC_SUB_TYPE__TABLE){ if(ctx->subType == TOPIC_SUB_TYPE__TABLE){
if ((me.uid != ctx->suid && me.type == TSDB_SUPER_TABLE) || if ((me.uid != ctx->suid && me.type == TSDB_SUPER_TABLE) ||
(me.ctbEntry.suid != ctx->suid && me.type == TSDB_CHILD_TABLE)){ (me.ctbEntry.suid != ctx->suid && me.type == TSDB_CHILD_TABLE)){
tDecoderClear(&dc);
continue; continue;
} }
} }

4
source/dnode/vnode/src/tq/tq.c
View File

@ -403,6 +403,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
code = -1; code = -1;
} }
goto OVER;
}else{ }else{
fetchOffsetNew = dataRsp.rspOffset; fetchOffsetNew = dataRsp.rspOffset;
} }
@ -434,7 +435,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
// TODO add push mgr // TODO add push mgr
tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer); tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer);
ASSERT(dataRsp.rspOffset.version >= dataRsp.reqOffset.version);
if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
code = -1; code = -1;
} }
@ -456,8 +456,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
// TODO continue scan until meeting batch requirement // TODO continue scan until meeting batch requirement
if (dataRsp.blockNum > 0 /* threshold */) { if (dataRsp.blockNum > 0 /* threshold */) {
tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer); tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer);
ASSERT(dataRsp.rspOffset.version >= dataRsp.reqOffset.version);
if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
code = -1; code = -1;
} }

61
source/dnode/vnode/src/tq/tqExec.c
View File

@ -120,36 +120,51 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp*
} }
} }
qStreamExtractOffset(task, &pRsp->rspOffset); if(pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN){
if (pRsp->blockNum == 0 && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
if (pDataBlock == NULL && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA){ tqDebug("vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode),
if(pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN && qStreamExtractPrepareUid(task) != 0){ pHandle->snapshotVer + 1);
tqOffsetResetToLog(pOffset, pHandle->snapshotVer);
qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType);
continue; continue;
} }
tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode), }else{
pHandle->snapshotVer + 1); if (pDataBlock == NULL && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA){
break; if(qStreamExtractPrepareUid(task) != 0){
continue;
}
tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode),
pHandle->snapshotVer + 1);
break;
}
if (pRsp->blockNum > 0){
tqDebug("tmqsnap task exec exited, get data");
break;
}
SMqMetaRsp* tmp = qStreamExtractMetaMsg(task);
if(tmp->rspOffset.type == TMQ_OFFSET__SNAPSHOT_DATA){
tqOffsetResetToData(pOffset, tmp->rspOffset.uid, tmp->rspOffset.ts);
qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType);
tmp->rspOffset.type = TMQ_OFFSET__SNAPSHOT_META;
tqDebug("tmqsnap task exec change to get data");
continue;
}
*pMetaRsp = *tmp;
tqDebug("tmqsnap task exec exited, get meta");
} }
if (pRsp->blockNum > 0){ tqDebug("task exec exited");
tqDebug("tmqsnap task exec exited, get data");
break;
}
SMqMetaRsp* tmp = qStreamExtractMetaMsg(task);
if(tmp->rspOffset.type == TMQ_OFFSET__SNAPSHOT_DATA){
tqOffsetResetToData(pOffset, tmp->rspOffset.uid, tmp->rspOffset.ts);
qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType);
tmp->rspOffset.type = TMQ_OFFSET__SNAPSHOT_META;
tqDebug("tmqsnap task exec change to get data");
continue;
}
*pMetaRsp = *tmp;
tqDebug("tmqsnap task exec exited, get meta");
break; break;
} }
if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) {
ASSERT(0);
}
ASSERT(pRsp->rspOffset.type != 0);
return 0; return 0;
} }

2
source/libs/executor/src/executor.c
View File

@ -847,6 +847,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
pInfo->dataReader = NULL; pInfo->dataReader = NULL;
cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
taosArrayDestroy(pTaskInfo->tableqinfoList.pTableList); taosArrayDestroy(pTaskInfo->tableqinfoList.pTableList);
if(mtInfo.uid == 0) return 0; // no data
initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, mtInfo); initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, mtInfo);
pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts; pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts;
pTaskInfo->tableqinfoList.pTableList = taosArrayInit(1, sizeof(STableKeyInfo)); pTaskInfo->tableqinfoList.pTableList = taosArrayInit(1, sizeof(STableKeyInfo));

2
source/libs/executor/src/scanoperator.c
View File

@ -1528,7 +1528,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
if(pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA){ if(pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA){
SSDataBlock* pBlock = &pInfo->pRes; SSDataBlock* pBlock = &pInfo->pRes;
if (tsdbNextDataBlock(pInfo->dataReader)) { if (pInfo->dataReader && tsdbNextDataBlock(pInfo->dataReader)) {
if (isTaskKilled(pTaskInfo)) { if (isTaskKilled(pTaskInfo)) {
longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
} }