xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'enh/TD-20043' of https://github.com/taosdata/TDengine into enh/TD-20043

This commit is contained in:
Shengliang Guan 2022-11-07 23:31:09 +08:00
parent 9914657cb0 70d60a764f
commit 05d0a0fe30
7 changed files with 129 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
source/dnode/vnode/src/inc/tsdb.h
View File

@ -155,15 +155,13 @@ int32_t tCmprBlockL(void const *lhs, void const *rhs);
int32_t tBlockDataCreate(SBlockData *pBlockData); int32_t tBlockDataCreate(SBlockData *pBlockData);
void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear); void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear);
int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid); int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid);
int32_t tBlockDataInitEx(SBlockData *pBlockData, SBlockData *pBlockDataFrom);
void tBlockDataReset(SBlockData *pBlockData); void tBlockDataReset(SBlockData *pBlockData);
int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid);
void tBlockDataClear(SBlockData *pBlockData); void tBlockDataClear(SBlockData *pBlockData);
SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx);
void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData);
int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest);
int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData);
int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); int32_t tBlockDataAddColData(SBlockData *pBlockData, SColData **ppColData);
int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[],
int32_t aBufN[]); int32_t aBufN[]);
int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]);
@ -473,7 +471,7 @@ struct SBlockData {
int64_t *aUid; // uids of each row, only exist in block data in .last file (uid == 0) int64_t *aUid; // uids of each row, only exist in block data in .last file (uid == 0)
int64_t *aVersion; // versions of each row int64_t *aVersion; // versions of each row
TSKEY *aTSKEY; // timestamp of each row TSKEY *aTSKEY; // timestamp of each row
SArray *aIdx; // SArray<int32_t> int32_t nColData;
SArray *aColData; // SArray<SColData> SArray *aColData; // SArray<SColData>
}; };

61
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -185,11 +185,13 @@ static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STabl
SRowMerger* pMerger, SVersionRange* pVerRange); SRowMerger* pMerger, SVersionRange* pVerRange);
static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger,
STsdbReader* pReader); STsdbReader* pReader);
static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, STableBlockScanInfo* pInfo); static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow,
STableBlockScanInfo* pInfo);
static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData,
int32_t rowIndex); int32_t rowIndex);
static void setComposedBlockFlag(STsdbReader* pReader, bool composed); static void setComposedBlockFlag(STsdbReader* pReader, bool composed);
static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, SVersionRange* pVerRange); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order,
SVersionRange* pVerRange);
static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList,
STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow); STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow);
@ -244,7 +246,7 @@ static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {
pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES); pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES);
} }
for(int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
char* p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo)); char* p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo));
if (p == NULL) { if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -266,7 +268,7 @@ static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {
static void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) { static void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) {
size_t num = taosArrayGetSize(pBuf->pData); size_t num = taosArrayGetSize(pBuf->pData);
for(int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
char** p = taosArrayGet(pBuf->pData, i); char** p = taosArrayGet(pBuf->pData, i);
taosMemoryFree(*p); taosMemoryFree(*p);
} }
@ -319,8 +321,8 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK
taosHashPut(pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info)); taosHashPut(pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info));
#endif #endif
tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReader, pScanInfo->uid, pScanInfo->lastKey, tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReader, pScanInfo->uid,
pTsdbReader->idStr); pScanInfo->lastKey, pTsdbReader->idStr);
} }
pTsdbReader->cost.createScanInfoList = (taosGetTimestampUs() - st) / 1000.0; pTsdbReader->cost.createScanInfoList = (taosGetTimestampUs() - st) / 1000.0;
@ -334,7 +336,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK
static void resetAllDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) { static void resetAllDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) {
STableBlockScanInfo** p = NULL; STableBlockScanInfo** p = NULL;
while ((p = taosHashIterate(pTableMap, p)) != NULL) { while ((p = taosHashIterate(pTableMap, p)) != NULL) {
STableBlockScanInfo* pInfo = *(STableBlockScanInfo**) p; STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p;
pInfo->iterInit = false; pInfo->iterInit = false;
pInfo->iiter.hasVal = false; pInfo->iiter.hasVal = false;
@ -708,7 +710,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
} }
SBlockIndex bIndex = {.ordinalIndex = j, .inFileOffset = block.aSubBlock->offset}; SBlockIndex bIndex = {.ordinalIndex = j, .inFileOffset = block.aSubBlock->offset};
bIndex.window = (STimeWindow) {.skey = block.minKey.ts, .ekey = block.maxKey.ts}; bIndex.window = (STimeWindow){.skey = block.minKey.ts, .ekey = block.maxKey.ts};
void* p = taosArrayPush(pScanInfo->pBlockList, &bIndex); void* p = taosArrayPush(pScanInfo->pBlockList, &bIndex);
if (p == NULL) { if (p == NULL) {
@ -969,7 +971,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
} }
int32_t colIndex = 0; int32_t colIndex = 0;
int32_t num = taosArrayGetSize(pBlockData->aIdx); int32_t num = pBlockData->nColData;
while (i < numOfOutputCols && colIndex < num) { while (i < numOfOutputCols && colIndex < num) {
rowIndex = 0; rowIndex = 0;
pColData = taosArrayGet(pResBlock->pDataBlock, i); pColData = taosArrayGet(pResBlock->pDataBlock, i);
@ -1300,8 +1302,8 @@ static bool getNeighborBlockOfSameTable(SFileDataBlockInfo* pBlockInfo, STableBl
int32_t step = asc ? 1 : -1; int32_t step = asc ? 1 : -1;
*nextIndex = pBlockInfo->tbBlockIdx + step; *nextIndex = pBlockInfo->tbBlockIdx + step;
*pBlockIndex = *(SBlockIndex*) taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); *pBlockIndex = *(SBlockIndex*)taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex);
// tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, pIndex->ordinalIndex, pBlock, tGetDataBlk); // tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, pIndex->ordinalIndex, pBlock, tGetDataBlk);
return true; return true;
} }
@ -1365,7 +1367,8 @@ static bool keyOverlapFileBlock(TSDBKEY key, SDataBlk* pBlock, SVersionRange* pV
(pBlock->minVer <= pVerRange->maxVer); (pBlock->minVer <= pVerRange->maxVer);
} }
static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SDataBlk* pBlock, int32_t startIndex) { static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SDataBlk* pBlock,
int32_t startIndex) {
size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);
for (int32_t i = startIndex; i < num; i += 1) { for (int32_t i = startIndex; i < num; i += 1) {
@ -2313,7 +2316,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
goto _end; goto _end;
} }
pBlockScanInfo = *(STableBlockScanInfo**) p; pBlockScanInfo = *(STableBlockScanInfo**)p;
SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader);
@ -2324,7 +2327,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
copyBlockDataToSDataBlock(pReader, pBlockScanInfo); copyBlockDataToSDataBlock(pReader, pBlockScanInfo);
// record the last key value // record the last key value
pBlockScanInfo->lastKey = asc? pBlock->maxKey.ts:pBlock->minKey.ts; pBlockScanInfo->lastKey = asc ? pBlock->maxKey.ts : pBlock->minKey.ts;
goto _end; goto _end;
} }
} }
@ -2553,7 +2556,7 @@ static void extractOrderedTableUidList(SUidOrderCheckInfo* pOrderCheckInfo, SRea
void* p = taosHashIterate(pStatus->pTableMap, NULL); void* p = taosHashIterate(pStatus->pTableMap, NULL);
while (p != NULL) { while (p != NULL) {
STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**) p; STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)p;
pOrderCheckInfo->tableUidList[index++] = pScanInfo->uid; pOrderCheckInfo->tableUidList[index++] = pScanInfo->uid;
p = taosHashIterate(pStatus->pTableMap, p); p = taosHashIterate(pStatus->pTableMap, p);
} }
@ -2627,7 +2630,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
while (1) { while (1) {
// load the last data block of current table // load the last data block of current table
STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**) pStatus->pTableIter; STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter;
bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo, pReader); bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo, pReader);
if (!hasVal) { if (!hasVal) {
bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus);
@ -2665,7 +2668,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader;
if (pBlockInfo != NULL) { if (pBlockInfo != NULL) {
pScanInfo = *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); pScanInfo =
*(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
} else { } else {
pScanInfo = *pReader->status.pTableIter; pScanInfo = *pReader->status.pTableIter;
} }
@ -2717,7 +2721,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlock->maxKey.ts, pReader->order); setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlock->maxKey.ts, pReader->order);
// update the last key for the corresponding table // update the last key for the corresponding table
pScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->order)? pInfo->window.ekey:pInfo->window.skey; pScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->order) ? pInfo->window.ekey : pInfo->window.skey;
} }
} }
@ -3414,7 +3418,8 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, STableBlockScanInfo* pScanInfo) { int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow,
STableBlockScanInfo* pScanInfo) {
int32_t numOfRows = pBlock->info.rows; int32_t numOfRows = pBlock->info.rows;
int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
int64_t uid = pScanInfo->uid; int64_t uid = pScanInfo->uid;
@ -3474,7 +3479,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
} }
SColVal cv = {0}; SColVal cv = {0};
int32_t numOfInputCols = pBlockData->aIdx->size; int32_t numOfInputCols = pBlockData->nColData;
int32_t numOfOutputCols = pResBlock->pDataBlock->size; int32_t numOfOutputCols = pResBlock->pDataBlock->size;
while (i < numOfOutputCols && j < numOfInputCols) { while (i < numOfOutputCols && j < numOfInputCols) {
@ -3555,8 +3560,8 @@ int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t n
taosHashClear(pReader->status.pTableMap); taosHashClear(pReader->status.pTableMap);
STableKeyInfo* pList = (STableKeyInfo*) pTableList; STableKeyInfo* pList = (STableKeyInfo*)pTableList;
for(int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
STableBlockScanInfo* pInfo = getPosInBlockInfoBuf(&pReader->blockInfoBuf, i); STableBlockScanInfo* pInfo = getPosInBlockInfoBuf(&pReader->blockInfoBuf, i);
pInfo->uid = pList[i].uid; pInfo->uid = pList[i].uid;
taosHashPut(pReader->status.pTableMap, &pInfo->uid, sizeof(uint64_t), &pInfo, POINTER_BYTES); taosHashPut(pReader->status.pTableMap, &pInfo->uid, sizeof(uint64_t), &pInfo, POINTER_BYTES);
@ -3714,7 +3719,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr);
return code; return code;
_err: _err:
tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr); tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr);
return code; return code;
} }
@ -3882,7 +3887,8 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
} }
bool tsdbTableNextDataBlock(STsdbReader* pReader, uint64_t uid) { bool tsdbTableNextDataBlock(STsdbReader* pReader, uint64_t uid) {
STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &uid, sizeof(uid)); STableBlockScanInfo* pBlockScanInfo =
*(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &uid, sizeof(uid));
if (pBlockScanInfo == NULL) { // no data block for the table of given uid if (pBlockScanInfo == NULL) { // no data block for the table of given uid
return false; return false;
} }
@ -3929,7 +3935,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter);
SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
// int64_t stime = taosGetTimestampUs(); // int64_t stime = taosGetTimestampUs();
SBlockLoadSuppInfo* pSup = &pReader->suppInfo; SBlockLoadSuppInfo* pSup = &pReader->suppInfo;
@ -3995,7 +4001,8 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) {
} }
SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter);
STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); STableBlockScanInfo* pBlockScanInfo =
*(STableBlockScanInfo**)taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
if (pBlockScanInfo == NULL) { if (pBlockScanInfo == NULL) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid, tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid,
@ -4260,7 +4267,7 @@ int32_t tsdbTakeReadSnap(STsdb* pTsdb, STsdbReadSnap** ppSnap, const char* idStr
} }
tsdbTrace("vgId:%d, take read snapshot, %s", TD_VID(pTsdb->pVnode), idStr); tsdbTrace("vgId:%d, take read snapshot, %s", TD_VID(pTsdb->pVnode), idStr);
_exit: _exit:
return code; return code;
} }

49
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -514,7 +514,7 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData,
pSmaInfo->size = 0; pSmaInfo->size = 0;
// encode // encode
for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) {
SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type)) continue; if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type)) continue;
@ -1112,7 +1112,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo
ASSERT(p - pReader->aBuf[0] == pBlkInfo->szKey); ASSERT(p - pReader->aBuf[0] == pBlkInfo->szKey);
// read and decode columns // read and decode columns
if (taosArrayGetSize(pBlockData->aIdx) == 0) goto _exit; if (pBlockData->nColData == 0) goto _exit;
if (hdr.szBlkCol > 0) { if (hdr.szBlkCol > 0) {
int64_t offset = pBlkInfo->offset + pBlkInfo->szKey; int64_t offset = pBlkInfo->offset + pBlkInfo->szKey;
@ -1128,7 +1128,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo
SBlockCol *pBlockCol = &blockCol; SBlockCol *pBlockCol = &blockCol;
int32_t n = 0; int32_t n = 0;
for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) {
SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
while (pBlockCol && pBlockCol->cid < pColData->cid) { while (pBlockCol && pBlockCol->cid < pColData->cid) {
@ -1212,49 +1212,6 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData
ASSERT(pDataBlk->nSubBlock == 1); ASSERT(pDataBlk->nSubBlock == 1);
#if 0
if (pDataBlk->nSubBlock > 1) {
SBlockData bData1;
SBlockData bData2;
// create
code = tBlockDataCreate(&bData1);
if (code) goto _err;
code = tBlockDataCreate(&bData2);
if (code) goto _err;
// init
tBlockDataInitEx(&bData1, pBlockData);
tBlockDataInitEx(&bData2, pBlockData);
for (int32_t iSubBlock = 1; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) {
code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[iSubBlock], &bData1);
if (code) {
tBlockDataDestroy(&bData1, 1);
tBlockDataDestroy(&bData2, 1);
goto _err;
}
code = tBlockDataCopy(pBlockData, &bData2);
if (code) {
tBlockDataDestroy(&bData1, 1);
tBlockDataDestroy(&bData2, 1);
goto _err;
}
code = tBlockDataMerge(&bData1, &bData2, pBlockData);
if (code) {
tBlockDataDestroy(&bData1, 1);
tBlockDataDestroy(&bData2, 1);
goto _err;
}
}
tBlockDataDestroy(&bData1, 1);
tBlockDataDestroy(&bData2, 1);
}
#endif
return code; return code;
_err: _err:

115
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -613,7 +613,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
return &pIter->colVal; return &pIter->colVal;
} }
} else { } else {
if (pIter->i < taosArrayGetSize(pIter->pRow->pBlockData->aIdx)) { if (pIter->i < pIter->pRow->pBlockData->nColData) {
SColData *pColData = tBlockDataGetColDataByIdx(pIter->pRow->pBlockData, pIter->i); SColData *pColData = tBlockDataGetColDataByIdx(pIter->pRow->pBlockData, pIter->i);
tColDataGetValue(pColData, pIter->pRow->iRow, &pIter->colVal); tColDataGetValue(pColData, pIter->pRow->iRow, &pIter->colVal);
@ -917,14 +917,9 @@ int32_t tBlockDataCreate(SBlockData *pBlockData) {
pBlockData->aUid = NULL; pBlockData->aUid = NULL;
pBlockData->aVersion = NULL; pBlockData->aVersion = NULL;
pBlockData->aTSKEY = NULL; pBlockData->aTSKEY = NULL;
pBlockData->aIdx = taosArrayInit(0, sizeof(int32_t)); pBlockData->nColData = 0;
if (pBlockData->aIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
pBlockData->aColData = taosArrayInit(0, sizeof(SColData)); pBlockData->aColData = taosArrayInit(0, sizeof(SColData));
if (pBlockData->aColData == NULL) { if (pBlockData->aColData == NULL) {
taosArrayDestroy(pBlockData->aIdx);
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
@ -937,12 +932,10 @@ void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) {
tFree((uint8_t *)pBlockData->aUid); tFree((uint8_t *)pBlockData->aUid);
tFree((uint8_t *)pBlockData->aVersion); tFree((uint8_t *)pBlockData->aVersion);
tFree((uint8_t *)pBlockData->aTSKEY); tFree((uint8_t *)pBlockData->aTSKEY);
taosArrayDestroy(pBlockData->aIdx);
taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataDestroy : NULL); taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataDestroy : NULL);
pBlockData->aUid = NULL; pBlockData->aUid = NULL;
pBlockData->aVersion = NULL; pBlockData->aVersion = NULL;
pBlockData->aTSKEY = NULL; pBlockData->aTSKEY = NULL;
pBlockData->aIdx = NULL;
pBlockData->aColData = NULL; pBlockData->aColData = NULL;
} }
@ -955,7 +948,7 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema,
pBlockData->uid = pId->uid; pBlockData->uid = pId->uid;
pBlockData->nRow = 0; pBlockData->nRow = 0;
taosArrayClear(pBlockData->aIdx); pBlockData->nColData = 0;
if (aCid) { if (aCid) {
int32_t iColumn = 1; int32_t iColumn = 1;
STColumn *pTColumn = &pTSchema->columns[iColumn]; STColumn *pTColumn = &pTSchema->columns[iColumn];
@ -969,7 +962,7 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema,
break; break;
} else if (pTColumn->colId == aCid[iCid]) { } else if (pTColumn->colId == aCid[iCid]) {
SColData *pColData; SColData *pColData;
code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); code = tBlockDataAddColData(pBlockData, &pColData);
if (code) goto _exit; if (code) goto _exit;
tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0); tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0);
@ -982,7 +975,7 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema,
STColumn *pTColumn = &pTSchema->columns[iColumn]; STColumn *pTColumn = &pTSchema->columns[iColumn];
SColData *pColData; SColData *pColData;
code = tBlockDataAddColData(pBlockData, iColumn - 1, &pColData); code = tBlockDataAddColData(pBlockData, &pColData);
if (code) goto _exit; if (code) goto _exit;
tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0); tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0);
@ -993,64 +986,36 @@ _exit:
return code; return code;
} }
int32_t tBlockDataInitEx(SBlockData *pBlockData, SBlockData *pBlockDataFrom) {
int32_t code = 0;
ASSERT(pBlockDataFrom->suid || pBlockDataFrom->uid);
pBlockData->suid = pBlockDataFrom->suid;
pBlockData->uid = pBlockDataFrom->uid;
pBlockData->nRow = 0;
taosArrayClear(pBlockData->aIdx);
for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockDataFrom->aIdx); iColData++) {
SColData *pColDataFrom = tBlockDataGetColDataByIdx(pBlockDataFrom, iColData);
SColData *pColData;
code = tBlockDataAddColData(pBlockData, iColData, &pColData);
if (code) goto _exit;
tColDataInit(pColData, pColDataFrom->cid, pColDataFrom->type, pColDataFrom->smaOn);
}
_exit:
return code;
}
void tBlockDataReset(SBlockData *pBlockData) { void tBlockDataReset(SBlockData *pBlockData) {
pBlockData->suid = 0; pBlockData->suid = 0;
pBlockData->uid = 0; pBlockData->uid = 0;
pBlockData->nRow = 0; pBlockData->nRow = 0;
taosArrayClear(pBlockData->aIdx); pBlockData->nColData = 0;
} }
void tBlockDataClear(SBlockData *pBlockData) { void tBlockDataClear(SBlockData *pBlockData) {
ASSERT(pBlockData->suid || pBlockData->uid); ASSERT(pBlockData->suid || pBlockData->uid);
pBlockData->nRow = 0; pBlockData->nRow = 0;
for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) {
SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
tColDataClear(pColData); tColDataClear(pColData);
} }
} }
int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData) { int32_t tBlockDataAddColData(SBlockData *pBlockData, SColData **ppColData) {
int32_t code = 0; int32_t code = 0;
SColData *pColData = NULL; SColData *pColData = NULL;
int32_t idx = taosArrayGetSize(pBlockData->aIdx);
if (idx >= taosArrayGetSize(pBlockData->aColData)) { if (pBlockData->nColData >= taosArrayGetSize(pBlockData->aColData)) {
if (taosArrayPush(pBlockData->aColData, &((SColData){0})) == NULL) { if (taosArrayPush(pBlockData->aColData, &((SColData){0})) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
} }
pColData = (SColData *)taosArrayGet(pBlockData->aColData, idx); pColData = (SColData *)taosArrayGet(pBlockData->aColData, pBlockData->nColData);
if (taosArrayInsert(pBlockData->aIdx, iColData, &idx) == NULL) { pBlockData->nColData++;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*ppColData = pColData; *ppColData = pColData;
return code; return code;
@ -1087,7 +1052,7 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS
tRowIterInit(&rIter, pRow, pTSchema); tRowIterInit(&rIter, pRow, pTSchema);
pColVal = tRowIterNext(&rIter); pColVal = tRowIterNext(&rIter);
for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) {
SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
while (pColVal && pColVal->cid < pColData->cid) { while (pColVal && pColVal->cid < pColData->cid) {
@ -1115,19 +1080,19 @@ int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFr
int32_t code = 0; int32_t code = 0;
int32_t iColData = 0; int32_t iColData = 0;
for (int32_t iColDataFrom = 0; iColDataFrom < taosArrayGetSize(pBlockDataFrom->aIdx); iColDataFrom++) { for (int32_t iColDataFrom = 0; iColDataFrom < pBlockDataFrom->nColData; iColDataFrom++) {
SColData *pColDataFrom = tBlockDataGetColDataByIdx(pBlockDataFrom, iColDataFrom); SColData *pColDataFrom = tBlockDataGetColDataByIdx(pBlockDataFrom, iColDataFrom);
while (true) { while (true) {
SColData *pColData; SColData *pColData;
if (iColData < taosArrayGetSize(pBlockData->aIdx)) { if (iColData < pBlockData->nColData) {
pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
} else { } else {
pColData = NULL; pColData = NULL;
} }
if (pColData == NULL || pColData->cid > pColDataFrom->cid) { if (pColData == NULL || pColData->cid > pColDataFrom->cid) {
code = tBlockDataAddColData(pBlockData, iColData, &pColData); code = tBlockDataAddColData(pBlockData, &pColData);
if (code) goto _exit; if (code) goto _exit;
tColDataInit(pColData, pColDataFrom->cid, pColDataFrom->type, pColDataFrom->smaOn); tColDataInit(pColData, pColDataFrom->cid, pColDataFrom->type, pColDataFrom->smaOn);
@ -1226,55 +1191,15 @@ _exit:
return code; return code;
} }
int32_t tBlockDataCopy(SBlockData *pSrc, SBlockData *pDest) {
int32_t code = 0;
tBlockDataClear(pDest);
ASSERT(pDest->suid == pSrc->suid);
ASSERT(pDest->uid == pSrc->uid);
ASSERT(taosArrayGetSize(pSrc->aIdx) == taosArrayGetSize(pDest->aIdx));
pDest->nRow = pSrc->nRow;
if (pSrc->uid == 0) {
code = tRealloc((uint8_t **)&pDest->aUid, sizeof(int64_t) * pDest->nRow);
if (code) goto _exit;
memcpy(pDest->aUid, pSrc->aUid, sizeof(int64_t) * pDest->nRow);
}
code = tRealloc((uint8_t **)&pDest->aVersion, sizeof(int64_t) * pDest->nRow);
if (code) goto _exit;
memcpy(pDest->aVersion, pSrc->aVersion, sizeof(int64_t) * pDest->nRow);
code = tRealloc((uint8_t **)&pDest->aTSKEY, sizeof(TSKEY) * pDest->nRow);
if (code) goto _exit;
memcpy(pDest->aTSKEY, pSrc->aTSKEY, sizeof(TSKEY) * pDest->nRow);
for (int32_t iColData = 0; iColData < taosArrayGetSize(pSrc->aIdx); iColData++) {
SColData *pColSrc = tBlockDataGetColDataByIdx(pSrc, iColData);
SColData *pColDest = tBlockDataGetColDataByIdx(pDest, iColData);
ASSERT(pColSrc->cid == pColDest->cid);
ASSERT(pColSrc->type == pColDest->type);
code = tColDataCopy(pColSrc, pColDest);
if (code) goto _exit;
}
_exit:
return code;
}
SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx) { SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx) {
ASSERT(idx >= 0 && idx < taosArrayGetSize(pBlockData->aIdx)); ASSERT(idx >= 0 && idx < pBlockData->nColData);
return (SColData *)taosArrayGet(pBlockData->aColData, *(int32_t *)taosArrayGet(pBlockData->aIdx, idx)); return (SColData *)taosArrayGet(pBlockData->aColData, idx);
} }
void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData) { void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData) {
ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID); ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID);
int32_t lidx = 0; int32_t lidx = 0;
int32_t ridx = taosArrayGetSize(pBlockData->aIdx) - 1; int32_t ridx = pBlockData->nColData - 1;
while (lidx <= ridx) { while (lidx <= ridx) {
int32_t midx = (lidx + ridx) / 2; int32_t midx = (lidx + ridx) / 2;
@ -1308,7 +1233,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut,
// encode ================= // encode =================
// columns AND SBlockCol // columns AND SBlockCol
aBufN[0] = 0; aBufN[0] = 0;
for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) {
SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
ASSERT(pColData->flag); ASSERT(pColData->flag);
@ -1431,7 +1356,7 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin
ASSERT(nt <= hdr.szBlkCol); ASSERT(nt <= hdr.szBlkCol);
SColData *pColData; SColData *pColData;
code = tBlockDataAddColData(pBlockData, taosArrayGetSize(pBlockData->aIdx), &pColData); code = tBlockDataAddColData(pBlockData, &pColData);
if (code) goto _exit; if (code) goto _exit;
tColDataInit(pColData, blockCol.cid, blockCol.type, blockCol.smaOn); tColDataInit(pColData, blockCol.cid, blockCol.type, blockCol.smaOn);

2
source/libs/sync/src/syncEnv.c
View File

@ -79,7 +79,7 @@ void syncNodeRemove(int64_t rid) { taosRemoveRef(gNodeRefId, rid); }
SSyncNode *syncNodeAcquire(int64_t rid) { SSyncNode *syncNodeAcquire(int64_t rid) {
SSyncNode *pNode = taosAcquireRef(gNodeRefId, rid); SSyncNode *pNode = taosAcquireRef(gNodeRefId, rid);
if (pNode == NULL) { if (pNode == NULL) {
sTrace("failed to acquire node from refId:%" PRId64, rid); sError("failed to acquire node from refId:%" PRId64, rid);
terrno = TSDB_CODE_SYN_INTERNAL_ERROR; terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
} }

36
source/libs/sync/src/syncMain.c
View File

@ -221,7 +221,11 @@ SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode) {
int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) {
SSyncNode* pSyncNode = syncNodeAcquire(rid); SSyncNode* pSyncNode = syncNodeAcquire(rid);
if (pSyncNode == NULL) return -1; if (pSyncNode == NULL) {
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
sError("sync begin snapshot error");
return -1;
}
int32_t code = 0; int32_t code = 0;
@ -330,7 +334,10 @@ _DEL_WAL:
int32_t syncEndSnapshot(int64_t rid) { int32_t syncEndSnapshot(int64_t rid) {
SSyncNode* pSyncNode = syncNodeAcquire(rid); SSyncNode* pSyncNode = syncNodeAcquire(rid);
if (pSyncNode == NULL) return -1; if (pSyncNode == NULL) {
sError("sync end snapshot error");
return -1;
}
int32_t code = 0; int32_t code = 0;
if (atomic_load_64(&pSyncNode->snapshottingIndex) != SYNC_INDEX_INVALID) { if (atomic_load_64(&pSyncNode->snapshottingIndex) != SYNC_INDEX_INVALID) {
@ -352,7 +359,10 @@ int32_t syncEndSnapshot(int64_t rid) {
int32_t syncStepDown(int64_t rid, SyncTerm newTerm) { int32_t syncStepDown(int64_t rid, SyncTerm newTerm) {
SSyncNode* pSyncNode = syncNodeAcquire(rid); SSyncNode* pSyncNode = syncNodeAcquire(rid);
if (pSyncNode == NULL) return -1; if (pSyncNode == NULL) {
sError("sync step down error");
return -1;
}
syncNodeStepDown(pSyncNode, newTerm); syncNodeStepDown(pSyncNode, newTerm);
syncNodeRelease(pSyncNode); syncNodeRelease(pSyncNode);
@ -361,7 +371,10 @@ int32_t syncStepDown(int64_t rid, SyncTerm newTerm) {
bool syncIsReadyForRead(int64_t rid) { bool syncIsReadyForRead(int64_t rid) {
SSyncNode* pSyncNode = syncNodeAcquire(rid); SSyncNode* pSyncNode = syncNodeAcquire(rid);
if (pSyncNode == NULL) return -1; if (pSyncNode == NULL) {
sError("sync ready for read error");
return false;
}
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER && pSyncNode->restoreFinish) { if (pSyncNode->state == TAOS_SYNC_STATE_LEADER && pSyncNode->restoreFinish) {
syncNodeRelease(pSyncNode); syncNodeRelease(pSyncNode);
@ -651,7 +664,10 @@ static void syncGetAndDelRespRpc(SSyncNode* pSyncNode, uint64_t index, SRpcHandl
int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) {
SSyncNode* pSyncNode = syncNodeAcquire(rid); SSyncNode* pSyncNode = syncNodeAcquire(rid);
if (pSyncNode == NULL) return -1; if (pSyncNode == NULL) {
sError("sync propose error");
return -1;
}
int32_t ret = syncNodePropose(pSyncNode, pMsg, isWeak); int32_t ret = syncNodePropose(pSyncNode, pMsg, isWeak);
syncNodeRelease(pSyncNode); syncNodeRelease(pSyncNode);
@ -2528,10 +2544,20 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SyncClientRequest* pMsg, SyncInd
// append entry // append entry
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
if (code != 0) { if (code != 0) {
if (ths->replicaNum == 1) {
if (h) {
taosLRUCacheRelease(ths->pLogStore->pCache, h, false);
} else {
syncEntryDestory(pEntry);
}
return -1;
} else {
// del resp mgr, call FpCommitCb // del resp mgr, call FpCommitCb
ASSERT(0); ASSERT(0);
return -1; return -1;
} }
}
// if mulit replica, start replicate right now // if mulit replica, start replicate right now
if (ths->replicaNum > 1) { if (ths->replicaNum > 1) {

2
source/libs/sync/src/syncRaftLog.c
View File

@ -236,6 +236,8 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
SWalReader* pWalHandle = pData->pWalHandle; SWalReader* pWalHandle = pData->pWalHandle;
if (pWalHandle == NULL) { if (pWalHandle == NULL) {
terrno = TSDB_CODE_SYN_INTERNAL_ERROR; terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
sError("vgId:%d, wal handle is NULL", pData->pSyncNode->vgId);
return -1; return -1;
} }