xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

adj stream function return

This commit is contained in:
54liuyao 2024-07-15 13:38:34 +08:00
parent 60b268bbbe
commit 9b067d075f
10 changed files with 181 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/libs/executor/storageapi.h
View File

@ -381,7 +381,7 @@ typedef struct SStateStore {
void** ppVal, int32_t* pVLen);
int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen);
int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol);
bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
@ -391,7 +391,7 @@ typedef struct SStateStore {
void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count);
int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count);
SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen);
int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo);
void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo);
int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo);

28
include/libs/stream/tstreamUpdate.h
View File

@ -15,29 +15,29 @@
#ifndef _TSTREAMUPDATE_H_
#define _TSTREAMUPDATE_H_
#include "storageapi.h"
#include "taosdef.h"
#include "tarray.h"
#include "tcommon.h"
#include "tmsg.h"
#include "storageapi.h"
#ifdef __cplusplus
extern "C" {
#endif
SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen);
SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen);
TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol, int32_t primaryKeyCol);
bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid);
void updateInfoDestroy(SUpdateInfo *pInfo);
void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo);
void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo);
int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo);
int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo);
void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count);
int32_t windowSBfAdd(SUpdateInfo *pInfo, uint64_t count);
bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol);
bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid);
void updateInfoDestroy(SUpdateInfo* pInfo);
void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo);
void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo);
int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo);
int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo);
void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count);
int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count);
bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
#ifdef __cplusplus
}

20
include/util/tbloomfilter.h
View File

@ -37,19 +37,19 @@ typedef struct SBloomFilter {
uint64_t size;
_hash_fn_t hashFn1;
_hash_fn_t hashFn2;
void *buffer;
void* buffer;
double errorRate;
} SBloomFilter;
int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter **ppBF);
int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2);
int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len);
int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2);
void tBloomFilterDestroy(SBloomFilter *pBF);
void tBloomFilterDump(const SBloomFilter *pBF);
bool tBloomFilterIsFull(const SBloomFilter *pBF);
int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder);
int32_t tBloomFilterDecode(SDecoder *pDecoder, SBloomFilter **ppBF);
int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter** ppBF);
int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2);
int32_t tBloomFilterPut(SBloomFilter* pBF, const void* keyBuf, uint32_t len);
int32_t tBloomFilterNoContain(const SBloomFilter* pBF, uint64_t h1, uint64_t h2);
void tBloomFilterDestroy(SBloomFilter* pBF);
void tBloomFilterDump(const SBloomFilter* pBF);
bool tBloomFilterIsFull(const SBloomFilter* pBF);
int32_t tBloomFilterEncode(const SBloomFilter* pBF, SEncoder* pEncoder);
int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF);
#ifdef __cplusplus
}

24
include/util/tscalablebf.h
View File

@ -23,22 +23,22 @@ extern "C" {
#endif
typedef struct SScalableBf {
SArray *bfArray; // array of bloom filters
uint32_t growth;
uint64_t numBits;
uint32_t maxBloomFilters;
int8_t status;
SArray* bfArray; // array of bloom filters
uint32_t growth;
uint64_t numBits;
uint32_t maxBloomFilters;
int8_t status;
_hash_fn_t hashFn1;
_hash_fn_t hashFn2;
} SScalableBf;
int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf);
int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len);
int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len);
int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len);
void tScalableBfDestroy(SScalableBf *pSBf);
int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder);
int32_t tScalableBfDecode(SDecoder *pDecoder, SScalableBf **ppSBf);
int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf);
int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len);
int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len);
int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len);
void tScalableBfDestroy(SScalableBf* pSBf);
int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder);
int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf);
#ifdef __cplusplus
}

2
source/libs/executor/src/groupoperator.c
View File

@ -1369,7 +1369,7 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup
pScanInfo->pPartScalarSup = pExpr;
pScanInfo->pPartTbnameSup = pTbnameExpr;
if (!pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen);
pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);
}
}

11
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -487,8 +487,11 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt
pScanInfo->windowSup.parentType = type;
pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup;
if (!pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo =
pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen);
int32_t code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate,
pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at since %s", __func__, __LINE__, tstrerror(code));
}
}
pScanInfo->interval = pInfo->interval;
@ -1769,8 +1772,8 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin
pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};
pScanInfo->pState = pAggSup->pState;
if (!pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark,
pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen);
pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark,
pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);
}
pScanInfo->twAggSup = *pTwSup;
pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo;

136
source/libs/stream/src/streamUpdate.c
View File

@ -18,6 +18,7 @@
#include "tencode.h"
#include "tstreamUpdate.h"
#include "ttime.h"
#include "tutil.h"
#define DEFAULT_FALSE_POSITIVE 0.01
#define DEFAULT_BUCKET_SIZE 131072
@ -34,7 +35,8 @@
static int64_t adjustExpEntries(int64_t entries) { return TMIN(DEFAULT_EXPECTED_ENTRIES, entries); }
int compareKeyTs(void* pTs1, void* pTs2, void* pPkVal, __compar_fn_t cmpPkFn) {
return compareInt64Val(pTs1, pTs2);;
return compareInt64Val(pTs1, pTs2);
;
}
int compareKeyTsAndPk(void* pValue1, void* pTs, void* pPkVal, __compar_fn_t cmpPkFn) {
@ -49,7 +51,7 @@ int compareKeyTsAndPk(void* pValue1, void* pTs, void* pPkVal, __compar_fn_t cmpP
int32_t getKeyBuff(TSKEY ts, int64_t tbUid, void* pVal, int32_t len, char* buff) {
*(TSKEY*)buff = ts;
memcpy(buff+ sizeof(TSKEY), &tbUid, sizeof(int64_t));
memcpy(buff + sizeof(TSKEY), &tbUid, sizeof(int64_t));
if (len == 0) {
return sizeof(TSKEY) + sizeof(int64_t);
}
@ -66,7 +68,7 @@ int32_t getValueBuff(TSKEY ts, char* pVal, int32_t len, char* buff) {
return sizeof(TSKEY) + len;
}
int32_t windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) {
int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pInfo->numSBFs < count) {
@ -74,10 +76,10 @@ int32_t windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) {
}
for (uint64_t i = 0; i < count; ++i) {
int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND);
SScalableBf *tsSBF = NULL;
SScalableBf* tsSBF = NULL;
code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &tsSBF);
TSDB_CHECK_CODE(code, lino, _error);
void *res = taosArrayPush(pInfo->pTsSBFs, &tsSBF);
void* res = taosArrayPush(pInfo->pTsSBFs, &tsSBF);
if (!res) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
@ -91,15 +93,15 @@ _error:
return code;
}
static void clearItemHelper(void *p) {
SScalableBf **pBf = p;
static void clearItemHelper(void* p) {
SScalableBf** pBf = p;
tScalableBfDestroy(*pBf);
}
void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) {
void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count) {
if (count < pInfo->numSBFs) {
for (uint64_t i = 0; i < count; ++i) {
SScalableBf *pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, 0);
SScalableBf* pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, 0);
tScalableBfDestroy(pTsSBFs);
taosArrayRemove(pInfo->pTsSBFs, 0);
}
@ -138,14 +140,19 @@ static int64_t adjustWatermark(int64_t adjInterval, int64_t originInt, int64_t w
return watermark;
}
SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen) {
return updateInfoInit(pInterval->interval, pInterval->precision, watermark, igUp, pkType, pkLen);
int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen,
SUpdateInfo** ppInfo) {
return updateInfoInit(pInterval->interval, pInterval->precision, watermark, igUp, pkType, pkLen, ppInfo);
}
SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen) {
SUpdateInfo *pInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo));
int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen,
SUpdateInfo** ppInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SUpdateInfo* pInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo));
if (pInfo == NULL) {
return NULL;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
pInfo->pTsBuckets = NULL;
pInfo->pTsSBFs = NULL;
@ -159,17 +166,19 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma
bfSize = (uint64_t)(pInfo->watermark / pInfo->interval);
pInfo->numSBFs = bfSize;
pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(void *));
pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(void*));
if (pInfo->pTsSBFs == NULL) {
updateInfoDestroy(pInfo);
return NULL;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
windowSBfAdd(pInfo, bfSize);
pInfo->pTsBuckets = taosArrayInit(DEFAULT_BUCKET_SIZE, sizeof(TSKEY));
if (pInfo->pTsBuckets == NULL) {
updateInfoDestroy(pInfo);
return NULL;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
TSKEY dumy = 0;
@ -181,22 +190,41 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma
}
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT);
pInfo->pMap = taosHashInit(DEFAULT_MAP_CAPACITY, hashFn, true, HASH_NO_LOCK);
if (!pInfo->pMap) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
pInfo->maxDataVersion = 0;
pInfo->pkColLen = pkLen;
pInfo->pkColType = pkType;
pInfo->pKeyBuff = taosMemoryCalloc(1, sizeof(TSKEY) + sizeof(int64_t) + pkLen);
if (!pInfo->pKeyBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
pInfo->pValueBuff = taosMemoryCalloc(1, sizeof(TSKEY) + pkLen);
if (!pInfo->pValueBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
if (pkLen != 0) {
pInfo->comparePkRowFn = compareKeyTsAndPk;
pInfo->comparePkCol = getKeyComparFunc(pkType, TSDB_ORDER_ASC);;
pInfo->comparePkCol = getKeyComparFunc(pkType, TSDB_ORDER_ASC);
;
} else {
pInfo->comparePkRowFn = compareKeyTs;
pInfo->comparePkCol = NULL;
}
return pInfo;
(*ppInfo) = pInfo;
_end:
if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) {
static SScalableBf* getSBf(SUpdateInfo* pInfo, TSKEY ts) {
int32_t code = 0;
if (ts <= 0) {
return NULL;
@ -214,7 +242,7 @@ static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) {
windowSBfAdd(pInfo, count);
index = pInfo->numSBFs - 1;
}
SScalableBf *res = taosArrayGetP(pInfo->pTsSBFs, index);
SScalableBf* res = taosArrayGetP(pInfo->pTsSBFs, index);
if (res == NULL) {
int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND);
code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &res);
@ -223,13 +251,13 @@ static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) {
return res;
}
bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid) {
void *pVal = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t));
bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid) {
void* pVal = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t));
if (pVal || taosHashGetSize(pInfo->pMap) >= DEFAULT_MAP_SIZE) return true;
return false;
}
TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) {
TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) {
if (pBlock == NULL || pBlock->info.rows == 0) return INT64_MIN;
TSKEY maxTs = INT64_MIN;
void* pPkVal = NULL;
@ -238,14 +266,14 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p
int32_t len = 0;
int64_t tbUid = pBlock->info.id.uid;
SColumnInfoData *pColDataInfo = taosArrayGet(pBlock->pDataBlock, primaryTsCol);
SColumnInfoData *pPkDataInfo = NULL;
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, primaryTsCol);
SColumnInfoData* pPkDataInfo = NULL;
if (primaryKeyCol >= 0) {
pPkDataInfo = taosArrayGet(pBlock->pDataBlock, primaryKeyCol);
}
for (int32_t i = 0; i < pBlock->info.rows; i++) {
TSKEY ts = ((TSKEY *)pColDataInfo->pData)[i];
TSKEY ts = ((TSKEY*)pColDataInfo->pData)[i];
if (maxTs < ts) {
maxTs = ts;
if (primaryKeyCol >= 0) {
@ -253,7 +281,7 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p
maxLen = colDataGetRowLength(pPkDataInfo, i);
}
}
SScalableBf *pSBf = getSBf(pInfo, ts);
SScalableBf* pSBf = getSBf(pInfo, ts);
if (pSBf) {
if (primaryKeyCol >= 0) {
pPkVal = colDataGetData(pPkDataInfo, i);
@ -263,7 +291,7 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p
tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen);
}
}
void *pMaxTs = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t));
void* pMaxTs = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t));
if (pMaxTs == NULL || pInfo->comparePkRowFn(pMaxTs, &maxTs, pMaxPkVal, pInfo->comparePkCol) == -1) {
int32_t valueLen = getValueBuff(maxTs, pMaxPkVal, maxLen, pInfo->pValueBuff);
taosHashPut(pInfo->pMap, &tbUid, sizeof(int64_t), pInfo->pValueBuff, valueLen);
@ -271,14 +299,14 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p
return maxTs;
}
bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) {
bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) {
int32_t res = TSDB_CODE_FAILED;
int32_t buffLen = 0;
buffLen = getKeyBuff(ts, tableId, pPkVal, len, pInfo->pKeyBuff);
void* *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t));
void** pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t));
uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets;
TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index);
TSKEY maxTs = *(TSKEY*)taosArrayGet(pInfo->pTsBuckets, index);
if (ts < maxTs - pInfo->watermark) {
// this window has been closed.
if (pInfo->pCloseWinSBF) {
@ -292,10 +320,11 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* p
return true;
}
SScalableBf *pSBf = getSBf(pInfo, ts);
SScalableBf* pSBf = getSBf(pInfo, ts);
int32_t size = taosHashGetSize(pInfo->pMap);
if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == -1 )) {
if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) ||
(pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == -1)) {
int32_t valueLen = getValueBuff(ts, pPkVal, len, pInfo->pValueBuff);
taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen);
// pSBf may be a null pointer
@ -324,7 +353,7 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* p
return true;
}
void updateInfoDestroy(SUpdateInfo *pInfo) {
void updateInfoDestroy(SUpdateInfo* pInfo) {
if (pInfo == NULL) {
return;
}
@ -332,7 +361,7 @@ void updateInfoDestroy(SUpdateInfo *pInfo) {
uint64_t size = taosArrayGetSize(pInfo->pTsSBFs);
for (uint64_t i = 0; i < size; i++) {
SScalableBf *pSBF = taosArrayGetP(pInfo->pTsSBFs, i);
SScalableBf* pSBF = taosArrayGetP(pInfo->pTsSBFs, i);
tScalableBfDestroy(pSBF);
}
@ -344,7 +373,7 @@ void updateInfoDestroy(SUpdateInfo *pInfo) {
taosMemoryFree(pInfo);
}
void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) {
void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo) {
if (pInfo->pCloseWinSBF) {
return;
}
@ -356,7 +385,7 @@ void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) {
}
}
void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) {
void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo) {
if (!pInfo || !pInfo->pCloseWinSBF) {
return;
}
@ -364,7 +393,7 @@ void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) {
pInfo->pCloseWinSBF = NULL;
}
int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) {
int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo) {
if (!pInfo) {
return 0;
}
@ -376,7 +405,7 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo)
int32_t size = taosArrayGetSize(pInfo->pTsBuckets);
if (tEncodeI32(&encoder, size) < 0) return -1;
for (int32_t i = 0; i < size; i++) {
TSKEY *pTs = (TSKEY *)taosArrayGet(pInfo->pTsBuckets, i);
TSKEY* pTs = (TSKEY*)taosArrayGet(pInfo->pTsBuckets, i);
if (tEncodeI64(&encoder, *pTs) < 0) return -1;
}
@ -385,7 +414,7 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo)
int32_t sBfSize = taosArrayGetSize(pInfo->pTsSBFs);
if (tEncodeI32(&encoder, sBfSize) < 0) return -1;
for (int32_t i = 0; i < sBfSize; i++) {
SScalableBf *pSBf = taosArrayGetP(pInfo->pTsSBFs, i);
SScalableBf* pSBf = taosArrayGetP(pInfo->pTsSBFs, i);
if (tScalableBfEncode(pSBf, &encoder) < 0) return -1;
}
@ -398,13 +427,13 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo)
int32_t mapSize = taosHashGetSize(pInfo->pMap);
if (tEncodeI32(&encoder, mapSize) < 0) return -1;
void *pIte = NULL;
void* pIte = NULL;
size_t keyLen = 0;
while ((pIte = taosHashIterate(pInfo->pMap, pIte)) != NULL) {
void *key = taosHashGetKey(pIte, &keyLen);
if (tEncodeU64(&encoder, *(uint64_t *)key) < 0) return -1;
void* key = taosHashGetKey(pIte, &keyLen);
if (tEncodeU64(&encoder, *(uint64_t*)key) < 0) return -1;
int32_t valueSize = taosHashGetValueSize(pIte);
if (tEncodeBinary(&encoder, (const uint8_t *)pIte, valueSize) < 0) return -1;
if (tEncodeBinary(&encoder, (const uint8_t*)pIte, valueSize) < 0) return -1;
}
if (tEncodeU64(&encoder, pInfo->maxDataVersion) < 0) return -1;
@ -419,7 +448,7 @@ int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo)
return tlen;
}
int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) {
int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
ASSERT(pInfo);
@ -440,9 +469,9 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) {
int32_t sBfSize = 0;
if (tDecodeI32(&decoder, &sBfSize) < 0) return -1;
pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void *));
pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void*));
for (int32_t i = 0; i < sBfSize; i++) {
SScalableBf *pSBf = NULL;
SScalableBf* pSBf = NULL;
code = tScalableBfDecode(&decoder, &pSBf);
TSDB_CHECK_CODE(code, lino, _error);
@ -462,8 +491,8 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) {
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT);
pInfo->pMap = taosHashInit(mapSize, hashFn, true, HASH_NO_LOCK);
uint64_t uid = 0;
void* pVal = NULL;
int32_t valSize = 0;
void* pVal = NULL;
int32_t valSize = 0;
for (int32_t i = 0; i < mapSize; i++) {
if (tDecodeU64(&decoder, &uid) < 0) return -1;
if (tDecodeBinary(&decoder, (uint8_t**)&pVal, &valSize) < 0) return -1;
@ -479,7 +508,8 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) {
pInfo->pValueBuff = taosMemoryCalloc(1, sizeof(TSKEY) + pInfo->pkColLen);
if (pInfo->pkColLen != 0) {
pInfo->comparePkRowFn = compareKeyTsAndPk;
pInfo->comparePkCol = getKeyComparFunc(pInfo->pkColType, TSDB_ORDER_ASC);;
pInfo->comparePkCol = getKeyComparFunc(pInfo->pkColType, TSDB_ORDER_ASC);
;
} else {
pInfo->comparePkRowFn = compareKeyTs;
pInfo->comparePkCol = NULL;
@ -496,8 +526,8 @@ _error:
return code;
}
bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) {
TSKEY *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t));
bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) {
TSKEY* pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t));
bool res = true;
if (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == 1) {
res = false;

28
source/util/src/tbloomfilter.c
View File

@ -22,27 +22,27 @@
#define UNIT_NUM_BITS 64ULL
#define UNIT_ADDR_NUM_BITS 6ULL
static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) {
static FORCE_INLINE bool setBit(uint64_t* buf, uint64_t index) {
uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS;
uint64_t old = buf[unitIndex];
buf[unitIndex] |= (1ULL << (index % UNIT_NUM_BITS));
return buf[unitIndex] != old;
}
static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) {
static FORCE_INLINE bool getBit(uint64_t* buf, uint64_t index) {
uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS;
uint64_t mask = 1ULL << (index % UNIT_NUM_BITS);
return buf[unitIndex] & mask;
}
int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter **ppBF) {
int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter** ppBF) {
int32_t code = 0;
int32_t lino = 0;
if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter));
SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter));
if (pBF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
@ -77,7 +77,7 @@ _error:
return code;
}
int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) {
int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2) {
ASSERT(!tBloomFilterIsFull(pBF));
bool hasChange = false;
const register uint64_t size = pBF->numBits;
@ -93,7 +93,7 @@ int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) {
return TSDB_CODE_FAILED;
}
int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) {
int32_t tBloomFilterPut(SBloomFilter* pBF, const void* keyBuf, uint32_t len) {
uint64_t h1 = (uint64_t)pBF->hashFn1(keyBuf, len);
uint64_t h2 = (uint64_t)pBF->hashFn2(keyBuf, len);
bool hasChange = false;
@ -110,7 +110,7 @@ int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) {
return TSDB_CODE_FAILED;
}
int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) {
int32_t tBloomFilterNoContain(const SBloomFilter* pBF, uint64_t hash1, uint64_t hash2) {
const register uint64_t size = pBF->numBits;
uint64_t cbHash = hash1;
for (uint32_t i = 0; i < pBF->hashFunctions; ++i) {
@ -122,7 +122,7 @@ int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t hash1, uint64_t
return TSDB_CODE_FAILED;
}
void tBloomFilterDestroy(SBloomFilter *pBF) {
void tBloomFilterDestroy(SBloomFilter* pBF) {
if (pBF == NULL) {
return;
}
@ -130,24 +130,24 @@ void tBloomFilterDestroy(SBloomFilter *pBF) {
taosMemoryFree(pBF);
}
int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder) {
int32_t tBloomFilterEncode(const SBloomFilter* pBF, SEncoder* pEncoder) {
if (tEncodeU32(pEncoder, pBF->hashFunctions) < 0) return -1;
if (tEncodeU64(pEncoder, pBF->expectedEntries) < 0) return -1;
if (tEncodeU64(pEncoder, pBF->numUnits) < 0) return -1;
if (tEncodeU64(pEncoder, pBF->numBits) < 0) return -1;
if (tEncodeU64(pEncoder, pBF->size) < 0) return -1;
for (uint64_t i = 0; i < pBF->numUnits; i++) {
uint64_t *pUnits = (uint64_t *)pBF->buffer;
uint64_t* pUnits = (uint64_t*)pBF->buffer;
if (tEncodeU64(pEncoder, pUnits[i]) < 0) return -1;
}
if (tEncodeDouble(pEncoder, pBF->errorRate) < 0) return -1;
return 0;
}
int32_t tBloomFilterDecode(SDecoder *pDecoder, SBloomFilter **ppBF) {
int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF) {
int32_t code = 0;
int32_t lino = 0;
SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter));
SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter));
if (!pBF) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
@ -175,7 +175,7 @@ int32_t tBloomFilterDecode(SDecoder *pDecoder, SBloomFilter **ppBF) {
}
pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t));
for (int32_t i = 0; i < pBF->numUnits; i++) {
uint64_t *pUnits = (uint64_t *)pBF->buffer;
uint64_t* pUnits = (uint64_t*)pBF->buffer;
if (tDecodeU64(pDecoder, pUnits + i) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
@ -195,4 +195,4 @@ _error:
return TSDB_CODE_FAILED;
}
bool tBloomFilterIsFull(const SBloomFilter *pBF) { return pBF->size >= pBF->expectedEntries; }
bool tBloomFilterIsFull(const SBloomFilter* pBF) { return pBF->size >= pBF->expectedEntries; }

40
source/util/src/tscalablebf.c
View File

@ -25,10 +25,10 @@
#define SBF_INVALID -1
#define SBF_VALID 0
static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate,
SBloomFilter **ppNormalBf);
static int32_t tScalableBfAddFilter(SScalableBf* pSBf, uint64_t expectedEntries, double errorRate,
SBloomFilter** ppNormalBf);
int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf **ppSBf) {
int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
const uint32_t defaultSize = 8;
@ -36,7 +36,7 @@ int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf));
SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf));
if (pSBf == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
@ -44,13 +44,13 @@ int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf
pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS;
pSBf->status = SBF_VALID;
pSBf->numBits = 0;
pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void *));
pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void*));
if (!pSBf->bfArray) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
SBloomFilter *pNormalBf = NULL;
SBloomFilter* pNormalBf = NULL;
code = tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf);
if (code != TSDB_CODE_SUCCESS) {
tScalableBfDestroy(pSBf);
@ -67,7 +67,7 @@ _error:
return code;
}
int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pSBf->status == SBF_INVALID) {
@ -75,7 +75,7 @@ int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t le
TSDB_CHECK_CODE(code, lino, _error);
}
int32_t size = taosArrayGetSize(pSBf->bfArray);
SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1);
SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1);
if (!pNormalBf) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
@ -97,7 +97,7 @@ _error:
return code;
}
int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pSBf->status == SBF_INVALID) {
@ -113,7 +113,7 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
}
}
SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1);
SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1);
ASSERT(pNormalBf);
if (tBloomFilterIsFull(pNormalBf)) {
code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth,
@ -132,7 +132,7 @@ _error:
return code;
}
int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len) {
if (pSBf->status == SBF_INVALID) {
return TSDB_CODE_FAILED;
}
@ -147,8 +147,8 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32
return TSDB_CODE_SUCCESS;
}
static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate,
SBloomFilter **ppNormalBf) {
static int32_t tScalableBfAddFilter(SScalableBf* pSBf, uint64_t expectedEntries, double errorRate,
SBloomFilter** ppNormalBf) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) {
@ -156,7 +156,7 @@ static int32_t tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries,
TSDB_CHECK_CODE(code, lino, _error);
}
SBloomFilter *pNormalBf = NULL;
SBloomFilter* pNormalBf = NULL;
code = tBloomFilterInit(expectedEntries, errorRate, &pNormalBf);
TSDB_CHECK_CODE(code, lino, _error);
@ -175,7 +175,7 @@ _error:
return code;
}
void tScalableBfDestroy(SScalableBf *pSBf) {
void tScalableBfDestroy(SScalableBf* pSBf) {
if (pSBf == NULL) {
return;
}
@ -185,7 +185,7 @@ void tScalableBfDestroy(SScalableBf *pSBf) {
taosMemoryFree(pSBf);
}
int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) {
int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder) {
if (!pSBf) {
if (tEncodeI32(pEncoder, 0) < 0) return -1;
return 0;
@ -193,7 +193,7 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) {
int32_t size = taosArrayGetSize(pSBf->bfArray);
if (tEncodeI32(pEncoder, size) < 0) return -1;
for (int32_t i = 0; i < size; i++) {
SBloomFilter *pBF = taosArrayGetP(pSBf->bfArray, i);
SBloomFilter* pBF = taosArrayGetP(pSBf->bfArray, i);
if (tBloomFilterEncode(pBF, pEncoder) < 0) return -1;
}
if (tEncodeU32(pEncoder, pSBf->growth) < 0) return -1;
@ -203,10 +203,10 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) {
return 0;
}
int32_t tScalableBfDecode(SDecoder *pDecoder, SScalableBf **ppSBf) {
int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf));
SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf));
if (!pSBf) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
@ -230,7 +230,7 @@ int32_t tScalableBfDecode(SDecoder *pDecoder, SScalableBf **ppSBf) {
}
for (int32_t i = 0; i < size; i++) {
SBloomFilter *pBF = NULL;
SBloomFilter* pBF = NULL;
code = tBloomFilterDecode(pDecoder, &pBF);
TSDB_CHECK_CODE(code, lino, _error);
taosArrayPush(pSBf->bfArray, &pBF);

36
source/util/test/bloomFilterTest.cpp
View File

@ -8,15 +8,14 @@ using namespace std;
TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) {
int64_t ts1 = 1650803518000;
SBloomFilter *pBFTmp = NULL;
SBloomFilter* pBFTmp = NULL;
GTEST_ASSERT_EQ(0, tBloomFilterInit(100, 0, &pBFTmp));
GTEST_ASSERT_EQ(0, tBloomFilterInit(100, 1, &pBFTmp));
GTEST_ASSERT_EQ(0, tBloomFilterInit(100, -0.1, &pBFTmp));
GTEST_ASSERT_EQ(0, tBloomFilterInit(0, 0.01, &pBFTmp));
SBloomFilter *pBF1 = NULL,
int32_t code = tBloomFilterInit(100, 0.005, &pBF1);
GTEST_ASSERT_EQ(0,code);
SBloomFilter *pBF1 = NULL, int32_t code = tBloomFilterInit(100, 0.005, &pBF1);
GTEST_ASSERT_EQ(0, code);
GTEST_ASSERT_EQ(pBF1->numBits, 1152);
GTEST_ASSERT_EQ(pBF1->numUnits, 1152 / 64);
int64_t count = 0;
@ -28,18 +27,18 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) {
}
ASSERT_TRUE(tBloomFilterIsFull(pBF1));
SBloomFilter *pBF2 = NULL;
SBloomFilter* pBF2 = NULL;
GTEST_ASSERT_EQ(0, tBloomFilterInit(1000 * 10000, 0.1, &pBF2));
GTEST_ASSERT_EQ(pBF2->numBits, 47925312);
GTEST_ASSERT_EQ(pBF2->numUnits, 47925312 / 64);
SBloomFilter *pBF3 = NULL;
SBloomFilter* pBF3 = NULL;
GTEST_ASSERT_EQ(0, tBloomFilterInit(10000 * 10000, 0.001));
GTEST_ASSERT_EQ(pBF3->numBits, 1437758784);
GTEST_ASSERT_EQ(pBF3->numUnits, 1437758784 / 64);
int64_t size = 10000;
SBloomFilter *pBF4 = NULL;
SBloomFilter* pBF4 = NULL;
GTEST_ASSERT_EQ(0, tBloomFilterInit(size, 0.001));
for (int64_t i = 0; i < 1000; i++) {
int64_t ts = i + ts1;
@ -48,17 +47,17 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) {
ASSERT_TRUE(!tBloomFilterIsFull(pBF4));
for (int64_t i = 0; i < 1000; i++) {
int64_t ts = i + ts1;
uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t));
uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t));
int64_t ts = i + ts1;
uint64_t h1 = (uint64_t)pBF4->hashFn1((const char*)&ts, sizeof(int64_t));
uint64_t h2 = (uint64_t)pBF4->hashFn2((const char*)&ts, sizeof(int64_t));
GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_FAILED);
}
for (int64_t i = 2000; i < 3000; i++) {
int64_t ts = i + ts1;
uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t));
uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t));
GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_SUCCESS);
int64_t ts = i + ts1;
uint64_t h1 = (uint64_t)pBF4->hashFn1((const char*)&ts, sizeof(int64_t));
uint64_t h2 = (uint64_t)pBF4->hashFn2((const char*)&ts, sizeof(int64_t));
GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_SUCCESS);
}
tBloomFilterDestroy(pBF1);
@ -70,13 +69,13 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) {
TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) {
int64_t ts1 = 1650803518000;
SScalableBf *tsSBF = NULL;
SScalableBf* tsSBF = NULL;
GTEST_ASSERT_NE(0, tScalableBfInit(100, 0, &tsSBF));
GTEST_ASSERT_NE(0, tScalableBfInit(100, 1, &tsSBF));
GTEST_ASSERT_NE(0, tScalableBfInit(100, -0.1, &tsSBF));
GTEST_ASSERT_NE(0, tScalableBfInit(0, 0.01, &tsSBF));
SScalableBf *pSBF1 = NULL;
SScalableBf* pSBF1 = NULL;
GTEST_ASSERT_EQ(0, tScalableBfInit(100, 0.01, &pSBF1));
GTEST_ASSERT_EQ(pSBF1->numBits, 1152);
int64_t count = 0;
@ -116,7 +115,7 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) {
int32_t aSize = taosArrayGetSize(pSBF1->bfArray);
int64_t totalBits = 0;
for (int64_t i = 0; i < aSize; i++) {
SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF1->bfArray, i);
SBloomFilter* pBF = (SBloomFilter*)taosArrayGetP(pSBF1->bfArray, i);
ASSERT_TRUE(tBloomFilterIsFull(pBF));
totalBits += pBF->numBits;
}
@ -128,7 +127,8 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) {
}
int64_t size = 10000;
SScalableBf *pSBF4 = tScalableBfInit(size, 0.001);
SScalableBf* pSBF4 = NULL;
GTEST_ASSERT_EQ(0, tScalableBfInit(size, 0.001, pSBF4));
for (int64_t i = 0; i < 1000; i++) {
int64_t ts = i + ts1;
GTEST_ASSERT_EQ(tScalableBfPut(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS);